618 files changed, 105483 insertions, 41343 deletions

diff --git a/erts/Makefile.in b/erts/Makefile.in
index fabf86db7c..5df6d71ef3 100644
--- a/erts/Makefile.in
+++ b/erts/Makefile.in
@@ -1,82 +1,61 @@
 # 
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2006-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2006-2012. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 # 
+
+.NOTPARALLEL:
+
 include $(ERL_TOP)/make/target.mk
 include vsn.mk
 
 # ----------------------------------------------------------------------
 
 # Other erts dirs than the emulator dir...
-ERTSDIRS = doc/src etc epmd lib_src
+ERTSDIRS = etc epmd lib_src
 XINSTDIRS = preloaded
 ifeq ($(NO_START_SCRIPTS),)
 ERTSDIRS += start_scripts
 endif
 
-# Until hybrid is nofrag, don't build it.
-#BUILD_HYBRID_EMU=@ERTS_BUILD_HYBRID_EMU@
-BUILD_HYBRID_EMU=no
-
 EXTRA_FLAVORS=smp
-ifeq ($(BUILD_HYBRID_EMU),yes)
-EXTRA_FLAVORS += hybrid
-endif
 
-#
-# Some byggy 'make's get confused when a directory is created and used
-# for storing files which other files depend on during the same "make
-# session". As a workaround we do a 'make generate' (which creates
-# all directories) before doing 'make opt', etc...
-#
-
-ifneq ($(BUILD_HYBRID_EMU),yes)
+.PHONY: all
 all: smp opt
-else
-all: hybrid smp opt
-endif
 
-debug opt docs clean:
-	@ case $@ in \
-	    docs|clean) ;; \
-	    *) ( cd emulator && $(MAKE) generate TYPE=$@ FLAVOR=$(FLAVOR)) ;; \
-	  esac
-	@ ( cd emulator && $(MAKE) $@ FLAVOR=$(FLAVOR))
-	@for d in $(ERTSDIRS); do \
+.PHONY: docs
+docs:
+	( cd doc/src && $(MAKE) $@ )
+
+.PHONY: debug opt clean
+debug opt clean:
+	for d in emulator $(ERTSDIRS); do \
 		if test -d $$d; then \
-			( cd $$d && $(MAKE) $@ ) || exit $$? ; \
+			( cd $$d && $(MAKE) $@ FLAVOR=$(FLAVOR) ) || exit $$? ; \
 		fi ; \
-	 done
+	done
 
 # ----------------------------------------------------------------------
 # These are "convenience targets", provided as shortcuts for developers
 # - don't use them in scripts or assume they will always stay like this!
 #
 
+.PHONY: $(EXTRA_FLAVORS)
 $(EXTRA_FLAVORS):
-	@ ( cd emulator \
-	    && $(MAKE) generate TYPE=opt FLAVOR=$@ \
-	    && $(MAKE) opt FLAVOR=$@ )
-
-ifneq ($(BUILD_HYBRID_EMU),yes)
-hybrid:
-	@echo '*** Omitted build of hybrid heap emulator'
-	@echo '*** since target is $(TARGET)'
-endif
+	( cd emulator && $(MAKE) opt FLAVOR=$@ )
 
 # Make erl script and erlc in $(ERL_TOP)/bin which runs the compiled version
 # Note that erlc is not a script and requires extra handling on cygwin.
@@ -85,25 +64,31 @@ endif
 
 # The copying of beam.dll should be removed when the beam dll need no longer be
 # in the same directory...
+.PHONY: local_setup
 local_setup:
 	@cd start_scripts && $(MAKE)
+	@echo `ls $(ERL_TOP)/bin/`
 	@rm -f $(ERL_TOP)/bin/erl $(ERL_TOP)/bin/erlc $(ERL_TOP)/bin/cerl \
 		$(ERL_TOP)/bin/erl.exe $(ERL_TOP)/bin/erlc.exe \
 		$(ERL_TOP)/bin/escript $(ERL_TOP)/bin/escript.exe \
 		$(ERL_TOP)/bin/dialyzer $(ERL_TOP)/bin/dialyzer.exe \
 		$(ERL_TOP)/bin/typer $(ERL_TOP)/bin/typer.exe \
+		$(ERL_TOP)/bin/run_test $(ERL_TOP)/bin/run_test.exe \
+		$(ERL_TOP)/bin/ct_run $(ERL_TOP)/bin/ct_run.exe \
 		$(ERL_TOP)/bin/start*.boot $(ERL_TOP)/bin/start*.script
 	@if [ "X$(TARGET)" = "Xwin32" ]; then \
 		cp $(ERL_TOP)/bin/$(TARGET)/dialyzer.exe $(ERL_TOP)/bin/dialyzer.exe; \
 		cp $(ERL_TOP)/bin/$(TARGET)/typer.exe $(ERL_TOP)/bin/typer.exe; \
+		cp $(ERL_TOP)/bin/$(TARGET)/ct_run.exe $(ERL_TOP)/bin/ct_run.exe; \
+		cp $(ERL_TOP)/bin/$(TARGET)/ct_run.exe $(ERL_TOP)/bin/run_test.exe; \
 		cp $(ERL_TOP)/bin/$(TARGET)/erlc.exe $(ERL_TOP)/bin/erlc.exe; \
 		cp $(ERL_TOP)/bin/$(TARGET)/erl.exe $(ERL_TOP)/bin/erl.exe; \
 		cp $(ERL_TOP)/bin/$(TARGET)/werl.exe $(ERL_TOP)/bin/werl.exe; \
 		cp $(ERL_TOP)/bin/$(TARGET)/escript.exe $(ERL_TOP)/bin/escript.exe; \
 	 	chmod 755 $(ERL_TOP)/bin/erl.exe $(ERL_TOP)/bin/erlc.exe \
 			 $(ERL_TOP)/bin/werl.exe; \
-		$(ERL_TOP)/erts/etc/win32/cygwin_tools/make_local_ini.sh \
-			$(ERL_TOP); \
+		make_local_ini.sh $(ERL_TOP); \
+		cp $(ERL_TOP)/bin/erl.ini $(ERL_TOP)/bin/$(TARGET)/erl.ini; \
 	else \
 	sed	-e "s;%FINAL_ROOTDIR%;$(ERL_TOP);"   \
 		-e "s;erts-.*/bin;bin/$(TARGET);"    \
@@ -115,6 +100,8 @@ local_setup:
 	        $(ERL_TOP)/erts/etc/unix/cerl.src > $(ERL_TOP)/bin/cerl; \
 		cp $(ERL_TOP)/bin/$(TARGET)/dialyzer $(ERL_TOP)/bin/dialyzer; \
 		cp $(ERL_TOP)/bin/$(TARGET)/typer $(ERL_TOP)/bin/typer; \
+		cp $(ERL_TOP)/bin/$(TARGET)/ct_run $(ERL_TOP)/bin/ct_run; \
+		ln -s $(ERL_TOP)/bin/ct_run $(ERL_TOP)/bin/run_test; \
 		cp $(ERL_TOP)/bin/$(TARGET)/erlc $(ERL_TOP)/bin/erlc; \
 		cp $(ERL_TOP)/bin/$(TARGET)/escript $(ERL_TOP)/bin/escript; \
 	 	chmod 755 $(ERL_TOP)/bin/erl $(ERL_TOP)/bin/erlc \
@@ -125,11 +112,13 @@ local_setup:
 		$(ERL_TOP)/bin/start_clean.script
 
 # Run the configure script
+.PHONY: configure
 configure:
 	@set -e ; cd autoconf && $(MAKE)
 
 # Remake the makefiles, if you already have configured but you have edited
 # a "Makefile.in".
+.PHONY: makefiles
 makefiles:
 	@set -e ; cd autoconf && $(MAKE) $@
 
@@ -137,21 +126,17 @@ makefiles:
 # Release targets
 #
 
-release release_docs:
-ifeq ($(BUILD_HYBRID_EMU),yes)
-	@if test $@ = release; then ( cd emulator && $(MAKE) $@ FLAVOR=hybrid) fi
-else
-	@if test $@ = release; then ( $(MAKE) hybrid ) fi
-endif
-	@if test $@ = release; then ( cd emulator && $(MAKE) $@ FLAVOR=smp) fi
-	@ (cd emulator && $(MAKE) $@ FLAVOR=plain)
-	@for d in $(ERTSDIRS) $(XINSTDIRS); do \
+.PHONY: release
+release:
+	for f in plain $(EXTRA_FLAVORS) ; do \
+		( cd emulator && $(MAKE) release FLAVOR=$$f ) \
+	done
+	for d in $(ERTSDIRS) $(XINSTDIRS); do \
 		if test -d $$d; then \
 			( cd $$d && $(MAKE) $@ ) || exit $$? ; \
 		fi ; \
 	 done
 
-# ----------------------------------------------------------------------
-
-.PHONY: debug opt docs clean local_setup configure release \
-	release_docs run_test_cases hybrid smp
+.PHONY: release_docs
+release_docs:
+	( cd doc/src && $(MAKE) $@ )
diff --git a/erts/aclocal.m4 b/erts/aclocal.m4
index 3d935b7295..339a15a2bb 100644
--- a/erts/aclocal.m4
+++ b/erts/aclocal.m4
@@ -1,7 +1,7 @@
 dnl
 dnl %CopyrightBegin%
 dnl
-dnl Copyright Ericsson AB 1998-2010. All Rights Reserved.
+dnl Copyright Ericsson AB 1998-2011. All Rights Reserved.
 dnl
 dnl The contents of this file are subject to the Erlang Public License,
 dnl Version 1.1, (the "License"); you may not use this file except in
@@ -112,6 +112,94 @@ fi
 
 dnl ----------------------------------------------------------------------
 dnl
+dnl LM_WINDOWS_ENVIRONMENT
+dnl
+dnl
+dnl Tries to determine thw windows build environment, i.e. 
+dnl MIXED_CYGWIN_VC or MIXED_MSYS_VC 
+dnl
+
+AC_DEFUN(LM_WINDOWS_ENVIRONMENT,
+[
+MIXED_CYGWIN=no
+MIXED_MSYS=no
+
+AC_MSG_CHECKING(for mixed cygwin or msys and native VC++ environment)
+if test "X$host" = "Xwin32" -a "x$GCC" != "xyes"; then
+	if test -x /usr/bin/cygpath; then
+		CFLAGS="-O2"
+		MIXED_CYGWIN=yes
+		AC_MSG_RESULT([Cygwin and VC])
+		MIXED_CYGWIN_VC=yes
+		CPPFLAGS="$CPPFLAGS -DERTS_MIXED_CYGWIN_VC"
+	elif test -x /usr/bin/msysinfo; then
+	        CFLAGS="-O2"
+		MIXED_MSYS=yes
+		AC_MSG_RESULT([MSYS and VC])
+		MIXED_MSYS_VC=yes
+		CPPFLAGS="$CPPFLAGS -DERTS_MIXED_MSYS_VC"
+	else		    
+		AC_MSG_RESULT([undeterminable])
+		AC_MSG_ERROR(Seems to be mixed windows but not with cygwin, cannot handle this!)
+	fi
+else
+	AC_MSG_RESULT([no])
+	MIXED_CYGWIN_VC=no
+	MIXED_MSYS_VC=no
+fi
+AC_SUBST(MIXED_CYGWIN_VC)
+AC_SUBST(MIXED_MSYS_VC)
+
+MIXED_VC=no
+if test "x$MIXED_MSYS_VC" = "xyes" -o  "x$MIXED_CYGWIN_VC" = "xyes" ; then
+   MIXED_VC=yes
+fi
+
+AC_SUBST(MIXED_VC)
+
+if test "x$MIXED_MSYS" != "xyes"; then
+   AC_MSG_CHECKING(for mixed cygwin and native MinGW environment)
+   if test "X$host" = "Xwin32" -a "x$GCC" = x"yes"; then
+	if test -x /usr/bin/cygpath; then
+		CFLAGS="-O2"
+		MIXED_CYGWIN=yes
+		AC_MSG_RESULT([yes])
+		MIXED_CYGWIN_MINGW=yes
+		CPPFLAGS="$CPPFLAGS -DERTS_MIXED_CYGWIN_MINGW"
+	else
+		AC_MSG_RESULT([undeterminable])
+		AC_MSG_ERROR(Seems to be mixed windows but not with cygwin, cannot handle this!)
+	fi
+    else
+	AC_MSG_RESULT([no])
+	MIXED_CYGWIN_MINGW=no
+    fi
+else
+	MIXED_CYGWIN_MINGW=no
+fi	
+AC_SUBST(MIXED_CYGWIN_MINGW)
+
+AC_MSG_CHECKING(if we mix cygwin with any native compiler)
+if test "X$MIXED_CYGWIN" = "Xyes"; then
+	AC_MSG_RESULT([yes])	
+else
+	AC_MSG_RESULT([no])
+fi
+
+AC_SUBST(MIXED_CYGWIN)
+	
+AC_MSG_CHECKING(if we mix msys with another native compiler)
+if test "X$MIXED_MSYS" = "Xyes" ; then
+	AC_MSG_RESULT([yes])	
+else
+	AC_MSG_RESULT([no])
+fi
+
+AC_SUBST(MIXED_MSYS)
+])		
+	
+dnl ----------------------------------------------------------------------
+dnl
 dnl LM_FIND_EMU_CC
 dnl
 dnl
@@ -125,6 +213,11 @@ AC_DEFUN(LM_FIND_EMU_CC,
 			ac_cv_prog_emu_cc,
 			[
 AC_TRY_COMPILE([],[
+#if defined(__clang_major__) && __clang_major__ >= 3
+    /* clang 3.x or later is fine */
+#elif defined(__llvm__)
+#error "this version of llvm is unable to correctly compile beam_emu.c"
+#endif
     __label__ lbl1;
     __label__ lbl2;
     int x = magic();
@@ -140,7 +233,7 @@ lbl2:
 ],ac_cv_prog_emu_cc=$CC,ac_cv_prog_emu_cc=no)
 
 if test $ac_cv_prog_emu_cc = no; then
-	for ac_progname in emu_cc.sh gcc; do
+	for ac_progname in emu_cc.sh gcc-4.2 gcc; do
   		IFS="${IFS= 	}"; ac_save_ifs="$IFS"; IFS=":"
   		ac_dummy="$PATH"
   		for ac_dir in $ac_dummy; do
@@ -165,6 +258,11 @@ if test $ac_cv_prog_emu_cc != no; then
 	CFLAGS=""
 	CPPFLAGS=""
 	AC_TRY_COMPILE([],[
+#if defined(__clang_major__) && __clang_major__ >= 3
+    /* clang 3.x or later is fine */
+#elif defined(__llvm__)
+#error "this version of llvm is unable to correctly compile beam_emu.c"
+#endif
     	__label__ lbl1;
     	__label__ lbl2;
     	int x = magic();
@@ -386,14 +484,24 @@ AC_DEFUN(LM_SYS_IPV6,
 AC_CACHE_VAL(ac_cv_sys_ipv6_support,
 [ok_so_far=yes
  AC_TRY_COMPILE([#include <sys/types.h>
-#include <netinet/in.h>],
+#ifdef __WIN32__
+#include <winsock2.h>
+#include <ws2tcpip.h>
+#else
+#include <netinet/in.h>
+#endif],
    [struct in6_addr a6; struct sockaddr_in6 s6;], ok_so_far=yes, ok_so_far=no)
 
 if test $ok_so_far = yes; then
   ac_cv_sys_ipv6_support=yes
 else
   AC_TRY_COMPILE([#include <sys/types.h>
-#include <netinet/in.h>],
+#ifdef __WIN32__
+#include <winsock2.h>
+#include <ws2tcpip.h>
+#else
+#include <netinet/in.h>
+#endif],
     [struct in_addr6 a6; struct sockaddr_in6 s6;],
     ac_cv_sys_ipv6_support=in_addr6, ac_cv_sys_ipv6_support=no)
 fi
@@ -512,6 +620,8 @@ dnl
 AC_DEFUN(LM_CHECK_THR_LIB,
 [
 
+NEED_NPTL_PTHREAD_H=no
+
 dnl win32?
 AC_MSG_CHECKING([for native win32 threads])
 if test "X$host_os" = "Xwin32"; then
@@ -519,11 +629,13 @@ if test "X$host_os" = "Xwin32"; then
     THR_DEFS="-DWIN32_THREADS"
     THR_LIBS=
     THR_LIB_NAME=win32_threads
+    THR_LIB_TYPE=win32_threads
 else
     AC_MSG_RESULT(no)
     THR_DEFS=
     THR_LIBS=
     THR_LIB_NAME=
+    THR_LIB_TYPE=posix_unknown
 
 dnl Try to find POSIX threads
 
@@ -584,8 +696,11 @@ dnl On ofs1 the '-pthread' switch should be used
 		    AC_MSG_WARN([result yes guessed because of cross compilation])
 		fi
 		if test $nptl = yes; then
+		    THR_LIB_TYPE=posix_nptl
 		    need_nptl_incldir=no
-		    AC_CHECK_HEADER(nptl/pthread.h, need_nptl_incldir=yes)
+		    AC_CHECK_HEADER(nptl/pthread.h,
+				    [need_nptl_incldir=yes
+				     NEED_NPTL_PTHREAD_H=yes])
 		    if test $need_nptl_incldir = yes; then
 			# Ahh...
 			nptl_path="$C_INCLUDE_PATH:$CPATH"
@@ -649,6 +764,68 @@ fi
 
 ])
 
+AC_DEFUN(ERL_INTERNAL_LIBS,
+[
+
+ERTS_INTERNAL_X_LIBS=
+
+AC_CHECK_LIB(kstat, kstat_open,
+[AC_DEFINE(HAVE_KSTAT, 1, [Define if you have kstat])
+ERTS_INTERNAL_X_LIBS="$ERTS_INTERNAL_X_LIBS -lkstat"])
+
+AC_SUBST(ERTS_INTERNAL_X_LIBS)
+
+])
+
+AC_DEFUN(ETHR_CHK_SYNC_OP,
+[
+    AC_MSG_CHECKING([for $3-bit $1()])
+    case "$2" in
+	"1") sync_call="$1(&var);";;
+	"2") sync_call="$1(&var, ($4) 0);";;
+	"3") sync_call="$1(&var, ($4) 0, ($4) 0);";;
+    esac
+    have_sync_op=no
+    AC_TRY_LINK([],
+	[
+	    $4 res;
+	    volatile $4 var;
+	    res = $sync_call
+	],
+	[have_sync_op=yes])
+    test $have_sync_op = yes && $5
+    AC_MSG_RESULT([$have_sync_op])
+])
+
+AC_DEFUN(ETHR_CHK_INTERLOCKED,
+[
+    ilckd="$1"
+    AC_MSG_CHECKING([for ${ilckd}()])
+    case "$2" in
+	"1") ilckd_call="${ilckd}(var);";;
+	"2") ilckd_call="${ilckd}(var, ($3) 0);";;
+	"3") ilckd_call="${ilckd}(var, ($3) 0, ($3) 0);";;
+	"4") ilckd_call="${ilckd}(var, ($3) 0, ($3) 0, arr);";;
+    esac
+    have_interlocked_op=no
+    AC_TRY_LINK(
+	[
+	#define WIN32_LEAN_AND_MEAN
+	#include <windows.h>
+	#include <intrin.h>
+	],
+	[
+	    volatile $3 *var;
+	    volatile $3 arr[2];
+
+	    $ilckd_call
+	    return 0;
+	],
+	[have_interlocked_op=yes])
+    test $have_interlocked_op = yes && $4
+    AC_MSG_RESULT([$have_interlocked_op])
+])
+
 dnl ----------------------------------------------------------------------
 dnl
 dnl ERL_FIND_ETHR_LIB
@@ -672,10 +849,14 @@ AC_DEFUN(ERL_FIND_ETHR_LIB,
 [
 
 LM_CHECK_THR_LIB
+ERL_INTERNAL_LIBS
 
+ethr_have_native_atomics=no
+ethr_have_native_spinlock=no
 ETHR_THR_LIB_BASE="$THR_LIB_NAME"
+ETHR_THR_LIB_BASE_TYPE="$THR_LIB_TYPE"
 ETHR_DEFS="$THR_DEFS"
-ETHR_X_LIBS="$THR_LIBS"
+ETHR_X_LIBS="$THR_LIBS $ERTS_INTERNAL_X_LIBS"
 ETHR_LIBS=
 ETHR_LIB_NAME=
 
@@ -687,6 +868,7 @@ ethr_lib_name=ethread
 case "$THR_LIB_NAME" in
 
     win32_threads)
+	ETHR_THR_LIB_BASE_DIR=win
 	# * _WIN32_WINNT >= 0x0400 is needed for
 	#   TryEnterCriticalSection
 	# * _WIN32_WINNT >= 0x0403 is needed for
@@ -712,10 +894,48 @@ case "$THR_LIB_NAME" in
         if test $found_win32_winnt = no; then
 	    AC_MSG_ERROR([-D_WIN32_WINNT missing in CPPFLAGS])
         fi
+
 	AC_DEFINE(ETHR_WIN32_THREADS, 1, [Define if you have win32 threads])
+
+	ETHR_CHK_INTERLOCKED([_InterlockedDecrement], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT, 1, [Define if you have _InterlockedDecrement()]))
+	ETHR_CHK_INTERLOCKED([_InterlockedDecrement_rel], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT_REL, 1, [Define if you have _InterlockedDecrement_rel()]))
+	ETHR_CHK_INTERLOCKED([_InterlockedIncrement], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT, 1, [Define if you have _InterlockedIncrement()]))
+	ETHR_CHK_INTERLOCKED([_InterlockedIncrement_acq], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ, 1, [Define if you have _InterlockedIncrement_acq()]))
+	ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD, 1, [Define if you have _InterlockedExchangeAdd()]))
+	ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd_acq], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ, 1, [Define if you have _InterlockedExchangeAdd_acq()]))
+	ETHR_CHK_INTERLOCKED([_InterlockedAnd], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDAND, 1, [Define if you have _InterlockedAnd()]))
+	ETHR_CHK_INTERLOCKED([_InterlockedOr], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDOR, 1, [Define if you have _InterlockedOr()]))
+	ETHR_CHK_INTERLOCKED([_InterlockedExchange], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGE, 1, [Define if you have _InterlockedExchange()]))
+	ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE, 1, [Define if you have _InterlockedCompareExchange()]))
+	test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes
+	ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange_acq], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ, 1, [Define if you have _InterlockedCompareExchange_acq()]))
+	test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes
+	ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange_rel], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL, 1, [Define if you have _InterlockedCompareExchange_rel()]))
+	test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes
+
+	ETHR_CHK_INTERLOCKED([_InterlockedDecrement64], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT64, 1, [Define if you have _InterlockedDecrement64()]))
+	ETHR_CHK_INTERLOCKED([_InterlockedDecrement64_rel], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT64_REL, 1, [Define if you have _InterlockedDecrement64_rel()]))
+	ETHR_CHK_INTERLOCKED([_InterlockedIncrement64], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT64, 1, [Define if you have _InterlockedIncrement64()]))
+	ETHR_CHK_INTERLOCKED([_InterlockedIncrement64_acq], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ, 1, [Define if you have _InterlockedIncrement64_acq()]))
+	ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64, 1, [Define if you have _InterlockedExchangeAdd64()]))
+	ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd64_acq], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ, 1, [Define if you have _InterlockedExchangeAdd64_acq()]))
+	ETHR_CHK_INTERLOCKED([_InterlockedAnd64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDAND64, 1, [Define if you have _InterlockedAnd64()]))
+	ETHR_CHK_INTERLOCKED([_InterlockedOr64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDOR64, 1, [Define if you have _InterlockedOr64()]))
+	ETHR_CHK_INTERLOCKED([_InterlockedExchange64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGE64, 1, [Define if you have _InterlockedExchange64()]))
+	ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64, 1, [Define if you have _InterlockedCompareExchange64()]))
+	test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes
+	ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64_acq], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ, 1, [Define if you have _InterlockedCompareExchange64_acq()]))
+	test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes
+	ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64_rel], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL, 1, [Define if you have _InterlockedCompareExchange64_rel()]))
+	test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes
+
+	ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange128], [4], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE128, 1, [Define if you have _InterlockedCompareExchange128()]))
+
+	test "$ethr_have_native_atomics" = "yes" && ethr_have_native_spinlock=yes
 	;;
 
     pthread)
+	ETHR_THR_LIB_BASE_DIR=pthread
     	AC_DEFINE(ETHR_PTHREADS, 1, [Define if you have pthreads])
 	case $host_os in
 	    openbsd*)
@@ -771,9 +991,7 @@ case "$THR_LIB_NAME" in
 		if test $usable_sigaltstack = no; then
 		    ETHR_DEFS="$ETHR_DEFS -DETHR_UNUSABLE_SIGALTSTACK"
 		fi
-
-		AC_DEFINE(ETHR_INIT_MUTEX_IN_CHILD_AT_FORK, 1, \
-[Define if mutexes should be reinitialized (instead of unlocked) in child at fork.]) ;;
+		;;
 	    *) ;;
 	esac
 
@@ -799,6 +1017,15 @@ case "$THR_LIB_NAME" in
 			AC_DEFINE(ETHR_HAVE_MIT_PTHREAD_H, 1, \
 [Define if the pthread.h header file is in pthread/mit directory.]))
 
+	if test $NEED_NPTL_PTHREAD_H = yes; then
+	    AC_DEFINE(ETHR_NEED_NPTL_PTHREAD_H, 1, \
+[Define if you need the <nptl/pthread.h> header file.])
+	fi
+
+	AC_CHECK_HEADER(sched.h, \
+			AC_DEFINE(ETHR_HAVE_SCHED_H, 1, \
+[Define if you have the <sched.h> header file.]))
+
 	AC_CHECK_HEADER(sys/time.h, \
 			AC_DEFINE(ETHR_HAVE_SYS_TIME_H, 1, \
 [Define if you have the <sys/time.h> header file.]))
@@ -814,39 +1041,256 @@ case "$THR_LIB_NAME" in
 	dnl Check for functions
 	dnl
 
-	AC_CHECK_FUNC(pthread_atfork, \
-			AC_DEFINE(ETHR_HAVE_PTHREAD_ATFORK, 1, \
-[Define if you have the pthread_atfork function.]))
-	AC_CHECK_FUNC(pthread_mutexattr_settype, \
-			AC_DEFINE(ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE, 1, \
-[Define if you have the pthread_mutexattr_settype function.]))
-	AC_CHECK_FUNC(pthread_mutexattr_setkind_np, \
-			AC_DEFINE(ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP, 1, \
-[Define if you have the pthread_mutexattr_setkind_np function.]))
 	AC_CHECK_FUNC(pthread_spin_lock, \
-			AC_DEFINE(ETHR_HAVE_PTHREAD_SPIN_LOCK, 1, \
-[Define if you have the pthread_spin_lock function.]))
-	test "$force_linux_pthread_rwlocks" = "yes" || {
-	    force_linux_pthread_rwlocks=no
-	}
-	case "$force_linux_pthread_rwlocks-$host_os" in
-		no-linux*) # Writers may get starved
-			# TODO: write a test that tests the implementation
-			;;
-		*)
-			AC_CHECK_FUNC(pthread_rwlock_init, \
-				AC_DEFINE(ETHR_HAVE_PTHREAD_RWLOCK_INIT, 1, \
-[Define if you have a pthread_rwlock implementation that can be used.]))
-			;;
-	esac
+			[ethr_have_native_spinlock=yes \
+			 AC_DEFINE(ETHR_HAVE_PTHREAD_SPIN_LOCK, 1, \
+[Define if you have the pthread_spin_lock function.])])
+
+	have_sched_yield=no
+	have_librt_sched_yield=no
+	AC_CHECK_FUNC(sched_yield, [have_sched_yield=yes])
+	if test $have_sched_yield = no; then
+	    AC_CHECK_LIB(rt, sched_yield,
+			 [have_librt_sched_yield=yes
+			  ETHR_X_LIBS="$ETHR_X_LIBS -lrt"])
+	fi
+	if test $have_sched_yield = yes || test $have_librt_sched_yield = yes; then
+	    AC_DEFINE(ETHR_HAVE_SCHED_YIELD, 1, [Define if you have the sched_yield() function.])
+	    AC_MSG_CHECKING([whether sched_yield() returns an int])
+	    sched_yield_ret_int=no
+	    AC_TRY_COMPILE([
+				#ifdef ETHR_HAVE_SCHED_H
+				#include <sched.h>
+				#endif
+			   ],
+			   [int sched_yield();],
+			   [sched_yield_ret_int=yes])
+	    AC_MSG_RESULT([$sched_yield_ret_int])
+	    if test $sched_yield_ret_int = yes; then
+		AC_DEFINE(ETHR_SCHED_YIELD_RET_INT, 1, [Define if sched_yield() returns an int.])
+	    fi
+	fi
+
+	have_pthread_yield=no
+	AC_CHECK_FUNC(pthread_yield, [have_pthread_yield=yes])
+	if test $have_pthread_yield = yes; then
+	    AC_DEFINE(ETHR_HAVE_PTHREAD_YIELD, 1, [Define if you have the pthread_yield() function.])
+	    AC_MSG_CHECKING([whether pthread_yield() returns an int])
+	    pthread_yield_ret_int=no
+	    AC_TRY_COMPILE([
+				#if defined(ETHR_NEED_NPTL_PTHREAD_H)
+				#include <nptl/pthread.h>
+				#elif defined(ETHR_HAVE_MIT_PTHREAD_H)
+				#include <pthread/mit/pthread.h>
+				#elif defined(ETHR_HAVE_PTHREAD_H)
+				#include <pthread.h>
+				#endif
+			   ],
+			   [int pthread_yield();],
+			   [pthread_yield_ret_int=yes])
+	    AC_MSG_RESULT([$pthread_yield_ret_int])
+	    if test $pthread_yield_ret_int = yes; then
+		AC_DEFINE(ETHR_PTHREAD_YIELD_RET_INT, 1, [Define if pthread_yield() returns an int.])
+	    fi
+	fi
+
+	have_pthread_rwlock_init=no
+	AC_CHECK_FUNC(pthread_rwlock_init, [have_pthread_rwlock_init=yes])
+	if test $have_pthread_rwlock_init = yes; then
+
+	    ethr_have_pthread_rwlockattr_setkind_np=no
+	    AC_CHECK_FUNC(pthread_rwlockattr_setkind_np,
+			  [ethr_have_pthread_rwlockattr_setkind_np=yes])
+
+	    if test $ethr_have_pthread_rwlockattr_setkind_np = yes; then
+		AC_DEFINE(ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP, 1, \
+[Define if you have the pthread_rwlockattr_setkind_np() function.])
+
+		AC_MSG_CHECKING([for PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP])
+		ethr_pthread_rwlock_writer_nonrecursive_initializer_np=no
+		AC_TRY_LINK([
+				#if defined(ETHR_NEED_NPTL_PTHREAD_H)
+				#include <nptl/pthread.h>
+				#elif defined(ETHR_HAVE_MIT_PTHREAD_H)
+				#include <pthread/mit/pthread.h>
+				#elif defined(ETHR_HAVE_PTHREAD_H)
+				#include <pthread.h>
+				#endif
+			    ],
+			    [
+				pthread_rwlockattr_t *attr;
+				return pthread_rwlockattr_setkind_np(attr,
+				    PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
+			    ],
+			    [ethr_pthread_rwlock_writer_nonrecursive_initializer_np=yes])
+		AC_MSG_RESULT([$ethr_pthread_rwlock_writer_nonrecursive_initializer_np])
+		if test $ethr_pthread_rwlock_writer_nonrecursive_initializer_np = yes; then
+		    AC_DEFINE(ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP, 1, \
+[Define if you have the PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP rwlock attribute.])
+		fi
+	    fi
+	fi
+
+	if test "$force_pthread_rwlocks" = "yes"; then
+
+	    AC_DEFINE(ETHR_FORCE_PTHREAD_RWLOCK, 1, \
+[Define if you want to force usage of pthread rwlocks])
+
+	    if test $have_pthread_rwlock_init = yes; then
+		AC_MSG_WARN([Forced usage of pthread rwlocks. Note that this implementation may suffer from starvation issues.])
+	    else
+		AC_MSG_ERROR([User forced usage of pthread rwlock, but no such implementation was found])
+	    fi
+	fi
+
 	AC_CHECK_FUNC(pthread_attr_setguardsize, \
 			AC_DEFINE(ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE, 1, \
 [Define if you have the pthread_attr_setguardsize function.]))
 
+	linux_futex=no
+	AC_MSG_CHECKING([for Linux futexes])
+	AC_TRY_LINK([
+			#include <sys/syscall.h>
+			#include <unistd.h>
+			#include <linux/futex.h>
+			#include <sys/time.h>
+		    ],
+		    [
+			int i = 1;
+			syscall(__NR_futex, (void *) &i, FUTEX_WAKE, 1,
+				(void*)0,(void*)0, 0);
+			syscall(__NR_futex, (void *) &i, FUTEX_WAIT, 0,
+				(void*)0,(void*)0, 0);
+			return 0;
+		    ],
+		    linux_futex=yes)
+	AC_MSG_RESULT([$linux_futex])
+	test $linux_futex = yes && AC_DEFINE(ETHR_HAVE_LINUX_FUTEX, 1, [Define if you have a linux futex implementation.])
+
+	AC_CHECK_SIZEOF(int)
+	AC_CHECK_SIZEOF(long)
+	AC_CHECK_SIZEOF(long long)
+	AC_CHECK_SIZEOF(__int128_t)
+
+	if test "$ac_cv_sizeof_int" = "4"; then
+	    int32="int"
+	elif test "$ac_cv_sizeof_long" = "4"; then
+	    int32="long"
+	elif test "$ac_cv_sizeof_long_long" = "4"; then
+	    int32="long long"
+	else
+	    AC_MSG_ERROR([No 32-bit type found])
+	fi
+
+	if test "$ac_cv_sizeof_int" = "8"; then
+	    int64="int"
+	elif test "$ac_cv_sizeof_long" = "8"; then
+	    int64="long"
+	elif test "$ac_cv_sizeof_long_long" = "8"; then
+	    int64="long long"
+	else
+	    AC_MSG_ERROR([No 64-bit type found])
+	fi
+
+	int128=no
+	if test "$ac_cv_sizeof___int128_t" = "16"; then
+	    int128="__int128_t"
+	fi
+
+	ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32, 1, [Define if you have __sync_val_compare_and_swap() for 32-bit integers]))
+	test "$have_sync_op" = "yes" && ethr_have_native_atomics=yes
+	ETHR_CHK_SYNC_OP([__sync_add_and_fetch], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_ADD_AND_FETCH32, 1, [Define if you have __sync_add_and_fetch() for 32-bit integers]))
+	ETHR_CHK_SYNC_OP([__sync_fetch_and_and], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_AND32, 1, [Define if you have __sync_fetch_and_and() for 32-bit integers]))
+	ETHR_CHK_SYNC_OP([__sync_fetch_and_or], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_OR32, 1, [Define if you have __sync_fetch_and_or() for 32-bit integers]))
+
+	ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64, 1, [Define if you have __sync_val_compare_and_swap() for 64-bit integers]))
+	test "$have_sync_op" = "yes" && ethr_have_native_atomics=yes
+	ETHR_CHK_SYNC_OP([__sync_add_and_fetch], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_ADD_AND_FETCH64, 1, [Define if you have __sync_add_and_fetch() for 64-bit integers]))
+	ETHR_CHK_SYNC_OP([__sync_fetch_and_and], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_AND64, 1, [Define if you have __sync_fetch_and_and() for 64-bit integers]))
+	ETHR_CHK_SYNC_OP([__sync_fetch_and_or], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_OR64, 1, [Define if you have __sync_fetch_and_or() for 64-bit integers]))
+
+	if test $int128 != no; then
+	    ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [128], [$int128], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP128, 1, [Define if you have __sync_val_compare_and_swap() for 128-bit integers]))
+	fi
+
+	AC_MSG_CHECKING([for a usable libatomic_ops implementation])
+	case "x$with_libatomic_ops" in
+	    xno | xyes | x)
+		libatomic_ops_include=
+		;;
+	    *)
+		if test -d "${with_libatomic_ops}/include"; then
+		    libatomic_ops_include="-I$with_libatomic_ops/include"
+		    CPPFLAGS="$CPPFLAGS $libatomic_ops_include"
+		else
+		    AC_MSG_ERROR([libatomic_ops include directory $with_libatomic_ops/include not found])
+		fi;;
+	esac
+	ethr_have_libatomic_ops=no
+	AC_TRY_LINK([#include "atomic_ops.h"],
+		    [
+			volatile AO_t x;
+			AO_t y;
+			int z;
+
+			AO_nop_full();
+			AO_store(&x, (AO_t) 0);
+			z = AO_load(&x);
+			z = AO_compare_and_swap_full(&x, (AO_t) 0, (AO_t) 1);
+		    ],
+		    [ethr_have_native_atomics=yes
+		     ethr_have_libatomic_ops=yes])
+	AC_MSG_RESULT([$ethr_have_libatomic_ops])
+	if test $ethr_have_libatomic_ops = yes; then
+	    AC_CHECK_SIZEOF(AO_t, ,
+			    [
+				#include <stdio.h>
+				#include "atomic_ops.h"
+			    ])
+	    AC_DEFINE_UNQUOTED(ETHR_SIZEOF_AO_T, $ac_cv_sizeof_AO_t, [Define to the size of AO_t if libatomic_ops is used])
+
+	    AC_DEFINE(ETHR_HAVE_LIBATOMIC_OPS, 1, [Define if you have libatomic_ops atomic operations])
+	    if test "x$with_libatomic_ops" != "xno" && test "x$with_libatomic_ops" != "x"; then
+		AC_DEFINE(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS, 1, [Define if you prefer libatomic_ops native ethread implementations])
+	    fi
+	    ETHR_DEFS="$ETHR_DEFS $libatomic_ops_include"
+	elif test "x$with_libatomic_ops" != "xno" && test "x$with_libatomic_ops" != "x"; then
+	    AC_MSG_ERROR([No usable libatomic_ops implementation found])
+	fi
+
+	case "$host_cpu" in
+	  sparc | sun4u | sparc64 | sun4v)
+		case "$with_sparc_memory_order" in
+		    "TSO")
+			AC_DEFINE(ETHR_SPARC_TSO, 1, [Define if only run in Sparc TSO mode]);;
+		    "PSO")
+			AC_DEFINE(ETHR_SPARC_PSO, 1, [Define if only run in Sparc PSO, or TSO mode]);;
+		    "RMO"|"")
+			AC_DEFINE(ETHR_SPARC_RMO, 1, [Define if run in Sparc RMO, PSO, or TSO mode]);;
+		    *)
+			AC_MSG_ERROR([Unsupported Sparc memory order: $with_sparc_memory_order]);;
+		esac
+		ethr_have_native_atomics=yes;; 
+	  i86pc | i*86 | x86_64 | amd64)
+		if test "$enable_x86_out_of_order" = "yes"; then
+			AC_DEFINE(ETHR_X86_OUT_OF_ORDER, 1, [Define if x86/x86_64 out of order instructions should be synchronized])
+		fi
+		ethr_have_native_atomics=yes;;
+	  macppc | ppc | "Power Macintosh")
+		ethr_have_native_atomics=yes;;
+	  tile)
+		ethr_have_native_atomics=yes;;
+	  *)
+		;;
+	esac
+
+	test ethr_have_native_atomics = "yes" && ethr_have_native_spinlock=yes
+
 	dnl Restore LIBS
 	LIBS=$saved_libs
 	dnl restore CPPFLAGS
 	CPPFLAGS=$saved_cppflags
+
 	;;
     *)
 	;;
@@ -862,17 +1306,161 @@ fi
 
 if test "x$ETHR_THR_LIB_BASE" != "x"; then
 	ETHR_DEFS="-DUSE_THREADS $ETHR_DEFS"
-	ETHR_LIBS="-l$ethr_lib_name $ETHR_X_LIBS"
+	ETHR_LIBS="-l$ethr_lib_name -lerts_internal_r $ETHR_X_LIBS"
 	ETHR_LIB_NAME=$ethr_lib_name
 fi
 
 AC_CHECK_SIZEOF(void *)
 AC_DEFINE_UNQUOTED(ETHR_SIZEOF_PTR, $ac_cv_sizeof_void_p, [Define to the size of pointers])
 
-if test "X$disable_native_ethr_impls" = "Xyes"; then
-	AC_DEFINE(ETHR_DISABLE_NATIVE_IMPLS, 1, [Define if you want to disable native ethread implementations])
+AC_CHECK_SIZEOF(int)
+AC_DEFINE_UNQUOTED(ETHR_SIZEOF_INT, $ac_cv_sizeof_int, [Define to the size of int])
+AC_CHECK_SIZEOF(long)
+AC_DEFINE_UNQUOTED(ETHR_SIZEOF_LONG, $ac_cv_sizeof_long, [Define to the size of long])
+AC_CHECK_SIZEOF(long long)
+AC_DEFINE_UNQUOTED(ETHR_SIZEOF_LONG_LONG, $ac_cv_sizeof_long_long, [Define to the size of long long])
+AC_CHECK_SIZEOF(__int64)
+AC_DEFINE_UNQUOTED(ETHR_SIZEOF___INT64, $ac_cv_sizeof___int64, [Define to the size of __int64])
+AC_CHECK_SIZEOF(__int128_t)
+AC_DEFINE_UNQUOTED(ETHR_SIZEOF___INT128_T, $ac_cv_sizeof___int128_t, [Define to the size of __int128_t])
+
+
+case X$erl_xcomp_bigendian in
+    X) ;;
+    Xyes|Xno) ac_cv_c_bigendian=$erl_xcomp_bigendian;;
+    *) AC_MSG_ERROR([Bad erl_xcomp_bigendian value: $erl_xcomp_bigendian]);;
+esac
+
+AC_C_BIGENDIAN
+
+if test "$ac_cv_c_bigendian" = "yes"; then
+    AC_DEFINE(ETHR_BIGENDIAN, 1, [Define if bigendian])
 fi
 
+AC_ARG_ENABLE(native-ethr-impls,
+	      AS_HELP_STRING([--disable-native-ethr-impls],
+                             [disable native ethread implementations]),
+[ case "$enableval" in
+    no) disable_native_ethr_impls=yes ;;
+    *)  disable_native_ethr_impls=no ;;
+  esac ], disable_native_ethr_impls=no)
+
+AC_ARG_ENABLE(x86-out-of-order,
+	      AS_HELP_STRING([--enable-x86-out-of-order],
+                             [enable x86/x84_64 out of order support (default disabled)]))
+
+test "X$disable_native_ethr_impls" = "Xyes" &&
+  AC_DEFINE(ETHR_DISABLE_NATIVE_IMPLS, 1, [Define if you want to disable native ethread implementations])
+
+AC_ARG_ENABLE(prefer-gcc-native-ethr-impls,
+	      AS_HELP_STRING([--enable-prefer-gcc-native-ethr-impls],
+			     [prefer gcc native ethread implementations]),
+[ case "$enableval" in
+    yes) enable_prefer_gcc_native_ethr_impls=yes ;;
+    *)  enable_prefer_gcc_native_ethr_impls=no ;;
+  esac ], enable_prefer_gcc_native_ethr_impls=no)
+
+test $enable_prefer_gcc_native_ethr_impls = yes &&
+  AC_DEFINE(ETHR_PREFER_GCC_NATIVE_IMPLS, 1, [Define if you prefer gcc native ethread implementations])
+
+AC_ARG_WITH(libatomic_ops,
+	    AS_HELP_STRING([--with-libatomic_ops=PATH],
+			   [specify and prefer usage of libatomic_ops in the ethread library]))
+
+AC_ARG_WITH(with_sparc_memory_order,
+	    AS_HELP_STRING([--with-sparc-memory-order=TSO|PSO|RMO],
+			   [specify sparc memory order (defaults to RMO)]))
+
+ETHR_X86_SSE2_ASM=no
+case "$GCC-$ac_cv_sizeof_void_p-$host_cpu" in
+  yes-4-i86pc | yes-4-i*86 | yes-4-x86_64 | yes-4-amd64)
+    AC_MSG_CHECKING([for gcc sse2 asm support])
+    save_CFLAGS="$CFLAGS"
+    CFLAGS="$CFLAGS -msse2"
+    gcc_sse2_asm=no
+    AC_TRY_COMPILE([],
+	[
+		long long x, *y;
+		__asm__ __volatile__("movq %1, %0\n\t" : "=x"(x) : "m"(*y) : "memory");
+	],
+	[gcc_sse2_asm=yes])
+    CFLAGS="$save_CFLAGS"
+    AC_MSG_RESULT([$gcc_sse2_asm])
+    if test "$gcc_sse2_asm" = "yes"; then
+      AC_DEFINE(ETHR_GCC_HAVE_SSE2_ASM_SUPPORT, 1, [Define if you use a gcc that supports -msse2 and understand sse2 specific asm statements])
+      ETHR_X86_SSE2_ASM=yes
+    fi
+    ;;
+  *)
+    ;;
+esac
+
+case "$GCC-$host_cpu" in
+  yes-i86pc | yes-i*86 | yes-x86_64 | yes-amd64)
+    gcc_dw_cmpxchg_asm=no
+    AC_MSG_CHECKING([for gcc double word cmpxchg asm support])    
+    AC_TRY_COMPILE([],
+	[
+    char xchgd;
+    long new[2], xchg[2], *p;		  
+    __asm__ __volatile__(
+#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__
+	"pushl %%ebx\n\t"
+	"movl %8, %%ebx\n\t"
+#endif
+#if ETHR_SIZEOF_PTR == 4
+	"lock; cmpxchg8b %0\n\t"
+#else
+	"lock; cmpxchg16b %0\n\t"
+#endif
+	"setz %3\n\t"
+#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__
+	"popl %%ebx\n\t"
+#endif
+	: "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=c"(xchgd)
+	: "m"(*p), "1"(xchg[1]), "2"(xchg[0]), "3"(new[1]),
+#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__
+	  "r"(new[0])
+#else
+	  "b"(new[0])
+#endif
+	: "cc", "memory");
+
+	],
+	[gcc_dw_cmpxchg_asm=yes])
+    if test $gcc_dw_cmpxchg_asm = no && test $ac_cv_sizeof_void_p = 4; then
+      AC_TRY_COMPILE([],
+	  [
+      char xchgd;
+      long new[2], xchg[2], *p;
+#if !defined(__PIC__) || !__PIC__
+#  error nope
+#endif
+      __asm__ __volatile__(
+    	  "pushl %%ebx\n\t"
+	  "movl (%7), %%ebx\n\t"
+	  "movl 4(%7), %%ecx\n\t"
+	  "lock; cmpxchg8b %0\n\t"
+  	  "setz %3\n\t"
+	  "popl %%ebx\n\t"
+	  : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=c"(xchgd)
+	  : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), "3"(new)
+	: "cc", "memory");
+
+	],
+	[gcc_dw_cmpxchg_asm=yes])
+      if test "$gcc_dw_cmpxchg_asm" = "yes"; then
+        AC_DEFINE(ETHR_CMPXCHG8B_REGISTER_SHORTAGE, 1, [Define if you get a register shortage with cmpxchg8b and position independent code])
+      fi
+    fi
+    AC_MSG_RESULT([$gcc_dw_cmpxchg_asm])
+    if test "$gcc_dw_cmpxchg_asm" = "yes"; then
+      AC_DEFINE(ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT, 1, [Define if you use a gcc that supports the double word cmpxchg instruction])
+    fi;;
+  *)
+    ;;
+esac
+
 AC_DEFINE(ETHR_HAVE_ETHREAD_DEFINES, 1, \
 [Define if you have all ethread defines])
 
@@ -881,6 +1469,8 @@ AC_SUBST(ETHR_LIBS)
 AC_SUBST(ETHR_LIB_NAME)
 AC_SUBST(ETHR_DEFS)
 AC_SUBST(ETHR_THR_LIB_BASE)
+AC_SUBST(ETHR_THR_LIB_BASE_DIR)
+AC_SUBST(ETHR_X86_SSE2_ASM)
 
 ])
 
@@ -1151,11 +1741,11 @@ dnl Freely inspired by AC_TRY_LINK. (Maybe better to create a
 dnl AC_LANG_JAVA instead...)
 AC_DEFUN(ERL_TRY_LINK_JAVA,
 [java_link='$JAVAC conftest.java 1>&AC_FD_CC'
-changequote(�, �)dnl
+changequote(, )dnl
 cat > conftest.java <<EOF
-�$1�
+$1
 class conftest { public static void main(String[] args) {
-   �$2�
+   $2
    ; return; }}
 EOF
 changequote([, ])dnl
diff --git a/erts/autoconf/configure.vxworks b/erts/autoconf/configure.vxworks
index 70d7bdbaf2..23a93faa31 100755
--- a/erts/autoconf/configure.vxworks
+++ b/erts/autoconf/configure.vxworks
@@ -2,7 +2,7 @@
 # 
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 1997-2009. All Rights Reserved.
+# Copyright Ericsson AB 1997-2011. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -101,7 +101,6 @@ epmd_dir=${ERL_TOP}/erts/epmd/src
 os_mon_dir=${ERL_TOP}/lib/os_mon/c_src
 orber_dir=${ERL_TOP}/lib/orber/c_src
 ic_dir=${ERL_TOP}/lib/ic/c_src
-asn1_dir=${ERL_TOP}/lib/asn1/c_src
 internal_tools_dir=${ERL_TOP}
 libdir=${ERL_TOP}/lib
 tsdir=$libdir/test_server/src
@@ -122,7 +121,6 @@ CONFIG_FILES="${ERL_TOP}/erts/emulator/$host/Makefile
 	    $os_mon_dir/$host/Makefile
 	    $zlibdir/$host/Makefile
 	    $ic_dir/$host/Makefile
-	    $asn1_dir/$host/Makefile
 	    $runtime_tools_dir/$host/Makefile
 	    $tools_dir/$host/Makefile
 	    $orber_dir/$host/Makefile"
diff --git a/erts/autoconf/vxworks/sed.general b/erts/autoconf/vxworks/sed.general
index f725a6f9ca..ac792b44b5 100644
--- a/erts/autoconf/vxworks/sed.general
+++ b/erts/autoconf/vxworks/sed.general
@@ -1,7 +1,7 @@
 # 
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 1997-2009. All Rights Reserved.
+# Copyright Ericsson AB 1997-2011. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -55,18 +55,21 @@ s|@ETHR_LIBS@||
 s|@ETHR_LIB_NAME@||
 s|@ETHR_DEFS@||
 s|@ETHR_THR_LIB_BASE@||
+s|@ETHR_THR_LIB_BASE_DIR@||
 s|@EMU_THR_DEFS@||
 s|@EMU_THR_LIBS@||
 s|@EMU_THR_LIB_NAME@|ethread|
 s|@ERTS_ENABLE_KERNEL_POLL@|no|
+s|@ERTS_INTERNAL_X_LIBS@||
 s|@cc_root@|/clearcase/otp/|
 # Define VxWorks even though cross-compiling. 
+s|@CROSS_COMPILING|yes|
+
 s|@HCFLAGS@|-DVXWORKS|
 s|@HCLIBS@||
 s|@ENABLE_ALLOC_TYPE_VARS@||
 s|@TERMCAP_LIB@||
 s|@ERTS_BUILD_SMP_EMU@|no|
-s|@ERTS_BUILD_HYBRID_EMU@|no|
 s|@HAVE_VALGRIND@|no|
 s|@EXEEXT@||
 s|@WITH_SCTP@||
diff --git a/erts/autoconf/win32.config.cache b/erts/autoconf/win32.config.cache.static
index 31dfe510cd..b387db2b22 100755
--- a/erts/autoconf/win32.config.cache
+++ b/erts/autoconf/win32.config.cache.static
@@ -61,7 +61,6 @@ ac_cv_func_fork=${ac_cv_func_fork=no}
 ac_cv_func_fork_works=${ac_cv_func_fork_works=no}
 ac_cv_func_fpsetmask=${ac_cv_func_fpsetmask=no}
 ac_cv_func_fstat=${ac_cv_func_fstat=yes}
-ac_cv_func_getaddrinfo=${ac_cv_func_getaddrinfo=no}
 ac_cv_func_gethostbyaddr=${ac_cv_func_gethostbyaddr=no}
 ac_cv_func_gethostbyaddr_r=${ac_cv_func_gethostbyaddr_r=no}
 ac_cv_func_gethostbyname=${ac_cv_func_gethostbyname=no}
@@ -71,7 +70,6 @@ ac_cv_func_gethostname=${ac_cv_func_gethostname=no}
 ac_cv_func_gethrtime=${ac_cv_func_gethrtime=no}
 ac_cv_func_getipnodebyaddr=${ac_cv_func_getipnodebyaddr=no}
 ac_cv_func_getipnodebyname=${ac_cv_func_getipnodebyname=no}
-ac_cv_func_getnameinfo=${ac_cv_func_getnameinfo=no}
 ac_cv_func_getpagesize=${ac_cv_func_getpagesize=no}
 ac_cv_func_gettimeofday=${ac_cv_func_gettimeofday=no}
 ac_cv_func_gmtime_r=${ac_cv_func_gmtime_r=no}
@@ -98,7 +96,6 @@ ac_cv_func_sbrk=${ac_cv_func_sbrk=no}
 ac_cv_func_select=${ac_cv_func_select=no}
 ac_cv_func_setlocale=${ac_cv_func_setlocale=yes}
 ac_cv_func_setsid=${ac_cv_func_setsid=no}
-ac_cv_func_setvbuf_reversed=${ac_cv_func_setvbuf_reversed=yes}
 ac_cv_func_socket=${ac_cv_func_socket=no}
 ac_cv_func_strchr=${ac_cv_func_strchr=yes}
 ac_cv_func_strerror=${ac_cv_func_strerror=yes}
@@ -126,7 +123,6 @@ ac_cv_header_ieeefp_h=${ac_cv_header_ieeefp_h=no}
 ac_cv_header_inttypes_h=${ac_cv_header_inttypes_h=no}
 ac_cv_header_langinfo_h=${ac_cv_header_langinfo_h=no}
 ac_cv_header_limits_h=${ac_cv_header_limits_h=yes}
-ac_cv_header_mach_o_dyld_h=${ac_cv_header_mach_o_dyld_h=no}
 ac_cv_header_malloc_h=${ac_cv_header_malloc_h=yes}
 ac_cv_header_memory_h=${ac_cv_header_memory_h=yes}
 ac_cv_header_net_errno_h=${ac_cv_header_net_errno_h=no}
@@ -212,7 +208,6 @@ ac_cv_sizeof_void_p=${ac_cv_sizeof_void_p=4}
 ac_cv_struct_exception=${ac_cv_struct_exception=no}
 ac_cv_struct_sockaddr_sa_len=${ac_cv_struct_sockaddr_sa_len=no}
 ac_cv_struct_tm=${ac_cv_struct_tm=time.h}
-ac_cv_sys_ipv6_support=${ac_cv_sys_ipv6_support=no}
 ac_cv_sys_multicast_support=${ac_cv_sys_multicast_support=no}
 ac_cv_type_char=${ac_cv_type_char=yes}
 ac_cv_type_int=${ac_cv_type_int=yes}
diff --git a/erts/autoconf/win64.config.cache.static b/erts/autoconf/win64.config.cache.static
new file mode 100755
index 0000000000..a8a2bfb59c
--- /dev/null
+++ b/erts/autoconf/win64.config.cache.static
@@ -0,0 +1,271 @@
+# This file is a shell script that caches the results of configure
+# tests run on this system so they can be shared between configure
+# scripts and configure runs, see configure's option --config-cache.
+# It is not useful on other systems.  If it contains results you don't
+# want to keep, you may remove or edit it.
+#
+# config.status only pays attention to the cache file if you give it
+# the --recheck option to rerun configure.
+#
+# `ac_cv_env_foo' variables (set or unset) will be overridden when
+# loading this file, other *unset* `ac_cv_foo' will be assigned the
+# following values.
+
+ac_cv_c_bigendian=${ac_cv_c_bigendian=no}
+ac_cv_c_compiler_gnu=${ac_cv_c_compiler_gnu=no}
+ac_cv_c_const=${ac_cv_c_const=yes}
+ac_cv_cxx_compiler_gnu=${ac_cv_cxx_compiler_gnu=no}
+ac_cv_decl_h_errno=${ac_cv_decl_h_errno=no}
+ac_cv_decl_inaddr_loopback=${ac_cv_decl_inaddr_loopback=no}
+ac_cv_decl_inaddr_loopback_rpc=${ac_cv_decl_inaddr_loopback_rpc=no}
+ac_cv_decl_inaddr_loopback_winsock2=${ac_cv_decl_inaddr_loopback_winsock2=yes}
+ac_cv_decl_so_bsdcompat=${ac_cv_decl_so_bsdcompat=no}
+ac_cv_decl_sys_errlist=${ac_cv_decl_sys_errlist=no}
+ac_cv_env_AR_set=set
+ac_cv_env_AR_value=ar.sh
+ac_cv_env_CCC_set=
+ac_cv_env_CCC_value=
+ac_cv_env_CC_set=set
+ac_cv_env_CC_value=cc.sh
+ac_cv_env_CFLAGS_set=
+ac_cv_env_CFLAGS_value=
+ac_cv_env_CPPFLAGS_set=
+ac_cv_env_CPPFLAGS_value=
+ac_cv_env_CPP_set=
+ac_cv_env_CPP_value=
+ac_cv_env_CXXFLAGS_set=
+ac_cv_env_CXXFLAGS_value=
+ac_cv_env_CXX_set=set
+ac_cv_env_CXX_value=cc.sh
+ac_cv_env_LDFLAGS_set=
+ac_cv_env_LDFLAGS_value=
+ac_cv_env_build_alias_set=set
+ac_cv_env_build_alias_value=win32
+ac_cv_env_host_alias_set=set
+ac_cv_env_host_alias_value=win32
+ac_cv_env_target_alias_set=set
+ac_cv_env_target_alias_value=win32
+ac_cv_exeext=${ac_cv_exeext=.exe}
+ac_cv_func___brk=${ac_cv_func___brk=no}
+ac_cv_func___sbrk=${ac_cv_func___sbrk=no}
+ac_cv_func__brk=${ac_cv_func__brk=no}
+ac_cv_func__doprnt=${ac_cv_func__doprnt=no}
+ac_cv_func__sbrk=${ac_cv_func__sbrk=no}
+ac_cv_func_accept=${ac_cv_func_accept=no}
+ac_cv_func_alloca_works=${ac_cv_func_alloca_works=yes}
+ac_cv_func_brk=${ac_cv_func_brk=no}
+ac_cv_func_clock_gettime=${ac_cv_func_clock_gettime=no}
+ac_cv_func_connect=${ac_cv_func_connect=no}
+ac_cv_func_decl_fread=${ac_cv_func_decl_fread=yes}
+ac_cv_func_dlopen=${ac_cv_func_dlopen=no}
+ac_cv_func_dup2=${ac_cv_func_dup2=yes}
+ac_cv_func_fdatasync=${ac_cv_func_fdatasync=no}
+ac_cv_func_finite=${ac_cv_func_finite=no}
+ac_cv_func_flockfile=${ac_cv_func_flockfile=no}
+ac_cv_func_fork=${ac_cv_func_fork=no}
+ac_cv_func_fork_works=${ac_cv_func_fork_works=no}
+ac_cv_func_fpsetmask=${ac_cv_func_fpsetmask=no}
+ac_cv_func_fstat=${ac_cv_func_fstat=yes}
+ac_cv_func_gethostbyaddr=${ac_cv_func_gethostbyaddr=no}
+ac_cv_func_gethostbyaddr_r=${ac_cv_func_gethostbyaddr_r=no}
+ac_cv_func_gethostbyname=${ac_cv_func_gethostbyname=yes}
+ac_cv_func_gethostbyname2=${ac_cv_func_gethostbyname2=no}
+ac_cv_func_gethostbyname_r=${ac_cv_func_gethostbyname_r=no}
+ac_cv_func_gethostname=${ac_cv_func_gethostname=no}
+ac_cv_func_gethrtime=${ac_cv_func_gethrtime=no}
+ac_cv_func_getifaddrs=${ac_cv_func_getifaddrs=no}
+ac_cv_func_getipnodebyaddr=${ac_cv_func_getipnodebyaddr=no}
+ac_cv_func_getipnodebyname=${ac_cv_func_getipnodebyname=no}
+ac_cv_func_getpagesize=${ac_cv_func_getpagesize=no}
+ac_cv_func_gettimeofday=${ac_cv_func_gettimeofday=no}
+ac_cv_func_gmtime_r=${ac_cv_func_gmtime_r=no}
+ac_cv_func_ieee_handler=${ac_cv_func_ieee_handler=no}
+ac_cv_func_inet_ntoa=${ac_cv_func_inet_ntoa=no}
+ac_cv_func_inet_pton=${ac_cv_func_inet_pton=yes}
+ac_cv_func_isinf=${ac_cv_func_isinf=no}
+ac_cv_func_isnan=${ac_cv_func_isnan=no}
+ac_cv_func_localtime_r=${ac_cv_func_localtime_r=no}
+ac_cv_func_mallopt=${ac_cv_func_mallopt=no}
+ac_cv_func_memchr=${ac_cv_func_memchr=yes}
+ac_cv_func_memcmp_working=${ac_cv_func_memcmp_working=yes}
+ac_cv_func_memcpy=${ac_cv_func_memcpy=no}
+ac_cv_func_memmove=${ac_cv_func_memmove=yes}
+ac_cv_func_memset=${ac_cv_func_memset=yes}
+ac_cv_func_mmap=${ac_cv_func_mmap=no}
+ac_cv_func_mmap_fixed_mapped=${ac_cv_func_mmap_fixed_mapped=no}
+ac_cv_func_mremap=${ac_cv_func_mremap=no}
+ac_cv_func_nl_langinfo=${ac_cv_func_nl_langinfo=no}
+ac_cv_func_openpty=${ac_cv_func_openpty=no}
+ac_cv_func_poll=${ac_cv_func_poll=no}
+ac_cv_func_posix2time=${ac_cv_func_posix2time=no}
+ac_cv_func_posix_fadvise=${ac_cv_func_posix_fadvise=no}
+ac_cv_func_pread=${ac_cv_func_pread=no}
+ac_cv_func_pwrite=${ac_cv_func_pwrite=no}
+ac_cv_func_res_gethostbyname=${ac_cv_func_res_gethostbyname=no}
+ac_cv_func_sbrk=${ac_cv_func_sbrk=no}
+ac_cv_func_sctp_bindx=${ac_cv_func_sctp_bindx=no}
+ac_cv_func_sctp_peeloff=${ac_cv_func_sctp_peeloff=no}
+ac_cv_func_select=${ac_cv_func_select=no}
+ac_cv_func_setlocale=${ac_cv_func_setlocale=yes}
+ac_cv_func_setsid=${ac_cv_func_setsid=no}
+ac_cv_func_setvbuf_reversed=${ac_cv_func_setvbuf_reversed=yes}
+ac_cv_func_socket=${ac_cv_func_socket=no}
+ac_cv_func_strchr=${ac_cv_func_strchr=yes}
+ac_cv_func_strerror=${ac_cv_func_strerror=yes}
+ac_cv_func_strerror_r=${ac_cv_func_strerror_r=no}
+ac_cv_func_strlcat=${ac_cv_func_strlcat=no}
+ac_cv_func_strlcpy=${ac_cv_func_strlcpy=no}
+ac_cv_func_strncasecmp=${ac_cv_func_strncasecmp=no}
+ac_cv_func_strrchr=${ac_cv_func_strrchr=yes}
+ac_cv_func_strstr=${ac_cv_func_strstr=yes}
+ac_cv_func_uname=${ac_cv_func_uname=no}
+ac_cv_func_vfork=${ac_cv_func_vfork=no}
+ac_cv_func_vfork_works=${ac_cv_func_vfork_works=no}
+ac_cv_func_vprintf=${ac_cv_func_vprintf=yes}
+ac_cv_func_writev=${ac_cv_func_writev=no}
+ac_cv_have_decl_SCTPS_BOUND=${ac_cv_have_decl_SCTPS_BOUND=no}
+ac_cv_have_decl_SCTPS_COOKIE_ECHOED=${ac_cv_have_decl_SCTPS_COOKIE_ECHOED=no}
+ac_cv_have_decl_SCTPS_COOKIE_WAIT=${ac_cv_have_decl_SCTPS_COOKIE_WAIT=no}
+ac_cv_have_decl_SCTPS_ESTABLISHED=${ac_cv_have_decl_SCTPS_ESTABLISHED=no}
+ac_cv_have_decl_SCTPS_IDLE=${ac_cv_have_decl_SCTPS_IDLE=no}
+ac_cv_have_decl_SCTPS_LISTEN=${ac_cv_have_decl_SCTPS_LISTEN=no}
+ac_cv_have_decl_SCTPS_SHUTDOWN_ACK_SENT=${ac_cv_have_decl_SCTPS_SHUTDOWN_ACK_SENT=no}
+ac_cv_have_decl_SCTPS_SHUTDOWN_PENDING=${ac_cv_have_decl_SCTPS_SHUTDOWN_PENDING=no}
+ac_cv_have_decl_SCTPS_SHUTDOWN_RECEIVED=${ac_cv_have_decl_SCTPS_SHUTDOWN_RECEIVED=no}
+ac_cv_have_decl_SCTPS_SHUTDOWN_SENT=${ac_cv_have_decl_SCTPS_SHUTDOWN_SENT=no}
+ac_cv_have_decl_SCTP_ABORT=${ac_cv_have_decl_SCTP_ABORT=no}
+ac_cv_have_decl_SCTP_ADDR_CONFIRMED=${ac_cv_have_decl_SCTP_ADDR_CONFIRMED=no}
+ac_cv_have_decl_SCTP_ADDR_OVER=${ac_cv_have_decl_SCTP_ADDR_OVER=no}
+ac_cv_have_decl_SCTP_BOUND=${ac_cv_have_decl_SCTP_BOUND=no}
+ac_cv_have_decl_SCTP_CLOSED=${ac_cv_have_decl_SCTP_CLOSED=no}
+ac_cv_have_decl_SCTP_COOKIE_ECHOED=${ac_cv_have_decl_SCTP_COOKIE_ECHOED=no}
+ac_cv_have_decl_SCTP_COOKIE_WAIT=${ac_cv_have_decl_SCTP_COOKIE_WAIT=no}
+ac_cv_have_decl_SCTP_DELAYED_ACK_TIME=${ac_cv_have_decl_SCTP_DELAYED_ACK_TIME=no}
+ac_cv_have_decl_SCTP_EMPTY=${ac_cv_have_decl_SCTP_EMPTY=no}
+ac_cv_have_decl_SCTP_EOF=${ac_cv_have_decl_SCTP_EOF=no}
+ac_cv_have_decl_SCTP_ESTABLISHED=${ac_cv_have_decl_SCTP_ESTABLISHED=no}
+ac_cv_have_decl_SCTP_LISTEN=${ac_cv_have_decl_SCTP_LISTEN=no}
+ac_cv_have_decl_SCTP_SENDALL=${ac_cv_have_decl_SCTP_SENDALL=no}
+ac_cv_have_decl_SCTP_SHUTDOWN_ACK_SENT=${ac_cv_have_decl_SCTP_SHUTDOWN_ACK_SENT=no}
+ac_cv_have_decl_SCTP_SHUTDOWN_PENDING=${ac_cv_have_decl_SCTP_SHUTDOWN_PENDING=no}
+ac_cv_have_decl_SCTP_SHUTDOWN_RECEIVED=${ac_cv_have_decl_SCTP_SHUTDOWN_RECEIVED=no}
+ac_cv_have_decl_SCTP_SHUTDOWN_SENT=${ac_cv_have_decl_SCTP_SHUTDOWN_SENT=no}
+ac_cv_have_decl_SCTP_UNORDERED=${ac_cv_have_decl_SCTP_UNORDERED=no}
+ac_cv_have_decl_posix2time=${ac_cv_have_decl_posix2time=no}
+ac_cv_header_arpa_inet_h=${ac_cv_header_arpa_inet_h=no}
+ac_cv_header_arpa_nameser_h=${ac_cv_header_arpa_nameser_h=no}
+ac_cv_header_dirent_dirent_h=${ac_cv_header_dirent_dirent_h=no}
+ac_cv_header_dirent_ndir_h=${ac_cv_header_dirent_ndir_h=no}
+ac_cv_header_dirent_sys_dir_h=${ac_cv_header_dirent_sys_dir_h=no}
+ac_cv_header_dirent_sys_ndir_h=${ac_cv_header_dirent_sys_ndir_h=no}
+ac_cv_header_dlfcn_h=${ac_cv_header_dlfcn_h=no}
+ac_cv_header_fcntl_h=${ac_cv_header_fcntl_h=yes}
+ac_cv_header_gl_gl_h=${ac_cv_header_gl_gl_h=yes}
+ac_cv_header_ieeefp_h=${ac_cv_header_ieeefp_h=no}
+ac_cv_header_ifaddrs_h=${ac_cv_header_ifaddrs_h=no}
+ac_cv_header_inttypes_h=${ac_cv_header_inttypes_h=no}
+ac_cv_header_langinfo_h=${ac_cv_header_langinfo_h=no}
+ac_cv_header_limits_h=${ac_cv_header_limits_h=yes}
+ac_cv_header_mach_o_dyld_h=${ac_cv_header_mach_o_dyld_h=no}
+ac_cv_header_malloc_h=${ac_cv_header_malloc_h=yes}
+ac_cv_header_memory_h=${ac_cv_header_memory_h=yes}
+ac_cv_header_net_errno_h=${ac_cv_header_net_errno_h=no}
+ac_cv_header_net_if_dl_h=${ac_cv_header_net_if_dl_h=no}
+ac_cv_header_netdb_h=${ac_cv_header_netdb_h=no}
+ac_cv_header_netinet_in_h=${ac_cv_header_netinet_in_h=no}
+ac_cv_header_netpacket_packet_h=${ac_cv_header_netpacket_packet_h=no}
+ac_cv_header_poll_h=${ac_cv_header_poll_h=no}
+ac_cv_header_pty_h=${ac_cv_header_pty_h=no}
+ac_cv_header_stdc=${ac_cv_header_stdc=yes}
+ac_cv_header_stddef_h=${ac_cv_header_stddef_h=yes}
+ac_cv_header_stdint_h=${ac_cv_header_stdint_h=yes}
+ac_cv_header_stdlib_h=${ac_cv_header_stdlib_h=yes}
+ac_cv_header_string_h=${ac_cv_header_string_h=yes}
+ac_cv_header_strings_h=${ac_cv_header_strings_h=no}
+ac_cv_header_sys_devpoll_h=${ac_cv_header_sys_devpoll_h=no}
+ac_cv_header_sys_epoll_h=${ac_cv_header_sys_epoll_h=no}
+ac_cv_header_sys_event_h=${ac_cv_header_sys_event_h=no}
+ac_cv_header_sys_ioctl_h=${ac_cv_header_sys_ioctl_h=no}
+ac_cv_header_sys_param_h=${ac_cv_header_sys_param_h=no}
+ac_cv_header_sys_resource_h=${ac_cv_header_sys_resource_h=no}
+ac_cv_header_sys_select_h=${ac_cv_header_sys_select_h=no}
+ac_cv_header_sys_socket_h=${ac_cv_header_sys_socket_h=no}
+ac_cv_header_sys_socketio_h=${ac_cv_header_sys_socketio_h=no}
+ac_cv_header_sys_sockio_h=${ac_cv_header_sys_sockio_h=no}
+ac_cv_header_sys_stat_h=${ac_cv_header_sys_stat_h=yes}
+ac_cv_header_sys_stropts_h=${ac_cv_header_sys_stropts_h=no}
+ac_cv_header_sys_sysctl_h=${ac_cv_header_sys_sysctl_h=no}
+ac_cv_header_sys_time_h=${ac_cv_header_sys_time_h=no}
+ac_cv_header_sys_types_h=${ac_cv_header_sys_types_h=yes}
+ac_cv_header_sys_uio_h=${ac_cv_header_sys_uio_h=no}
+ac_cv_header_sys_wait_h=${ac_cv_header_sys_wait_h=no}
+ac_cv_header_syslog_h=${ac_cv_header_syslog_h=no}
+ac_cv_header_time=${ac_cv_header_time=no}
+ac_cv_header_unistd_h=${ac_cv_header_unistd_h=no}
+ac_cv_header_util_h=${ac_cv_header_util_h=no}
+ac_cv_header_utmp_h=${ac_cv_header_utmp_h=no}
+ac_cv_header_valgrind_valgrind_h=${ac_cv_header_valgrind_valgrind_h=no}
+ac_cv_header_vfork_h=${ac_cv_header_vfork_h=no}
+ac_cv_header_windows_h=${ac_cv_header_windows_h=yes}
+ac_cv_header_winsock2_h=${ac_cv_header_winsock2_h=yes}
+ac_cv_header_ws2tcpip_h=${ac_cv_header_ws2tcpip_h=yes}
+ac_cv_lib_dl_dlopen=${ac_cv_lib_dl_dlopen=no}
+ac_cv_lib_inet_main=${ac_cv_lib_inet_main=no}
+ac_cv_lib_kstat_kstat_open=${ac_cv_lib_kstat_kstat_open=no}
+ac_cv_lib_m_sin=${ac_cv_lib_m_sin=no}
+ac_cv_lib_nsl_gethostbyname=${ac_cv_lib_nsl_gethostbyname=no}
+ac_cv_lib_nsl_main=${ac_cv_lib_nsl_main=no}
+ac_cv_lib_resolv_res_gethostbyname=${ac_cv_lib_resolv_res_gethostbyname=no}
+ac_cv_lib_rt_clock_gettime=${ac_cv_lib_rt_clock_gettime=no}
+ac_cv_lib_socket_getpeername=${ac_cv_lib_socket_getpeername=no}
+ac_cv_lib_socket_main=${ac_cv_lib_socket_main=yes}
+ac_cv_lib_socket_socket=${ac_cv_lib_socket_socket=no}
+ac_cv_lib_util_openpty=${ac_cv_lib_util_openpty=no}
+ac_cv_lib_ws2_32_main=${ac_cv_lib_ws2_32_main=yes}
+ac_cv_member_struct_ErlDrvEntry_stop_select=${ac_cv_member_struct_ErlDrvEntry_stop_select=no}
+ac_cv_member_struct_sctp_paddrparams_spp_flags=${ac_cv_member_struct_sctp_paddrparams_spp_flags=no}
+ac_cv_member_struct_sctp_paddrparams_spp_pathmtu=${ac_cv_member_struct_sctp_paddrparams_spp_pathmtu=no}
+ac_cv_member_struct_sctp_paddrparams_spp_sackdelay=${ac_cv_member_struct_sctp_paddrparams_spp_sackdelay=no}
+ac_cv_member_struct_sctp_remote_error_sre_data=${ac_cv_member_struct_sctp_remote_error_sre_data=no}
+ac_cv_member_struct_sctp_send_failed_ssf_data=${ac_cv_member_struct_sctp_send_failed_ssf_data=no}
+ac_cv_objext=${ac_cv_objext=o}
+ac_cv_path_MKDIR=${ac_cv_path_MKDIR=/bin/mkdir}
+ac_cv_path_RM=${ac_cv_path_RM=/bin/rm}
+ac_cv_prog_AR=${ac_cv_prog_AR=ar.sh}
+ac_cv_prog_CC=${ac_cv_prog_CC=cc.sh}
+ac_cv_prog_CPP=${ac_cv_prog_CPP='cc.sh -E'}
+ac_cv_prog_CXX=${ac_cv_prog_CXX=cc.sh}
+ac_cv_prog_DED_LD=${ac_cv_prog_DED_LD=ld.sh}
+ac_cv_prog_JAVAC=${ac_cv_prog_JAVAC=javac.sh}
+ac_cv_prog_M4=${ac_cv_prog_M4=m4}
+ac_cv_prog_RANLIB=${ac_cv_prog_RANLIB=true}
+ac_cv_prog_cc_c89=${ac_cv_prog_cc_c89=}
+ac_cv_prog_cc_g=${ac_cv_prog_cc_g=yes}
+ac_cv_search_fdatasync=${ac_cv_search_fdatasync=no}
+ac_cv_search_opendir=${ac_cv_search_opendir=no}
+ac_cv_search_strerror=${ac_cv_search_strerror='none required'}
+ac_cv_sizeof___int64=${ac_cv_sizeof___int64=8}
+ac_cv_sizeof_char=${ac_cv_sizeof_char=1}
+ac_cv_sizeof_int=${ac_cv_sizeof_int=4}
+ac_cv_sizeof_long=${ac_cv_sizeof_long=4}
+ac_cv_sizeof_long_long=${ac_cv_sizeof_long_long=8}
+ac_cv_sizeof_off_t=${ac_cv_sizeof_off_t=4}
+ac_cv_sizeof_short=${ac_cv_sizeof_short=2}
+ac_cv_sizeof_size_t=${ac_cv_sizeof_size_t=8}
+ac_cv_sizeof_void_p=${ac_cv_sizeof_void_p=8}
+ac_cv_struct_exception=${ac_cv_struct_exception=no}
+ac_cv_struct_sockaddr_sa_len=${ac_cv_struct_sockaddr_sa_len=no}
+ac_cv_struct_tm=${ac_cv_struct_tm=time.h}
+ac_cv_type_off_t=${ac_cv_type_off_t=yes}
+ac_cv_type_pid_t=${ac_cv_type_pid_t=no}
+ac_cv_type_signal=${ac_cv_type_signal=void}
+ac_cv_type_size_t=${ac_cv_type_size_t=yes}
+ac_cv_type_uid_t=${ac_cv_type_uid_t=no}
+ac_cv_working_alloca_h=${ac_cv_working_alloca_h=no}
+erl_cv_time_correction=${erl_cv_time_correction=none}
+erts_cv___after_morecore_hook_can_track_malloc=${erts_cv___after_morecore_hook_can_track_malloc=no}
+erts_cv_fwrite_unlocked=${erts_cv_fwrite_unlocked=no}
+erts_cv_have__end_symbol=${erts_cv_have__end_symbol=no}
+erts_cv_have_end_symbol=${erts_cv_have_end_symbol=no}
+erts_cv_putc_unlocked=${erts_cv_putc_unlocked=no}
+erts_cv_windows_h_includes_winsock2_h=${erts_cv_windows_h_includes_winsock2_h=no}
diff --git a/erts/configure.in b/erts/configure.in
index 5fa1245b13..b3289bf84c 100644
--- a/erts/configure.in
+++ b/erts/configure.in
@@ -2,7 +2,7 @@ dnl Process this file with autoconf to produce a configure script. -*-m4-*-
 
 dnl %CopyrightBegin%
 dnl
-dnl Copyright Ericsson AB 1997-2010. All Rights Reserved.
+dnl Copyright Ericsson AB 1997-2012. All Rights Reserved.
 dnl
 dnl The contents of this file are subject to the Erlang Public License,
 dnl Version 1.1, (the "License"); you may not use this file except in
@@ -61,6 +61,9 @@ if test x"${ERL_TOP}/erts" != x"$srcdir"; then
 fi
 erl_top=${ERL_TOP}
 
+# Remove old configuration information
+/bin/rm -f "$ERL_TOP/erts/CONF_INFO"
+
 # echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 # echo X
 # echo "X srcdir  = $srcdir"
@@ -87,6 +90,13 @@ else
     host_os=$host
 fi
 
+if test "$cross_compiling" = "yes"; then
+    CROSS_COMPILING=yes
+else
+    CROSS_COMPILING=no
+fi
+AC_SUBST(CROSS_COMPILING)
+
 ERL_XCOMP_SYSROOT_INIT
 
 AC_ISC_POSIX
@@ -100,7 +110,8 @@ ENABLE_ALLOC_TYPE_VARS=
 AC_SUBST(ENABLE_ALLOC_TYPE_VARS)
 
 AC_ARG_ENABLE(bootstrap-only,
-[  --enable-bootstrap-only enable bootstrap only configuration],
+AS_HELP_STRING([--enable-bootstrap-only],
+               [enable bootstrap only configuration]),
 [ if test "X$enableval" = "Xyes"; then
 	# Disable stuff not necessary in a bootstrap only system in order
 	# to speed up things by reducing the amount of stuff needing to be
@@ -116,38 +127,38 @@ AC_ARG_ENABLE(bootstrap-only,
 ])
 
 AC_ARG_ENABLE(threads,
-[  --enable-threads        enable async thread support
-  --disable-threads       disable async thread support],
+AS_HELP_STRING([--enable-threads], [enable async thread support])
+AS_HELP_STRING([--disable-threads], [disable async thread support]),
 [ case "$enableval" in
     no) enable_threads=no ;;
     *)  enable_threads=yes ;;
   esac ], enable_threads=unknown)
 
+AC_ARG_ENABLE(halfword-emulator,
+AS_HELP_STRING([--enable-halfword-emulator],
+               [enable halfword emulator (only for 64bit builds)]),
+[ case "$enableval" in
+    no) enable_halfword_emualtor=no ;;
+    *)  enable_halfword_emulator=yes ;;
+  esac ], enable_halfword_emulator=unknown)
+
 AC_ARG_ENABLE(smp-support,
-[  --enable-smp-support    enable smp support
-  --disable-smp-support   disable smp support],
+AS_HELP_STRING([--enable-smp-support], [enable smp support])
+AS_HELP_STRING([--disable-smp-support], [disable smp support]),
 [ case "$enableval" in
     no) enable_smp_support=no ;;
     *)  enable_smp_support=yes ;;
   esac ], enable_smp_support=unknown)
 
 AC_ARG_WITH(termcap,
-[  --with-termcap          use termcap (default)
-  --without-termcap       do not use any termcap libraries (ncurses,curses,termcap,termlib)],
+AS_HELP_STRING([--with-termcap], [use termcap (default)])
+AS_HELP_STRING([--without-termcap],
+               [do not use any termcap libraries (ncurses,curses,termcap,termlib)]),
 [],
 [with_termcap=yes])
 
-
-AC_ARG_ENABLE(hybrid-heap,
-[  --enable-hybrid-heap    enable hybrid heap
-  --disable-hybrid-heap   disable hybrid heap],
-[ case "$enableval" in
-    no) enable_hybrid_heap=no ;;
-    *)  enable_hybrid_heap=yes ;;
-  esac ], enable_hybrid_heap=unknown)
-
 AC_ARG_ENABLE(lock-checking,
-[  --enable-lock-checking  enable lock checking],
+AS_HELP_STRING([--enable-lock-checking], [enable lock checking]),
 [ case "$enableval" in
     no) enable_lock_check=no ;;
     *)  enable_lock_check=yes ;;
@@ -156,16 +167,15 @@ AC_ARG_ENABLE(lock-checking,
   enable_lock_check=no)
 
 AC_ARG_ENABLE(lock-counter,
-[  --enable-lock-counter    enable lock counters
-  --disable-lock-counter   disable lock counters],
+AS_HELP_STRING([--enable-lock-counter], [enable lock counters]),
 [ case "$enableval" in
     no) enable_lock_count=no ;;
     *)  enable_lock_count=yes ;;
   esac ], enable_lock_count=no)
 
 AC_ARG_ENABLE(kernel-poll,
-[  --enable-kernel-poll    enable kernel poll support
-  --disable-kernel-poll   disable kernel poll support],
+AS_HELP_STRING([--enable-kernel-poll], [enable kernel poll support])
+AS_HELP_STRING([--disable-kernel-poll], [disable kernel poll support]),
 [ case "$enableval" in
     no) enable_kernel_poll=no ;;
     *)  enable_kernel_poll=yes ;;
@@ -173,28 +183,27 @@ AC_ARG_ENABLE(kernel-poll,
 
 
 AC_ARG_ENABLE(sctp,
-[  --enable-sctp           enable sctp support
-  --disable-sctp          disable sctp support],
+AS_HELP_STRING([--enable-sctp], [enable sctp support (default)])
+AS_HELP_STRING([--disable-sctp], [disable sctp support]),
 [ case "$enableval" in
     no) enable_sctp=no ;;
     *)  enable_sctp=yes ;;
   esac ], enable_sctp=unknown)
 
 AC_ARG_ENABLE(hipe,
-[  --enable-hipe           enable hipe support
-  --disable-hipe          disable hipe support])
+AS_HELP_STRING([--enable-hipe], [enable hipe support])
+AS_HELP_STRING([--disable-hipe], [disable hipe support]))
 
 AC_ARG_ENABLE(native-libs,
-[  --enable-native-libs    compile Erlang libraries to native code])
+AS_HELP_STRING([--enable-native-libs],
+               [compile Erlang libraries to native code]))
 
 AC_ARG_ENABLE(tsp,
-[  --enable-tsp            compile tsp app])
-
-AC_ARG_ENABLE(elib-malloc,
-[  --enable-elib-malloc    use elib_malloc instead of normal malloc])
+AS_HELP_STRING([--enable-tsp], [compile tsp app]))
 
 AC_ARG_ENABLE(fp-exceptions,
-[  --enable-fp-exceptions    Use hardware floating point exceptions (default if hipe enabled)],
+AS_HELP_STRING([--enable-fp-exceptions],
+               [use hardware floating point exceptions (default if hipe enabled)]),
 [ case "$enableval" in
     no) enable_fp_exceptions=no ;;
     *)  enable_fp_exceptions=yes ;;
@@ -202,7 +211,8 @@ AC_ARG_ENABLE(fp-exceptions,
 ],enable_fp_exceptions=auto)
 
 AC_ARG_ENABLE(darwin-universal,
-[  --enable-darwin-universal    build universal binaries on darwin i386],
+AS_HELP_STRING([--enable-darwin-universal],
+               [build universal binaries on darwin i386]),
 [ case "$enableval" in
     no) enable_darwin_universal=no ;;
     *)  enable_darwin_univeral=yes ;;
@@ -211,7 +221,7 @@ AC_ARG_ENABLE(darwin-universal,
 
 
 AC_ARG_ENABLE(darwin-64bit,
-[  --enable-darwin-64bit    build 64bit binaries on darwin],
+AS_HELP_STRING([--enable-darwin-64bit], [build 64bit binaries on darwin]),
 [ case "$enableval" in
     no) enable_darwin_64bit=no ;;
     *)  enable_darwin_64bit=yes ;;
@@ -219,7 +229,8 @@ AC_ARG_ENABLE(darwin-64bit,
 ],enable_darwin_64bit=no)
 
 AC_ARG_ENABLE(m64-build,
-[  --enable-m64-build    build 64bit binaries using the -m64 flag to (g)cc],
+AS_HELP_STRING([--enable-m64-build],
+               [build 64bit binaries using the -m64 flag to (g)cc]),
 [ case "$enableval" in
     no) enable_m64_build=no ;;
     *)  enable_m64_build=yes ;;
@@ -227,7 +238,8 @@ AC_ARG_ENABLE(m64-build,
 ],enable_m64_build=no)
 
 AC_ARG_ENABLE(m32-build,
-[  --enable-m32-build    build 32bit binaries using the -m32 flag to (g)cc],
+AS_HELP_STRING([--enable-m32-build],
+               [build 32bit binaries using the -m32 flag to (g)cc]),
 [ case "$enableval" in
     no) enable_m32_build=no ;;
     *)
@@ -239,17 +251,11 @@ AC_ARG_ENABLE(m32-build,
   esac
 ],enable_m32_build=no)
 
-AC_ARG_ENABLE(fixalloc,
-[  --disable-fixalloc      disable the use of fix_alloc])
-if test x${enable_fixalloc} = xno ; then
-  AC_DEFINE(NO_FIX_ALLOC,[],
-	    [Define if you don't want the fix allocator in Erlang])
-fi
-
 AC_SUBST(PERFCTR_PATH)
 AC_ARG_WITH(perfctr,
-[  --with-perfctr=PATH     specify location of perfctr include and lib
-  --without-perfctr       don't use perfctr (default)])
+AS_HELP_STRING([--with-perfctr=PATH],
+               [specify location of perfctr include and lib])
+AS_HELP_STRING([--without-perfctr], [don't use perfctr (default)]))
 
 if test "x$with_perfctr" = "xno" -o "x$with_perfctr" = "x" ; then
     PERFCTR_PATH=
@@ -262,27 +268,69 @@ else
               [Define to enable hrvtime() on Linux systems with perfctr extension])
 fi
 
+
+AC_ARG_WITH(dynamic-trace,
+AS_HELP_STRING([--with-dynamic-trace={dtrace|systemtap}],
+	       [specify use of dynamic trace framework, dtrace or systemtap])
+AS_HELP_STRING([--without-dynamic-trace], 
+               [don't enable any dynamic tracing (default)]))
+
+if test X"$with_dynamic_trace" = X""; then
+   with_dynamic_trace=no
+fi
+
+case "$with_dynamic_trace" in
+     no) DYNAMIC_TRACE_FRAMEWORK=;;
+     dtrace) 
+	    AC_DEFINE(USE_DTRACE,[1],
+		[Define if you want to use dtrace for dynamic tracing])
+     	     DYNAMIC_TRACE_FRAMEWORK=dtrace;;
+     systemtap) 
+	    AC_DEFINE(USE_SYSTEMTAP,[1],
+		[Define if you want to use systemtap for dynamic tracing])
+	    DYNAMIC_TRACE_FRAMEWORK=systemtap;;
+     *)	
+     	    AC_MSG_ERROR(Unknown dynamic tracing framework specified with --with-dynamic-trace!);;
+esac
+
+if test X"$DYNAMIC_TRACE_FRAMEWORK" != X""; then
+   AC_DEFINE(USE_DYNAMIC_TRACE,[1],
+		[Define if you want to use dynamic tracing])
+fi
+
+AC_ARG_ENABLE(vm-probes,
+AS_HELP_STRING([--enable-vm-probes],
+               [add dynamic trace probes to the Beam VM (only possible if --with-dynamic-trace is enabled, and then default)]),
+	       [ case "$enableval" in
+    	       	 no) use_vm_probes=no ;;
+    		 *)
+			if  test X"$DYNAMIC_TRACE_FRAMEWORK" != X""; then
+		 	    use_vm_probes=yes ;
+			else
+			    AC_MSG_ERROR(Can not enable VM probes without any dynamic tracing framework!);
+			fi;;  
+  		 esac ], if  test X"$DYNAMIC_TRACE_FRAMEWORK" != X""; then
+		 	    use_vm_probes=yes ;
+			else
+			    use_vm_probes=no
+			fi)
+
+AC_SUBST(USE_VM_PROBES)
+if test X"$use_vm_probes" = X"yes"; then
+   USE_VM_PROBES=yes
+   AC_DEFINE(USE_VM_PROBES,[1],
+	     [Define to enable VM dynamic trace probes])
+fi
+
+
 AC_ARG_ENABLE(clock-gettime,
-[  --enable-clock-gettime  Use clock-gettime for time correction],
+AS_HELP_STRING([--enable-clock-gettime],
+               [use clock-gettime for time correction]),
 [ case "$enableval" in
     no) clock_gettime_correction=no ;;
     *) clock_gettime_correction=yes ;;
   esac ], clock_gettime_correction=unknown)
 
-AC_ARG_ENABLE(native-ethr-impls,
-[  --enable-native-ethr-impls enable native ethread implementations
-  --disable-native-ethr-impls disable native ethread implementations],
-[ case "$enableval" in
-    no) disable_native_ethr_impls=yes ;;
-    *)  disable_native_ethr_impls=no ;;
-  esac ], disable_native_ethr_impls=no)
-
-dnl Defined in libraries/megaco/configure.in but we need it here
-dnl also in order to show it to the "top user"
-
-AC_ARG_ENABLE(megaco_flex_scanner_lineno,
-[  --disable-megaco-flex-scanner-lineno disable megaco flex scanner lineno])
-
 dnl Magic test for clearcase.
 OTP_RELEASE=
 if test "${ERLANG_COMMERCIAL_BUILD}" != ""; then
@@ -382,12 +430,6 @@ case $host_os in
 	# The ethread library requires _WIN32_WINNT of at least 0x0403.
 	# -D_WIN32_WINNT=* from CPPFLAGS is saved in ETHR_DEFS.
 	CPPFLAGS="$CPPFLAGS -D_WIN32_WINNT=0x0500 -DWINVER=0x0500"
-	# _USE_32BIT_TIME_T is needed when using VC++ 2005 (ctime() will fail
-	# otherwise since we pass it a 32-bit value).
-	#
-	# FIXME: Use time_t all the way and remove _USE_32BIT_TIME_T.
-	AC_MSG_WARN([Reverting to 32-bit time_t])
-	CPPFLAGS="$CPPFLAGS -D_USE_32BIT_TIME_T"
 	;;
     darwin*)
         CPPFLAGS="$CPPFLAGS -D_XOPEN_SOURCE"
@@ -397,53 +439,7 @@ case $host_os in
 esac
 
 
-
-MIXED_CYGWIN=no
-
-AC_MSG_CHECKING(for mixed cygwin and native VC++ environment)
-if test "X$host" = "Xwin32" -a "x$GCC" != x"yes"; then
-	if test -x /usr/bin/cygpath; then
-		CFLAGS="-O2"
-		MIXED_CYGWIN=yes
-		AC_MSG_RESULT([yes])
-		MIXED_CYGWIN_VC=yes
-		CPPFLAGS="$CPPFLAGS -DERTS_MIXED_CYGWIN_VC"
-	else
-		AC_MSG_RESULT([undeterminable])
-		AC_MSG_ERROR(Seems to be mixed windows but not with cygwin, cannot handle this!)
-	fi
-else
-	AC_MSG_RESULT([no])
-	MIXED_CYGWIN_VC=no
-fi
-AC_SUBST(MIXED_CYGWIN_VC)
-
-AC_MSG_CHECKING(for mixed cygwin and native MinGW environment)
-if test "X$host" = "Xwin32" -a "x$GCC" = x"yes"; then
-	if test -x /usr/bin/cygpath; then
-		CFLAGS="-O2"
-		MIXED_CYGWIN=yes
-		AC_MSG_RESULT([yes])
-		MIXED_CYGWIN_MINGW=yes
-		CPPFLAGS="$CPPFLAGS -DERTS_MIXED_CYGWIN_MINGW"
-	else
-		AC_MSG_RESULT([undeterminable])
-		AC_MSG_ERROR(Seems to be mixed windows but not with cygwin, cannot handle this!)
-	fi
-else
-	AC_MSG_RESULT([no])
-	MIXED_CYGWIN_MINGW=no
-fi
-AC_SUBST(MIXED_CYGWIN_MINGW)
-
-AC_MSG_CHECKING(if we mix cygwin with any native compiler)
-if test "X$MIXED_CYGWIN" = "Xyes" ; then
-	AC_MSG_RESULT([yes])	
-else
-	AC_MSG_RESULT([no])
-fi
-
-AC_SUBST(MIXED_CYGWIN)
+LM_WINDOWS_ENVIRONMENT
 	
 dnl
 dnl Flags to the C compiler
@@ -489,7 +485,6 @@ CFLAG_RUNTIME_LIBRARY_PATH="-Wl,-R"
 case $host_os in
   darwin*)
 	CFLAG_RUNTIME_LIBRARY_PATH=
-	CFLAGS="$CFLAGS -no-cpp-precomp"
 	;;
   win32)
 	CFLAG_RUNTIME_LIBRARY_PATH=
@@ -572,6 +567,11 @@ AC_SUBST(WFLAGS)
 AC_SUBST(CFLAG_RUNTIME_LIBRARY_PATH)
 
 AC_CHECK_SIZEOF(void *) # Needed for ARCH and smp checks below
+if test "x$ac_cv_sizeof_void_p" = x8; then
+  AC_SUBST(EXTERNAL_WORD_SIZE, 64)
+else
+  AC_SUBST(EXTERNAL_WORD_SIZE, 32)
+fi
 
 dnl
 dnl Figure out operating system and cpu architecture
@@ -618,6 +618,8 @@ case $chk_arch_ in
     armv5b)	ARCH=arm;;
     armv5teb)	ARCH=arm;;
     armv5tel)	ARCH=arm;;
+    armv5tejl)	ARCH=arm;;
+    armv7l)	ARCH=arm;;
     tile)	ARCH=tile;;
     *)	 	ARCH=noarch;;
 esac
@@ -749,6 +751,26 @@ esac
 	
 AC_SUBST(LIBCARBON)
 
+dnl Check if we should/can build a halfword emulator
+
+AC_MSG_CHECKING(if we are building a halfword emulator (32bit heap on 64bit machine))
+if test "$enable_halfword_emulator" = "yes"; then
+	if test "$ARCH" = "amd64"; then
+		AC_DEFINE(HALFWORD_HEAP_EMULATOR, [1],
+			  [Define if building a halfword-heap 64bit emulator])
+		ENABLE_ALLOC_TYPE_VARS="$ENABLE_ALLOC_TYPE_VARS halfword"
+		AC_MSG_RESULT([yes])
+	else
+		AC_MSG_ERROR(no; halfword emulator not supported on this architecture)
+	fi
+else
+	AC_MSG_RESULT([no])
+fi
+
+
+
+
+
 dnl some tests below will call this if we haven't already - and autoconf
 dnl can't handle those tests being done conditionally at runtime
 AC_PROG_CPP
@@ -794,13 +816,14 @@ fi
 AC_CHECK_PROGS(XSLTPROC, xsltproc)
 if test -z "$XSLTPROC"; then
   echo "xsltproc" >> doc/CONF_INFO
-  AC_MSG_WARN([No 'xsltproc' command found: the documentation can not be built])
+  AC_MSG_WARN([No 'xsltproc' command found: the documentation cannot be built])
 fi
 
 AC_CHECK_PROGS(FOP, fop)
 if test -z "$FOP"; then
+  FOP="$ERL_TOP/make/fakefop"
   echo "fop" >> doc/CONF_INFO
-  AC_MSG_WARN([No 'fop' command found: the documentation can not be built])
+  AC_MSG_WARN([No 'fop' command found: going to generate placeholder PDF files])
 fi
 
 dnl
@@ -880,16 +903,6 @@ fi
 
 AC_SUBST(ERLANG_OSTYPE)
 
-dnl Which sysv4 would this be, and what is it for???
-dnl XXX: replace with feature tests.
-case $host_os in
-  sysv4*)
-    AC_DEFINE(SOCKOPT_CONNECT_STAT,[],[Obscure SYSV feature])
-    AC_DEFINE(NO_PRAGMA_WEAK,[],[Obscure SYSV feature])
-    LIBS="$LIBS -lgen -lc -L /usr/ucblib -lucb"
-    ;;
-esac
-
 # Check how to export functions from the emulator executable, needed
 # when dynamically loaded drivers are loaded (so that they can find
 # emulator functions).
@@ -1021,11 +1034,48 @@ if test $ERTS_BUILD_SMP_EMU = yes; then
 	fi
 
 	AC_DEFINE(ERTS_HAVE_SMP_EMU, 1, [Define if the smp emulator is built])
+
+	case "$ethr_have_native_atomics-$ethr_have_native_spinlock" in
+	  yes-*)
+		;;
+
+	  no-yes)
+
+		test -f "$ERL_TOP/erts/CONF_INFO" ||
+		    echo "" > "$ERL_TOP/erts/CONF_INFO"
+		cat >> $ERL_TOP/erts/CONF_INFO <<EOF
+
+                 No native atomic implementation available.
+                 Fallbacks implemented using spinlocks will be
+                 used. Note that the performance of the SMP
+                 runtime system will suffer due to this.
+
+EOF
+		;;
+
+	  no-no)
+
+		test -f "$ERL_TOP/erts/CONF_INFO" ||
+		    echo "" > "$ERL_TOP/erts/CONF_INFO"
+		cat >> "$ERL_TOP/erts/CONF_INFO" <<EOF
+
+                 No native atomic implementation, nor no native
+                 spinlock implementation available. Fallbacks
+                 implemented using mutexes will be used. Note
+                 that the performance of the SMP runtime system
+                 will suffer much due to this.
+
+EOF
+		;;
+
+	esac
+
 	enable_threads=force
 fi
 
 AC_SUBST(ERTS_BUILD_SMP_EMU)
 
+AC_CHECK_FUNCS([posix_fadvise])
 
 
 #
@@ -1120,17 +1170,15 @@ else
 		enable_child_waiter_thread=yes
 		;;
 	    linux*)
-		AC_DEFINE(USE_RECURSIVE_MALLOC_MUTEX,[1],
-			[Define if malloc should use a recursive mutex])
 		AC_MSG_CHECKING([whether dlopen() needs to be called before first call to dlerror()])
-		if test "x$ETHR_THR_LIB_BASE_NAME" != "xnptl"; then
+		if test "x$ETHR_THR_LIB_BASE_TYPE" != "xposix_nptl"; then
 		    AC_DEFINE(ERTS_NEED_DLOPEN_BEFORE_DLERROR,[1],
 			[Define if dlopen() needs to be called before first call to dlerror()])
 			AC_MSG_RESULT(yes)
 		else
 			AC_MSG_RESULT(no)
 		fi
-		if test "x$ETHR_THR_LIB_BASE_NAME" != "xnptl"; then
+		if test "x$ETHR_THR_LIB_BASE_TYPE" != "xposix_nptl"; then
 		    # Child waiter thread cannot be enabled
 		    disable_child_waiter_thread=yes
 		    enable_child_waiter_thread=no
@@ -1192,13 +1240,7 @@ fi
 
 AC_SUBST(EMU_LOCK_CHECKING)
 
-ERTS_INTERNAL_X_LIBS=
-
-AC_CHECK_LIB(kstat, kstat_open,
-[AC_DEFINE(HAVE_KSTAT, 1, [Define if you have kstat])
-ERTS_INTERNAL_X_LIBS="$ERTS_INTERNAL_X_LIBS -lkstat"])
-
-AC_SUBST(ERTS_INTERNAL_X_LIBS)
+ERL_INTERNAL_LIBS
 
 dnl THR_LIBS and THR_DEFS are only used by odbc
 THR_LIBS=$ETHR_X_LIBS
@@ -1243,8 +1285,7 @@ dnl zlib
 dnl -------------
 
 AC_ARG_ENABLE(shared-zlib,
-[  --enable-shared-zlib        enable using shared zlib library
-  --disable-shared-zlib       disable shared zlib, compile own zlib source (default)],
+AS_HELP_STRING([--enable-shared-zlib], [enable using shared zlib library]),
 [ case "$enableval" in
     no) enable_shared_zlib=no ;;
     *)  enable_shared_zlib=yes ;;
@@ -1422,8 +1463,70 @@ AC_CHECK_HEADERS(fcntl.h limits.h unistd.h syslog.h dlfcn.h ieeefp.h \
                  sys/types.h sys/stropts.h sys/sysctl.h \
                  sys/ioctl.h sys/time.h sys/uio.h \
                  sys/socket.h sys/sockio.h sys/socketio.h \
-                 net/errno.h malloc.h mach-o/dyld.h arpa/nameser.h \
-		 pty.h util.h utmp.h langinfo.h poll.h)
+                 net/errno.h malloc.h arpa/nameser.h libdlpi.h \
+		 pty.h util.h utmp.h langinfo.h poll.h sdkddkver.h)
+
+AC_CHECK_MEMBERS([struct ifreq.ifr_hwaddr], [], [],
+	[#ifdef __WIN32__
+	 #else
+	 #ifdef VXWORKS
+	 #else
+	 #include <net/if.h>
+	 #endif
+	 #endif
+	])
+
+AC_CHECK_MEMBERS([struct ifreq.ifr_enaddr], [], [],
+	[#ifdef __WIN32__
+	 #else
+	 #ifdef VXWORKS
+	 #else
+	 #include <net/if.h>
+	 #endif
+	 #endif
+	])
+
+dnl ----------------------------------------------------------------------
+dnl Check the availability for libdlpi
+dnl ----------------------------------------------------------------------
+AC_CHECK_LIB(dlpi, dlpi_open)
+if test x"$ac_cv_lib_dlpi_dlpi_open" = x"no"; then
+   unset -v ac_cv_lib_dlpi_dlpi_open
+   dnl Try again now with -L/lib (or ditto 64) as argument to linker since
+   dnl gcc makes /usr/ccs/bin/ld ignore the crle configured linker default paths
+   dnl typically causing dlpi not being found on Solaris et.al
+   save_ldflags="$LDFLAGS"
+   try_dlpi_lib=/lib
+   if test x"$ac_cv_sizeof_void_p" = x"8"; then
+      if test -d /lib64; then
+	 try_dlpi_lib=/lib64
+      elif test -d /lib/64; then
+	 try_dlpi_lib=/lib/64
+      fi
+   fi
+   if test ! -f "$try_dlpi_lib/libdlpi.so" && \
+      test -f "$try_dlpi_lib/libdlpi.so.1"
+   then
+      dnl It looks like there is a missing symlink
+      dnl - let's be helpful and notify the user
+      dnl NOTE this help is far from perfect e.g if there would be no
+      dnl *.so.1 but a *.so.1.123 or *.so.2 this will be no help
+      AC_MSG_ERROR(
+	[Your OS installation is missing a symbolic link.
+	Maybe it lacks some development package(s)...
+	It can anyhow be fixed with the following command:
+	# ln -s libdlpi.so.1 $try_dlpi_lib/libdlpi.so
+	])
+   fi
+   LDFLAGS="-L$try_dlpi_lib -R$try_dlpi_lib $LDFLAGS"
+   unset -v try_dlpi_lib
+   AC_MSG_NOTICE([Extending the search to include /lib])
+   AC_CHECK_LIB(dlpi, dlpi_open)
+   if test x"$ac_cv_lib_dlpi_dlpi_open" = x"no"; then
+      LDFLAGS="$save_ldflags"
+   fi
+   unset -v save_ldflags
+fi
 
 AC_CHECK_HEADER(sys/resource.h,
 	[AC_DEFINE(HAVE_SYS_RESOURCE_H, 1,
@@ -1441,7 +1544,7 @@ AC_CHECK_HEADER(sys/devpoll.h, have_kernel_poll=/dev/poll)
 
 dnl Check for kernel SCTP support
 AC_SUBST(LIBSCTP)
-if test "x$enable_sctp" = "xyes" ; then
+if test "x$enable_sctp" != "xno" ; then
     AC_CHECK_HEADER(netinet/sctp.h,
         [LIBSCTP=libsctp.so.1
 	 AC_DEFINE(HAVE_SCTP_H, [1],
@@ -1451,8 +1554,24 @@ if test "x$enable_sctp" = "xyes" ; then
 	 #include <sys/socket.h>
 	 #endif
 	])
+fi
+
+if test x"$ac_cv_header_netinet_sctp_h" = x"yes"; then
+    AC_CHECK_FUNCS([sctp_bindx sctp_peeloff])
     AC_CHECK_DECLS([SCTP_UNORDERED, SCTP_ADDR_OVER, SCTP_ABORT,
-                    SCTP_EOF, SCTP_SENDALL, SCTP_ADDR_CONFIRMED], [], [],
+                    SCTP_EOF, SCTP_SENDALL, SCTP_ADDR_CONFIRMED,
+		    SCTP_DELAYED_ACK_TIME,
+		    SCTP_EMPTY,
+		    SCTP_CLOSED, SCTPS_IDLE,
+		    SCTP_BOUND, SCTPS_BOUND,
+		    SCTP_LISTEN, SCTPS_LISTEN,
+		    SCTP_COOKIE_WAIT, SCTPS_COOKIE_WAIT,
+		    SCTP_COOKIE_ECHOED, SCTPS_COOKIE_ECHOED,
+		    SCTP_ESTABLISHED, SCTPS_ESTABLISHED,
+		    SCTP_SHUTDOWN_PENDING, SCTPS_SHUTDOWN_PENDING,
+		    SCTP_SHUTDOWN_SENT, SCTPS_SHUTDOWN_SENT,
+		    SCTP_SHUTDOWN_RECEIVED, SCTPS_SHUTDOWN_RECEIVED,
+		    SCTP_SHUTDOWN_ACK_SENT, SCTPS_SHUTDOWN_ACK_SENT], [], [],
         [#if HAVE_SYS_SOCKET_H
          #include <sys/socket.h>
          #endif
@@ -1519,6 +1638,7 @@ AC_CHECK_SIZEOF(void *)
 AC_CHECK_SIZEOF(long long)
 AC_CHECK_SIZEOF(size_t)
 AC_CHECK_SIZEOF(off_t)
+AC_CHECK_SIZEOF(time_t)
 
 BITS64=
 
@@ -1611,6 +1731,33 @@ esac
 
 AC_C_BIGENDIAN
 
+dnl fdatasync syscall (Unix only)
+AC_CHECK_FUNCS([fdatasync])
+
+dnl Find which C libraries are required to use fdatasync
+dnl TODO: Remove check once SunOS >= 5.11 is required by erts.
+dnl       fdatasync requires linking against -lrt on SunOS <= 5.10.
+dnl       OpenSolaris 2009.06 is SunOS 5.11 and does not require -lrt.
+AC_SEARCH_LIBS(fdatasync, [rt])
+
+
+dnl sendfile syscall
+case $host_os in
+    linux*|freebsd*|dragonfly*|darwin*)
+		AC_CHECK_FUNCS([sendfile])
+		;;
+    solaris*)
+		AC_SEARCH_LIBS(sendfilev, sendfile,
+			AC_DEFINE([HAVE_SENDFILEV],[1],
+		           [Define to 1 if you have the `sendfilev' function.]))
+		;;
+    win32)
+		LIBS="$LIBS -lmswsock"
+		;;
+    *)
+		;;
+esac
+
 dnl ----------------------------------------------------------------------
 dnl Checks for library functions.
 dnl ----------------------------------------------------------------------
@@ -1623,18 +1770,62 @@ LIBS="$LIBS $EMU_THR_X_LIBS"
 
 dnl Check if we have these, in which case we'll try to build
 dnl inet_gethost with ipv6 support.
-AC_CHECK_FUNC(getaddrinfo, have_getaddrinfo=yes, have_getaddrinfo=no)
+AC_CHECK_HEADERS(windows.h)
+AC_CHECK_HEADERS(winsock2.h)
+AC_CHECK_HEADERS(ws2tcpip.h,[],[],[
+#ifdef HAVE_WINSOCK2_H
+#include <winsock2.h>
+#endif
+#ifdef HAVE_WINDOWS_H
+#include <windows.h>
+#endif
+])
+dnl AC_CHECK_FUNC(getaddrinfo, have_getaddrinfo=yes, have_getaddrinfo=no)
+AC_MSG_CHECKING(for getaddrinfo)
+AC_TRY_LINK([
+#include <stdlib.h>
+#include <string.h>
+#ifdef HAVE_WINSOCK2_H
+#include <winsock2.h>
+#endif
+#ifdef HAVE_WINDOWS_H
+#include <windows.h>
+#endif
+#ifdef HAVE_WS2TCPIP_H
+#include <ws2tcpip.h>
+#endif
+#ifndef __WIN32__
+#include <sys/socket.h>
+#include <netdb.h>
+#endif
+], 
+[
+getaddrinfo("","",NULL,NULL);
+],have_getaddrinfo=yes, have_getaddrinfo=no)
 if test $have_getaddrinfo = yes; then
+        AC_MSG_RESULT([yes])
         AC_MSG_CHECKING([whether getaddrinfo accepts enough flags])
-        AC_TRY_RUN([
+        AC_TRY_COMPILE([
 #include <stdlib.h>
 #include <string.h>
+#ifdef HAVE_WINSOCK2_H
+#include <winsock2.h>
+#endif
+#ifdef HAVE_WINDOWS_H
+#include <windows.h>
+#endif
+#ifdef HAVE_WS2TCPIP_H
+#include <ws2tcpip.h>
+#endif
+#ifndef __WIN32__
 #include <sys/socket.h>
 #include <netdb.h>
-int main(int argc, char **argv) {
+#endif
+],
+[
     struct addrinfo hints, *ai;
     memset(&hints, 0, sizeof(hints));
-    hints.ai_flags = (AI_CANONNAME|AI_V4MAPPED|AI_ADDRCONFIG);
+    hints.ai_flags = AI_CANONNAME;
     hints.ai_socktype = SOCK_STREAM;
     hints.ai_family = AF_INET6;
     if (getaddrinfo("::", NULL, &hints, &ai) == 0) {
@@ -1643,36 +1834,56 @@ int main(int argc, char **argv) {
     } else {
 	exit(1);
     }
-}
-	],, have_getaddrinfo=no,
-	[
-	case X$erl_xcomp_getaddrinfo in
-	    X) have_getaddrinfo=cross;;
-	    Xyes|Xno) have_getaddrinfo=$erl_xcomp_getaddrinfo;;
-	    *) AC_MSG_ERROR([Bad erl_xcomp_getaddrinfo value: $erl_xcomp_getaddrinfo]);;
-	esac
-	])
+],, have_getaddrinfo=no)
 	AC_MSG_RESULT($have_getaddrinfo)
 	case $have_getaddrinfo in
 	    yes)
 		AC_DEFINE(HAVE_GETADDRINFO, [1], 
 		          [Define to 1 if you have a good `getaddrinfo' function.]);;
-	    cross)
-		AC_MSG_WARN([result no guessed because of cross compilation]);;
 	    *) ;;
 	esac
+else
+    AC_MSG_RESULT([no])
+fi
+AC_MSG_CHECKING(for getnameinfo)
+AC_TRY_LINK([
+#include <stdlib.h>
+#include <string.h>
+#ifdef HAVE_WINSOCK2_H
+#include <winsock2.h>
+#endif
+#ifdef HAVE_WINDOWS_H
+#include <windows.h>
+#endif
+#ifdef HAVE_WS2TCPIP_H
+#include <ws2tcpip.h>
+#endif
+#ifndef __WIN32__
+#include <sys/socket.h>
+#include <netdb.h>
+#endif
+], 
+[
+getnameinfo(NULL,0,NULL,0,NULL,0,0);
+],have_getnameinfo=yes, have_getnameinfo=no)
+if test $have_getnameinfo = yes; then
+        AC_MSG_RESULT([yes])
+	AC_DEFINE(HAVE_GETNAMEINFO, [1], 
+		  [Define to 1 if you have a good `getnameinfo' function.])
+else
+	AC_MSG_RESULT([no])	  
 fi
-AC_CHECK_FUNCS([getnameinfo getipnodebyname getipnodebyaddr gethostbyname2])
+
+
+AC_CHECK_FUNCS([getipnodebyname getipnodebyaddr gethostbyname2])
 
 AC_CHECK_FUNCS([ieee_handler fpsetmask finite isnan isinf res_gethostbyname dlopen \
 		pread pwrite writev memmove strerror strerror_r strncasecmp \
-		gethrtime localtime_r gmtime_r mremap memcpy mallopt \
+		gethrtime localtime_r gmtime_r inet_pton mmap mremap memcpy mallopt \
 		sbrk _sbrk __sbrk brk _brk __brk \
 		flockfile fstat strlcpy strlcat setsid posix2time setlocale nl_langinfo poll])
-if test "X$host" = "Xwin32"; then
-	ac_cv_func_setvbuf_reversed=yes
-fi
-AC_FUNC_SETVBUF_REVERSED	
+
+AC_CHECK_DECLS([posix2time],,,[#include <time.h>])
 
 disable_vfork=false
 if test "x$EMU_THR_LIB_NAME" != "x"; then
@@ -1698,7 +1909,6 @@ if test $disable_vfork = true; then
 fi
 
 AC_FUNC_VPRINTF
-AC_FUNC_MMAP
 
 dnl The AC_DEFINEs are necessary for autoheader to work. :-(
 dnl for gzio
@@ -1730,6 +1940,9 @@ fi
 dnl Need by run_erl.
 AC_CHECK_FUNCS([openpty])
 
+AC_CHECK_HEADERS(net/if_dl.h ifaddrs.h netpacket/packet.h)
+AC_CHECK_FUNCS([getifaddrs])
+
 dnl ----------------------------------------------------------------------
 dnl Checks for features/quirks in the system that affects Erlang.
 dnl ----------------------------------------------------------------------
@@ -1791,6 +2004,27 @@ if test $processor_bind_functionality = yes; then
 	AC_DEFINE(HAVE_PROCESSOR_BIND, 1, [Define if you have processor_bind functionality])
 fi
 
+AC_MSG_CHECKING([for cpuset_getaffinity/cpuset_setaffinity])
+AC_TRY_COMPILE([
+#include <sys/param.h>
+#include <sys/cpuset.h>
+],
+[
+	int res;
+	cpuset_t cpuset;
+	CPU_ZERO(&cpuset);
+	CPU_SET(1, &cpuset);
+	res = cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, sizeof(cpuset_t), &cpuset);
+	res = cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, sizeof(cpuset_t), &cpuset);
+	res = CPU_ISSET(1, &cpuset);
+	CPU_CLR(1, &cpuset);
+],
+		cpuset_xetaffinity=yes,
+		cpuset_xetaffinity=no)
+AC_MSG_RESULT([$cpuset_xetaffinity])
+if test $cpuset_xetaffinity = yes; then
+	AC_DEFINE(HAVE_CPUSET_xETAFFINITY, 1, [Define if you have cpuset_getaffinity/cpuset_setaffinity])
+fi
 
 AC_CACHE_CHECK([for 'end' symbol],
 		erts_cv_have_end_symbol,
@@ -2240,9 +2474,15 @@ if test "$cross_compiling" != "yes" && test X${enable_hipe} != Xno; then
   if test -z "$M4"; then
 	enable_hipe=no
    	AC_MSG_NOTICE([HiPE disabled as no valid m4 is found in PATH])
+  elif test "$enable_halfword_emulator" = "yes"; then
+       if test X${enable_hipe} = Xyes; then
+              AC_MSG_ERROR([HiPE can not be combined with halfword emulator (yet)])
+       else		    
+   	      AC_MSG_NOTICE([HiPE auto-disabled on halfword emulator])
+       fi	
   else
   	case "$ARCH-$OPSYS" in
-	    x86-linux|amd64-linux|x86-darwin*|amd64-darwin*|ppc-linux|ppc-darwin|arm-linux|amd64-freebsd|x86-freebsd|x86-sol2|amd64-sol2|ultrasparc-linux)
+	    x86-linux|amd64-linux|x86-darwin*|amd64-darwin*|ppc-linux|ppc64-linux|ppc-darwin|arm-linux|amd64-freebsd|x86-freebsd|x86-sol2|amd64-sol2|ultrasparc-linux)
       		enable_hipe=yes
       		;;	
   	esac
@@ -2286,6 +2526,17 @@ case $ARCH-$OPSYS in
 		darwin_mcontext_leopard=no
 		;;
 esac
+
+if test X${enable_fp_exceptions} = Xauto ; then
+   case $host_os in
+	darwin*)
+	      enable_fp_exceptions=no
+	      AC_MSG_NOTICE([Floating point exceptions disabled by default on MacOS X]) ;;
+	      *)
+	      ;;
+    esac
+fi
+
 if test X${enable_fp_exceptions} = Xauto ; then
    if test X${enable_hipe} = Xyes; then
       enable_fp_exceptions=yes
@@ -2333,7 +2584,7 @@ static void new_fp_exception(void)
  * Implement unmask_fpe() and check_fpe() based on CPU/OS combination
  */
 
-#if (defined(__i386__) || defined(__x86_64__)) && defined(__GNUC__) && !defined(__CYGWIN__)
+#if (defined(__i386__) || defined(__x86_64__)) && defined(__GNUC__) && !defined(__CYGWIN__) && !defined(__MINGW32__)
 
 static void unmask_x87(void)
 {
@@ -2615,7 +2866,7 @@ static __inline__ int check_fpe(double f)
  * Implement SIGFPE handler based on CPU/OS combination
  */
 
-#if (defined(__linux__) && (defined(__i386__) || defined(__x86_64__) || defined(__sparc__) || defined(__powerpc__))) || (defined(__DARWIN__) && (defined(__i386__) || defined(__x86_64__) || defined(__ppc__))) || (defined(__FreeBSD__) && (defined(__i386__) || defined(__x86_64__))) || (defined(__OpenBSD__) && defined(__x86_64__)) || (defined(__sun__) && defined(__x86_64__))
+#if (defined(__linux__) && (defined(__i386__) || defined(__x86_64__) || defined(__sparc__) || defined(__powerpc__))) || (defined(__DARWIN__) && (defined(__i386__) || defined(__x86_64__) || defined(__ppc__))) || (defined(__FreeBSD__) && (defined(__i386__) || defined(__x86_64__))) || ((defined(__OpenBSD__) || defined(__NetBSD__)) && defined(__x86_64__)) || (defined(__sun__) && defined(__x86_64__))
 
 #if defined(__linux__) && defined(__i386__)
 #if !defined(X86_FXSR_MAGIC)
@@ -2729,6 +2980,11 @@ static void fpe_sig_action(int sig, siginfo_t *si, void *puc)
     struct fxsave64 *fxsave = uc->sc_fpstate;
     fxsave->fx_mxcsr = 0x1F80;
     fxsave->fx_fsw &= ~0xFF;
+#elif defined(__NetBSD__) && defined(__x86_64__)
+    mcontext_t *mc = &uc->uc_mcontext;
+    struct fxsave64 *fxsave = (struct fxsave64 *)&mc->__fpregs;
+    fxsave->fx_mxcsr = 0x1F80;
+    fxsave->fx_fsw &= ~0xFF;
 #elif defined(__sun__) && defined(__x86_64__)
     mcontext_t *mc = &uc->uc_mcontext;
     struct fpchip_state *fpstate = &mc->fpregs.fp_reg_set.fpchip_state;
@@ -2872,33 +3128,6 @@ fi
 AC_SUBST(TSP_APP)
 
 #
-# Check if we should build hybrid emulator
-#
-
-AC_MSG_CHECKING([whether a hybrid heap emulator should be built])
-case $enable_hybrid_heap-$host_os in
-    yes-*)
-	AC_MSG_RESULT([yes; enabled by user])
-	ERTS_BUILD_HYBRID_EMU=yes;;
-    no-*)
-	AC_MSG_RESULT([no; disabled by user])
-	ERTS_BUILD_HYBRID_EMU=no;;
-    *-win32|*-vxworks|*-ose) # vxworks and ose have their own "configure scripts"...
-	AC_MSG_RESULT([no; default on this platform])
-	ERTS_BUILD_HYBRID_EMU=no;;
-    *)
-	AC_MSG_RESULT([yes; default on this platform])
-	ERTS_BUILD_HYBRID_EMU=yes;;
-esac
-
-
-if test $ERTS_BUILD_HYBRID_EMU = yes; then
-    AC_DEFINE(ERTS_HAVE_HYBRID_EMU, 1, [Define if the hybrid emulator is built])
-fi
-
-AC_SUBST(ERTS_BUILD_HYBRID_EMU)
-
-#
 # Check if we should enable HiPE.
 #
 
@@ -2911,8 +3140,6 @@ dnl done up where floating point is checked, need to descide there already...
 if test X${enable_hipe} = Xyes; then
   if test X$ac_cv_sizeof_void_p != X4 -a X$ARCH != Xamd64 -a X$ARCH != Xppc64; then
 	AC_MSG_WARN([HiPE is not supported in 64-bit builds])
-  elif test X$FPE != Xreliable -a X$ARCH != Xarm; then
-	AC_MSG_WARN([HiPE is not supported on $ARCH without reliable floating-point exceptions])
   else
      HIPE_ENABLED=yes
      AC_DEFINE(HIPE,[1],[Define to enable HiPE])
@@ -2946,17 +3173,6 @@ if test "x$HIPE_ENABLED" = "xyes" ; then
 fi
 
 #
-# Check if we should use elib_malloc.
-#
-
-if test X${enable_elib_malloc} = Xyes; then
- AC_DEFINE(ENABLE_ELIB_MALLOC,[],[Define to enable use of elib_malloc (a malloc() replacement)])
- AC_DEFINE(ELIB_HEAP_SBRK,[],[Elib sbrk])
- AC_DEFINE(ELIB_ALLOC_IS_CLIB,[],[Use elib malloc as clib])
- AC_DEFINE(ELIB_SORTED_BLOCKS,[],[Define to enable the use of sorted blocks when using elib_malloc])
-fi
-
-#
 # Check for working poll().
 #
 AC_MSG_CHECKING([for working poll()])
@@ -3352,6 +3568,74 @@ dnl
 LM_FIND_EMU_CC
 
 dnl
+dnl DTrace
+dnl
+case $DYNAMIC_TRACE_FRAMEWORK in
+     dtrace|systemtap)
+        AC_CHECK_TOOL(DTRACE, dtrace, none)
+        test "$DTRACE" = "none" && AC_MSG_ERROR([No dtrace utility found.]);
+	enable_dtrace_test=yes;;
+     *) enable_dtrace_test=no;;
+esac        
+		     
+AC_SUBST(DTRACE)
+
+AC_SUBST(DTRACE_CPP)
+AC_SUBST(DTRACE_ENABLED)
+AC_SUBST(DTRACE_ENABLED_2STEP)
+DTRACE_CPP=-C
+DTRACE_ENABLED=
+DTRACE_ENABLED_2STEP=
+DTRACE_2STEP_TEST=./dtrace-test.o
+DTRACE_BITS_FLAG=
+case $OPSYS in
+    freebsd)
+	if test "$BITS64" = "yes" ; then
+		DTRACE_BITS_FLAG=-64
+	else
+		DTRACE_BITS_FLAG=-32
+	fi
+    ;;
+    *)
+	: # Nothing to do
+    ;;
+esac
+if test "$enable_dtrace_test" = "yes" ; then
+        if test "$DTRACE" = "dtrace" ; then
+                AC_CHECK_HEADERS(sys/sdt.h)
+                # The OS X version of dtrace prints a spurious line here.
+                if ! dtrace -h $DTRACE_CPP -Iemulator/beam -o ./foo-dtrace.h -s emulator/beam/erlang_dtrace.d; then
+                        AC_MSG_ERROR([Could not precompile erlang_dtrace.d: dtrace -h failed])
+                fi
+                rm -f foo-dtrace.h
+
+                $RM -f $DTRACE_2STEP_TEST
+                if dtrace -G $DTRACE_CPP $DTRACE_BITS_FLAG -Iemulator/beam -o $DTRACE_2STEP_TEST -s emulator/beam/erlang_dtrace.d 2> /dev/null && \
+                   test -f $DTRACE_2STEP_TEST ; then
+                        rm $DTRACE_2STEP_TEST
+                        DTRACE_ENABLED_2STEP=yes
+                        AC_MSG_NOTICE([dtrace precompilation for 2-stage DTrace successful])
+                else
+                        AC_MSG_NOTICE([dtrace precompilation for 1-stage DTrace successful])
+                fi
+                DTRACE_ENABLED=yes
+                case $OPSYS in
+                    linux)
+                        : # No extra libs to add to LIBS
+                    ;;
+                    freebsd)
+                        LIBS="$LIBS -lelf"
+                    ;;
+                    *)
+                        LIBS="$LIBS -ldtrace"
+                    ;;
+                esac
+        else
+                AC_MSG_ERROR([Dtrace preprocessing test failed.])
+        fi
+fi
+
+dnl
 dnl SSL, SSH and CRYPTO need the OpenSSL libraries
 dnl
 dnl Check flags --with-ssl, --without-ssl --with-ssl=PATH.
@@ -3364,6 +3648,8 @@ dnl use "PATH/include" and "PATH/lib".
 AC_SUBST(SSL_INCLUDE)
 AC_SUBST(SSL_ROOT)
 AC_SUBST(SSL_LIBDIR)
+AC_SUBST(SSL_CRYPTO_LIBNAME)
+AC_SUBST(SSL_SSL_LIBNAME)
 AC_SUBST(SSL_CC_RUNTIME_LIBRARY_PATH)
 AC_SUBST(SSL_LD_RUNTIME_LIBRARY_PATH)
 AC_SUBST(SSL_DED_LD_RUNTIME_LIBRARY_PATH)
@@ -3372,14 +3658,16 @@ AC_SUBST(SSL_LINK_WITH_KERBEROS)
 AC_SUBST(STATIC_KERBEROS_LIBS)
 AC_SUBST(SSL_LINK_WITH_ZLIB)
 AC_SUBST(STATIC_ZLIB_LIBS)
-AC_SUBST(OPENSSL_CMD)
 
-std_ssl_locations="/usr/local /usr/sfw /opt/local /usr /usr/pkg /usr/local/openssl /usr/lib/openssl /usr/openssl /usr/local/ssl /usr/lib/ssl /usr/ssl"
+std_ssl_locations="/usr/local /usr/sfw /usr /opt/local /usr/pkg /usr/local/openssl /usr/lib/openssl /usr/openssl /usr/local/ssl /usr/lib/ssl /usr/ssl"
 
 AC_ARG_WITH(ssl-zlib,
-[  --with-ssl-zlib=PATH    specify location of ZLib to be used by OpenSSL
-  --with-ssl-zlib         link SSL with  Zlib (default if found)
-  --without-ssl-zlib      don't link SSL with ZLib])
+AS_HELP_STRING([--with-ssl-zlib=PATH],
+               [specify location of ZLib to be used by OpenSSL])
+AS_HELP_STRING([--with-ssl-zlib],
+               [link SSL with  Zlib (default if found)])
+AS_HELP_STRING([--without-ssl-zlib],
+               [don't link SSL with ZLib]))
 
 
 if  test "x$with_ssl_zlib" = "xno"; then
@@ -3390,7 +3678,7 @@ elif test "x$with_ssl_zlib" = "xyes" || test "x$with_ssl_zlib" = "x"; then
 		AC_MSG_WARN([Cannot search for zlib; missing cross system root (erl_xcomp_sysroot).])
 		SSL_LINK_WITH_ZLIB=no	
 		STATIC_ZLIB_LIBS=	
-	elif  test "x$MIXED_CYGWIN" = "xyes"; then
+	elif  test "x$MIXED_CYGWIN" = "xyes" -o "x$MIXED_MSYS" = "xyes"; then
 		SSL_LINK_WITH_ZLIB=no	
 		STATIC_ZLIB_LIBS=	
 	else
@@ -3448,13 +3736,13 @@ fi
 		
 			
 AC_ARG_WITH(ssl,
-[  --with-ssl=PATH         specify location of OpenSSL include and lib
-  --with-ssl              use SSL (default)
-  --without-ssl           don't use SSL])
+AS_HELP_STRING([--with-ssl=PATH], [specify location of OpenSSL include and lib])
+AS_HELP_STRING([--with-ssl], [use SSL (default)])
+AS_HELP_STRING([--without-ssl], [don't use SSL]))
 
 AC_ARG_ENABLE(dynamic-ssl-lib,
-[  --enable-dynamic-ssl-lib        enable using dynamic openssl libraries
-  --disable-dynamic-ssl-lib       disable using dynamic openssl libraries],
+AS_HELP_STRING([--disable-dynamic-ssl-lib],
+               [disable using dynamic openssl libraries]),
 [ case "$enableval" in
     no) enable_dynamic_ssl=no ;;
     *)  enable_dynamic_ssl=yes ;;
@@ -3502,10 +3790,23 @@ case "$erl_xcomp_without_sysroot-$with_ssl" in
     	AC_CHECK_PROG(REGTOOL, regtool, regtool, false)
 	if test "$ac_cv_prog_REGTOOL" != false; then
 		wrp="/machine/software/microsoft/windows/currentversion/"
-		urp="uninstall/openssl_is1/inno setup: app path"
+	   	if test "x$ARCH" = "xamd64"; then
+		   urp="uninstall/openssl (64-bit)_is1/inno setup: app path"
+		   regtool_subsystem=-w
+		else
+		   urp="uninstall/openssl (32-bit)_is1/inno setup: app path"
+		   regtool_subsystem=-W
+		fi	
 		rp="$wrp$urp"
-		if regtool -q get "$rp" > /dev/null; then
-			ssl_install_dir=`regtool -q get "$rp"`
+		if regtool -q $regtool_subsystem get "$rp" > /dev/null; then
+		   true
+		else
+		   # Fallback to unspecified wordlength
+		   urp="uninstall/openssl_is1/inno setup: app path"
+		   rp="$wrp$urp"
+		fi	
+		if regtool -q $regtool_subsystem get "$rp" > /dev/null; then
+			ssl_install_dir=`regtool -q $regtool_subsystem get "$rp"`
 			# Try hard to get rid of spaces...
 			if cygpath -d "$ssl_install_dir" > /dev/null 2>&1; then
 				ssl_install_dir=`cygpath -d "$ssl_install_dir"`
@@ -3513,25 +3814,72 @@ case "$erl_xcomp_without_sysroot-$with_ssl" in
 			extra_dir=`cygpath $ssl_install_dir`
 		fi
 	fi
+    elif test "x$MIXED_MSYS" = "xyes"; then
+    	AC_CHECK_PROG(REGTOOL, reg_query.sh, reg_query.sh, false)
+	if test "$ac_cv_prog_REGTOOL" != false; then
+	   	if test "x$ARCH" = "xamd64"; then
+		   rp="HKLM/SOFTWARE/Microsoft/Windows/CurrentVersion/Uninstall/OpenSSL (64-bit)_is1"
+		else
+		   rp="HKLM/SOFTWARE/Microsoft/Windows/CurrentVersion/Uninstall/OpenSSL_is1"
+		fi	
+		key="Inno Setup: App Path"
+		if "$ac_cv_prog_REGTOOL" "$rp" "$key" > /dev/null; then
+			ssl_install_dir=`"$ac_cv_prog_REGTOOL" "$rp" "$key"`
+			extra_dir=`win2msys_path.sh "$ssl_install_dir"`
+		fi
+	fi
     fi
     # We search for OpenSSL in the common OS standard locations.
     SSL_APP=ssl
     CRYPTO_APP=crypto
     SSH_APP=ssh
 
-    AC_MSG_CHECKING(for OpenSSL >= 0.9.7 in standard locations) 
-    for rdir in $extra_dir /cygdrive/c/OpenSSL $std_ssl_locations; do
+    SSL_CRYPTO_LIBNAME=crypto
+    SSL_SSL_LIBNAME=ssl
+    
+    if  test "x$MIXED_CYGWIN" = "xyes" -o "x$MIXED_MSYS" = "xyes"; then
+    	if test "x$ARCH" = "xamd64"; then
+	  std_win_ssl_locations="/cygdrive/c/OpenSSL-Win64 /c/OpenSSL-Win64 /opt/local64/pgm/OpenSSL"
+	else
+	  std_win_ssl_locations="/cygdrive/c/OpenSSL-Win32 /c/OpenSSL-Win32 /cygdrive/c/OpenSSL /c/OpenSSL /opt/local/pgm/OpenSSL"
+	fi
+    else
+        std_win_ssl_locations=""
+    fi
+
+
+    AC_MSG_CHECKING(for OpenSSL >= 0.9.7 in standard locations)
+    for rdir in $extra_dir $std_win_ssl_locations $std_ssl_locations; do
 	dir="$erl_xcomp_sysroot$rdir"
 	if test -f "$erl_xcomp_isysroot$rdir/include/openssl/opensslv.h"; then
 		is_real_ssl=yes
 		SSL_ROOT="$dir"
-		if test "x$MIXED_CYGWIN" = "xyes" ; then
-			if test -f "$dir/lib/VC/ssleay32.lib" || \
-			   test -f "$dir/lib/VC/openssl.lib"; then 
+		if test "x$MIXED_CYGWIN" = "xyes" -o "x$MIXED_MSYS" = "xyes"; then
+			if test -f "$dir/lib/VC/libeay32.lib"; then
+				SSL_RUNTIME_LIBDIR="$rdir/lib/VC"
+				SSL_LIBDIR="$dir/lib/VC"
+				SSL_CRYPTO_LIBNAME=libeay32
+				SSL_SSL_LIBNAME=ssleay32
+			elif test -f "$dir/lib/VC/openssl.lib"; then 
 				SSL_RUNTIME_LIBDIR="$rdir/lib/VC"
 				SSL_LIBDIR="$dir/lib/VC"
-			elif test -f "$dir/lib/ssleay32.lib" || \
-			     test -f "$dir/lib/openssl.lib"; then
+			elif test -f $dir/lib/VC/libeay32MD.lib; then
+		            SSL_CRYPTO_LIBNAME=libeay32MD
+		            SSL_SSL_LIBNAME=ssleay32MD
+			    if test "x$enable_dynamic_ssl" = "xno" && \
+	                       test -f  $dir/lib/VC/static/libeay32MD.lib; then
+				  SSL_RUNTIME_LIBDIR="$rdir/lib/VC/static"
+				  SSL_LIBDIR="$dir/lib/VC/static"
+		            else
+			          SSL_RUNTIME_LIBDIR="$rdir/lib/VC"
+				  SSL_LIBDIR="$dir/lib/VC"
+                            fi 
+			elif test -f "$dir/lib/libeay32.lib"; then
+				SSL_RUNTIME_LIBDIR="$rdir/lib"
+				SSL_LIBDIR="$dir/lib"
+				SSL_CRYPTO_LIBNAME=libeay32
+				SSL_SSL_LIBNAME=ssleay32
+			elif test -f "$dir/lib/openssl.lib"; then
 				SSL_RUNTIME_LIBDIR="$rdir/lib"
 				SSL_LIBDIR="$dir/lib"
 			else
@@ -3564,21 +3912,6 @@ case "$erl_xcomp_without_sysroot-$with_ssl" in
 			SSL_DYNAMIC_ONLY=yes
 		fi
 		SSL_BINDIR="$rdir/bin"
-dnl 		Should one use EXEEXT or ac_exeext?
-		if test -f "$erl_xcomp_sysroot$SSL_BINDIR/openssl$EXEEXT"; then
-		    if test "$cross_compiling" = "yes"; then
-			dnl Cannot test it; hope it is working...
-			OPENSSL_CMD="$SSL_BINDIR/openssl"
-		    else
-			if "$SSL_BINDIR/openssl" version > /dev/null 2>&1; then
-				OPENSSL_CMD="$SSL_BINDIR/openssl"
-			else
-				is_real_ssl=no
-			fi
-		    fi
-		else
-			is_real_ssl=no
-		fi
 		if test "x$is_real_ssl" = "xyes" ; then
 			SSL_INCLUDE="-I$dir/include"
 			old_CPPFLAGS=$CPPFLAGS
@@ -3596,7 +3929,7 @@ dnl 		Should one use EXEEXT or ac_exeext?
          		])
          		CPPFLAGS=$old_CPPFLAGS
 			if test "x$ssl_found" = "xyes"; then
-			   	if test "x$MIXED_CYGWIN" = "xyes" ; then
+			   	if test "x$MIXED_CYGWIN" = "xyes" -o "x$MIXED_MSYS" = "xyes"; then
 				   	ssl_linkable=yes
 				else
 					saveCFLAGS="$CFLAGS"
@@ -3642,7 +3975,6 @@ dnl 		Should one use EXEEXT or ac_exeext?
 		  SSL_RUNTIME_LIB="/usr/lib"
 		  SSL_LIB="$erl_xcomp_sysroot/usr/lib"
 		  SSL_BINDIR="/usr/sbin"
-		  OPENSSL_CMD="$SSL_BINDIR/openssl"
 		  dnl OpenBSD requires us to link with -L and -l
 		  SSL_DYNAMIC_ONLY="yes" 
 		fi
@@ -3702,20 +4034,37 @@ dnl		so it is - be adoptable
     fi
     ;;
   *)
-    if test "$cross_compiling" = "yes"; then
-	case "$with_ssl" in
-	    "$erl_xcomp_sysroot"*) ;;
-	    *) AC_MSG_ERROR([Invalid path to option --with-ssl=PATH (not a subdirectory to cross system root)]);;
-	esac
-    fi
-
     # Option given with PATH to package
     if test ! -d "$with_ssl" ; then
        AC_MSG_ERROR(Invalid path to option --with-ssl=PATH)
     fi
     SSL_ROOT="$with_ssl"
-    if test "x$MIXED_CYGWIN" = "xyes" && test -d "$with_ssl/lib/VC"; then
-	SSL_LIBDIR="$with_ssl/lib/VC"
+    SSL_CRYPTO_LIBNAME=crypto
+    SSL_SSL_LIBNAME=ssl
+    if test "x$MIXED_CYGWIN" = "xyes" -o "x$MIXED_MSYS" = "xyes" && test -d "$with_ssl/lib/VC"; then
+	if test -f "$with_ssl/lib/VC/libeay32.lib"; then
+	    SSL_LIBDIR="$with_ssl/lib/VC"
+	    SSL_CRYPTO_LIBNAME=libeay32
+	    SSL_SSL_LIBNAME=ssleay32
+	elif test -f "$with_ssl/lib/VC/openssl.lib"; then 
+	    SSL_LIBDIR="$with_ssl/lib/VC"
+	elif test -f $with_ssl/lib/VC/libeay32MD.lib; then
+	    SSL_CRYPTO_LIBNAME=libeay32MD
+            SSL_SSL_LIBNAME=ssleay32MD
+	    if test "x$enable_dynamic_ssl" = "xno" && \
+	       test -f  $with_ssl/lib/VC/static/libeay32MD.lib; then
+		SSL_LIBDIR="$with_ssl/lib/VC/static"
+	    else
+		SSL_LIBDIR="$with_ssl/lib/VC"
+            fi 
+	elif test -f "$with_ssl/lib/libeay32.lib"; then
+	    SSL_LIBDIR="$with_ssl/lib"
+	    SSL_CRYPTO_LIBNAME=libeay32
+	    SSL_SSL_LIBNAME=ssleay32
+	else
+	    # This probably wont work, but that's what the user said, so...
+	    SSL_LIBDIR="$with_ssl/lib"
+	fi
     elif test "x$ac_cv_sizeof_void_p" = "x8"; then
 	if test -f "$with_ssl/lib64/libcrypto.a"; then
 		SSL_LIBDIR="$with_ssl/lib64"
@@ -3735,7 +4084,6 @@ dnl		so it is - be adoptable
 	SSL_DYNAMIC_ONLY=yes
     fi
     SSL_INCLUDE="-I$with_ssl/include"
-    OPENSSL_CMD="$with_ssl/bin/openssl"
     SSL_APP=ssl
     CRYPTO_APP=crypto
     SSH_APP=ssh
@@ -3941,9 +4289,9 @@ esac
 
 			
 AC_ARG_WITH(javac,
-[  --with-javac=JAVAC      specify Java compiler to use
-  --with-javac            use a Java compiler if found (default)
-  --without-javac         don't use any Java compiler])
+AS_HELP_STRING([--with-javac=JAVAC], [specify Java compiler to use])
+AS_HELP_STRING([--with-javac], [use a Java compiler if found (default)])
+AS_HELP_STRING([--without-javac], [don't use any Java compiler]))
 
 dnl
 dnl Then there are a number of apps which needs a java compiler...
@@ -4107,7 +4455,6 @@ AH_BOTTOM([
 #endif
 ])
 
-
 dnl ----------------------------------------------------------------------
 dnl Output the result.
 dnl ----------------------------------------------------------------------
@@ -4116,8 +4463,6 @@ dnl  Note that the output files are relative to $srcdir
 
 AC_OUTPUT(
   emulator/$host/Makefile:emulator/Makefile.in
-  emulator/zlib/$host/Makefile:emulator/zlib/Makefile.in
-  emulator/pcre/$host/Makefile:emulator/pcre/Makefile.in
   epmd/src/$host/Makefile:epmd/src/Makefile.in
   etc/common/$host/Makefile:etc/common/Makefile.in
   include/internal/$host/ethread.mk:include/internal/ethread.mk.in
@@ -4131,8 +4476,7 @@ dnl The ones below should be moved to their respective lib
 dnl
   ../lib/ic/c_src/$host/Makefile:../lib/ic/c_src/Makefile.in
   ../lib/os_mon/c_src/$host/Makefile:../lib/os_mon/c_src/Makefile.in
-  ../lib/ssl/c_src/$host/Makefile:../lib/ssl/c_src/Makefile.in
-  ../lib/ssl/examples/certs/$host/Makefile:../lib/ssl/examples/certs/Makefile.in
+dnl  ../lib/ssl/c_src/$host/Makefile:../lib/ssl/c_src/Makefile.in
   ../lib/crypto/c_src/$host/Makefile:../lib/crypto/c_src/Makefile.in
   ../lib/orber/c_src/$host/Makefile:../lib/orber/c_src/Makefile.in
   ../lib/runtime_tools/c_src/$host/Makefile:../lib/runtime_tools/c_src/Makefile.in
diff --git a/erts/doc/specs/.gitignore b/erts/doc/specs/.gitignore
new file mode 100644
index 0000000000..322eebcb06
--- /dev/null
+++ b/erts/doc/specs/.gitignore
@@ -0,0 +1 @@
+specs_*.xml
diff --git a/erts/doc/src/Makefile b/erts/doc/src/Makefile
index 3dfefa2001..13cf003aad 100644
--- a/erts/doc/src/Makefile
+++ b/erts/doc/src/Makefile
@@ -1,21 +1,24 @@
 # 
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 1997-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 1997-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 # 
+
+SPECS_ESRC = ../../preloaded/src/
+
 include $(ERL_TOP)/make/target.mk
 include $(ERL_TOP)/make/$(TARGET)/otp.mk
 
@@ -43,8 +46,15 @@ XML_REF1_FILES = epmd.xml \
 	run_erl.xml \
 	start.xml
 
+XML_REF3_EFILES = \
+	erl_prim_loader.xml \
+	erlang.xml \
+	init.xml \
+	zlib.xml
+
 XML_REF3_FILES = \
 	driver_entry.xml \
+	erl_nif.xml \
 	erl_set_memory_block.xml \
 	erl_driver.xml \
 	erl_prim_loader.xml \
@@ -97,18 +107,26 @@ HTML_REF_MAN_FILE = $(HTMLDIR)/index.html
 
 TOP_PDF_FILE = $(PDFDIR)/$(APPLICATION)-$(VSN).pdf
 
+SPECS_FILES = $(XML_REF3_EFILES:%.xml=$(SPECDIR)/specs_%.xml)
+
+TOP_SPECS_FILE = specs.xml
+
 # ----------------------------------------------------
 # FLAGS 
 # ----------------------------------------------------
 XML_FLAGS += 
 
+KERNEL_SRC=$(ERL_TOP)/lib/kernel/src
+KERNEL_INCLUDE=$(ERL_TOP)/lib/kernel/include
+SPECS_FLAGS = -I$(KERNEL_SRC) -I$(KERNEL_INCLUDE)
+
 # ----------------------------------------------------
 # Targets
 # ----------------------------------------------------
 $(HTMLDIR)/%.gif: %.gif
 	$(INSTALL_DATA) $< $@
 
-docs: pdf html man $(INFO_FILE) 
+docs: man pdf html $(INFO_FILE)
 
 $(TOP_PDF_FILE): $(XML_FILES)
 
@@ -131,23 +149,40 @@ clean:
 	rm -f $(MAN1DIR)/*
 	rm -f $(MAN3DIR)/*
 	rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo)
+	rm -f $(SPECDIR)/*
 	rm -f errs core *~ 
 
+$(SPECDIR)/specs_driver_entry.xml:
+	escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \
+                -o$(dir $@) -module driver_entry
+$(SPECDIR)/specs_erl_nif.xml:
+	escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \
+                -o$(dir $@) -module erl_nif
+$(SPECDIR)/specs_erl_set_memory_block.xml:
+	escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \
+                -o$(dir $@) -module erl_set_memory_block
+$(SPECDIR)/specs_erl_driver.xml:
+	escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \
+                -o$(dir $@) -module erl_driver
+$(SPECDIR)/specs_erts_alloc.xml:
+	escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \
+                -o$(dir $@) -module erts_alloc
+
 # ----------------------------------------------------
 # Release Target
 # ---------------------------------------------------- 
 include $(ERL_TOP)/make/otp_release_targets.mk
 
 release_docs_spec: docs
-	$(INSTALL_DIR) $(RELSYSDIR)/doc/pdf
-	$(INSTALL_DATA) $(TOP_PDF_FILE) $(RELSYSDIR)/doc/pdf
-	$(INSTALL_DIR) $(RELSYSDIR)/doc/html
+	$(INSTALL_DIR) "$(RELSYSDIR)/doc/pdf"
+	$(INSTALL_DATA) $(TOP_PDF_FILE) "$(RELSYSDIR)/doc/pdf"
+	$(INSTALL_DIR) "$(RELSYSDIR)/doc/html"
 	$(INSTALL_DATA) $(HTMLDIR)/* \
-		$(RELSYSDIR)/doc/html
-	$(INSTALL_DATA) $(INFO_FILE) $(RELSYSDIR)
-	$(INSTALL_DIR) $(RELEASE_PATH)/man/man3
-	$(INSTALL_DATA) $(MAN3DIR)/* $(RELEASE_PATH)/man/man3
-	$(INSTALL_DIR) $(RELEASE_PATH)/man/man1
-	$(INSTALL_DATA) $(MAN1_FILES) $(RELEASE_PATH)/man/man1
+		"$(RELSYSDIR)/doc/html"
+	$(INSTALL_DATA) $(INFO_FILE) "$(RELSYSDIR)"
+	$(INSTALL_DIR) "$(RELEASE_PATH)/man/man3"
+	$(INSTALL_DATA) $(MAN3DIR)/* "$(RELEASE_PATH)/man/man3"
+	$(INSTALL_DIR) "$(RELEASE_PATH)/man/man1"
+	$(INSTALL_DATA) $(MAN1_FILES) "$(RELEASE_PATH)/man/man1"
 
 release_spec:
diff --git a/erts/doc/src/absform.xml b/erts/doc/src/absform.xml
index 4c84412dd6..4455d0ac92 100644
--- a/erts/doc/src/absform.xml
+++ b/erts/doc/src/absform.xml
@@ -4,7 +4,7 @@
 <chapter>
   <header>
     <copyright>
-      <year>2001</year><year>2009</year>
+      <year>2001</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -285,7 +285,8 @@
       <item>If E is <c><![CDATA[fun Name / Arity]]></c>, then
        Rep(E) = <c><![CDATA[{'fun',LINE,{function,Name,Arity}}]]></c>.</item>
       <item>If E is <c><![CDATA[fun Module:Name/Arity]]></c>, then
-       Rep(E) = <c><![CDATA[{'fun',LINE,{function,Module,Name,Arity}}]]></c>.</item>
+       Rep(E) = <c><![CDATA[{'fun',LINE,{function,Rep(Module),Rep(Name),Rep(Arity)}}]]></c>.
+      (Before the R15 release: Rep(E) = <c><![CDATA[{'fun',LINE,{function,Module,Name,Arity}}]]></c>.)</item>
       <item>If E is <c><![CDATA[fun Fc_1 ; ... ; Fc_k end]]></c> 
        where each <c><![CDATA[Fc_i]]></c> is a function clause then Rep(E) =
       <c><![CDATA[{'fun',LINE,{clauses,[Rep(Fc_1), ..., Rep(Fc_k)]}}]]></c>.</item>
diff --git a/erts/doc/src/alt_dist.xml b/erts/doc/src/alt_dist.xml
index a929aec97f..038950b54d 100644
--- a/erts/doc/src/alt_dist.xml
+++ b/erts/doc/src/alt_dist.xml
@@ -4,7 +4,7 @@
 <chapter>
   <header>
     <copyright>
-      <year>2000</year><year>2010</year>
+      <year>2000</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -377,7 +377,7 @@
 ( 1) typedef enum { 
 ( 2)     portTypeUnknown,      /* An uninitialized port */
 ( 3)     portTypeListener,     /* A listening port/socket */
-( 4)     portTypeAcceptor,     /* An intermidiate stage when accepting 
+( 4)     portTypeAcceptor,     /* An intermediate stage when accepting
 ( 5)                              on a listen port */
 ( 6)     portTypeConnector,    /* An intermediate stage when connecting */
 ( 7)     portTypeCommand,      /* A connected open port in command mode */
@@ -434,7 +434,7 @@
 (13)     Word received;            /* Bytes received */
 (14)     struct uds_data *partner; /* The partner in an accept/listen pair */
 (15)     struct uds_data *next;    /* Next structure in list */
-(16)     /* The input buffer and it's data */
+(16)     /* The input buffer and its data */
 (17)     int buffer_size;          /* The allocated size of the input buffer */
 (18)     int buffer_pos;           /* Current position in input buffer */
 (19)     int header_pos;           /* Where the current header is in the 
@@ -825,7 +825,7 @@
         I/O vector itself. One can use this to allocate the binaries
         for the queue "manually" in the driver, but we'll just fill
         the binary array with NULL values (line 7) , which will make
-        the runtime system allocate it's own buffers when we call
+        the runtime system allocate its own buffers when we call
         <c><![CDATA[driver_enqv]]></c> (line 37).</p>
       <p></p>
       <p>The routine builds an I/O vector containing the header bytes
@@ -942,7 +942,7 @@
          between invocations of Erlang nodes with the same name.</item>
       </list>
       <p>The control interface gets a buffer to return its value in,
-        but is free to allocate it's own buffer is the provided one is
+        but is free to allocate its own buffer is the provided one is
         to small. Here is the code for <c><![CDATA[uds_control]]></c>:</p>
       <code type="none"><![CDATA[
 ( 1) static int uds_control(ErlDrvData handle, unsigned int command, 
@@ -1042,7 +1042,7 @@
       <c><![CDATA[net_kernel:start/1]]></c> function, which is useful as it starts
       the distribution on a running system, where tracing/debugging
       can be performed. The <c><![CDATA[net_kernel:start/1]]></c> routine takes a
-      list as it's single argument. The lists first element should be
+      list as its single argument. The lists first element should be
       the node name (without the "@hostname") as an atom, and the second (and
       last) element should be one of the atoms <c><![CDATA[shortnames]]></c> or 
       <c><![CDATA[longnames]]></c>. In the example case <c><![CDATA[shortnames]]></c> is
diff --git a/erts/doc/src/driver.xml b/erts/doc/src/driver.xml
index 12c79aee90..52283879c7 100644
--- a/erts/doc/src/driver.xml
+++ b/erts/doc/src/driver.xml
@@ -4,7 +4,7 @@
 <chapter>
   <header>
     <copyright>
-      <year>2001</year><year>2010</year>
+      <year>2001</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -30,11 +30,11 @@
   </header>
 
   <note><p>This document was written a long time ago. A lot of it is still
-           valid, but some things have changed since it was first written.
-	   Updates of this document are planned for the future. The reader
-	   is encouraged to also read the
-	   <seealso marker="erl_driver">erl_driver</seealso>, and the
-	   <seealso marker="erl_driver">driver_entry</seealso> documentation.
+           interesting since it explains important concepts, but it was
+	   written for an older driver interface so the examples do not
+	   work anymore. The reader is encouraged to read
+	   <seealso marker="erl_driver">erl_driver</seealso> and the
+	   <seealso marker="driver_entry">driver_entry</seealso> documentation.
    </p></note>
 
   <section>
@@ -63,7 +63,8 @@
     <p>This is a simple driver for accessing a postgres
       database using the libpq C client library. Postgres
       is used because it's free and open source. For information
-      on postgres, refer to the website www.postgres.org.</p>
+      on postgres, refer to the website
+      <url href="http://www.postgres.org">www.postgres.org</url>.</p>
     <p>The driver is synchronous, it uses the synchronous calls of
       the client library. This is only for simplicity, and is
       generally not good, since it will
@@ -196,18 +197,21 @@ static ErlDrvData start(ErlDrvPort port, char *command)
     <p>We call disconnect to log out from the database.
       (This should have been done from Erlang, but just in case.)</p>
     <code type="none"><![CDATA[
- static int do_disconnect(our_data_t* data, ei_x_buff* x);
+static int do_disconnect(our_data_t* data, ei_x_buff* x);
 
 static void stop(ErlDrvData drv_data)
 {
-    do_disconnect((our_data_t*)drv_data, NULL);
+    our_data_t* data = (our_data_t*)drv_data;
+
+    do_disconnect(data, NULL);
+    driver_free(data);
 }
     ]]></code>
     <p>We use the binary format only to return data to the emulator;
       input data is a string paramater for <c><![CDATA[connect]]></c> and
       <c><![CDATA[select]]></c>. The returned data consists of Erlang terms.</p>
     <p>The functions <c><![CDATA[get_s]]></c> and <c><![CDATA[ei_x_to_new_binary]]></c> are
-      utilities that is used to make the code shorter. <c><![CDATA[get_s]]></c>
+      utilities that are used to make the code shorter. <c><![CDATA[get_s]]></c>
       duplicates the string and zero-terminates it, since the
       postgres client library wants that. <c><![CDATA[ei_x_to_new_binary]]></c>
       takes an <c><![CDATA[ei_x_buff]]></c> buffer and allocates a binary and
@@ -241,7 +245,7 @@ static int control(ErlDrvData drv_data, unsigned int command, char *buf,
     return r;
 }
     ]]></code>
-    <p>In <c><![CDATA[do_connect]]></c> is where we log in to the database. If the connection
+    <p><c><![CDATA[do_connect]]></c> is where we log in to the database. If the connection
       was successful we store the connection handle in our driver
       data, and return ok. Otherwise, we return the error message
       from postgres, and store <c><![CDATA[NULL]]></c> in the driver data.</p>
@@ -261,7 +265,7 @@ static int do_connect(const char *s, our_data_t* data, ei_x_buff* x)
 }
     ]]></code>
     <p>If we are connected (if the connection handle is not <c><![CDATA[NULL]]></c>),
-      we log out from the database. We need to check if a we should
+      we log out from the database. We need to check if we should
       encode an ok, since we might get here from the <c><![CDATA[stop]]></c>
       function, which doesn't return data to the emulator.</p>
     <code type="none"><![CDATA[
@@ -276,7 +280,7 @@ static int do_disconnect(our_data_t* data, ei_x_buff* x)
     return 0;
 }
     ]]></code>
-    <p>We execute a query and encodes the result. Encoding is done
+    <p>We execute a query and encode the result. Encoding is done
       in another C module, <c><![CDATA[pg_encode.c]]></c> which is also provided
       as sample code.</p>
     <code type="none"><![CDATA[
@@ -288,7 +292,7 @@ static int do_select(const char* s, our_data_t* data, ei_x_buff* x)
     return 0;
 }
     ]]></code>
-    <p>Here we simply checks the result from postgres, and
+    <p>Here we simply check the result from postgres, and
       if it's data we encode it as lists of lists with
       column data. Everything from postgres is C strings,
       so we just use <c><![CDATA[ei_x_encode_string]]></c> to send
@@ -389,7 +393,7 @@ disconnect(Port) ->
 select(Port, Query) ->
     binary_to_term(port_control(Port, ?DRV_SELECT, Query)).
     ]]></code>
-    <p>The api is simple: <c><![CDATA[connect/1]]></c> loads the driver, opens it
+    <p>The API is simple: <c><![CDATA[connect/1]]></c> loads the driver, opens it
       and logs on to the database, returning the Erlang port
       if successful, <c><![CDATA[select/2]]></c> sends a query to the driver,
       and returns the result, <c><![CDATA[disconnect/1]]></c> closes the
@@ -413,8 +417,8 @@ select(Port, Query) ->
     <title>Sample asynchronous driver</title>
     <p>Sometimes database queries can take long time to
       complete, in our <c><![CDATA[pg_sync]]></c> driver, the emulator
-      halts while the driver is doing it's job. This is
-      often not acceptable, since no other Erlang processes
+      halts while the driver is doing its job. This is
+      often not acceptable, since no other Erlang process
       gets a chance to do anything. To improve on our
       postgres driver, we reimplement it using the asynchronous
       calls in LibPQ.</p>
@@ -469,7 +473,7 @@ typedef struct our_data_t {
       whether the driver is waiting for a connection or waiting
       for the result of a query. (This is needed since the entry
       <c><![CDATA[ready_io]]></c> will be called both when connecting and
-      when there is query result.)</p>
+      when there is a query result.)</p>
     <code type="none"><![CDATA[
 static int do_connect(const char *s, our_data_t* data)
 {
@@ -568,7 +572,7 @@ static void ready_io(ErlDrvData drv_data, ErlDrvEvent event)
       connection is successful, or error if it's not. If the
       connection is not yet established, we simply return; <c><![CDATA[ready_io]]></c>
       will be called again.</p>
-    <p>If we have result from a connect, indicated that we have data in
+    <p>If we have a result from a connect, indicated by having data in
       the <c><![CDATA[x]]></c> buffer, we no longer need to select on
       output (<c><![CDATA[ready_output]]></c>), so we remove this by calling
       <c><![CDATA[driver_select]]></c>.</p>
@@ -627,9 +631,9 @@ return_port_data(Port) ->
       message queue. The function <c><![CDATA[return_port_data]]></c> above
       receives data from the port. Since the data is in
       binary format, we use <c><![CDATA[binary_to_term/1]]></c> to convert
-      it to Erlang term. Note that the driver is opened in
-      binary mode, <c><![CDATA[open_port/2]]></c> is called with the option
-      <c><![CDATA[[binary]]]></c>. This means that data sent from the driver
+      it to an Erlang term. Note that the driver is opened in
+      binary mode (<c><![CDATA[open_port/2]]></c> is called with the option
+      <c><![CDATA[[binary]]]></c>). This means that data sent from the driver
       to the emulator is sent as binaries. Without the <c><![CDATA[binary]]></c>
       option, they would have been lists of integers.</p>
   </section>
@@ -643,15 +647,15 @@ return_port_data(Port) ->
       of a list of integers. For large lists (more than 100000
       elements), this will take some time, so we will perform this
       as an asynchronous task.</p>
-    <p>The asynchronous api for drivers are quite complicated. First
+    <p>The asynchronous API for drivers is quite complicated. First
       of all, the work must be prepared. In our example we do this
       in <c><![CDATA[output]]></c>. We could have used <c><![CDATA[control]]></c> just as well,
       but we want some variation in our examples. In our driver, we allocate
-      a structure that contains all needed for the asynchronous task
+      a structure that contains anything that's needed for the asynchronous task
       to do the work. This is done in the main emulator thread.
       Then the asynchronous function is called from a driver thread,
-      separate from the main emulator thread. Note that the driver-
-      functions are not reentrant, so they shouldn't be used.
+      separate from the main emulator thread. Note that the driver-functions
+      are not reentrant, so they shouldn't be used.
       Finally, after the function is completed, the driver callback
       <c><![CDATA[ready_async]]></c> is called from the main emulator thread,
       this is where we return the result to Erlang. (We can't
@@ -689,7 +693,7 @@ static ErlDrvEntry next_perm_driver_entry = {
       be sent later from the <c><![CDATA[ready_async]]></c> call-back.</p>
     <p>The <c><![CDATA[async_data]]></c> will be passed to the <c><![CDATA[do_perm]]></c> function.
       We do not use a <c><![CDATA[async_free]]></c> function (the last argument to
-      <c><![CDATA[driver_async]]></c>, it's only used if the task is cancelled
+      <c><![CDATA[driver_async]]></c>), it's only used if the task is cancelled
       programmatically.</p>
     <code type="none"><![CDATA[
 struct our_async_data {
@@ -740,7 +744,7 @@ static void ready_async(ErlDrvData drv_data, ErlDrvThreadData async_data)
     ErlDrvPort port = reinterpret_cast<ErlDrvPort>(drv_data);
     our_async_data* d = reinterpret_cast<our_async_data*>(async_data);
     int n = d->data.size(), result_n = n*2 + 3;
-    ErlDrvTermData* result = new ErlDrvTermData[result_n], * rp = result;
+    ErlDrvTermData *result = new ErlDrvTermData[result_n], *rp = result;
     for (vector<int>::iterator i = d->data.begin();
          i != d->data.end(); ++i) {
         *rp++ = ERL_DRV_INT;
diff --git a/erts/doc/src/driver_entry.xml b/erts/doc/src/driver_entry.xml
index e71b48bd92..a2efdf3ebc 100644
--- a/erts/doc/src/driver_entry.xml
+++ b/erts/doc/src/driver_entry.xml
@@ -4,7 +4,7 @@
 <cref>
   <header>
     <copyright>
-      <year>2001</year><year>2010</year>
+      <year>2001</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -34,32 +34,36 @@
   <lib>driver_entry</lib>
   <libsummary>The driver-entry structure used by erlang drivers.</libsummary>
   <description>
-    <p>As of erts version 5.5.3 the driver interface has been extended
-      (see <seealso marker="driver_entry#extended_marker">extended marker</seealso>).
-      The extended interface introduce
-      <seealso marker="erl_driver#version_management">version management</seealso>,
-      the possibility to pass capability flags
-      (see <seealso marker="driver_entry#driver_flags">driver flags</seealso>)
-      to the runtime system at driver initialization, and some new
-      driver API functions.      </p>
+    <p>
+      As of erts version 5.9 (OTP release R15B) the driver interface
+      has been changed with larger types for the callbacks
+      <seealso marker="#output">output</seealso>,
+      <seealso marker="#control">control</seealso> and
+      <seealso marker="#call">call</seealso>.
+      See driver <seealso marker="erl_driver#version_management">
+      version management</seealso> in
+      <seealso marker="erl_driver">erl_driver</seealso>.
+    </p>
     <note>
       <p>Old drivers (compiled with an <c>erl_driver.h</c> from an
-        earlier erts version than 5.5.3) have to be recompiled
-        (but does not have to use the extended interface).</p>
+      earlier erts version than 5.9) have to be updated and have
+      to use the extended interface (with
+      <seealso marker="erl_driver#version_management">version management
+      </seealso>).</p>
     </note>
     <p>The <c>driver_entry</c> structure is a C struct that all erlang
-      drivers defines. It contains entry points for the erlang driver
+      drivers define. It contains entry points for the erlang driver
       that are called by the erlang emulator when erlang code accesses
       the driver.</p>
     <p>      
       <marker id="emulator"></marker>
-      The <seealso marker="driver_entry">erl_driver</seealso> driver
-      API functions needs a port handle
+      The <seealso marker="erl_driver">erl_driver</seealso> driver
+      API functions need a port handle
       that identifies the driver instance (and the port in the
       emulator). This is only passed to the <c>start</c> function, but
       not to the other functions. The <c>start</c> function returns a
       driver-defined handle that is passed to the other functions. A
-      common practice is to have the <c>start</c> function allocating
+      common practice is to have the <c>start</c> function allocate
       some application-defined structure and stash the <c>port</c>
       handle in it, to use it later with the driver API functions.</p>
     <p>The driver call-back functions are called synchronously from the
@@ -84,7 +88,7 @@
       the emulator, the driver is <em>not</em> allowed to modify the
       <c>driver_entry</c>.</p>
     <note>
-      <p>Do <em>not</em> declare the <c>driver_entry</c><c>const</c>. This since the emulator needs to
+      <p>Do <em>not</em> declare the <c>driver_entry</c> <c>const</c>. This since the emulator needs to
         modify the <c>handle</c>, and the <c>handle2</c>
         fields. A statically allocated, and <c>const</c>
         declared <c>driver_entry</c> may be located in
@@ -116,12 +120,12 @@ typedef struct erl_drv_entry {
     void (*stop)(ErlDrvData drv_data);
                                 /* called when port is closed, and when the
                                    emulator is halted. */
-    void (*output)(ErlDrvData drv_data, char *buf, int len);
+    void (*output)(ErlDrvData drv_data, char *buf, ErlDrvSizeT len);
                                 /* called when we have output from erlang to 
                                    the port */
     void (*ready_input)(ErlDrvData drv_data, ErlDrvEvent event); 
                                 /* called when we have input from one of 
-                                   the driver's handles) */
+                                   the driver's handles */
     void (*ready_output)(ErlDrvData drv_data, ErlDrvEvent event);  
                                 /* called when output is possible to one of 
                                    the driver's handles */
@@ -130,10 +134,11 @@ typedef struct erl_drv_entry {
     void (*finish)(void);       /* called before unloading the driver -
                                    DYNAMIC DRIVERS ONLY */
     void *handle;               /* Reserved -- Used by emulator internally */
-    int (*control)(ErlDrvData drv_data, unsigned int command, char *buf, 
-                   int len, char **rbuf, int rlen); 
+    ErlDrvSSizeT (*control)(ErlDrvData drv_data, unsigned int command,
+                            char *buf, ErlDrvSizeT len,
+			    char **rbuf, ErlDrvSizeT rlen);
                                 /* "ioctl" for drivers - invoked by 
-                                   port_control/3) */
+                                   port_control/3 */
     void (*timeout)(ErlDrvData drv_data);        /* Handling of timeout in driver */
     void (*outputv)(ErlDrvData drv_data, ErlIOVec *ev);
                                 /* called when we have output from erlang
@@ -144,9 +149,10 @@ typedef struct erl_drv_entry {
                                    closed, and there is data in the 
                                    driver queue that needs to be flushed
                                    before 'stop' can be called */
-    int (*call)(ErlDrvData drv_data, unsigned int command, char *buf, 
-                   int len, char **rbuf, int rlen, unsigned int *flags); 
-                                /* Works mostly like 'control', a syncronous
+    ErlDrvSSizeT (*call)(ErlDrvData drv_data, unsigned int command,
+                         char *buf, ErlDrvSizeT len,
+			 char **rbuf, ErlDrvSizeT rlen, unsigned int *flags);
+                                /* Works mostly like 'control', a synchronous
                                    call into the driver. */
     void (*event)(ErlDrvData drv_data, ErlDrvEvent event,
                   ErlDrvEventData event_data);
@@ -172,7 +178,7 @@ typedef struct erl_drv_entry {
           added to the driver list.) The driver should return 0, or if
           the driver can't initialize, -1.</p>
       </item>
-      <tag><marker id="start"/>int (*start)(ErlDrvPort port, char* command)</tag>
+      <tag><marker id="start"/>ErlDrvData (*start)(ErlDrvPort port, char* command)</tag>
       <item>
         <p>This is called when the driver is instantiated, when
           <c>open_port/2</c> is called. The driver should return a
@@ -188,9 +194,11 @@ typedef struct erl_drv_entry {
         <p>This is called when the port is closed, with
           <c>port_close/1</c> or <c>Port ! {self(), close}</c>. Note
           that terminating the port owner process also closes the
-          port.</p>
+          port. If <c>drv_data</c> is a pointer to memory allocated in
+	  <c>start</c>, then <c>stop</c> is the place to deallocate that
+	  memory.</p>
       </item>
-      <tag><marker id="output"/>void (*output)(ErlDrvData drv_data, char *buf, int len)</tag>
+      <tag><marker id="output"/>void (*output)(ErlDrvData drv_data, char *buf, ErlDrvSizeT len)</tag>
       <item>
         <p>This is called when an erlang process has sent data to the
           port. The data is pointed to by <c>buf</c>, and is
@@ -217,6 +225,10 @@ typedef struct erl_drv_entry {
           completes, write to the pipe (use <c>SetEvent</c> on
           Windows), this will make the emulator call
           <c>ready_input</c> or <c>ready_output</c>.</p>
+          <p>Spurious events may happen. That is, calls to <c>ready_input</c>
+          or <c>ready_output</c> even though no real events are signaled. In
+          reality it should be rare (and OS dependant), but a robust driver
+          must nevertheless be able to handle such cases.</p>
       </item>
       <tag><marker id="driver_name"/>char *driver_name</tag>
       <item>
@@ -233,11 +245,11 @@ typedef struct erl_drv_entry {
       </item>
       <tag>void *handle</tag>
       <item>
-        <p>This field is reserved for the emulators internal use. The
+        <p>This field is reserved for the emulator's internal use. The
           emulator will modify this field; therefore, it is important
           that the <c>driver_entry</c> isn't declared <c>const</c>.</p> 
       </item>
-      <tag><marker id="control"></marker>int (*control)(ErlDrvData drv_data, unsigned int command, char *buf,  int len, char **rbuf, int rlen)</tag>
+      <tag><marker id="control"></marker>ErlDrvSSizeT (*control)(ErlDrvData drv_data, unsigned int command, char *buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen)</tag>
       <item>
         <p>This is a special routine invoked with the erlang function
           <c>port_control/3</c>. It works a little like an "ioctl" for
@@ -310,7 +322,7 @@ typedef struct erl_drv_entry {
           opposed to the asynchronous function, which is called in
           some thread (if multithreading is enabled).</p>
       </item>
-      <tag><marker id="call"/>int (*call)(ErlDrvData drv_data, unsigned int command, char *buf, int len, char **rbuf, int rlen, unsigned int *flags)</tag>
+      <tag><marker id="call"/>ErlDrvSSizeT (*call)(ErlDrvData drv_data, unsigned int command, char *buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen, unsigned int *flags)</tag>
       <item>
         <p>This function is called from <c>erlang:port_call/3</c>. It
           works a lot like the <c>control</c> call-back, but uses the
@@ -397,7 +409,7 @@ typedef struct erl_drv_entry {
       <tag>void *handle2</tag>
       <item>
         <p>
-          This field is reserved for the emulators internal use. The
+          This field is reserved for the emulator's internal use. The
           emulator will modify this field; therefore, it is important
           that the <c>driver_entry</c> isn't declared <c>const</c>.
         </p>
@@ -446,7 +458,7 @@ typedef struct erl_drv_entry {
     <title>SEE ALSO</title>
     <p><seealso marker="erl_driver">erl_driver(3)</seealso>, 
       <seealso marker="kernel:erl_ddll">erl_ddll(3)</seealso>,
-      <seealso marker="erts:erlang">erlang(3)</seealso>,
+      <seealso marker="erlang">erlang(3)</seealso>,
       kernel(3)</p>
   </section>
 </cref>
diff --git a/erts/doc/src/epmd.xml b/erts/doc/src/epmd.xml
index 796ab3820b..3e7005410f 100644
--- a/erts/doc/src/epmd.xml
+++ b/erts/doc/src/epmd.xml
@@ -4,7 +4,7 @@
 <comref>
   <header>
     <copyright>
-      <year>1996</year><year>2009</year>
+      <year>1996</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -31,9 +31,23 @@
     <rev>A</rev>
     <file>epmd.xml</file>
   </header>
+
   <com>epmd</com>
-  <comsummary>Erlang Port Mapper Daemon </comsummary>
+  <comsummary>
+  <p>Erlang Port Mapper Daemon</p>
+  <taglist>
+  <tag><c><![CDATA[epmd [-d|-debug] [DbgExtra...] [-port No] [-daemon] [-relaxed_command_check]]]></c></tag>
+  <item>
+  <p>Starts the port mapper daemon</p>
+  </item>
+  <tag><c><![CDATA[epmd [-d|-debug] [-port No] [-names|-kill|-stop Name]]]></c></tag>
+  <item>
+  <p>Communicates with a running port mapper daemon</p>
+  </item>
+  </taglist>
+  </comsummary>
   <description>
+
     <p>This daemon acts as a name server on all hosts  involved  in
       distributed  Erlang computations. When an Erlang node 
       starts, the node has a name and it obtains an address from the host 
@@ -46,51 +60,193 @@
       The job of the <c><![CDATA[epmd]]></c> daemon is to keep track of which
       node name listens on which address. Hence, <c><![CDATA[epmd]]></c> map
       symbolic node names to machine addresses.</p>
-    <p>The daemon is started automatically by the Erlang start-up script.</p>
-    <p>The program <c><![CDATA[epmd]]></c> can also be used for a variety of other 
-      purposes, for example checking the DNS (Domain Name System) 
-      configuration of a host.</p>
+
+      <p>The TCP/IP <c>epmd</c> daemon actually only keeps track of
+      the <c>Name</c> (first) part of an Erlang node name, the <c>Host</c>
+      part (whatever is after the <c><![CDATA[@]]></c> is implicit in the
+      node name where the <c>epmd</c> daemon was actually contacted,
+      as is the IP address where the Erlang node can be
+      reached. Consistent and correct TCP naming services are
+      therefore required for an Erlang network to function
+      correctly.</p>
+
+    <taglist>
+    <tag>Starting the port mapper daemon</tag>
+    <item>
+
+        <p>The daemon is started automatically by the <c>erl</c>
+        command if the node is to be distributed and there is no
+        running instance present. If automatically launched,
+        environment variables has to be used to alter the behavior of
+        the daemon. See the <seealso
+        marker="#environment_variables">Environment
+        variables</seealso> section below.</p>
+
+        <p>If the -daemon argument is not given, the
+        <c><![CDATA[epmd]]></c> runs as a normal program with the
+	controlling terminal of the shell in which it is
+	started. Normally, it should run as a daemon.</p>
+
+	<p>Regular start-up options are described in the
+	<seealso marker="#daemon_flags">Regular options</seealso>
+	section below.</p>
+
+	<p>The <c>DbgExtra</c> options are described in the
+	<seealso marker="#debug_flags">DbgExtra options</seealso>
+        section below.</p>
+
+     </item>
+     <tag>Communicating with a running port mapper daemon</tag>
+     <item>
+
+	<p>Communicating with the running epmd daemon by means of the
+	<c>epmd</c> program is done primarily for debugging
+	purposes.</p>
+
+	<p>The different queries are described in the <seealso
+        marker="#interactive_flags">Interactive options</seealso>
+        section below.</p>
+
+     </item>
+     </taglist>
   </description>
-  <funcs>
-    <func>
-      <name>epmd [-daemon] </name>
-      <fsummary>Start a name server as a daemon</fsummary>
-      <desc>
-        <p>Starts a name server as a daemon. If it has no argument, the
-          <c><![CDATA[epmd]]></c> runs as a normal program with the controlling terminal 
-          of the shell in which it is started. Normally, it should run as a 
-          daemon.</p>
-      </desc>
-    </func>
-    <func>
-      <name>epmd -names</name>
-      <fsummary>Request the names of the registered Erlang nodes on  this host</fsummary>
-      <desc>
-        <p>Requests the names of the local Erlang nodes <c><![CDATA[epmd]]></c> has 
-          registered.</p>
-      </desc>
-    </func>
-    <func>
-      <name>epmd -kill</name>
-      <fsummary>Kill the <c><![CDATA[epmd]]></c>process</fsummary>
-      <desc>
-        <p>Kills the <c><![CDATA[epmd]]></c> process.</p>
-      </desc>
-    </func>
-    <func>
-      <name>epmd -help</name>
-      <fsummary>List options</fsummary>
-      <desc>
-        <p>Write short info about the usage including some debugging
-          options not listed here.</p>
-      </desc>
-    </func>
-  </funcs>
+  <section>
+    <marker id="daemon_flags"></marker>
+    <title>Regular options</title>
+
+    <p>These options are available when starting the actual name server. The name server is normally started automatically by the <c>erl</c> command (if not already available), but it can also be started at i.e. system start-up.</p>
+    <taglist>
+    <tag><c><![CDATA[-address List]]></c></tag>
+    <item>
+    <p>Let this instance of <c>epmd</c> listen only on the
+      comma-separated list of IP addresses and on the loopback address
+      (which is implicitly added to the list if it has not been
+      specified). This can also be set using the
+      <c><![CDATA[ERL_EPMD_ADDRESS]]></c> environment variable, see the
+      section <seealso marker="#environment_variables">Environment
+      variables</seealso> below.</p>
+    </item>
+    <tag><c><![CDATA[-port No]]></c></tag>
+    <item>
+      <p>Let this instance of epmd listen to another TCP port than
+      default 4369. This can also be set using the
+      <c><![CDATA[ERL_EPMD_PORT]]></c> environment variable, see the
+      section <seealso marker="#environment_variables">Environment
+      variables</seealso> below</p>
+    </item>
+    <tag><c><![CDATA[-d | -debug]]></c></tag>
+    <item>
+
+      <p>Enable debug output. The more <c>-d</c> flags given, the more
+      debug output you will get (to a certain limit). This option is
+      most useful when the epmd daemon is not started as a daemon.</p>
+    </item>
+    <tag><c><![CDATA[-daemon]]></c></tag>
+    <item>
+      <p>Start epmd detached from the controlling terminal. Logging will end up in syslog when available and correctly configured. If the epmd daemon is started at boot, this option should definitely be used. It is also used when the <c>erl</c> command automatically starts <c>epmd</c>.</p>
+    </item>
+    <tag><c><![CDATA[-relaxed_command_check]]></c></tag>
+    <item>
+      <p>Start the epmd program with relaxed command checking (mostly for backward compatibility). This affects the following:</p>
+      <list type="bulleted">
+          <item>
+            <p>With relaxed command checking, the <c>epmd</c> daemon can be killed from the localhost with i.e. <c>epmd -kill</c> even if there are active nodes registered. Normally only daemons with an empty node database can be killed with the <c>epmd -kill</c> command.</p>
+	  </item>
+	  <item>
+	    <p>The <c>epmd -stop</c> command (and the corresponding messages to epmd, as can be given using <c>erl_interface/ei</c>) is normally always ignored, as it opens up for strange situation when two nodes of the same name can be alive at the same time. A node unregisters itself by just closing the connection to epmd, why the <c>stop</c> command was only intended for use in debugging situations.</p>
+	    <p>With relaxed command checking enabled, you can forcibly unregister live nodes.</p>
+	  </item>
+        </list>
+	<p>Relaxed command checking can also be enabled by setting the environment variable <c>ERL_EPMD_RELAXED_COMMAND_CHECK</c> prior to starting <c>epmd</c>.</p>
+	<p>Only use relaxed command checking on systems with very limited interactive usage.</p>
+    </item>
+  </taglist>
+  </section>
 
   <section>
+    <marker id="debug_flags"></marker>
+    <title>DbgExtra options</title>
+    <p>These options are purely for debugging and testing epmd clients, they should not be used in normal operation.</p>
+
+    <taglist>
+    <tag><c><![CDATA[-packet_timeout Seconds]]></c></tag>
+    <item>
+    <p>Set the number of seconds a connection can be
+    inactive before epmd times out and closes the
+    connection (default 60).</p>
+    </item>
+    <tag><c><![CDATA[-delay_accept Seconds]]></c></tag>
+    <item>
+    <p>To simulate a busy server you can insert a delay between epmd
+    gets notified about that a new connection is requested and
+    when the connections gets accepted.</p>
+    </item>
+    <tag><c><![CDATA[-delay_write Seconds]]></c></tag>
+    <item>
+    <p>Also a simulation of a busy server. Inserts
+    a delay before a reply is sent.</p>
+    </item>
+    </taglist>
+  </section>
+  <section>
+    <marker id="interactive_flags"></marker>
+    <title>Interactive options</title>
+    <p>These options make <c>epmd</c> run as an interactive command displaying the results of sending queries ta an already running instance of <c>epmd</c>. The epmd contacted is always on the local node, but the <c>-port</c> option can be used to select between instances if several are running using different port on the host.</p>
+    <taglist>
+    <tag><c><![CDATA[-port No]]></c></tag>
+    <item>
+      <p>Contacts the <c>epmd</c> listening on the given TCP port number
+      (default 4369). This can also be set using the
+      <c><![CDATA[ERL_EPMD_PORT]]></c> environment variable, see the
+      section <seealso marker="#environment_variables">Environment
+      variables</seealso> below</p>
+    </item>
+    <tag><c><![CDATA[-names]]></c></tag>
+    <item>
+      <p>List names registered with the currently running epmd</p>
+    </item>
+    <tag><c><![CDATA[-kill]]></c></tag>
+    <item>
+      <p>Kill the currently running <c>epmd</c>.</p>
+
+      <p>Killing the running <c>epmd</c> is only allowed if <c>epmd
+        -names</c> show an empty database or
+        <c>-relaxed_command_check</c> was given when the running
+        instance of <c>epmd</c> was started. Note that
+        <c>-relaxed_command_check</c> is given when starting the
+        daemon that is to accept killing when it has live nodes
+        registered. When running epmd interactively,
+        <c>-relaxed_command_check</c> has no effect. A daemon that is
+        started without relaxed command checking has to be killed
+        using i.e. signals or some other OS specific method if it has
+        active clients registered.</p>
+    </item>
+    <tag><c><![CDATA[-stop Name]]></c></tag>
+    <item>
+       <p>Forcibly unregister a live node from <c>epmd</c>'s database</p>
+
+       <p>This command can only be used when contacting <c>epmd</c>
+       instances started with the <c>-relaxed_command_check</c>
+       flag. Note that relaxed command checking has to be enabled for
+       the <c>epmd</c> daemon contacted, When running epmd
+       interactively,
+        <c>-relaxed_command_check</c> has no effect.</p>
+    </item>
+  </taglist>
+  </section>
+  <section>
     <marker id="environment_variables"></marker>
     <title>Environment variables</title>
     <taglist>
+      <tag><c><![CDATA[ERL_EPMD_ADDRESS]]></c></tag>
+      <item>
+        <p>This environment variable may be set to a comma-separated
+          list of IP addresses, in which case the <c>epmd</c> daemon
+          will listen only on the specified address(es) and on the
+          loopback address (which is implicitly added to the list if it
+          has not been specified). The default behaviour is to listen on
+          all available IP addresses.</p>
+      </item>
       <tag><c><![CDATA[ERL_EPMD_PORT]]></c></tag>
       <item>
         <p>This environment variable can contain the port number epmd will use.
@@ -99,6 +255,15 @@
           independent clusters of nodes, to co-exist on the same host.
           All nodes in a cluster must use the same epmd port number.</p>
       </item>
+      <tag><c><![CDATA[ERL_EPMD_RELAXED_COMMAND_CHECK]]></c></tag>
+      <item>
+        <p>If set prior to start, the <c>epmd</c> daemon will behave
+        as if the <c>-relaxed_command_check</c> option was given at
+        start-up. If consequently setting this option before starting
+        the Erlang virtual machine, the automatically started
+        <c>epmd</c> will accept the <c>-kill</c> and <c>-stop</c>
+        commands without restrictions.</p>
+      </item>
     </taglist>
   </section>
 
@@ -116,5 +281,29 @@
       silently be ignored.
       </p>
   </section>
+  <section>
+  <title>Access restrictions</title>
+  <p>The <c>epmd</c> daemon accepts messages from both localhost and
+  remote hosts. However, only the query commands are answered (and
+  acted upon) if the query comes from a remote host. It is always an
+  error to try to register a nodename if the client is not a process
+  located on the same host as the <c>epmd</c> instance is running on,
+  why such requests are considered hostile and the connection is
+  immediately closed.</p>
+
+  <p>The queries accepted from remote nodes are:</p>
+      <list type="bulleted">
+          <item>
+	  <p>Port queries - i.e. on which port does the node with a given
+	  name listen</p>
+	  </item>
+	  <item>
+	  <p>Name listing - i.e. give a list of all names registered on
+	  the host</p>
+	  </item>
+     </list>
+  <p>To restrict access further, firewall software has to be used.</p>
+  </section>
+
 </comref>
 
diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml
index bb741c7836..f8a92e1ec4 100644
--- a/erts/doc/src/erl.xml
+++ b/erts/doc/src/erl.xml
@@ -4,7 +4,7 @@
 <comref>
   <header>
     <copyright>
-      <year>1996</year><year>2010</year>
+      <year>1996</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -41,6 +41,12 @@
       to scroll back to text which has scrolled off the screen.
       The <c><![CDATA[erl]]></c> program must be used, however, in pipelines or if
       you want to redirect standard input or output.</p>
+      <note><p>As of ERTS version 5.9 (OTP-R15B) the runtime system will by
+               default <em>not</em> bind schedulers to logical processors.
+	       For more information see documentation of the
+	       <seealso marker="#+sbt">+sbt</seealso> system flag.
+	    </p>
+      </note>
   </description>
   <funcs>
     <func>
@@ -211,7 +217,8 @@
       <tag><c><![CDATA[-detached]]></c></tag>
       <item>
         <p>Starts the Erlang runtime system detached from the system
-          console. Useful for running daemons and backgrounds processes.</p>
+          console. Useful for running daemons and backgrounds processes. Implies
+          <c><![CDATA[-noinput]]></c>.</p>
       </item>
       <tag><c><![CDATA[-emu_args]]></c></tag>
       <item>
@@ -435,11 +442,7 @@
 	  system with SMP support is available. <c>-smp auto</c> starts
 	  the Erlang runtime system with SMP support enabled if it is
 	  available and more than one logical processor are detected.
-	  <c>-smp disable</c> starts a runtime system without SMP support.
-	  By default <c>-smp auto</c> will be used unless a conflicting
-	  parameter has been passed, then <c>-smp disable</c> will be
-	  used. Currently only the <c>-hybrid</c> parameter conflicts
-	  with <c>-smp auto</c>.</p>
+	  <c>-smp disable</c> starts a runtime system without SMP support.</p>
         <p><em>NOTE</em>: The runtime system with SMP support will not
           be available on all supported platforms. See also the
           <seealso marker="#+S">+S</seealso> flag.</p>
@@ -521,6 +524,28 @@
 	<p>Calling <c>erlang:halt/1</c> with a string argument will still
 	produce a crash dump.</p>
       </item>
+      <tag><c><![CDATA[+e Number]]></c></tag>
+      <item>
+        <p>Set max number of ETS tables.</p>
+      </item>
+      <tag><c><![CDATA[+ec]]></c></tag>
+      <item>
+        <p>Force the <c>compressed</c> option on all ETS tables.
+              Only intended for test and evaluation.</p>
+      </item>
+      <tag><c><![CDATA[+fnl]]></c></tag>
+      <item>
+        <p>The VM works with file names as if they are encoded using the ISO-latin-1 encoding, disallowing Unicode characters with codepoints beyond 255. This is default on operating systems that have transparent file naming, i.e. all Unixes except MacOSX.</p>
+      </item>
+      <tag><c><![CDATA[+fnu]]></c></tag>
+      <item>
+        <p>The VM works with file names as if they are encoded using UTF-8 (or some other system specific Unicode encoding). This is the default on operating systems that enforce Unicode encoding, i.e. Windows and MacOSX.</p>
+	<p>By enabling Unicode file name translation on systems where this is not default, you open up to the possibility that some file names can not be interpreted by the VM and therefore will be returned to the program as raw binaries. The option is therefore considered experimental.</p>
+      </item>
+      <tag><c><![CDATA[+fna]]></c></tag>
+      <item>
+        <p>Selection between <c>+fnl</c> and <c>+fnu</c> is done based on the current locale settings in the OS, meaning that if you have set your terminal for UTF-8 encoding, the filesystem is expected to use the same encoding for filenames (use with care).</p>
+      </item>
       <tag><c><![CDATA[+hms Size]]></c></tag>
       <item>
         <p>Sets the default heap size of processes to the size
@@ -544,6 +569,13 @@
         <p>Enables auto load tracing, displaying info while loading
           code.</p>
       </item>
+      <tag><c><![CDATA[+L]]></c></tag>
+      <item>
+        <p>Don't load information about source filenames and line numbers.
+	  This will save some memory, but exceptions will not contain
+	  information about the filenames and line numbers.
+          </p>
+      </item>
       <tag><marker id="erts_alloc"><c><![CDATA[+MFlag Value]]></c></marker></tag>
       <item>
         <p>Memory allocator specific flags, see
@@ -583,6 +615,24 @@
       <item>
         <p>Force ets memory block to be moved on realloc.</p>
       </item>
+      <tag><marker id="+rg"><c><![CDATA[+rg ReaderGroupsLimit]]></c></marker></tag>
+      <item>
+        <p>Limits the amount of reader groups used by read/write locks
+	   optimized for read operations in the Erlang runtime system. By
+	   default the reader groups limit equals 8.</p>
+	<p>When the amount of schedulers is less than or equal to the reader
+	   groups limit, each scheduler has its own reader group. When the
+	   amount of schedulers is larger than the reader groups limit,
+	   schedulers share reader groups. Shared reader groups degrades
+	   read lock and read unlock performance while a large amount of
+	   reader groups degrades write lock performance, so the limit is a
+	   tradeoff between performance for read operations and performance
+	   for write operations. Each reader group currently consumes 64 byte
+	   in each read/write lock. Also note that a runtime system using
+	   shared reader groups benefits from <seealso marker="#+sbt">binding
+	   schedulers to logical processors</seealso>, since the reader groups
+	   are distributed better between schedulers.</p>
+      </item>
       <tag><marker id="+S"><c><![CDATA[+S Schedulers:SchedulerOnline]]></c></marker></tag>
       <item>
         <p>Sets the amount of scheduler threads to create and scheduler
@@ -611,44 +661,69 @@
 	    </p>
 	    <taglist>
 	      <tag><c>u</c></tag>
-	      <item><p>Same as
-	      <seealso marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, unbound)</seealso>.
-	      </p></item>
+              <item>
+                <p><c>unbound</c> - Schedulers will not be bound to logical
+		processors, i.e., the operating system decides where the
+		scheduler threads execute, and when to migrate them. This is
+		the default.</p>
+              </item>
 	      <tag><c>ns</c></tag>
-	      <item><p>Same as
-	      <seealso marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, no_spread)</seealso>.
-	      </p></item>
+              <item>
+                <p><c>no_spread</c> - Schedulers with close scheduler
+		identifiers will be bound as close as possible in hardware.</p>
+              </item>
 	      <tag><c>ts</c></tag>
-	      <item><p>Same as
-	      <seealso marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, thread_spread)</seealso>.
-	      </p></item>
+              <item>
+                <p><c>thread_spread</c> - Thread refers to hardware threads
+		(e.g. Intel's hyper-threads). Schedulers with low scheduler
+		identifiers, will be bound to the first hardware thread of
+		each core, then schedulers with higher scheduler identifiers
+		will be bound to the second hardware thread of each core,
+		etc.</p>
+              </item>
 	      <tag><c>ps</c></tag>
-	      <item><p>Same as
-	      <seealso marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, processor_spread)</seealso>.
-	      </p></item>
+              <item>
+                <p><c>processor_spread</c> - Schedulers will be spread like
+		<c>thread_spread</c>, but also over physical processor chips.</p>
+              </item>
 	      <tag><c>s</c></tag>
-	      <item><p>Same as
-	      <seealso marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, spread)</seealso>.
-	      </p></item>
+              <item>
+                <p><c>spread</c> - Schedulers will be spread as much as
+		possible.</p>
+              </item>
 	      <tag><c>nnts</c></tag>
-	      <item><p>Same as
-	      <seealso marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, no_node_thread_spread)</seealso>.
-	      </p></item>
+              <item>
+                <p><c>no_node_thread_spread</c> - Like <c>thread_spread</c>,
+		but if multiple NUMA (Non-Uniform Memory Access) nodes exists,
+		schedulers will be spread over one NUMA node at a time,
+		i.e., all logical processors of one NUMA node will be bound
+		to schedulers in sequence.</p>
+              </item>
 	      <tag><c>nnps</c></tag>
-	      <item><p>Same as
-	      <seealso marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, no_node_processor_spread)</seealso>.
-	      </p></item>
+              <item>
+                <p><c>no_node_processor_spread</c> - Like
+		<c>processor_spread</c>, but if multiple NUMA nodes exists,
+		schedulers will be spread over one NUMA node at a time, i.e.,
+		all logical processors of one NUMA node will be bound to
+		schedulers in sequence.</p>
+              </item>
 	      <tag><c>tnnps</c></tag>
-	      <item><p>Same as
-	      <seealso marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, thread_no_node_processor_spread)</seealso>.
-	      </p></item>
+              <item>
+                <p><c>thread_no_node_processor_spread</c> - A combination of
+		<c>thread_spread</c>, and <c>no_node_processor_spread</c>.
+		Schedulers will be spread over hardware threads across NUMA
+		nodes, but schedulers will only be spread over processors
+		internally in one NUMA node at a time.</p>
+              </item>
 	      <tag><c>db</c></tag>
-	      <item><p>Same as
-	      <seealso marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, default_bind)</seealso>.
-	      </p></item>
+              <item>
+                <p><c>default_bind</c> - Binds schedulers the default way.
+		Currently the default is <c>thread_no_node_processor_spread</c>
+		(which might change in the future).</p>
+              </item>
 	    </taglist>
-	    <p>Binding of schedulers are currently only supported on newer
-	       Linux and Solaris systems.</p>
+	    <p>Binding of schedulers is currently only supported on newer
+	       Linux, Solaris, FreeBSD, and Windows systems.</p>
 	    <p>If no CPU topology is available when the <c>+sbt</c> flag
 	       is processed and <c>BindType</c> is any other type than
 	       <c>u</c>, the runtime system will fail to start. CPU
@@ -657,8 +732,57 @@
 	       that the <c>+sct</c> flag may have to be passed before the
 	       <c>+sbt</c> flag on the command line (in case no CPU topology
 	       has been automatically detected).</p>
-	    <p>For more information, see
-	    <seealso marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, SchedulerBindType)</seealso>.
+            <p>The runtime system will by default <em>not</em> bind schedulers
+	       to logical processors.
+	    </p>
+	    <p><em>NOTE:</em> If the Erlang runtime system is the only operating system
+	       process that binds threads to logical processors, this
+	       improves the performance of the runtime system. However,
+	       if other operating system processes (as for example
+	       another Erlang runtime system) also bind threads to
+	       logical processors, there might be a performance penalty
+	       instead. In some cases this performance penalty might be
+	       severe. If this is the case, you are advised to not
+	       bind the schedulers.</p>
+            <p>How schedulers are bound matters. For example, in
+	       situations when there are fewer running processes than
+	       schedulers online, the runtime system tries to migrate
+	       processes to schedulers with low scheduler identifiers.
+	       The more the schedulers are spread over the hardware,
+	       the more resources will be available to the runtime
+	       system in such situations.
+	    </p>
+	    <p>
+	       <em>NOTE:</em> If a scheduler fails to bind, this
+	       will often be silently ignored. This since it isn't always
+	       possible to verify valid logical processor identifiers. If
+	       an error is reported, it will be reported to the
+	       <c>error_logger</c>. If you want to verify that the
+	       schedulers actually have bound as requested, call
+	       <seealso marker="erlang#system_info_scheduler_bindings">erlang:system_info(scheduler_bindings)</seealso>.
+	    </p>
+          </item>
+	  <tag><marker id="+sbwt"><c>+sbwt none|very_short|short|medium|long|very_long</c></marker></tag>
+	  <item>
+             <p>Set scheduler busy wait threshold. Default is <c>medium</c>.
+	        The threshold determines how long schedulers should busy
+		wait when running out of work before going to sleep.
+	     </p>
+             <p><em>NOTE:</em> This flag may be removed or changed at any time
+	        without prior notice.
+	     </p>
+	  </item>
+          <tag><marker id="+scl"><c>+scl true|false</c></marker></tag>
+          <item>
+	    <p>Enable or disable scheduler compaction of load. By default
+	    scheduler compaction of load is enabled. When enabled, load
+	    balancing will strive for a load distribution which causes
+	    as many scheduler threads as possible to be fully loaded (i.e.,
+	    not run out of work). This is accomplished by migrating load
+	    (e.g. runnable processes) into a smaller set of schedulers
+	    when schedulers frequently run out of work. When disabled,
+	    the frequency with which schedulers run out of work will
+	    not be taken into account by the load balancing logic.
 	    </p>
           </item>
           <tag><marker id="+sct"><c>+sct CpuTopology</c></marker></tag>
@@ -676,6 +800,12 @@
               <item><c><![CDATA[<IdDefs> = <LogicalIds><ThreadIds><CoreIds><ProcessorIds><NodeIds> | <LogicalIds><ThreadIds><CoreIds><NodeIds><ProcessorIds>]]></c></item>
               <item><c><![CDATA[CpuTopology = <IdDefs>:<IdDefs> | <IdDefs>]]></c></item>
             </list>
+            <p>Set a user defined CPU topology. The user defined
+	       CPU topology will override any automatically detected
+	       CPU topology. The CPU topology is used when
+	       <seealso marker="#+sbt">binding schedulers to logical
+	       processors</seealso>.
+	    </p>
             <p>Upper-case letters signify real identifiers and lower-case
 	       letters signify fake identifiers only used for description
 	       of the topology. Identifiers passed as real identifiers may
@@ -775,16 +905,41 @@
 	        how the real CPU topology looks like is likely to
 		decrease the performance of the runtime system.</p>
 	    <p>For more information, see
-	    <seealso marker="erlang#system_flag_cpu_topology">erlang:system_flag(cpu_topology, CpuTopology)</seealso>.</p>
+	    <seealso marker="erlang#system_info_cpu_topology">erlang:system_info(cpu_topology)</seealso>.</p>
           </item>
+	  <tag><marker id="+sws"><c>+sws default|legacy|proposal</c></marker></tag>
+	  <item>
+             <p>Set scheduler wakeup strategy. Default is <c>legacy</c> (has been
+	        used since OTP-R13B). The <c>proposal</c> strategy is the currently
+		proposed strategy for OTP-R16. Note that the <c>proposal</c> strategy
+		might change during OTP-R15.
+	     </p>
+             <p><em>NOTE:</em> This flag may be removed or changed at any time
+	        without prior notice.
+	     </p>
+	  </item>
+	  <tag><marker id="+swt"><c>+swt very_low|low|medium|high|very_high</c></marker></tag>
+	  <item>
+             <p>Set scheduler wakeup threshold. Default is <c>medium</c>.
+	        The threshold determines when to wake up sleeping schedulers
+		when more work than can be handled by currently awake schedulers
+		exist. A low threshold will cause earlier wakeups, and a high
+		threshold will cause later wakeups. Early wakeups will
+		distribute work over multiple schedulers faster, but work will
+		more easily bounce between schedulers.
+	     </p>
+             <p><em>NOTE:</em> This flag may be removed or changed at any time
+	        without prior notice.
+	     </p>
+	  </item>
+	  <tag><marker id="sched_thread_stack_size"><c><![CDATA[+sss size]]></c></marker></tag>
+	  <item>
+	    <p>Suggested stack size, in kilowords, for scheduler threads.
+            Valid range is 4-8192 kilowords. The default stack size
+            is OS dependent.</p>
+	  </item>
         </taglist>
       </item>
-      <tag><marker id="sched_thread_stack_size"><c><![CDATA[+sss size]]></c></marker></tag>
-      <item>
-        <p>Suggested stack size, in kilowords, for scheduler threads.
-          Valid range is 4-8192 kilowords. The default stack size
-          is OS dependent.</p>
-      </item>
       <tag><marker id="+t"><c><![CDATA[+t size]]></c></marker></tag>
       <item>
         <p>Set the maximum number of atoms the VM can handle. Default is 1048576.</p>
@@ -838,6 +993,25 @@
           <seealso marker="kernel:error_logger#warning_map/0">error_logger(3)</seealso>
           for further information.</p>
       </item>
+      <tag><c><![CDATA[+zFlag Value]]></c></tag>
+      <item>
+        <p>Miscellaneous flags.</p>
+        <taglist>
+          <tag><marker id="+zdbbl"><c>+zdbbl size</c></marker></tag>
+          <item>
+            <p>Set the distribution buffer busy limit
+	    (<seealso marker="erlang#system_info_dist_buf_busy_limit">dist_buf_busy_limit</seealso>)
+	    in kilobytes. Valid range is 1-2097151. Default is 1024.</p>
+            <p>A larger buffer limit will allow processes to buffer
+            more outgoing messages over the distribution. When the
+            buffer limit has been reached, sending processes will be
+            suspended until the buffer size has shrunk. The buffer
+            limit is per distribution channel. A higher limit will
+            give lower latency and higher throughput at the expense
+            of higher memory usage.</p>
+          </item>
+        </taglist>
+      </item>
     </taglist>
   </section>
 
@@ -877,7 +1051,7 @@
           the <c><![CDATA[-extra]]></c> section, i.e. the end of the command line
           following after an <c><![CDATA[-extra]]></c> flag.</p>
       </item>
-      <tag><c><![CDATA[ERL_ZFLAGS]]></c>and <c><![CDATA[ERL_FLAGS]]></c></tag>
+      <tag><c><![CDATA[ERL_ZFLAGS]]></c> and <c><![CDATA[ERL_FLAGS]]></c></tag>
       <item>
         <p>The content of these environment variables will be added to the
           end of the command line for <c><![CDATA[erl]]></c>.</p>
@@ -894,6 +1068,15 @@
 	  add to the code path.
   	  See <seealso marker="kernel:code">code(3)</seealso>.</p>
       </item>
+      <tag><c><![CDATA[ERL_EPMD_ADDRESS]]></c></tag>
+      <item>
+        <p>This environment variable may be set to a comma-separated
+          list of IP addresses, in which case the
+          <seealso marker="epmd">epmd</seealso> daemon
+          will listen only on the specified address(es) and on the
+          loopback address (which is implicitly added to the list if it
+          has not been specified).</p>
+      </item>
       <tag><c><![CDATA[ERL_EPMD_PORT]]></c></tag>
       <item>
         <p>This environment variable can contain the port number to use when
@@ -906,6 +1089,49 @@
   </section>
 
   <section>
+    <marker id="configuration"></marker>
+    <title>Configuration</title>
+    <p>The standard Erlang/OTP system can be re-configured to change the default
+       behavior on start-up.</p>
+    <taglist>
+       <tag>The .erlang Start-up File</tag>
+       <item>
+         <p>When Erlang/OTP is started, the system searches for a file named .erlang
+            in the directory where Erlang/OTP is started. If not found, the user's home
+            directory is searched for an .erlang file.</p>
+         <p>If an .erlang file is found, it is assumed to contain valid Erlang expressions.
+            These expressions are evaluated as if they were input to the shell.</p>
+         <p>A typical .erlang file contains a set of search paths, for example:</p>
+    <code type="none"><![CDATA[
+        io:format("executing user profile in HOME/.erlang\n",[]).
+        code:add_path("/home/calvin/test/ebin").
+        code:add_path("/home/hobbes/bigappl-1.2/ebin").
+        io:format(".erlang rc finished\n",[]).
+    ]]></code>
+       </item>
+       <tag>user_default and shell_default</tag>
+       <item>
+         <p>Functions in the shell which are not prefixed by a module name are assumed
+           to be functional objects (Funs), built-in functions (BIFs), or belong to the
+           module user_default or shell_default.</p>
+         <p>To include private shell commands, define them in a module user_default and
+           add the following argument as the first line in the .erlang file.</p>
+    <code type="none"><![CDATA[
+         code:load_abs("..../user_default").
+    ]]></code>
+       </item>
+       <tag>erl</tag>
+       <item>
+         <p>If the contents of .erlang are changed and a private version of
+           user_default is defined, it is possible to customize the Erlang/OTP environment.
+           More powerful changes can be made by supplying command line arguments in the
+           start-up script erl. Refer to erl(1) and <seealso marker="init">init(3)</seealso>
+           for further information.</p>
+       </item>
+    </taglist>
+  </section>
+
+   <section>
     <title>SEE ALSO</title>
     <p><seealso marker="init">init(3)</seealso>,
       <seealso marker="erl_prim_loader">erl_prim_loader(3)</seealso>,
diff --git a/erts/doc/src/erl_dist_protocol.xml b/erts/doc/src/erl_dist_protocol.xml
index 5978af178a..6c725fc82d 100644
--- a/erts/doc/src/erl_dist_protocol.xml
+++ b/erts/doc/src/erl_dist_protocol.xml
@@ -5,7 +5,7 @@
   <header>
     <copyright>
       <year>2007</year>
-      <year>2007</year>
+      <year>2011</year>
       <holder>Ericsson AB, All Rights Reserved</holder>
     </copyright>
     <legalnotice>
@@ -206,7 +206,7 @@ By default EPMD listens on port 4369.
       <section>
 	<title>Unregister a node from the EPMD</title>
 	<p>
-	  A node unregister itself from the EPMD by simply closing the
+	  A node unregisters itself from the EPMD by simply closing the
 	  TCP connection towards EPMD established when the node was registered.
 	</p>
       </section>
diff --git a/erts/doc/src/erl_driver.xml b/erts/doc/src/erl_driver.xml
index 5061230a33..187c263b60 100644
--- a/erts/doc/src/erl_driver.xml
+++ b/erts/doc/src/erl_driver.xml
@@ -4,7 +4,7 @@
 <cref>
   <header>
     <copyright>
-      <year>2001</year><year>2010</year>
+      <year>2001</year><year>2012</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -37,15 +37,18 @@
     <p>As of erts version 5.5.3 the driver interface has been extended
       (see <seealso marker="driver_entry#extended_marker">extended marker</seealso>).
       The extended interface introduce
-      <seealso marker="erl_driver#version_management">version management</seealso>,
+      <seealso marker="#version_management">version management</seealso>,
       the possibility to pass capability flags
       (see <seealso marker="driver_entry#driver_flags">driver flags</seealso>)
       to the runtime system at driver initialization, and some new
       driver API functions.      </p>
     <note>
-      <p>Old drivers (compiled with an <c>erl_driver.h</c> from an
-        earlier erts version than 5.5.3) have to be recompiled
-        (but does not have to use the extended interface).</p>
+      <p>As of erts version 5.9 old drivers have to be recompiled
+      and have to use the extended interface. They also have to be
+      adjusted to the
+      <seealso marker="#rewrites_for_64_bits">64-bit capable driver interface.
+      </seealso>
+      </p>
     </note>
     <p>The driver calls back to the emulator, using the API
       functions declared in <c>erl_driver.h</c>. They are used for
@@ -56,16 +59,16 @@
       instance is connected to an Erlang port. Every port has a port
       owner process. Communication with the port is normally done
       through the port owner process.</p>
-    <p>Most of the functions takes the <c>port</c> handle as an
+    <p>Most of the functions take the <c>port</c> handle as an
       argument. This identifies the driver instance. Note that this
       port handle must be stored by the driver, it is not given when
       the driver is called from the emulator (see
       <seealso marker="driver_entry#emulator">driver_entry</seealso>).</p>
-    <p>Some of the functions takes a parameter of type
+    <p>Some of the functions take a parameter of type
       <c>ErlDrvBinary</c>, a driver binary. It should be both
-      allocated and freed by the caller. Using a binary directly avoid
+      allocated and freed by the caller. Using a binary directly avoids
       one extra copying of data.</p>
-    <p>Many of the output functions has a "header buffer", with
+    <p>Many of the output functions have a "header buffer", with
       <c>hbuf</c> and <c>hlen</c> parameters. This buffer is sent as a
       list before the binary (or list, depending on port mode) that is
       sent. This is convenient when matching on messages received from
@@ -92,7 +95,7 @@
       with SMP support without being rewritten if driver
       level locking is used.</p>
     <note>
-      <p>It is assumed that drivers does not access other drivers. If
+      <p>It is assumed that drivers do not access other drivers. If
         drivers should access each other they have to provide their own
         mechanism for thread safe synchronization. Such "inter driver
         communication" is strongly discouraged.</p>
@@ -113,12 +116,12 @@
         call-backs may be made from different threads.</p>
     </note>
     <p>Most functions in this API are <em>not</em> thread-safe, i.e.,
-      they may <em>not</em> be called from an arbitrary thread. Function
+      they may <em>not</em> be called from an arbitrary thread. Functions
       that are not documented as thread-safe may only be called from
       driver call-backs or function calls descending from a driver
       call-back call. Note that driver call-backs may be called from
       different threads. This, however, is not a problem for any
-      functions in this API, since the emulator have control over
+      function in this API, since the emulator has control over
       these threads.</p>
     <note>
       <p>Functions not explicitly documented as thread-safe are
@@ -155,10 +158,10 @@
           more information.</p>
       </item>
       <tag>Output functions</tag>
-      <item>With the output functions, the driver sends data back
+      <item>With the output functions, the driver sends data back to
        the emulator. They will be received as messages by the port owner
        process, see <c>open_port/2</c>. The vector function and the
-       function taking a driver binary is faster, because that avoid
+       function taking a driver binary are faster, because they avoid
        copying the data buffer. There is also a fast way of sending
        terms from the driver, without going through the binary term
        format.</item>
@@ -193,14 +196,14 @@
 	 use functionality from the POSIX thread API or the Windows
 	 native thread API.
       </p>
-      <p>The Erlang driver thread API only return error codes when it is
+      <p>The Erlang driver thread API only returns error codes when it is
          reasonable to recover from an error condition. If it isn't reasonable
 	 to recover from an error condition, the whole runtime system is
 	 terminated. For example, if a create mutex operation fails, an error
 	 code is returned, but if a lock operation on a mutex fails, the
 	 whole runtime system is terminated.
       </p>
-      <p>Note that there exist no "condition variable wait with timeout" in
+      <p>Note that there exists no "condition variable wait with timeout" in
          the Erlang driver thread API. This is due to issues with
 	 <c>pthread_cond_timedwait()</c>. When the system clock suddenly
 	 is changed, it isn't always guaranteed that you will wake up from
@@ -241,10 +244,10 @@
 	 to give you better error reports.
       </p>
       </item>
-      <tag>Adding / remove drivers</tag>
-      <item>A driver can add and later remove drivers.</item>
+      <tag>Adding / removing drivers</tag>
+      <item><p>A driver can add and later remove drivers.</p></item>
       <tag>Monitoring processes</tag>
-      <item>A driver can monitor a process that does not own a port.</item>
+      <item><p>A driver can monitor a process that does not own a port.</p></item>
       <tag><marker id="version_management">Version management</marker></tag>
       <item>
         <p>Version management is enabled for drivers that have set the
@@ -262,21 +265,209 @@
           could, under rare circumstances, mean that drivers have to
           be slightly modified. If so, this will of course be documented.
           <c>ERL_DRV_EXTENDED_MINOR_VERSION</c> will be incremented when
-          new features are added. The runtime system use the minor version
+          new features are added. The runtime system uses the minor version
           of the driver to determine what features to use.
           The runtime system will refuse to load a driver if the major
           versions differ, or if the major versions are equal and the
           minor version used by the driver is greater than the one used
           by the runtime system.</p>
-        <p>The emulator tries to check that a driver that doesn't use the
-          extended driver interface isn't incompatible when loading it.
-          It can, however, not make sure that it isn't incompatible. Therefore,
-          when loading a driver that doesn't use the extended driver
-          interface, there is a risk that it will be loaded also when
-          the driver is incompatible. When the driver use the extended driver
-          interface, the emulator can verify that it isn't of an incompatible
-          driver version. You are therefore advised to use the extended driver
-          interface.</p>
+	<p>The emulator will refuse to load a driver that does not use
+	  the extended driver interface since,
+	  to allow for 64-bit capable drivers,
+	  incompatible type changes for the callbacks
+	  <seealso marker="driver_entry#output">output</seealso>,
+	  <seealso marker="driver_entry#control">control</seealso> and
+	  <seealso marker="driver_entry#call">call</seealso>
+	  were introduced in release R15B. A driver written
+	  with the old types would compile with warnings and when
+	  called return garbage sizes to the emulator causing it
+	  to read random memory and create huge incorrect result blobs.</p>
+	<p>Therefore it is not enough to just recompile drivers written with
+	  version management for pre-R15B types; the types have to be changed
+	  in the driver suggesting other rewrites especially regarding
+	  size variables. Investigate all warnings when recompiling!</p>
+	<p>Also, the API driver functions <c>driver_output*</c>,
+	  <c>driver_vec_to_buf</c>, <c>driver_alloc/realloc*</c>
+	  and the <c>driver_*</c> queue functions were changed to have
+	  larger length arguments and return values. This is a
+	  lesser problem since code that passes smaller types
+	  will get them auto converted in the calls and as long as
+	  the driver does not handle sizes that overflow an <c>int</c>
+	  all will work as before.</p>
+      </item>
+    </taglist>
+  </section>
+
+  <section>
+    <marker id="rewrites_for_64_bits"/>
+    <title>
+      REWRITES FOR 64-BIT DRIVER INTERFACE
+    </title>
+    <p>
+      For erts-5.9 two new integer types
+      <seealso marker="#ErlDrvSizeT">ErlDrvSizeT</seealso> and
+      <seealso marker="#ErlDrvSSizeT">ErlDrvSSizeT</seealso>
+      were introduced that can hold 64-bit sizes if necessary.
+    </p>
+    <p>
+      To not update a driver and just recompile it probably works
+      when building for a 32-bit machine creating a false sense of security.
+      Hopefully that will generate many important warnings.
+      But when recompiling the same driver later on for a 64-bit machine
+      there <em>will</em> be warnings and almost certainly crashes.
+      So it is a BAD idea to postpone updating the driver and
+      not fixing the warnings!
+    </p>
+    <p>
+      When recompiling with <c>gcc</c> use the <c>-Wstrict-prototypes</c>
+      flag to get better warnings. Try to find a similar flag if you
+      are using some other compiler.
+    </p>
+    <p>
+      Here follows a checklist for rewriting a pre erts-5.9 driver,
+      most important first.
+    </p>
+    <taglist>
+      <tag>Return types for driver callbacks</tag>
+      <item>
+	<p>
+	  Rewrite driver callback
+	  <c><seealso marker="driver_entry#control">control</seealso></c>
+	  to use return type <c>ErlDrvSSizeT</c> instead of <c>int</c>.
+	</p>
+	<p>
+	  Rewrite driver callback
+	  <c><seealso marker="driver_entry#call">call</seealso></c>
+	  to use return type <c>ErlDrvSSizeT</c> instead of <c>int</c>.
+	</p>
+	<note>
+	  <p>
+	    These changes are essential to not crash the emulator
+	    or worse cause malfunction.
+	    Without them a driver may return garbage in the high 32 bits
+	    to the emulator causing it to build a huge result from random
+	    bytes either crashing on memory allocation or succeeding with
+	    a random result from the driver call.
+	  </p>
+	</note>
+      </item>
+      <tag>Arguments to driver callbacks</tag>
+      <item>
+	<p>
+	  Driver callback
+	  <c><seealso marker="driver_entry#output">output</seealso></c>
+	  now gets <c>ErlDrvSizeT</c> as 3rd argument instead
+	  of previously <c>int</c>.
+	</p>
+	<p>
+	  Driver callback
+	  <c><seealso marker="driver_entry#control">control</seealso></c>
+	  now gets <c>ErlDrvSizeT</c> as 4th and 6th arguments instead
+	  of previously <c>int</c>.
+	</p>
+	<p>
+	  Driver callback
+	  <c><seealso marker="driver_entry#call">call</seealso></c>
+	  now gets <c>ErlDrvSizeT</c> as 4th and 6th arguments instead
+	  of previously <c>int</c>.
+	</p>
+	<p>
+	  Sane compiler's calling conventions probably make these changes
+	  necessary only for a driver to handle data chunks that require
+	  64-bit size fields (mostly larger than 2 GB since that is what
+	  an <c>int</c> of 32 bits can hold). But it is possible to think
+	  of non-sane calling conventions that would make the driver
+	  callbacks mix up the arguments causing malfunction.
+	</p>
+	<note>
+	  <p>
+	    The argument type change is from signed to unsigned which
+	    may cause problems for e.g. loop termination conditions or
+	    error conditions if you just change the types all over the place.
+	  </p>
+	</note>
+      </item>
+      <tag>Larger <c>size</c> field in <c>ErlIOVec</c></tag>
+      <item>
+	<p>
+	  The <c>size</c> field in
+	  <seealso marker="#ErlIOVec"><c>ErlIOVec</c></seealso>
+	  has been changed to <c>ErlDrvSizeT</c> from <c>int</c>.
+	  Check all code that use that field.
+	</p>
+	<p>
+	  Automatic type casting probably makes these changes necessary only
+	  for a driver that encounters sizes larger than 32 bits.
+	</p>
+	<note>
+	  <p>
+	    The <c>size</c> field changed from signed to unsigned which
+	    may cause problems for e.g. loop termination conditions or
+	    error conditions if you just change the types all over the place.
+	  </p>
+	</note>
+      </item>
+      <tag>Arguments and return values in the driver API</tag>
+      <item>
+	<p>
+	  Many driver API functions have changed argument type
+	  and/or return value to <c>ErlDrvSizeT</c> from mostly <c>int</c>.
+	  Automatic type casting probably makes these changes necessary only
+	  for a driver that encounters sizes larger than 32 bits.
+	</p>
+	<taglist>
+	  <tag><seealso marker="#driver_output">driver_output</seealso></tag>
+	  <item>3rd argument</item>
+	  <tag><seealso marker="#driver_output2">driver_output2</seealso></tag>
+	  <item>3rd and 5th arguments</item>
+	  <tag>
+	  <seealso marker="#driver_output_binary">driver_output_binary</seealso>
+	  </tag>
+	  <item>3rd 5th and 6th arguments</item>
+	  <tag><seealso marker="#driver_outputv">driver_outputv</seealso></tag>
+	  <item>3rd and 5th arguments</item>
+	  <tag>
+	  <seealso marker="#driver_vec_to_buf">driver_vec_to_buf</seealso>
+	  </tag>
+	  <item>3rd argument and return value</item>
+	  <tag><seealso marker="#driver_alloc">driver_alloc</seealso></tag>
+	  <item>1st argument</item>
+	  <tag><seealso marker="#driver_realloc">driver_realloc</seealso></tag>
+	  <item>2nd argument</item>
+	  <tag>
+	  <seealso marker="#driver_alloc_binary">driver_alloc_binary</seealso>
+	  </tag>
+	  <item>1st argument</item>
+	  <tag>
+	  <seealso marker="#driver_realloc_binary">driver_realloc_binary</seealso>
+	  </tag>
+	  <item>2nd argument</item>
+	  <tag><seealso marker="#driver_enq">driver_enq</seealso></tag>
+	  <item>3rd argument</item>
+	  <tag><seealso marker="#driver_pushq">driver_pushq</seealso></tag>
+	  <item>3rd argument</item>
+	  <tag><seealso marker="#driver_deq">driver_deq</seealso></tag>
+	  <item>2nd argument and return value</item>
+	  <tag><seealso marker="#driver_sizeq">driver_sizeq</seealso></tag>
+	  <item>return value</item>
+	  <tag><seealso marker="#driver_enq_bin">driver_enq_bin</seealso></tag>
+	  <item>3rd and 4th argument</item>
+	  <tag><seealso marker="#driver_pushq_bin">driver_pushq_bin</seealso></tag>
+	  <item>3rd and 4th argument</item>
+	  <tag><seealso marker="#driver_enqv">driver_enqv</seealso></tag>
+	  <item>3rd argument</item>
+	  <tag><seealso marker="#driver_pushqv">driver_pushqv</seealso></tag>
+	  <item>3rd argument</item>
+	  <tag><seealso marker="#driver_peekqv">driver_peekqv</seealso></tag>
+	  <item>return value</item>
+	</taglist>
+	<note>
+	  <p>
+	    This is a change from signed to unsigned which
+	    may cause problems for e.g. loop termination conditions and
+	    error conditions if you just change the types all over the place.
+	  </p>
+	</note>
       </item>
     </taglist>
   </section>
@@ -285,7 +476,11 @@
     <title>DATA TYPES</title>
 
     <taglist>
-     <tag><marker id="ErlDrvSysInfo"/>ErlDrvSysInfo</tag>
+      <tag><marker id="ErlDrvSizeT"/>ErlDrvSizeT</tag>
+      <item><p>An unsigned integer type to be used as <c>size_t</c></p></item>
+      <tag><marker id="ErlDrvSSizeT"/>ErlDrvSSizeT</tag>
+      <item><p>A signed integer type the size of <c>ErlDrvSizeT</c></p></item>
+      <tag><marker id="ErlDrvSysInfo"/>ErlDrvSysInfo</tag>
       <item>
       <p/>
       <code type="none">
@@ -309,7 +504,7 @@ typedef struct ErlDrvSysInfo {
         <seealso marker="#driver_system_info">driver_system_info()</seealso>
         will write the system information when passed a reference to
         a <c>ErlDrvSysInfo</c> structure. A description of the
-        fields in the structure follow:
+        fields in the structure follows:
       </p>
       <taglist>
         <tag><c>driver_major_version</c></tag>
@@ -332,20 +527,12 @@ typedef struct ErlDrvSysInfo {
         <tag><c>erts_version</c></tag>
         <item>A string containing the version number of the runtime system
            (the same as returned by
-          <seealso marker="erts:erlang#system_info_version">erlang:system_info(version)</seealso>).
+          <seealso marker="erlang#system_info_version">erlang:system_info(version)</seealso>).
         </item>
         <tag><c>otp_release</c></tag>
         <item>A string containing the OTP release number
            (the same as returned by
-          <seealso marker="erts:erlang#system_info_otp_release">erlang:system_info(otp_release)</seealso>).
-        </item>
-        <tag><c>thread_support</c></tag>
-        <item>A value <c>!= 0</c> if the runtime system has thread support;
-           otherwise, <c>0</c>.
-        </item>
-        <tag><c>smp_support</c></tag>
-        <item>A value <c>!= 0</c> if the runtime system has SMP support;
-           otherwise, <c>0</c>.
+          <seealso marker="erlang#system_info_otp_release">erlang:system_info(otp_release)</seealso>).
         </item>
         <tag><c>thread_support</c></tag>
         <item>A value <c>!= 0</c> if the runtime system has thread support;
@@ -359,12 +546,12 @@ typedef struct ErlDrvSysInfo {
         <item>The number of async threads in the async thread pool used
            by <seealso marker="#driver_async">driver_async()</seealso>
            (the same as returned by
-          <seealso marker="erts:erlang#system_info_thread_pool_size">erlang:system_info(thread_pool_size)</seealso>).
+          <seealso marker="erlang#system_info_thread_pool_size">erlang:system_info(thread_pool_size)</seealso>).
         </item>
         <tag><c>scheduler_threads</c></tag>
         <item>The number of scheduler threads used by the runtime system
            (the same as returned by
-          <seealso marker="erts:erlang#system_info_schedulers">erlang:system_info(schedulers)</seealso>).
+          <seealso marker="erlang#system_info_schedulers">erlang:system_info(schedulers)</seealso>).
         </item>
         <tag><c>nif_major_version</c></tag>
         <item>The value of <c>ERL_NIF_MAJOR_VERSION</c> when the runtime system was compiled.
@@ -379,7 +566,7 @@ typedef struct ErlDrvSysInfo {
       <p/>
       <code type="none">
 typedef struct ErlDrvBinary {
-   int orig_size;
+   ErlDrvSint orig_size;
    char orig_bytes[];
 } ErlDrvBinary;
 </code>
@@ -401,8 +588,8 @@ typedef struct ErlDrvBinary {
             <seealso marker="#driver_binary_dec_refc">driver_binary_dec_refc()</seealso>.</p>
         </note>
         <p>Some driver calls, such as <c>driver_enq_binary</c>,
-          increments the driver reference count, and others, such as
-          <c>driver_deq</c> decrements it.</p>
+          increment the driver reference count, and others, such as
+          <c>driver_deq</c> decrement it.</p>
         <p>Using a driver binary instead of a normal buffer, is often
           faster, since the emulator doesn't need to copy the data,
           only the pointer is used.</p>
@@ -415,7 +602,7 @@ typedef struct ErlDrvBinary {
           <c>driver_outputv</c> calls, and in the queue. Also the
           driver call-back <seealso marker="driver_entry#outputv">outputv</seealso> uses driver
           binaries.</p>
-        <p>If the driver of some reason or another, wants to keep a
+        <p>If the driver for some reason or another, wants to keep a
           driver binary around, in a static variable for instance, the
           reference count should be incremented,
           and the binary can later be freed in the <seealso marker="driver_entry#stop">stop</seealso> call-back, with
@@ -423,7 +610,7 @@ typedef struct ErlDrvBinary {
         <p>Note that since a driver binary is shared by the driver and
           the emulator, a binary received from the emulator or sent to
           the emulator, must not be changed by the driver.</p>
-        <p>From erts version 5.5 (OTP release R11B), orig_bytes is 
+        <p>Since erts version 5.5 (OTP release R11B), orig_bytes is
           guaranteed to be properly aligned for storage of an array of
           doubles (usually 8-byte aligned).</p>
       </item>
@@ -431,7 +618,7 @@ typedef struct ErlDrvBinary {
       <item>
         <p>The <c>ErlDrvData</c> is a handle to driver-specific data,
           passed to the driver call-backs. It is a pointer, and is
-          most often casted to a specific pointer in the driver.</p>
+          most often type casted to a specific pointer in the driver.</p>
       </item>
       <tag>SysIOVec</tag>
       <item>
@@ -445,9 +632,9 @@ typedef struct ErlDrvBinary {
       <code type="none">
 typedef struct ErlIOVec {
   int vsize;
-  int size;
+  ErlDrvSizeT size;
   SysIOVec* iov;
-  >ErlDrvBinary** binv;
+  ErlDrvBinary** binv;
 } ErlIOVec;
 </code>
         <p>The I/O vector used by the emulator and drivers, is a list
@@ -495,17 +682,17 @@ typedef struct ErlIOVec {
           Currently, the only port specific data that the emulator
           associates with the port data lock is the driver queue.</p>
         <p>Normally a driver instance does not have a port data lock. If
-          the driver instance want to use a port data lock, it has to
+          the driver instance wants to use a port data lock, it has to
           create the port data lock by calling
           <seealso marker="#driver_pdl_create">driver_pdl_create()</seealso>.
           <em>NOTE</em>: Once the port data lock has been created, every
-          access to data associated with the port data lock have to be done
+          access to data associated with the port data lock has to be done
           while having the port data lock locked. The port data lock is
           locked, and unlocked, respectively, by use of
           <seealso marker="#driver_pdl_lock">driver_pdl_lock()</seealso>, and
           <seealso marker="#driver_pdl_unlock">driver_pdl_unlock()</seealso>.</p>
         <p>A port data lock is reference counted, and when the reference
-          count reach zero, it will be destroyed. The emulator will at
+          count reaches zero, it will be destroyed. The emulator will at
           least increment the reference count once when the lock is
           created and decrement it once when the port associated with
           the lock terminates. The emulator will also increment the
@@ -545,7 +732,7 @@ typedef struct ErlIOVec {
 	</p>
         <taglist>
           <tag>suggested_stack_size</tag>
-          <item>A suggestion, in kilo-words, on how large stack to use. A value less
+          <item>A suggestion, in kilo-words, on how large a stack to use. A value less
 	        than zero means default size.
 	  </item>
         </taglist>
@@ -638,7 +825,7 @@ typedef struct ErlIOVec {
       </desc>
     </func>
     <func>
-      <name><ret>int</ret><nametext>driver_output(ErlDrvPort port, char *buf, int len)</nametext></name>
+      <name><ret>int</ret><nametext>driver_output(ErlDrvPort port, char *buf, ErlDrvSizeT len)</nametext></name>
       <fsummary>Send data from driver to port owner</fsummary>
       <desc>
         <marker id="driver_output"></marker>
@@ -648,7 +835,7 @@ typedef struct ErlIOVec {
           opened.</p>
         <p>The data is queued in the port owner process' message
           queue. Note that this does not yield to the emulator. (Since
-          the driver and the emulator runs in the same thread.)</p>
+          the driver and the emulator run in the same thread.)</p>
         <p>The parameter <c>buf</c> points to the data to send, and
           <c>len</c> is the number of bytes.</p>
         <p>The return value for all output functions is 0. (Unless the
@@ -658,7 +845,7 @@ typedef struct ErlIOVec {
       </desc>
     </func>
     <func>
-      <name><ret>int</ret><nametext>driver_output2(ErlDrvPort port, char *hbuf, int hlen, char *buf, int len)</nametext></name>
+      <name><ret>int</ret><nametext>driver_output2(ErlDrvPort port, char *hbuf, ErlDrvSizeT hlen, char *buf, ErlDrvSizeT len)</nametext></name>
       <fsummary>Send data and binary data to port owner</fsummary>
       <desc>
         <marker id="driver_output2"></marker>
@@ -673,7 +860,7 @@ typedef struct ErlIOVec {
       </desc>
     </func>
     <func>
-      <name><ret>int</ret><nametext>driver_output_binary(ErlDrvPort port, char *hbuf, int hlen, ErlDrvBinary* bin, int offset, int len)</nametext></name>
+      <name><ret>int</ret><nametext>driver_output_binary(ErlDrvPort port, char *hbuf, ErlDrvSizeT hlen, ErlDrvBinary* bin, ErlDrvSizeT offset, ErlDrvSizeT len)</nametext></name>
       <fsummary>Send data from a driver binary to port owner</fsummary>
       <desc>
         <marker id="driver_output_binary"></marker>
@@ -696,7 +883,7 @@ typedef struct ErlIOVec {
       </desc>
     </func>
     <func>
-      <name><ret>int</ret><nametext>driver_outputv(ErlDrvPort port, char* hbuf, int hlen,  ErlIOVec *ev, int skip)</nametext></name>
+      <name><ret>int</ret><nametext>driver_outputv(ErlDrvPort port, char* hbuf, ErlDrvSizeT hlen,  ErlIOVec *ev, ErlDrvSizeT skip)</nametext></name>
       <fsummary>Send vectorized data to port owner</fsummary>
       <desc>
         <marker id="driver_outputv"></marker>
@@ -719,7 +906,7 @@ typedef struct ErlIOVec {
       </desc>
     </func>
     <func>
-      <name><ret>int</ret><nametext>driver_vec_to_buf(ErlIOVec *ev, char *buf, int len)</nametext></name>
+      <name><ret>ErlDrvSizeT</ret><nametext>driver_vec_to_buf(ErlIOVec *ev, char *buf, ErlDrvSizeT len)</nametext></name>
       <fsummary>Collect data segments into a buffer</fsummary>
       <desc>
         <marker id="driver_vec_to_buf"></marker>
@@ -746,10 +933,10 @@ typedef struct ErlIOVec {
           <c>time</c> parameter is the time in milliseconds before the
           timer expires.</p>
         <p>When the timer reaches 0 and expires, the driver entry
-          function <seealso marker="driver_entry#emulator">timeout</seealso> is called.</p>
+          function <seealso marker="driver_entry#timeout">timeout</seealso> is called.</p>
         <p>Note that there is only one timer on each driver instance;
           setting a new timer will replace an older one.</p>
-        <p>Return value i 0 (-1 only when the <c>timeout</c> driver
+        <p>Return value is 0 (-1 only when the <c>timeout</c> driver
           function is <c>NULL</c>).</p>
       </desc>
     </func>
@@ -781,7 +968,7 @@ typedef struct ErlIOVec {
         <marker id="driver_get_now"></marker>
         <p>This function reads a timestamp into the memory pointed to by 
           the parameter <c>now</c>. See the description of <seealso marker="#ErlDrvNowData">ErlDrvNowData</seealso> for
-          specification of it's fields.  </p>
+          specification of its fields.  </p>
         <p>The return value is 0 unless the <c>now</c> pointer is not
           valid, in which case it is &lt; 0.  </p>
       </desc>
@@ -799,20 +986,20 @@ typedef struct ErlIOVec {
           event object must be a socket or pipe (or other object that
           <c>select</c>/<c>poll</c> can use).
           On windows, the Win32 API function <c>WaitForMultipleObjects</c>
-          is used. This places other restriction on the event object.
+          is used. This places other restrictions on the event object.
           Refer to the Win32 SDK documentation.</p>
         <p>The <c>on</c> parameter should be <c>1</c> for setting events
           and <c>0</c> for clearing them.</p>
-        <p>The <c>mode</c> argument is bitwise-or combination of
+        <p>The <c>mode</c> argument is a bitwise-or combination of
           <c>ERL_DRV_READ</c>, <c>ERL_DRV_WRITE</c> and <c>ERL_DRV_USE</c>.
-          The first two specifies whether to wait for read events and/or write
+          The first two specify whether to wait for read events and/or write
           events. A fired read event will call
           <seealso marker="driver_entry#ready_input">ready_input</seealso>
           while a fired write event will call
           <seealso marker="driver_entry#ready_output">ready_output</seealso>.
           </p>
     <note>
-      <p>Some OS (Windows) does not differ between read and write events.
+      <p>Some OS (Windows) do not differentiate between read and write events.
          The call-back for a fired event then only depends on the value of <c>mode</c>.</p>
     </note>
         <p><c>ERL_DRV_USE</c> specifies if we are using the event object or if we want to close it.
@@ -834,13 +1021,13 @@ typedef struct ErlIOVec {
          as before. But it is recommended to update them to use <c>ERL_DRV_USE</c> and
          <c>stop_select</c> to make sure that event objects are closed in a safe way.</p>
     </note>           
-        <p>The return value is 0 (Failure, -1, only if the
+        <p>The return value is 0 (failure, -1, only if the
           <c>ready_input</c>/<c>ready_output</c> is
-          <c>NULL</c>.</p>
+          <c>NULL</c>).</p>
       </desc>
     </func>
     <func>
-      <name><ret>void *</ret><nametext>driver_alloc(size_t size)</nametext></name>
+      <name><ret>void *</ret><nametext>driver_alloc(ErlDrvSizeT size)</nametext></name>
       <fsummary>Allocate memory</fsummary>
       <desc>
         <marker id="driver_alloc"></marker>
@@ -854,7 +1041,7 @@ typedef struct ErlIOVec {
       </desc>
     </func>
     <func>
-      <name><ret>void *</ret><nametext>driver_realloc(void *ptr, size_t size)</nametext></name>
+      <name><ret>void *</ret><nametext>driver_realloc(void *ptr, ErlDrvSizeT size)</nametext></name>
       <fsummary>Resize an allocated memory block</fsummary>
       <desc>
         <marker id="driver_realloc"></marker>
@@ -880,7 +1067,7 @@ typedef struct ErlIOVec {
       </desc>
     </func>
     <func>
-      <name><ret>ErlDrvBinary*</ret><nametext>driver_alloc_binary(int size)</nametext></name>
+      <name><ret>ErlDrvBinary *</ret><nametext>driver_alloc_binary(ErlDrvSizeT size)</nametext></name>
       <fsummary>Allocate a driver binary</fsummary>
       <desc>
         <marker id="driver_alloc_binary"></marker>
@@ -900,7 +1087,7 @@ typedef struct ErlIOVec {
       </desc>
     </func>
     <func>
-      <name><ret>ErlDrvBinary*</ret><nametext>driver_realloc_binary(ErlDrvBinary *bin, int size)</nametext></name>
+      <name><ret>ErlDrvBinary *</ret><nametext>driver_realloc_binary(ErlDrvBinary *bin, ErlDrvSizeT size)</nametext></name>
       <fsummary>Resize a driver binary</fsummary>
       <desc>
         <marker id="driver_realloc_binary"></marker>
@@ -966,7 +1153,7 @@ typedef struct ErlIOVec {
       </desc>
     </func>
     <func>
-      <name><ret>int</ret><nametext>driver_enq(ErlDrvPort port, char* buf, int len)</nametext></name>
+      <name><ret>int</ret><nametext>driver_enq(ErlDrvPort port, char* buf, ErlDrvSizeT len)</nametext></name>
       <fsummary>Enqueue data in the driver queue</fsummary>
       <desc>
         <marker id="driver_enq"></marker>
@@ -990,7 +1177,7 @@ typedef struct ErlIOVec {
       </desc>
     </func>
     <func>
-      <name><ret>int</ret><nametext>driver_pushq(ErlDrvPort port, char* buf, int len)</nametext></name>
+      <name><ret>int</ret><nametext>driver_pushq(ErlDrvPort port, char* buf, ErlDrvSizeT len)</nametext></name>
       <fsummary>Push data at the head of the driver queue</fsummary>
       <desc>
         <marker id="driver_pushq"></marker>
@@ -1005,7 +1192,7 @@ typedef struct ErlIOVec {
       </desc>
     </func>
     <func>
-      <name><ret>int</ret><nametext>driver_deq(ErlDrvPort port, int size)</nametext></name>
+      <name><ret>ErlDrvSizeT</ret><nametext>driver_deq(ErlDrvPort port, ErlDrvSizeT size)</nametext></name>
       <fsummary>Dequeue data from the head of the driver queue</fsummary>
       <desc>
         <marker id="driver_deq"></marker>
@@ -1021,7 +1208,7 @@ typedef struct ErlIOVec {
       </desc>
     </func>
     <func>
-      <name><ret>int</ret><nametext>driver_sizeq(ErlDrvPort port)</nametext></name>
+      <name><ret>ErlDrvSizeT</ret><nametext>driver_sizeq(ErlDrvPort port)</nametext></name>
       <fsummary>Return the size of the driver queue</fsummary>
       <desc>
         <marker id="driver_sizeq"></marker>
@@ -1034,7 +1221,7 @@ typedef struct ErlIOVec {
       </desc>
     </func>
     <func>
-      <name><ret>int</ret><nametext>driver_enq_bin(ErlDrvPort port, ErlDrvBinary *bin, int offset,  int len)</nametext></name>
+      <name><ret>int</ret><nametext>driver_enq_bin(ErlDrvPort port, ErlDrvBinary *bin, ErlDrvSizeT offset, ErlDrvSizeT len)</nametext></name>
       <fsummary>Enqueue binary in the driver queue</fsummary>
       <desc>
         <marker id="driver_enq_bin"></marker>
@@ -1051,7 +1238,7 @@ typedef struct ErlIOVec {
       </desc>
     </func>
     <func>
-      <name><ret>int</ret><nametext>driver_pushq_bin(ErlDrvPort port, ErlDrvBinary *bin, int offset, int len)</nametext></name>
+      <name><ret>int</ret><nametext>driver_pushq_bin(ErlDrvPort port, ErlDrvBinary *bin, ErlDrvSizeT offset, ErlDrvSizeT len)</nametext></name>
       <fsummary>Push binary at the head of the driver queue</fsummary>
       <desc>
         <marker id="driver_pushq_bin"></marker>
@@ -1068,15 +1255,37 @@ typedef struct ErlIOVec {
       </desc>
     </func>
     <func>
-      <name><ret>SysIOVec*</ret><nametext>driver_peekq(ErlDrvPort port, int *vlen)</nametext></name>
+      <name><ret>ErlDrvSizeT</ret><nametext>driver_peekqv(ErlDrvPort port, ErlIOVec *ev)</nametext></name>
+      <fsummary>Get the driver queue as an IO vector</fsummary>
+      <desc>
+        <marker id="driver_peekqv"></marker>
+        <p>
+	  This function retrieves the driver queue into a supplied
+	  <c>ErlIOVec</c> <c>ev</c>. It also returns the queue size.
+	  This is one of two ways to get data out of the queue.
+	</p>
+	<p>
+	  If <c>ev</c> is <c>NULL</c> all ones i.e. <c>-1</c> type cast to
+	  <c>ErlDrvSizeT</c> is returned.
+	</p>
+        <p>Nothing is removed from the queue by this function, that must be done
+          with <c>driver_deq</c>.</p>
+        <p>This function can be called from an arbitrary thread if a
+          <seealso marker="#ErlDrvPDL">port data lock</seealso>
+          associated with the <c>port</c> is locked by the calling
+          thread during the call.</p>
+      </desc>
+    </func>
+    <func>
+      <name><ret>SysIOVec *</ret><nametext>driver_peekq(ErlDrvPort port, int *vlen)</nametext></name>
       <fsummary>Get the driver queue as a vector</fsummary>
       <desc>
         <marker id="driver_peekq"></marker>
         <p>This function retrieves the driver queue as a pointer to an
           array of <c>SysIOVec</c>s. It also returns the number of
-          elements in <c>vlen</c>. This is the only way to get data
+          elements in <c>vlen</c>. This is one of two ways to get data
           out of the queue.</p>
-        <p>Nothing is remove from the queue by this function, that must be done
+        <p>Nothing is removed from the queue by this function, that must be done
           with <c>driver_deq</c>.</p>
         <p>The returned array is suitable to use with the Unix system
           call <c>writev</c>.</p>
@@ -1087,7 +1296,7 @@ typedef struct ErlIOVec {
       </desc>
     </func>
     <func>
-      <name><ret>int</ret><nametext>driver_enqv(ErlDrvPort port, ErlIOVec *ev, int skip)</nametext></name>
+      <name><ret>int</ret><nametext>driver_enqv(ErlDrvPort port, ErlIOVec *ev, ErlDrvSizeT skip)</nametext></name>
       <fsummary>Enqueue vector in the driver queue</fsummary>
       <desc>
         <marker id="driver_enqv"></marker>
@@ -1103,7 +1312,7 @@ typedef struct ErlIOVec {
       </desc>
     </func>
     <func>
-      <name><ret>int</ret><nametext>driver_pushqv(ErlDrvPort port, ErlIOVec *ev, int skip)</nametext></name>
+      <name><ret>int</ret><nametext>driver_pushqv(ErlDrvPort port, ErlIOVec *ev, ErlDrvSizeT skip)</nametext></name>
       <fsummary>Push vector at the head of the driver queue</fsummary>
       <desc>
         <marker id="driver_pushqv"></marker>
@@ -1209,7 +1418,7 @@ typedef struct ErlIOVec {
       <fsummary>Stop monitoring a process from a driver</fsummary>
       <desc>
         <marker id="driver_demonitor_process"></marker>
-        <p>This function cancels an monitor created earlier. </p>
+        <p>This function cancels a monitor created earlier. </p>
         <p>The function returns 0 if a monitor was removed and &gt; 0
           if the monitor did no longer exist.</p>
       </desc>
@@ -1272,7 +1481,7 @@ typedef struct ErlIOVec {
       </desc>
     </func>
     <func>
-      <name><ret>char*</ret><nametext>erl_errno_id(int error)</nametext></name>
+      <name><ret>char *</ret><nametext>erl_errno_id(int error)</nametext></name>
       <fsummary>Get erlang error atom name from error number</fsummary>
       <desc>
         <marker id="erl_errno_id"></marker>
@@ -1326,7 +1535,7 @@ typedef struct ErlIOVec {
         <p>This function signals to erlang that the driver has
           encountered an EOF and should be closed, unless the port was
           opened with the <c>eof</c> option, in that case eof is sent
-          to the port. Otherwise, the port is close and an
+          to the port. Otherwise, the port is closed and an
           <c>'EXIT'</c> message is sent to the port owner process.</p>
         <p>The return value is 0.</p>
       </desc>
@@ -1349,8 +1558,8 @@ typedef struct ErlIOVec {
           (<c>driver_failure</c>).</p>
         <p>The driver should fail only when in severe error situations,
           when the driver cannot possibly keep open, for instance
-          buffer allocation gets out of memory. Normal errors is more
-          appropriate to handle with sending error codes with
+          buffer allocation gets out of memory. For normal errors
+          it is more appropriate to send error codes with
           <c>driver_output</c>.</p>
         <p>The return value is 0.</p>
       </desc>
@@ -1371,7 +1580,7 @@ typedef struct ErlIOVec {
         <p>This function returns the process id of the process that
 	  made the current call to the driver. The process id can be
 	  used with <c>driver_send_term</c> to send back data to the
-	  caller. <c>driver_caller()</c> only return valid data
+	  caller. <c>driver_caller()</c> only returns valid data
 	  when currently executing in one of the following driver
 	  callbacks:</p>
 	  <taglist>
@@ -1409,7 +1618,7 @@ typedef struct ErlIOVec {
           tuple, the elements are given first, and then the tuple
           term, with a count. Likewise for lists.</p>
         <p>A tuple must be specified with the number of elements. (The
-          elements precedes the <c>ERL_DRV_TUPLE</c> term.)</p>
+          elements precede the <c>ERL_DRV_TUPLE</c> term.)</p>
         <p>A list must be specified with the number of elements,
           including the tail, which is the last term preceding
           <c>ERL_DRV_LIST</c>.</p>
@@ -1502,7 +1711,7 @@ ERL_DRV_EXT2TERM     char *buf, ErlDrvUInt len
 	term encoded with the
 	<seealso marker="erl_ext_dist">external format</seealso>,
 	i.e., a term that has been encoded by
-	<seealso marker="erts:erlang#term_to_binary/2">erlang:term_to_binary</seealso>,
+	<seealso marker="erlang#term_to_binary/2">erlang:term_to_binary</seealso>,
 	<seealso marker="erl_interface:ei">erl_interface</seealso>, etc.
 	For example, if <c>binp</c> is a pointer to an <c>ErlDrvBinary</c>
 	that contains the term <c>{17, 4711}</c> encoded with the
@@ -1518,7 +1727,7 @@ ERL_DRV_EXT2TERM     char *buf, ErlDrvUInt len
     };
     driver_output_term(port, spec, sizeof(spec) / sizeof(spec[0]));
         ]]></code>
-	<p>If you want to pass a binary and doesn't already have the content
+	<p>If you want to pass a binary and don't already have the content
 	of the binary in an <c>ErlDrvBinary</c>, you can benefit from using
 	<c>ERL_DRV_BUF2BINARY</c> instead of creating an <c>ErlDrvBinary</c>
 	via <c>driver_alloc_binary()</c> and then pass the binary via
@@ -1565,7 +1774,7 @@ ERL_DRV_EXT2TERM     char *buf, ErlDrvUInt len
           <em>other</em> processes than the port owner process. The
           <c>receiver</c> parameter specifies the process to receive
           the data.</p>
-        <p>The parameters <c>term</c> and <c>n</c> does the same thing
+        <p>The parameters <c>term</c> and <c>n</c> do the same thing
           as in <seealso marker="#driver_output_term">driver_output_term</seealso>.</p>
         <p>This function is only thread-safe when the emulator with SMP
           support is used.</p>
@@ -1646,12 +1855,19 @@ ERL_DRV_EXT2TERM     char *buf, ErlDrvUInt len
       <fsummary>Cancel an asynchronous call</fsummary>
       <desc>
         <marker id="driver_async_cancel"></marker>
-        <p>This function cancels an asynchronous operation, by removing
-          it from the queue. Only functions in the queue can be
-          cancelled; if a function is executing, it's too late to
-          cancel it. The <c>async_free</c> function is also called.</p>
-        <p>The return value is 1 if the operation was removed from the
-          queue, otherwise 0.</p>
+        <p>This function used to cancel a scheduled asynchronous operation,
+	  if it was still in the queue. It returned 1 if it succeeded, and
+	  0 if it failed.</p>
+	<p>Since it could not guarantee success, it was more or less useless.
+	   The user had to implement synchronization of cancellation anyway.
+	   It also unnecessarily complicated the implementation. Therefore,
+	   as of OTP-R15B <c>driver_async_cancel()</c> is deprecated, and
+	   scheduled for removal in OTP-R16. It will currently always fail,
+	   and return 0.</p>
+	<warning><p><c>driver_async_cancel()</c> is deferred and will
+	            be removed in the OTP-R16 release.</p>
+	</warning>
+
       </desc>
     </func>
     <func>
@@ -1660,7 +1876,7 @@ ERL_DRV_EXT2TERM     char *buf, ErlDrvUInt len
       <desc>
         <marker id="driver_lock_driver"></marker>
         <p>This function locks the driver used by the port <c>port</c>
-          in memory for the rest of the emulator process
+          in memory for the rest of the emulator process'
           lifetime. After this call, the driver behaves as one of Erlang's 
           statically linked in drivers.</p>
       </desc>
@@ -1695,7 +1911,7 @@ ERL_DRV_EXT2TERM     char *buf, ErlDrvUInt len
            The driver defined handle is normally created in the
           <seealso marker="driver_entry#start">driver start call-back</seealso>
            when a port is created via
-          <seealso marker="erts:erlang#open_port/2">erlang:open_port/2</seealso>. </item>
+          <seealso marker="erlang#open_port/2">erlang:open_port/2</seealso>. </item>
         </taglist>
         <p>The caller of <c>driver_create_port()</c> is allowed to
           manipulate the newly created port when <c>driver_create_port()</c>
@@ -1904,7 +2120,7 @@ ERL_DRV_EXT2TERM     char *buf, ErlDrvUInt len
 		 corresponding to one of the involved thread identifiers
 		 has terminated since the thread identifier was saved,
 		 the result of <c>erl_drv_equal_tids()</c> might not give
-		 expected result.
+		 the expected result.
 	</p></note>
         <p>This function is thread-safe.</p>
       </desc>
@@ -2463,7 +2679,7 @@ ERL_DRV_EXT2TERM     char *buf, ErlDrvUInt len
     <title>SEE ALSO</title>
     <p><seealso marker="driver_entry">driver_entry(3)</seealso>,
       <seealso marker="kernel:erl_ddll">erl_ddll(3)</seealso>,
-      <seealso marker="erts:erlang">erlang(3)</seealso></p>
+      <seealso marker="erlang">erlang(3)</seealso></p>
     <p>An Alternative Distribution Driver (ERTS User's
       Guide Ch. 3)</p>
   </section>
diff --git a/erts/doc/src/erl_ext_dist.xml b/erts/doc/src/erl_ext_dist.xml
index c2d58d1ef1..fd2da2cfe3 100644
--- a/erts/doc/src/erl_ext_dist.xml
+++ b/erts/doc/src/erl_ext_dist.xml
@@ -5,7 +5,7 @@
   <header>
     <copyright>
       <year>2007</year>
-      <year>2007</year>
+      <year>2011</year>
       <holder>Ericsson AB, All Rights Reserved</holder>
     </copyright>
     <legalnotice>
diff --git a/erts/doc/src/erl_ext_fig.gif b/erts/doc/src/erl_ext_fig.gif
index 14d6bbc871..14d6bbc871 100755..100644
--- a/erts/doc/src/erl_ext_fig.gif
+++ b/erts/doc/src/erl_ext_fig.gif
diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml
index 0dd46a951a..f484e9eaf7 100644
--- a/erts/doc/src/erl_nif.xml
+++ b/erts/doc/src/erl_nif.xml
@@ -4,7 +4,7 @@
 <cref>
   <header>
     <copyright>
-      <year>2001</year><year>2009</year>
+      <year>2001</year><year>2012</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -34,18 +34,29 @@
   <lib>erl_nif</lib>
   <libsummary>API functions for an Erlang NIF library</libsummary>
   <description>
-     <warning><p>The NIF concept was introduced in R13B03 as an
-	EXPERIMENTAL feature. The interfaces may be changed in any way
-	in coming releases. The plan is however to lift the experimental label and
-  maintain interface backward compatibility from R14B.</p>
-  <p>Incompatible changes in <em>R13B04</em>:</p>
+     <note><p>The NIF concept is officially supported from R14B. NIF source code
+     written for earlier experimental versions might need adaption to run on R14B.</p>
+    <p>No incompatible changes between <em>R14B</em> and R14A.</p>
+    <p>Incompatible changes between <em>R14A</em> and R13B04:</p>
+  <list>
+    <item>Environment argument removed for <c>enif_alloc</c>,
+    <c>enif_realloc</c>, <c>enif_free</c>, <c>enif_alloc_binary</c>,
+    <c>enif_realloc_binary</c>, <c>enif_release_binary</c>,
+    <c>enif_alloc_resource</c>, <c>enif_release_resource</c>,
+    <c>enif_is_identical</c> and <c>enif_compare</c>.</item>
+    <item>Character encoding argument added to <c>enif_get_atom</c>
+    and <c>enif_make_existing_atom</c>.</item>
+    <item>Module argument added to <c>enif_open_resource_type</c>
+      while changing name spaces of resource types from global to module local.</item>
+  </list>
+  <p>Incompatible changes between <em>R13B04</em> and R13B03:</p>
   <list>
     <item>The function prototypes of the NIFs have  changed to expect <c>argc</c> and <c>argv</c>
 	  arguments. The arity of a NIF is by that no longer limited to 3.</item>
     <item><c>enif_get_data</c> renamed as <c>enif_priv_data</c>.</item>
     <item><c>enif_make_string</c> got a third argument for character encoding.</item>
   </list>
-  </warning>
+  </note>
 
     <p>A NIF library contains native implementation of some functions
     of an Erlang module. The native implemented functions (NIFs) are
@@ -109,9 +120,9 @@ ok
      the new directive <seealso
      marker="doc/reference_manual:code_loading#on_load">on_load</seealso> to automatically
      load the NIF library when the module is loaded.</p>
-     <note><p>A NIF must be exported or used locally by the module (or both).
-     An unused local stub function will be optimized away by the compiler
-      causing loading of the NIF library to fail.</p>
+     <note><p>A NIF does not have to be exported, it can be local to the module.
+     Note however that unused local stub functions will be optimized
+     away by the compiler causing loading of the NIF library to fail.</p>
     </note>
      <p>A loaded NIF library is tied to the Erlang module code version
      that loaded it. If the module is upgraded with a new version, the
@@ -122,12 +133,10 @@ ok
      will be shared as well. To avoid unintentionally shared static
      data, each Erlang module code can keep its own private data. This
      private data can be set when the NIF library is loaded and
-     then retrieved by calling <seealso marker="#enif_priv_data">enif_priv_data()</seealso>.</p>
+     then retrieved by calling <seealso marker="#enif_priv_data">enif_priv_data</seealso>.</p>
     <p>There is no way to explicitly unload a NIF library. A library will be
      automatically unloaded when the module code that it belongs to is purged
-     by the code server. A NIF library will also be unloaded if it is replaced
-     by another version of the library by a second call to
-     <c>erlang:load_nif/2</c> from the same module code.</p> 
+     by the code server.</p> 
   </description>
   <section>
   <title>FUNCTIONALITY</title>
@@ -137,14 +146,20 @@ ok
     <taglist>
       <tag>Read and write Erlang terms</tag>
       <item><p>Any Erlang terms can be passed to a NIF as function arguments and
-      be returned as function return values. The terms are of C-type <c>ERL_NIF_TERM</c>
+      be returned as function return values. The terms are of C-type
+      <seealso marker="#ERL_NIF_TERM">ERL_NIF_TERM</seealso>
       and can only be read or written using API functions. Most functions to read
       the content of a term are prefixed <c>enif_get_</c> and usually return
       true (or false) if the term was of the expected type (or not).    
       The functions to write terms are all prefixed <c>enif_make_</c> and usually
       return the created <c>ERL_NIF_TERM</c>. There are also some functions
       to query terms, like <c>enif_is_atom</c>, <c>enif_is_identical</c> and
-      <c>enif_compare</c>.</p></item>
+      <c>enif_compare</c>.</p>
+      <p>All terms of type <c>ERL_NIF_TERM</c> belong to an environment of type
+      <seealso marker="#ErlNifEnv">ErlNifEnv</seealso>. The lifetime of a term is
+      controlled by the lifetime of its environment object. All API functions that read
+      or write terms has the environment, that the term belongs to, as the first
+      function argument.</p></item>
       <tag>Binaries</tag>
       <item><p>Terms of type binary are accessed with the help of the struct type
       <seealso marker="#ErlNifBinary">ErlNifBinary</seealso>    
@@ -161,8 +176,10 @@ ok
       <seealso marker="#enif_release_binary">enif_release_binary</seealso>
       or made read-only by transferring it to an Erlang term with
       <seealso marker="#enif_make_binary">enif_make_binary</seealso>.
-      But it does not have do happen in the same NIF call. Read-only binaries
-      does not have to be released.</p>
+      But it does not have to happen in the same NIF call. Read-only binaries
+      do not have to be released.</p>
+      <p><seealso marker="#enif_make_new_binary">enif_make_new_binary</seealso>
+      can be used as a shortcut to allocate and return a binary in the same NIF call.</p>
       <p>Binaries are sequences of whole bytes. Bitstrings with an arbitrary
       bit length have no support yet.</p>
        </item>
@@ -170,67 +187,83 @@ ok
       <item><p>The use of resource objects is a way to return pointers to
       native data structures from a NIF in a safe way. A resource object is
       just a block of memory allocated with
-      <seealso marker="#enif_alloc_resource">enif_alloc_resource()</seealso>.
+      <seealso marker="#enif_alloc_resource">enif_alloc_resource</seealso>.
       A handle ("safe pointer") to this memory block can then be returned to Erlang by the use of
-      <seealso marker="#enif_make_resource">enif_make_resource()</seealso>.
+      <seealso marker="#enif_make_resource">enif_make_resource</seealso>.
       The term returned by <c>enif_make_resource</c>
-      is totally opaque in nature. It can be stored and passed between processses
-      on the same node, but the only real end usage is to pass it back as argument to a NIF.
-      The NIF can then do <seealso marker="#enif_get_resource">enif_get_resource()</seealso>
+      is totally opaque in nature. It can be stored and passed between processes
+      on the same node, but the only real end usage is to pass it back as an argument to a NIF.
+      The NIF can then call <seealso marker="#enif_get_resource">enif_get_resource</seealso>
       and get back a pointer to the memory block that is guaranteed to still be
       valid. A resource object will not be deallocated until the last handle term
       has been garbage collected by the VM and the resource has been
-      released with <seealso marker="#enif_release_resource">enif_release_resource()</seealso>
+      released with <seealso marker="#enif_release_resource">enif_release_resource</seealso>
       (not necessarily in that order).</p>      
       <p>All resource objects are created as instances of some <em>resource type</em>.
       This makes resources from different modules to be distinguishable. 
       A resource type is created by calling
-      <seealso marker="#enif_open_resource_type">enif_open_resource_type()</seealso>
+      <seealso marker="#enif_open_resource_type">enif_open_resource_type</seealso>
       when a library is loaded. Objects of that resource type can then later be allocated
       and <c>enif_get_resource</c> verifies that the resource is of the expected type.
       A resource type can have a user supplied destructor function that is
       automatically called when resources of that type are released (by either
       the garbage collector or <c>enif_release_resource</c>). Resource types
-      are uniquely identified by a supplied name string.</p>
-      <p>Resource types support upgrade in runtime by allowing a loaded NIF
-      library to takeover an already existing resource type and thereby
-      "inherit" all existing objects of that type. The destructor of the new
-      library will thereafter be called for the inherited objects and the
-      library with the old destructor function can be safely unloaded. Existing
-      resource objects, of a module that is upgraded, must either be deleted
-      or taken over by the new NIF library. The unloading of a library will be
-      postponed as long as it exists resource objects with a destructor
-      function in the library.
-      </p>
-      <p>Here is a template example of how to create and return a resource object.</p>
+      are uniquely identified by a supplied name string and the name of the
+      implementing module.</p>
+      <marker id="enif_resource_example"/><p>Here is a template example of how to create and return a resource object.</p>
       <p/>
       <code type="none">
     ERL_NIF_TERM term;
-    MyStruct* ptr = enif_alloc_resource(env, my_resource_type, sizeof(MyStruct));
+    MyStruct* obj = enif_alloc_resource(my_resource_type, sizeof(MyStruct));
 
     /* initialize struct ... */
 
-    term = enif_make_resource(env, ptr);
+    term = enif_make_resource(env, obj);
 
     if (keep_a_reference_of_our_own) {
-        /* store 'ptr' in static variable, private data or other resource object */
+        /* store 'obj' in static variable, private data or other resource object */
     }
     else {
-        enif_release_resource(env, obj);
+        enif_release_resource(obj);
         /* resource now only owned by "Erlang" */
     }
     return term;
-}
-</code>
-
+    </code>
+      <p>Note that once <c>enif_make_resource</c> creates the term to
+      return to Erlang, the code can choose to either keep its own
+      native pointer to the allocated struct and release it later, or
+      release it immediately and rely solely on the garbage collector
+      to eventually deallocate the resource object when it collects
+      the term.</p> 
+      <p>Another usage of resource objects is to create binary terms with
+      user defined memory management.
+      <seealso marker="#enif_make_resource_binary">enif_make_resource_binary</seealso>
+      will create a binary term that is connected to a resource object. The
+      destructor of the resource will be called when the binary is garbage
+      collected, at which time the binary data can be released. An example of
+      this can be a binary term consisting of data from a <c>mmap</c>'ed file.
+      The destructor can then do <c>munmap</c> to release the memory
+      region.</p>
+      <p>Resource types support upgrade in runtime by allowing a loaded NIF
+      library to takeover an already existing resource type and thereby
+      "inherit" all existing objects of that type. The destructor of the new
+      library will thereafter be called for the inherited objects and the
+      library with the old destructor function can be safely unloaded. Existing
+      resource objects, of a module that is upgraded, must either be deleted
+      or taken over by the new NIF library. The unloading of a library will be
+      postponed as long as there exist resource objects with a destructor
+      function in the library.
+      </p>
       </item>
       <tag>Threads and concurrency</tag>
       <item><p>A NIF is thread-safe without any explicit synchronization as
       long as it acts as a pure function and only reads the supplied
       arguments. As soon as you write towards a shared state either through
       static variables or <seealso marker="#enif_priv_data">enif_priv_data</seealso>
-      you need to supply your own explicit synchronization. Resource objects
-      will also require synchronization if you treat them as mutable.</p>
+      you need to supply your own explicit synchronization. This includes terms
+      in process independent environments that are shared between threads.
+      Resource objects will also require synchronization if you treat them as
+      mutable.</p>
       <p>The library initialization callbacks <c>load</c>, <c>reload</c> and
       <c>upgrade</c> are all thread-safe even for shared state data.</p>
       <p>Avoid doing lengthy work in NIF calls as that may degrade the
@@ -262,8 +295,8 @@ ok
        <item><p><c>load</c> is called when the NIF library is loaded
         and there is no previously loaded library for this module.</p>
         <p><c>*priv_data</c> can be set to point to some private data
-           that the library needs in able to keep a state between NIF 
-           calls. <c>enif_priv_data()</c> will return this pointer.
+           that the library needs in order to keep a state between NIF
+           calls. <c>enif_priv_data</c> will return this pointer.
            <c>*priv_data</c> will be initialized to NULL when <c>load</c> is
            called.</p> 
         <p><c>load_info</c> is the second argument to <seealso
@@ -273,21 +306,9 @@ ok
            initialization is needed.</p> 
        </item>
 
-      <tag><marker id="reload"/>int (*reload)(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)</tag>
-       <item><p><c>reload</c> is called when the NIF library is loaded
-        and there is already a previously loaded library for this
-        module code.</p>
-        <p>Works the same as <c>load</c>. The only difference is that
-        <c>*priv_data</c> already contains the value set by the
-        previous call to <c>load</c> or <c>reload</c>.</p>
-        <p>The library will fail to load if <c>reload</c> returns
-           anything other than 0 or if <c>reload</c> is NULL.</p> 
-      </item>
-
       <tag><marker id="upgrade"/>int (*upgrade)(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info)</tag>
        <item><p><c>upgrade</c> is called when the NIF library is loaded
-        and there is no previously loaded library for this module
-        code, BUT there is old code of this module with a loaded NIF library.</p>
+        and there is old code of this module with a loaded NIF library.</p>
         <p>Works the same as <c>load</c>. The only difference is that
         <c>*old_priv_data</c> already contains the value set by the
          last call to <c>load</c> or <c>reload</c> for the old module
@@ -304,6 +325,23 @@ ok
        called for a replaced library as a consequence of <c>reload</c>.</p>
       </item>
 
+      <tag><marker id="reload"/>int (*reload)(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)</tag>
+     <note><p>The reload mechanism is <em>deprecated</em>. It was only intended
+     as a development feature. Do not use it as an upgrade method for
+     live production systems. It might be removed in future releases. Be sure
+     to pass <c>reload</c> as <c>NULL</c> to <seealso marker="#ERL_NIF_INIT">ERL_NIF_INIT</seealso>
+     to disable it when not used.</p>
+     </note>
+       <item><p><c>reload</c> is called when the NIF library is loaded
+        and there is already a previously loaded library for this
+        module code.</p>
+        <p>Works the same as <c>load</c>. The only difference is that
+        <c>*priv_data</c> already contains the value set by the
+        previous call to <c>load</c> or <c>reload</c>.</p>
+        <p>The library will fail to load if <c>reload</c> returns
+           anything other than 0 or if <c>reload</c> is NULL.</p> 
+      </item>
+
     </taglist>
   </section>
 
@@ -315,19 +353,37 @@ ok
        <item>
         <p>Variables of type <c>ERL_NIF_TERM</c> can refer to any Erlang term.
         This is an opaque type and values of it can only by used either as
-        arguments to API functions or as return values from NIFs. A variable of
-        type <c>ERL_NIF_TERM</c> is only valid until the NIF call, where it was
-        obtained, returns.</p>
+        arguments to API functions or as return values from NIFs. All
+        <c>ERL_NIF_TERM</c>'s belong to an environment
+        (<seealso marker="#ErlNifEnv">ErlNifEnv</seealso>). A term can not be
+        destructed individually, it is valid until its environment is destructed.</p>
       </item>
      <tag><marker id="ErlNifEnv"/>ErlNifEnv</tag>
       <item>
-        <p><c>ErlNifEnv</c> contains information about the context in
-          which a NIF call is made. This pointer should not be
-          dereferenced in any way, but only passed on to API
-          functions. An <c>ErlNifEnv</c> pointer is only valid until
-          the function, where is what supplied as argument,
-          returns. There is thus useless and dangerous to store <c>ErlNifEnv</c>
-          pointers in between NIF calls.</p>
+        <p><c>ErlNifEnv</c> represents an environment that can host Erlang terms.
+        All terms in an environment are valid as long as the environment is valid.
+        <c>ErlNifEnv</c> is an opaque type and pointers to it can only be passed
+        on to API functions. There are two types of environments; process
+        bound and process independent.</p>
+        <p>A <em>process bound environment</em> is passed as the first argument to all NIFs.
+        All function arguments passed to a NIF will belong to that environment.
+        The return value from a NIF must also be a term belonging to the same
+        environment.
+        In addition a process bound environment contains transient information
+        about the calling Erlang process. The environment is only valid in the 
+        thread where it was supplied as argument until the NIF returns. It is
+        thus useless and dangerous to store pointers to process bound
+        environments between NIF calls. </p>
+        <p>A <em>process independent environment</em> is created by calling
+        <seealso marker="#enif_alloc_env">enif_alloc_env</seealso>. It can be
+        used to store terms between NIF calls and to send terms with
+        <seealso marker="#enif_send">enif_send</seealso>. A process
+        independent environment with all its terms is valid until you explicitly
+        invalidates it with <seealso marker="#enif_free_env">enif_free_env</seealso>
+        or <c>enif_send</c>.</p>
+        <p>All elements of a list/tuple must belong to the same environment as the
+        list/tuple itself. Terms can be copied between environments with
+        <seealso marker="#enif_make_copy">enif_make_copy</seealso>.</p>
       </item>
     <tag><marker id="ErlNifFunc"/>ErlNifFunc</tag>
      <item>
@@ -361,7 +417,18 @@ typedef struct {
         <p><c>ErlNifBinary</c> contains transient information about an
           inspected binary term. <c>data</c> is a pointer to a buffer
           of <c>size</c> bytes with the raw content of the binary.</p>
+          <p>Note that <c>ErlNifBinary</c> is a semi-opaque type and you are
+          only allowed to read fields <c>size</c> and <c>data</c>.</p>
       </item>
+      <tag><marker id="ErlNifPid"/>ErlNifPid</tag>
+       <item>
+          <p><c>ErlNifPid</c> is a process identifier (pid). In contrast to
+          pid terms (instances of <c>ERL_NIF_TERM</c>), <c>ErlNifPid</c>'s are self
+          contained and not bound to any
+          <seealso marker="#ErlNifEnv">environment</seealso>. <c>ErlNifPid</c>
+          is an opaque type.</p>
+        </item>
+
       <tag><marker id="ErlNifResourceType"/>ErlNifResourceType</tag>
        <item>
           <p>Each instance of <c>ErlNifResourceType</c> represent a class of
@@ -386,9 +453,9 @@ typedef enum {
     ERL_NIF_LATIN1
 }ErlNifCharEncoding;
 </code>
-              <p>The character encoding used in strings. The only supported
-              encoding is currently <c>ERL_NIF_LATIN1</c> for iso-latin-1
-              (8-bit ascii).</p>
+              <p>The character encoding used in strings and atoms. The only
+              supported encoding is currently <c>ERL_NIF_LATIN1</c> for
+              iso-latin-1 (8-bit ascii).</p>
             </item>
             <tag><marker id="ErlNifSysInfo"/>ErlNifSysInfo</tag>
              <item>
@@ -396,33 +463,53 @@ typedef enum {
                 to return information about the runtime system. Contains currently
                 the exact same content as <seealso marker="erl_driver#ErlDrvSysInfo">ErlDrvSysInfo</seealso>.</p>
               </item>
+              <tag><marker id="ErlNifSInt64"/>ErlNifSInt64</tag>
+               <item><p>A native signed 64-bit integer type.</p></item>
+              <tag><marker id="ErlNifUInt64"/>ErlNifUInt64</tag>
+               <item><p>A native unsigned 64-bit integer type.</p></item>
 
     </taglist>
   </section>
 
   <funcs>
-    <func><name><ret>void*</ret><nametext>enif_alloc(ErlNifEnv* env, size_t size)</nametext></name>
+    <func><name><ret>void *</ret><nametext>enif_alloc(size_t size)</nametext></name>
       <fsummary>Allocate dynamic memory.</fsummary>
       <desc><p>Allocate memory of <c>size</c> bytes. Return NULL if allocation failed.</p></desc>
     </func>
-    <func><name><ret>int</ret><nametext>enif_alloc_binary(ErlNifEnv* env, unsigned size, ErlNifBinary* bin)</nametext></name>
+    <func><name><ret>int</ret><nametext>enif_alloc_binary(size_t size, ErlNifBinary* bin)</nametext></name>
       <fsummary>Create a new binary.</fsummary>
-      <desc><p>Allocate a new binary of size of <c>size</c>
+      <desc><p>Allocate a new binary of size <c>size</c>
       bytes. Initialize the structure pointed to by <c>bin</c> to
       refer to the allocated binary. The binary must either be released by 
-      <seealso marker="#enif_release_binary">enif_release_binary()</seealso> 
+      <seealso marker="#enif_release_binary">enif_release_binary</seealso> 
       or ownership transferred to an Erlang term with 
-      <seealso marker="#enif_make_binary">enif_make_binary()</seealso>. 
+      <seealso marker="#enif_make_binary">enif_make_binary</seealso>. 
       An allocated (and owned) <c>ErlNifBinary</c> can be kept between NIF
       calls.</p>
-        <p>Return false if allocation failed.</p>
+        <p>Return true on success or false if allocation failed.</p>
       </desc>
     </func>
-    <func><name><ret>void*</ret><nametext>enif_alloc_resource(ErlNifEnv* env, ErlNifResourceType* type, unsigned size)</nametext></name>
+    <func><name><ret>ErlNifEnv *</ret><nametext>enif_alloc_env()</nametext></name>
+      <fsummary>Create a new environment</fsummary>
+      <desc><p>Allocate a new process independent environment. The environment can
+      be used to hold terms that is not bound to any process. Such terms can
+      later be copied to a process environment with
+      <seealso marker="#enif_make_copy">enif_make_copy</seealso>
+      or be sent to a process as a message with <seealso marker="#enif_send">enif_send</seealso>.</p>
+      <p>Return pointer to the new environment.</p>
+      </desc>
+    </func>
+    <func><name><ret>void *</ret><nametext>enif_alloc_resource(ErlNifResourceType* type, unsigned size)</nametext></name>
       <fsummary>Allocate a memory managed resource object</fsummary>
       <desc><p>Allocate a memory managed resource object of type <c>type</c> and size <c>size</c> bytes.</p></desc>
     </func>
-    <func><name><ret>int</ret><nametext>enif_compare(ErlNifEnv* env, ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)</nametext></name>
+    <func><name><ret>void</ret><nametext>enif_clear_env(ErlNifEnv* env)</nametext></name>
+      <fsummary>Clear an environment for reuse.</fsummary>
+      <desc><p>Free all terms in an environment and clear it for reuse. The environment must
+      have been allocated with <seealso marker="#enif_alloc_env">enif_alloc_env</seealso>.
+      </p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_compare(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)</nametext></name>
       <fsummary>Compare two terms</fsummary>
       <desc><p>Return an integer less than, equal to, or greater than
       zero if <c>lhs</c> is found, respectively, to be less than,
@@ -432,77 +519,104 @@ typedef enum {
     </func>
     <func><name><ret>void</ret><nametext>enif_cond_broadcast(ErlNifCond *cnd)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_broadcast">erl_drv_cond_broadcast()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_broadcast">erl_drv_cond_broadcast</seealso>.
           </p></desc>
     </func>
-    <func><name><ret>ErlNifCond*</ret><nametext>enif_cond_create(char *name)</nametext></name>
+    <func><name><ret>ErlNifCond *</ret><nametext>enif_cond_create(char *name)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_create">erl_drv_cond_create()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_create">erl_drv_cond_create</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_cond_destroy(ErlNifCond *cnd)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_destroy">erl_drv_cond_destroy()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_destroy">erl_drv_cond_destroy</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_cond_signal(ErlNifCond *cnd)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_signal">erl_drv_cond_signal()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_signal">erl_drv_cond_signal</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_cond_wait(ErlNifCond *cnd, ErlNifMutex *mtx)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_wait">erl_drv_cond_wait()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_wait">erl_drv_cond_wait</seealso>.
           </p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_equal_tids(ErlNifTid tid1, ErlNifTid tid2)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_equal_tids">erl_drv_equal_tids()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_equal_tids">erl_drv_equal_tids</seealso>.
           </p></desc>
     </func>
-    <func><name><ret>void</ret><nametext>enif_free(ErlNifEnv* env, void* ptr)</nametext></name>
+    <func><name><ret>void</ret><nametext>enif_free(void* ptr)</nametext></name>
       <fsummary>Free dynamic memory</fsummary>
       <desc><p>Free memory allocated by <c>enif_alloc</c>.</p></desc>
     </func>
-    <func><name><ret>int</ret><nametext>enif_get_atom(ErlNifEnv* env, 
-                  ERL_NIF_TERM term, char* buf, unsigned size)
-                             </nametext></name>
+    <func><name><ret>void</ret><nametext>enif_free_env(ErlNifEnv* env)</nametext></name>
+      <fsummary>Free an environment allocated with enif_alloc_env</fsummary>
+      <desc><p>Free an environment allocated with <seealso marker="#enif_alloc_env">enif_alloc_env</seealso>.
+      All terms created in the environment will be freed as well.</p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_get_atom(ErlNifEnv* env, ERL_NIF_TERM term, char* buf, unsigned size, ErlNifCharEncoding encode)</nametext></name>
       <fsummary>Get the text representation of an atom term</fsummary>
       <desc><p>Write a null-terminated string, in the buffer pointed to by
               <c>buf</c> of size <c>size</c>, consisting of the string 
-              representation of the atom <c>term</c>. Return the number of bytes
-              written (including terminating null character) or 0 if
+              representation of the atom <c>term</c> with encoding
+              <seealso marker="#ErlNifCharEncoding">encode</seealso>. Return
+              the number of bytes written (including terminating null character) or 0 if
               <c>term</c> is not an atom with maximum length of 
               <c>size-1</c>.</p></desc>
     </func>
+    <func><name><ret>int</ret><nametext>enif_get_atom_length(ErlNifEnv* env, ERL_NIF_TERM term, unsigned* len, ErlNifCharEncoding encode)</nametext></name>
+      <fsummary>Get the length of atom <c>term</c>.</fsummary>
+      <desc><p>Set <c>*len</c> to the length (number of bytes excluding
+      terminating null character) of the atom <c>term</c> with encoding
+      <c>encode</c>. Return true on success or false if <c>term</c> is not an
+      atom.</p></desc>
+    </func>
     <func><name><ret>int</ret><nametext>enif_get_double(ErlNifEnv* env, ERL_NIF_TERM term, double* dp)</nametext></name>
       <fsummary>Read a floating-point number term.</fsummary>
       <desc><p>Set <c>*dp</c> to the floating point value of
-      <c>term</c> or return false if <c>term</c> is not a float.</p></desc>
+      <c>term</c>. Return true on success or false if <c>term</c> is not a float.</p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_get_int(ErlNifEnv* env, ERL_NIF_TERM term, int* ip)</nametext></name>
-      <fsummary>Read an integer term.</fsummary>
+      <fsummary>Read an integer term</fsummary>
       <desc><p>Set <c>*ip</c> to the integer value of
-      <c>term</c> or return false if <c>term</c> is not an integer or is
-      outside the bounds of type <c>int</c></p></desc>
+      <c>term</c>. Return true on success or false if <c>term</c> is not an
+      integer or is outside the bounds of type <c>int</c>.</p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_get_int64(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifSInt64* ip)</nametext></name>
+      <fsummary>Read a 64-bit integer term</fsummary>
+      <desc><p>Set <c>*ip</c> to the integer value of
+      <c>term</c>. Return true on success or false if <c>term</c> is not an
+      integer or is outside the bounds of a signed 64-bit integer.</p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_get_local_pid(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifPid* pid)</nametext></name>
+    <fsummary>Read an local pid term</fsummary>
+    <desc><p>If <c>term</c> is the pid of a node local process, initialize the
+    pid variable <c>*pid</c> from it and return true. Otherwise return false.
+    No check if the process is alive is done.</p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_get_list_cell(ErlNifEnv* env, ERL_NIF_TERM list, ERL_NIF_TERM* head, ERL_NIF_TERM* tail)</nametext></name>
       <fsummary>Get head and tail from a list</fsummary>
       <desc><p>Set <c>*head</c> and <c>*tail</c> from
-      <c>list</c> or return false if <c>list</c> is not a non-empty
-      list.</p></desc>
+      <c>list</c> and return true, or return false if <c>list</c> is not a
+      non-empty list.</p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_get_list_length(ErlNifEnv* env, ERL_NIF_TERM term, unsigned* len)</nametext></name>
+      <fsummary>Get the length of list <c>term</c>.</fsummary>
+      <desc><p>Set <c>*len</c> to the length of list <c>term</c> and return true,
+      or return false if <c>term</c> is not a list.</p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_get_long(ErlNifEnv* env, ERL_NIF_TERM term, long int* ip)</nametext></name>
       <fsummary>Read an long integer term.</fsummary>
-      <desc><p>Set <c>*ip</c> to the long integer value of
-      <c>term</c> or return false if <c>term</c> is not an integer or is
+      <desc><p>Set <c>*ip</c> to the long integer value of <c>term</c> and
+      return true, or return false if <c>term</c> is not an integer or is
       outside the bounds of type <c>long int</c>.</p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp)</nametext></name>
       <fsummary>Get the pointer to a resource object</fsummary>
-      <desc><p>Set <c>*objp</c> to point to the resource object referred to by <c>term</c>.
-      The pointer is valid until the calling NIF returns and should not be released.</p>
-      <p>Return false if <c>term</c> is not a handle to a resource object
+      <desc><p>Set <c>*objp</c> to point to the resource object referred to by <c>term</c>.</p>
+      <p>Return true on success or false if <c>term</c> is not a handle to a resource object
       of type <c>type</c>.</p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_get_string(ErlNifEnv* env, 
@@ -527,27 +641,32 @@ typedef enum {
       <c>*arity</c> to the number of elements. Note that the array
       is read-only and <c>(*array)[N-1]</c> will be the Nth element of
       the tuple. <c>*array</c> is undefined if the arity of the tuple
-      is zero.</p><p>Return false if <c>term</c> is not a
+      is zero.</p><p>Return true on success or false if <c>term</c> is not a
       tuple.</p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_get_uint(ErlNifEnv* env, ERL_NIF_TERM term, unsigned int* ip)</nametext></name>
       <fsummary>Read an unsigned integer term.</fsummary>
-      <desc><p>Set <c>*ip</c> to the unsigned integer value of
-      <c>term</c> or return false if <c>term</c> is not an unsigned integer or is
-      outside the bounds of type <c>unsigned int</c></p></desc>
+      <desc><p>Set <c>*ip</c> to the unsigned integer value of <c>term</c> and
+      return true, or return false if <c>term</c> is not an unsigned integer or
+      is outside the bounds of type <c>unsigned int</c>.</p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_get_uint64(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifUInt64* ip)</nametext></name>
+      <fsummary>Read an unsigned 64-bit integer term.</fsummary>
+      <desc><p>Set <c>*ip</c> to the unsigned integer value of <c>term</c> and
+      return true, or return false if <c>term</c> is not an unsigned integer or
+      is outside the bounds of an unsigned 64-bit integer.</p></desc>
     </func>
-
     <func><name><ret>int</ret><nametext>enif_get_ulong(ErlNifEnv* env, ERL_NIF_TERM term, unsigned long* ip)</nametext></name>
       <fsummary>Read an unsigned integer term.</fsummary>
-      <desc><p>Set <c>*ip</c> to the unsigned long integer value of
-      <c>term</c> or return false if <c>term</c> is not an unsigned integer or is
-      outside the bounds of type <c>unsigned long</c></p></desc>
+      <desc><p>Set <c>*ip</c> to the unsigned long integer value of <c>term</c>
+      and return true, or return false if <c>term</c> is not an unsigned integer or is
+      outside the bounds of type <c>unsigned long</c>.</p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_inspect_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, ErlNifBinary* bin)</nametext></name>
       <fsummary>Inspect the content of a binary</fsummary>
       <desc><p>Initialize the structure pointed to by <c>bin</c> with 
               information about the binary term
-      <c>bin_term</c>. Return false if <c>bin_term</c> is not a binary.</p></desc> 
+      <c>bin_term</c>. Return true on success or false if <c>bin_term</c> is not a binary.</p></desc> 
     </func>
     <func><name><ret>int</ret><nametext>enif_inspect_iolist_as_binary(ErlNifEnv* 
                                 env, ERL_NIF_TERM term, ErlNifBinary* bin)
@@ -556,7 +675,7 @@ typedef enum {
       <desc><p>Initialize the structure pointed to by <c>bin</c> with one
       continuous buffer with the same byte content as <c>iolist</c>. As with 
       inspect_binary, the data pointed to by <c>bin</c> is transient and does 
-      not need to be released. Return false if <c>iolist</c> is not an
+      not need to be released. Return true on success or false if <c>iolist</c> is not an
       iolist.</p>
       </desc> 
     </func>
@@ -572,11 +691,19 @@ typedef enum {
       <fsummary>Determine if a term is an empty list</fsummary>
       <desc><p>Return true if <c>term</c> is an empty list.</p></desc>
     </func>
+    <marker id="enif_is_exception"/><func><name><ret>int</ret><nametext>enif_is_exception(ErlNifEnv* env, ERL_NIF_TERM term)</nametext></name>
+      <fsummary>Determine if a term is an exception</fsummary>
+      <desc><p>Return true if <c>term</c> is an exception.</p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_is_number(ErlNifEnv* env, ERL_NIF_TERM term)</nametext></name>
+      <fsummary>Determine if a term is a number (integer or float)</fsummary>
+      <desc><p>Return true if <c>term</c> is a number.</p></desc>
+    </func>
     <func><name><ret>int</ret><nametext>enif_is_fun(ErlNifEnv* env, ERL_NIF_TERM term)</nametext></name>
       <fsummary>Determine if a term is a fun</fsummary>
       <desc><p>Return true if <c>term</c> is a fun.</p></desc>
     </func>
-    <func><name><ret>int</ret><nametext>enif_is_identical(ErlNifEnv* env, ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)</nametext></name>
+    <func><name><ret>int</ret><nametext>enif_is_identical(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)</nametext></name>
       <fsummary>Erlang operator =:=</fsummary>
       <desc><p>Return true if the two terms are identical. Corresponds to the
       Erlang operators <c>=:=</c> and
@@ -590,19 +717,46 @@ typedef enum {
       <fsummary>Determine if a term is a port</fsummary>
       <desc><p>Return true if <c>term</c> is a port.</p></desc>
     </func>
-
     <func><name><ret>int</ret><nametext>enif_is_ref(ErlNifEnv* env, ERL_NIF_TERM term)</nametext></name>
       <fsummary>Determine if a term is a reference</fsummary>
       <desc><p>Return true if <c>term</c> is a reference.</p></desc>
     </func>
+    <func><name><ret>int</ret><nametext>enif_is_tuple(ErlNifEnv* env, ERL_NIF_TERM term)</nametext></name>
+      <fsummary>Determine if a term is a tuple</fsummary>
+      <desc><p>Return true if <c>term</c> is a tuple.</p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_is_list(ErlNifEnv* env, ERL_NIF_TERM term)</nametext></name>
+      <fsummary>Determine if a term is a list</fsummary>
+      <desc><p>Return true if <c>term</c> is a list.</p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_keep_resource(void* obj)</nametext></name>
+      <fsummary>Add a reference to a resource object</fsummary>
+      <desc><p>Add a reference to resource object <c>obj</c> obtained from
+      <seealso marker="#enif_alloc_resource">enif_alloc_resource</seealso>.
+      Each call to <c>enif_keep_resource</c> for an object must be balanced by
+      a call to <seealso marker="#enif_release_resource">enif_release_resource</seealso>
+      before the object will be destructed.</p></desc>
+    </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_atom(ErlNifEnv* env, const char* name)</nametext></name>
       <fsummary>Create an atom term</fsummary>
-      <desc><p>Create an atom term from the C-string <c>name</c>. Unlike other terms, atom
-      terms may be saved and used between NIF calls.</p></desc>
+      <desc><p>Create an atom term from the null-terminated C-string <c>name</c>
+      with iso-latin-1 encoding.</p></desc>
+    </func>
+    <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_atom_len(ErlNifEnv* env, const char* name, size_t len)</nametext></name>
+      <fsummary>Create an atom term</fsummary>
+      <desc><p>Create an atom term from the string <c>name</c> with length <c>len</c>.
+      Null-characters are treated as any other characters.</p></desc>
     </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_badarg(ErlNifEnv* env)</nametext></name>
       <fsummary>Make a badarg exception.</fsummary>
-      <desc><p>Make a badarg exception to be returned from a NIF.</p></desc>
+      <desc><p>Make a badarg exception to be returned from a NIF, and set
+      an associated exception reason in <c>env</c>. If
+      <c>enif_make_badarg</c> is called, the term it returns <em>must</em>
+      be returned from the function that called it. No other return value
+      is allowed. Also, the term returned from <c>enif_make_badarg</c> may
+      be passed only to
+      <seealso marker="#enif_is_exception">enif_is_exception</seealso> and
+      not to any other NIF API function.</p></desc>
     </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin)</nametext></name>
       <fsummary>Make a binary term.</fsummary>
@@ -611,21 +765,40 @@ typedef enum {
               <c>bin</c> should be considered read-only for the rest of the NIF
               call and then as released.</p></desc>
     </func>
+    <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term)</nametext></name>
+      <fsummary>Make a copy of a term.</fsummary>
+      <desc><p>Make a copy of term <c>src_term</c>. The copy will be created in
+      environment <c>dst_env</c>. The source term may be located in any
+      environment.</p></desc>        
+    </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_double(ErlNifEnv* env, double d)</nametext></name>
-      <fsummary>Create an floating-point term</fsummary>
-      <desc><p>Create an floating-point term from a <c>double</c>.</p></desc>
+      <fsummary>Create a floating-point term</fsummary>
+      <desc><p>Create a floating-point term from a <c>double</c>.</p></desc>
     </func>
-    <func><name><ret>int</ret><nametext>enif_make_existing_atom(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom)</nametext></name>
+    <func><name><ret>int</ret><nametext>enif_make_existing_atom(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom, ErlNifCharEncoding encode)</nametext></name>
       <fsummary>Create an existing atom term</fsummary>
       <desc><p>Try to create the term of an already existing atom from
-      the C-string <c>name</c>. If the atom already exist store the
-      term in <c>*atom</c> and return true, otherwise return
-      false.</p></desc>
+      the null-terminated C-string <c>name</c> with encoding
+      <seealso marker="#ErlNifCharEncoding">encode</seealso>. If the atom
+      already exists store the term in <c>*atom</c> and return true, otherwise
+      return false.</p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_make_existing_atom_len(ErlNifEnv* env, const char* name, size_t len, ERL_NIF_TERM* atom, ErlNifCharEncoding encoding)</nametext></name>
+      <fsummary>Create an existing atom term</fsummary>
+      <desc><p>Try to create the term of an already existing atom from the
+      string <c>name</c> with length <c>len</c> and encoding
+      <seealso marker="#ErlNifCharEncoding">encode</seealso>. Null-characters
+      are treated as any other characters. If the atom already exists store the term
+      in <c>*atom</c> and return true, otherwise return false.</p></desc>
     </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_int(ErlNifEnv* env, int i)</nametext></name>
       <fsummary>Create an integer term</fsummary>
       <desc><p>Create an integer term.</p></desc>
     </func>
+    <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_int64(ErlNifEnv* env, ErlNifSInt64 i)</nametext></name>
+      <fsummary>Create an integer term</fsummary>
+      <desc><p>Create an integer term from a signed 64-bit integer.</p></desc>
+    </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_list(ErlNifEnv* env, unsigned cnt, ...)</nametext></name>
       <fsummary>Create a list term.</fsummary>
       <desc><p>Create an ordinary list term of length <c>cnt</c>. Expects
@@ -644,7 +817,7 @@ typedef enum {
       <fsummary>Create a list term.</fsummary>
       <desc><p>Create an ordinary list term with length indicated by the
       function name. Prefer these functions (macros) over the variadic
-      <c>enif_make_list</c>  to get compile time error if the number of
+      <c>enif_make_list</c>  to get a compile time error if the number of
       arguments does not match.</p></desc>
     </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_list_cell(ErlNifEnv* env, ERL_NIF_TERM head, ERL_NIF_TERM tail)</nametext></name>
@@ -656,10 +829,31 @@ typedef enum {
       <desc><p>Create an ordinary list containing the elements of array <c>arr</c>
       of length <c>cnt</c>. An empty list is returned if <c>cnt</c> is 0.</p></desc>
     </func>
+    <func><name><ret>int</ret><nametext>enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list)</nametext></name>
+      <fsummary>Create the reverse list of the list <c>term</c>.</fsummary>
+      <desc><p>Set <c>*list</c> to the reverse list of the list <c>term</c> and return true,
+      or return false if <c>term</c> is not a list. This function should only be used on
+      short lists as a copy will be created of the list which will not be released until after the
+      nif returns.</p></desc>
+    </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_long(ErlNifEnv* env, long int i)</nametext></name>
       <fsummary>Create an integer term from a long int</fsummary>
       <desc><p>Create an integer term from a <c>long int</c>.</p></desc>
     </func>
+    <func><name><ret>unsigned char *</ret><nametext>enif_make_new_binary(ErlNifEnv* env, size_t size, ERL_NIF_TERM* termp)</nametext></name>
+      <fsummary>Allocate and create a new binary term</fsummary>
+      <desc><p>Allocate a binary of size <c>size</c> bytes and create an owning
+      term. The binary data is mutable until the calling NIF returns. This is a
+      quick way to create a new binary without having to use
+      <seealso marker="#ErlNifBinary">ErlNifBinary</seealso>. The drawbacks are
+      that the binary can not be kept between NIF calls and it can not be
+      reallocated.</p><p>Return a pointer to the raw binary data and set
+      <c>*termp</c> to the binary term.</p></desc>
+    </func>
+    <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_pid(ErlNifEnv* env, const ErlNifPid* pid)</nametext></name>
+      <fsummary>Make a pid term</fsummary>
+      <desc><p>Make a pid term from <c>*pid</c>.</p></desc>
+    </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_ref(ErlNifEnv* env)</nametext></name>
       <fsummary>Create a reference.</fsummary>
       <desc><p>Create a reference like <seealso marker="erlang#make_ref-0">erlang:make_ref/0</seealso>.</p></desc>
@@ -668,20 +862,50 @@ typedef enum {
       <fsummary>Create an opaque handle to a resource object</fsummary>
       <desc><p>Create an opaque handle to a memory managed resource object
       obtained by <seealso marker="#enif_alloc_resource">enif_alloc_resource</seealso>.
-      No ownership transfer is done, the resource object still needs to be released by
-      <seealso marker="#enif_release_resource">enif_release_resource</seealso>.</p>
-      <p>Note that the only defined behaviour when using of a resource term in
+      No ownership transfer is done, as the resource object still needs to be released by
+      <seealso marker="#enif_release_resource">enif_release_resource</seealso>,
+      but note that the call to <c>enif_release_resource</c> can occur
+      immediately after obtaining the term from <c>enif_make_resource</c>,
+      in which case the resource object will be deallocated when the
+      term is garbage collected. See the
+      <seealso marker="#enif_resource_example">example of creating and
+      returning a resource object</seealso> for more details.</p> 
+      <p>Note that the only defined behaviour of using a resource term in
       an Erlang program is to store it and send it between processes on the
       same node. Other operations such as matching or <c>term_to_binary</c>
       will have unpredictable (but harmless) results.</p></desc>
     </func>
+    <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_resource_binary(ErlNifEnv* env, void* obj, const void* data, size_t size)</nametext></name>
+      <fsummary>Create a custom binary term</fsummary>
+      <desc><p>Create a binary term that is memory managed by a resource object
+      <c>obj</c> obtained by <seealso marker="#enif_alloc_resource">enif_alloc_resource</seealso>.
+      The returned binary term will consist of <c>size</c> bytes pointed to
+      by <c>data</c>. This raw binary data must be kept readable and unchanged
+      until the destructor of the resource is called. The binary data may be
+      stored external to the resource object in which case it is the responsibility
+      of the destructor to release the data.</p>
+      <p>Several binary terms may be managed by the same resource object. The
+      destructor will not be called until the last binary is garbage collected.
+      This can be useful as a way to return different parts of a larger binary
+      buffer.</p>
+      <p>As with <seealso marker="#enif_make_resource">enif_make_resource</seealso>,
+      no ownership transfer is done. The resource still needs to be released with
+      <seealso marker="#enif_release_resource">enif_release_resource</seealso>.</p>
+      </desc>
+    </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_string(ErlNifEnv* env, const char* string, ErlNifCharEncoding encoding)</nametext></name>
       <fsummary>Create a string.</fsummary>
       <desc><p>Create a list containing the characters of the
       null-terminated string <c>string</c> with encoding <seealso marker="#ErlNifCharEncoding">encoding</seealso>.</p></desc>
     </func>
+    <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_string_len(ErlNifEnv* env, const char* string, size_t len, ErlNifCharEncoding encoding)</nametext></name>
+      <fsummary>Create a string.</fsummary>
+      <desc><p>Create a list containing the characters of the string <c>string</c> with
+      length <c>len</c> and encoding <seealso marker="#ErlNifCharEncoding">encoding</seealso>.
+      Null-characters are treated as any other characters.</p></desc>
+    </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_sub_binary(ErlNifEnv* 
-      env, ERL_NIF_TERM bin_term, unsigned pos, unsigned size)</nametext></name>
+      env, ERL_NIF_TERM bin_term, size_t pos, size_t size)</nametext></name>
       <fsummary>Make a subbinary term.</fsummary>
       <desc><p>Make a subbinary of binary <c>bin_term</c>, starting at 
               zero-based position <c>pos</c> with a length of <c>size</c> bytes. 
@@ -707,7 +931,7 @@ typedef enum {
       <fsummary>Create a tuple term.</fsummary>
       <desc><p>Create a tuple term with length indicated by the
       function name. Prefer these functions (macros) over the variadic
-      <c>enif_make_tuple</c>  to get compile time error if the number of
+      <c>enif_make_tuple</c>  to get a compile time error if the number of
       arguments does not match.</p></desc>
     </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_tuple_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[], unsigned cnt)</nametext></name>
@@ -719,36 +943,41 @@ typedef enum {
       <fsummary>Create an unsigned integer term</fsummary>
       <desc><p>Create an integer term from an <c>unsigned int</c>.</p></desc>
     </func>
+    <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_uint64(ErlNifEnv* env, ErlNifUInt64 i)</nametext></name>
+      <fsummary>Create an unsigned integer term</fsummary>
+      <desc><p>Create an integer term from an unsigned 64-bit integer.</p></desc>
+    </func>
     <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_ulong(ErlNifEnv* env, unsigned long i)</nametext></name>
       <fsummary>Create an integer term from an unsigned long int</fsummary>
       <desc><p>Create an integer term from an <c>unsigned long int</c>.</p></desc>
     </func>
-    <func><name><ret>ErlNifMutex*</ret><nametext>enif_mutex_create(char *name)</nametext></name>
+    <func><name><ret>ErlNifMutex *</ret><nametext>enif_mutex_create(char *name)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_create">erl_drv_mutex_create()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_create">erl_drv_mutex_create</seealso>.
     </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_mutex_destroy(ErlNifMutex *mtx)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_destroy">erl_drv_mutex_destroy()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_destroy">erl_drv_mutex_destroy</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_mutex_lock(ErlNifMutex *mtx)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_lock">erl_drv_mutex_lock()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_lock">erl_drv_mutex_lock</seealso>.
           </p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_mutex_trylock(ErlNifMutex *mtx)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_trylock">erl_drv_mutex_trylock()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_trylock">erl_drv_mutex_trylock</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_mutex_unlock(ErlNifMutex *mtx)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_unlock">erl_drv_mutex_unlock()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_unlock">erl_drv_mutex_unlock</seealso>.
           </p></desc>
     </func>
-    <func><name><ret>ErlNifResourceType*</ret><nametext>enif_open_resource_type(ErlNifEnv* env, const char* name,
+    <func><name><ret>ErlNifResourceType *</ret><nametext>enif_open_resource_type(ErlNifEnv* env,
+                             const char* module_str, const char* name,
                              ErlNifResourceDtor* dtor, ErlNifResourceFlags flags, ErlNifResourceFlags* tried)</nametext></name>
       <fsummary>Create or takeover a resource type</fsummary>
       <desc><p>Create or takeover a resource type identified by the string
@@ -762,10 +991,10 @@ typedef enum {
                The supplied destructor <c>dtor</c> will be called both for existing instances
                as well as new instances not yet created by the calling NIF library.</item>            
       </taglist>
-      <p>The two flag values can be combined with bitwise-or. To avoid unintentionally
-      name clashes a good practice is to include the module name as part of the
-      type <c>name</c>. The <c>dtor</c> may  be <c>NULL</c> in case no destructor
-      is needed.</p>
+      <p>The two flag values can be combined with bitwise-or. The name of the
+      resource type is local to the calling module. Argument <c>module_str</c>
+      is not (yet) used and must be NULL. The <c>dtor</c> may  be <c>NULL</c>
+      in case no destructor is needed.</p>
       <p>On success, return a pointer to the resource type and <c>*tried</c>
       will be set to either <c>ERL_NIF_RT_CREATE</c> or
       <c>ERL_NIF_RT_TAKEOVER</c> to indicate what was actually done.
@@ -776,135 +1005,169 @@ typedef enum {
        and <seealso marker="#upgrade">upgrade</seealso>.</p>
       </desc>
     </func>
-    <func><name><ret>void*</ret><nametext>enif_priv_data(ErlNifEnv* env)</nametext></name>
+    <func><name><ret>void *</ret><nametext>enif_priv_data(ErlNifEnv* env)</nametext></name>
       <fsummary>Get the private data of a NIF library</fsummary>
       <desc><p>Return the pointer to the private data that was set by <c>load</c>,
       <c>reload</c> or <c>upgrade</c>.</p>
       <p>Was previously named <c>enif_get_data</c>.</p></desc>
     </func>
-    <func><name><ret>void</ret><nametext>enif_realloc_binary(ErlNifEnv* env, ErlNifBinary* bin, unsigned size)</nametext></name>
+    <func><name><ret>int</ret><nametext>enif_realloc_binary(ErlNifBinary* bin, size_t size)</nametext></name>
       <fsummary>Change the size of a binary.</fsummary>
       <desc><p>Change the size of a binary <c>bin</c>. The source binary
       may be read-only, in which case it will be left untouched and
-      a mutable copy is allocated and assigned to <c>*bin</c>.</p></desc>
+      a mutable copy is allocated and assigned to <c>*bin</c>. Return true on success,
+      false if memory allocation failed.</p></desc>
     </func>
-    <func><name><ret>void</ret><nametext>enif_release_binary(ErlNifEnv* env, ErlNifBinary* bin)</nametext></name>
+    <func><name><ret>void</ret><nametext>enif_release_binary(ErlNifBinary* bin)</nametext></name>
       <fsummary>Release a binary.</fsummary>
-      <desc><p>Release a binary obtained 
-              from <c>enif_alloc_binary</c>.</p></desc>
+      <desc><p>Release a binary obtained from <c>enif_alloc_binary</c>.</p></desc>
     </func>
-    <func><name><ret>void</ret><nametext>enif_release_resource(ErlNifEnv* env, void* obj)</nametext></name>
+    <func><name><ret>void</ret><nametext>enif_release_resource(void* obj)</nametext></name>
       <fsummary>Release a resource object.</fsummary>
-      <desc><p>Release a resource objects obtained from <c>enif_alloc_resource</c>.
-      The object may still be alive if it is referred to by Erlang terms. Each call to
-      <c>enif_release_resource</c> must correspond to a previous call to <c>enif_alloc_resource</c>.
-      References made by <c>enif_make_resource</c> can only be released by the garbage collector.</p></desc>
-    </func>
-    <func><name><ret>ErlNifRWLock*</ret><nametext>enif_rwlock_create(char *name)</nametext></name>
+      <desc><p>Remove a reference to resource object <c>obj</c>obtained from
+      <seealso marker="#enif_alloc_resource">enif_alloc_resource</seealso>.
+      The resource object will be destructed when the last reference is removed.
+      Each call to <c>enif_release_resource</c> must correspond to a previous
+      call to <c>enif_alloc_resource</c> or
+      <seealso marker="#enif_keep_resource">enif_keep_resource</seealso>.
+      References made by <seealso marker="#enif_make_resource">enif_make_resource</seealso>
+      can only be removed by the garbage collector.</p></desc>
+    </func>
+    <func><name><ret>ErlNifRWLock *</ret><nametext>enif_rwlock_create(char *name)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_create">erl_drv_rwlock_create()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_create">erl_drv_rwlock_create</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_rwlock_destroy(ErlNifRWLock *rwlck)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_destroy">erl_drv_rwlock_destroy()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_destroy">erl_drv_rwlock_destroy</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_rwlock_rlock(ErlNifRWLock *rwlck)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_rlock">erl_drv_rwlock_rlock()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_rlock">erl_drv_rwlock_rlock</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_rwlock_runlock(ErlNifRWLock *rwlck)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_runlock">erl_drv_rwlock_runlock()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_runlock">erl_drv_rwlock_runlock</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_rwlock_rwlock(ErlNifRWLock *rwlck)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_rwlock">erl_drv_rwlock_rwlock()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_rwlock">erl_drv_rwlock_rwlock</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_rwlock_rwunlock(ErlNifRWLock *rwlck)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_rwunlock">erl_drv_rwlock_rwunlock()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_rwunlock">erl_drv_rwlock_rwunlock</seealso>.
           </p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_rwlock_tryrlock(ErlNifRWLock *rwlck)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_tryrlock">erl_drv_rwlock_tryrlock()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_tryrlock">erl_drv_rwlock_tryrlock</seealso>.
           </p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_rwlock_tryrwlock(ErlNifRWLock *rwlck)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_tryrwlock">erl_drv_rwlock_tryrwlock()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_tryrwlock">erl_drv_rwlock_tryrwlock</seealso>.
           </p></desc>
     </func>
-    <func><name><ret>unsigned</ret><nametext>enif_sizeof_resource(ErlNifEnv* env, void* obj)</nametext></name>
+    <func><name><ret>ErlNifPid *</ret><nametext>enif_self(ErlNifEnv* caller_env, ErlNifPid* pid)</nametext></name>
+      <fsummary>Get the pid of the calling process.</fsummary>
+      <desc><p>Initialize the pid variable <c>*pid</c> to represent the
+      calling process. Return <c>pid</c>.</p></desc>
+    </func>
+    <func><name><ret>int</ret><nametext>enif_send(ErlNifEnv* env, ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg)</nametext></name>
+      <fsummary>Send a message to a process.</fsummary>
+      <desc><p>Send a message to a process.</p>
+      <taglist>
+        <tag><c>env</c></tag>
+              <item>The environment of the calling process. Must be NULL if and
+              only if calling from a created thread.</item>
+        <tag><c>*to_pid</c></tag>
+              <item>The pid of the receiving process. The pid should refer to a process on the local node.</item>
+        <tag><c>msg_env</c></tag>
+              <item>The environment of the message term. Must be a process
+              independent environment allocated with
+               <seealso marker="#enif_alloc_env">enif_alloc_env</seealso>.</item>
+        <tag><c>msg</c></tag>
+              <item>The message term to send.</item>
+      </taglist>
+      <p>Return true on success, or false if <c>*to_pid</c> does not refer to an alive local process.</p>
+      <p>The message environment <c>msg_env</c> with all its terms (including
+      <c>msg</c>) will be invalidated by a successful call to <c>enif_send</c>. The environment
+      should either be freed with <seealso marker="#enif_free_env">enif_free_env</seealso>
+      of cleared for reuse with <seealso marker="#enif_clear_env">enif_clear_env</seealso>.</p>
+      <p>This function is only thread-safe when the emulator with SMP support is used.
+      It can only be used in a non-SMP emulator from a NIF-calling thread.</p>
+      </desc>
+    </func>
+    <func><name><ret>unsigned</ret><nametext>enif_sizeof_resource(void* obj)</nametext></name>
       <fsummary>Get the byte size of a resource object</fsummary>
       <desc><p>Get the byte size of a resource object <c>obj</c> obtained by
-      <c>enif_alloc_resource</c>.</p></desc>
+      <seealso marker="#enif_alloc_resource">enif_alloc_resource</seealso>.</p></desc>
     </func>
     <func>
       <name><ret>void</ret><nametext>enif_system_info(ErlNifSysInfo *sys_info_ptr, size_t size)</nametext></name>
       <fsummary>Get information about the Erlang runtime system</fsummary>
-      <desc><p>Same as <seealso marker="erl_driver#driver_system_info">driver_system_info()</seealso>.
+      <desc><p>Same as <seealso marker="erl_driver#driver_system_info">driver_system_info</seealso>.
             </p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_thread_create(char *name,ErlNifTid *tid,void * (*func)(void *),void *args,ErlNifThreadOpts *opts)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_create">erl_drv_thread_create()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_create">erl_drv_thread_create</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_thread_exit(void *resp)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_exit">erl_drv_thread_exit()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_exit">erl_drv_thread_exit</seealso>.
           </p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_thread_join(ErlNifTid, void **respp)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_join">erl_drv_thread_join ()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_join">erl_drv_thread_join </seealso>.
           </p></desc>
     </func>
-    <func><name><ret>ErlNifThreadOpts*</ret><nametext>enif_thread_opts_create(char *name)</nametext></name>
+    <func><name><ret>ErlNifThreadOpts *</ret><nametext>enif_thread_opts_create(char *name)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_opts_create">erl_drv_thread_opts_create()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_opts_create">erl_drv_thread_opts_create</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_thread_opts_destroy(ErlNifThreadOpts *opts)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_opts_destroy">erl_drv_thread_opts_destroy()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_opts_destroy">erl_drv_thread_opts_destroy</seealso>.
           </p></desc>
     </func>
     <func><name><ret>ErlNifTid</ret><nametext>enif_thread_self(void)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_self">erl_drv_thread_self()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_self">erl_drv_thread_self</seealso>.
           </p></desc>
     </func>
     <func><name><ret>int</ret><nametext>enif_tsd_key_create(char *name, ErlNifTSDKey *key)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_key_create">erl_drv_tsd_key_create()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_key_create">erl_drv_tsd_key_create</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_tsd_key_destroy(ErlNifTSDKey key)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_key_destroy">erl_drv_tsd_key_destroy()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_key_destroy">erl_drv_tsd_key_destroy</seealso>.
           </p></desc>
     </func>
-    <func><name><ret>void*</ret><nametext>enif_tsd_get(ErlNifTSDKey key)</nametext></name>
+    <func><name><ret>void *</ret><nametext>enif_tsd_get(ErlNifTSDKey key)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_get">erl_drv_tsd_get()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_get">erl_drv_tsd_get</seealso>.
           </p></desc>
     </func>
     <func><name><ret>void</ret><nametext>enif_tsd_set(ErlNifTSDKey key, void *data)</nametext></name>
     <fsummary></fsummary>
-    <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_set">erl_drv_tsd_set()</seealso>.
+    <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_set">erl_drv_tsd_set</seealso>.
           </p></desc>
     </func>
   </funcs>
   <section>
     <title>SEE ALSO</title>
-    <p><seealso marker="erlang#load_nif-2">load_nif(3)</seealso></p>
+    <p><seealso marker="erlang#load_nif-2">erlang:load_nif/2</seealso></p>
   </section>
 </cref>
 
diff --git a/erts/doc/src/erl_prim_loader.xml b/erts/doc/src/erl_prim_loader.xml
index ccaa9b725f..9f5b3f385b 100644
--- a/erts/doc/src/erl_prim_loader.xml
+++ b/erts/doc/src/erl_prim_loader.xml
@@ -4,7 +4,7 @@
 <erlref>
   <header>
     <copyright>
-      <year>1996</year><year>2009</year>
+      <year>1996</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -55,33 +55,31 @@
      <c>-loader_debug</c> are also experimental</p></warning>
 
   </description>
+  <datatypes>
+    <datatype>
+      <name name="host"/>
+    </datatype>
+  </datatypes>
+
   <funcs>
     <func>
-      <name>start(Id, Loader, Hosts) -> {ok, Pid} | {error, What}</name>
+      <name name="start" arity="3"/>
       <fsummary>Start the Erlang low level loader</fsummary>
-      <type>
-        <v>Id = term()</v>
-        <v>Loader = atom() | string()</v>
-        <v>Hosts = [Host]</v>
-        <v>Host = atom()</v>
-        <v>Pid = pid()</v>
-        <v>What = term()</v>
-      </type>
       <desc>
         <p>Starts the Erlang low level loader. This function is called
           by the <c>init</c> process (and module). The <c>init</c>
-          process reads the command line flags <c>-id Id</c>,
-          <c>-loader Loader</c>, and <c>-hosts Hosts</c>. These are
+          process reads the command line flags <c>-id <anno>Id</anno></c>,
+          <c>-loader <anno>Loader</anno></c>, and <c>-hosts <anno>Hosts</anno></c>. These are
           the arguments supplied to the <c>start/3</c> function.</p>
         <p>If <c>-loader</c> is not given, the default loader is
           <c>efile</c> which tells the system to read from the file
           system.</p>
-        <p>If <c>-loader</c> is <c>inet</c>, the <c>-id Id</c>,
-          <c>-hosts Hosts</c>, and <c>-setcookie Cookie</c> flags must
-          also be supplied. <c>Hosts</c> identifies hosts which this
+        <p>If <c>-loader</c> is <c>inet</c>, the <c>-id <anno>Id</anno></c>,
+          <c>-hosts <anno>Hosts</anno></c>, and <c>-setcookie Cookie</c> flags must
+          also be supplied. <c><anno>Hosts</anno></c> identifies hosts which this
           node can contact in order to load modules. One Erlang
           runtime system with a <c>erl_boot_server</c> process must be
-          started on each of hosts given in <c>Hosts</c> in order to
+          started on each of hosts given in <c><anno>Hosts</anno></c> in order to
           answer the requests. See <seealso
           marker="kernel:erl_boot_server">erl_boot_server(3)</seealso>.</p>
         <p>If <c>-loader</c> is something else, the given port program
@@ -90,35 +88,26 @@
       </desc>
     </func>
     <func>
-      <name>get_file(Filename) -> {ok, Bin, FullName} | error</name>
+      <name name="get_file" arity="1"/>
       <fsummary>Get a file</fsummary>
-      <type>
-        <v>Filename = string()</v>
-        <v>Bin = binary()</v>
-        <v>FullName = string()</v>
-      </type>
       <desc>
         <p>This function fetches a file using the low level loader.
-          <c>Filename</c> is either an absolute file name or just the name
+          <c><anno>Filename</anno></c> is either an absolute file name or just the name
           of the file, for example <c>"lists.beam"</c>. If an internal
           path is set to the loader, this path is used to find the file.
           If a user supplied loader is used, the path can be stripped
           off if it is obsolete, and the loader does not use a path.
-          <c>FullName</c> is the complete name of the fetched file.
-          <c>Bin</c> is the contents of the file as a binary.</p>
+          <c><anno>FullName</anno></c> is the complete name of the fetched file.
+          <c><anno>Bin</anno></c> is the contents of the file as a binary.</p>
 
-	  <p>The <c>Filename</c> can also be a file in an archive. For example
-          <c>/otp/root/lib/mnesia-4.4.7.ez/mnesia-4.4.7/ebin/mnesia_backup.beam</c>
+	  <p>The <c><anno>Filename</anno></c> can also be a file in an archive. For example
+          <c>$OTPROOT/lib/</c><c>mnesia-4.4.7.ez/mnesia-4.4.7/ebin/</c><c>mnesia.beam</c>.
           See <seealso marker="kernel:code">code(3)</seealso> about archive files.</p>
       </desc>
     </func>
     <func>
-      <name>get_path() -> {ok, Path}</name>
+      <name name="get_path" arity="0"/>
       <fsummary>Get the path set in the loader</fsummary>
-      <type>
-        <v>Path = [Dir]</v>
-        <v>Dir = string()</v>
-      </type>
       <desc>
         <p>This function gets the path set in the loader. The path is
           set by the <c>init</c> process according to information found
@@ -126,35 +115,26 @@
       </desc>
     </func>
     <func>
-      <name>list_dir(Dir) -> {ok, Filenames} | error</name>
+      <name name="list_dir" arity="1"/>
       <fsummary>List files in a directory</fsummary>
-      <type>
-        <v>Dir = name()</v>
-        <v>Filenames = [Filename]</v>
-        <v>Filename = string()</v>
-      </type>
       <desc>
         <p>Lists all the files in a directory. Returns
-          <c>{ok, Filenames}</c> if successful. Otherwise, it returns
-          <c>error</c>. <c>Filenames</c> is a list of
+          <c>{ok, <anno>Filenames</anno>}</c> if successful. Otherwise, it returns
+          <c>error</c>. <c><anno>Filenames</anno></c> is a list of
           the names of all the files in the directory. The names are
           not sorted.</p>
-	  <p>The <c>Dir</c> can also be a directory in an archive. For example
-          <c>/otp/root/lib/mnesia-4.4.7.ez/mnesia-4.4.7/ebin</c>
+	  <p>The <c><anno>Dir</anno></c> can also be a directory in an archive. For example
+          <c>$OTPROOT/lib/</c><c>mnesia-4.4.7.ez/mnesia-4.4.7/ebin</c>.
            See <seealso marker="kernel:code">code(3)</seealso> about archive files.</p>
       </desc>
     </func>
     <func>
-      <name>read_file_info(Filename) -> {ok, FileInfo} | error</name>
+      <name name="read_file_info" arity="1"/>
       <fsummary>Get information about a file</fsummary>
-      <type>
-        <v>Filename = name()</v>
-        <v>FileInfo = #file_info{}</v>
-      </type>
       <desc>
         <p>Retrieves information about a file. Returns
-          <c>{ok, FileInfo}</c> if successful, otherwise
-          <c>error</c>. <c>FileInfo</c> is a record
+          <c>{ok, <anno>FileInfo</anno>}</c> if successful, otherwise
+          <c>error</c>. <c><anno>FileInfo</anno></c> is a record
           <c>file_info</c>, defined in the Kernel include file
           <c>file.hrl</c>. Include the following directive in the module
           from which the function is called:</p>
@@ -162,18 +142,14 @@
 -include_lib("kernel/include/file.hrl").</code>
         <p>See <seealso marker="kernel:file">file(3)</seealso> for more info about
           the record <c>file_info</c>.</p>
-	<p>The <c>Filename</c> can also be a file in an archive. For example
-          <c>/otp/root/lib/mnesia-4.4.7.ez/mnesia-4.4.7/ebin/mnesia_backup.beam</c>
+	<p>The <c><anno>Filename</anno></c> can also be a file in an archive. For example
+          <c>$OTPROOT/lib/</c><c>mnesia-4.4.7.ez/mnesia-4.4.7/ebin/</c><c>mnesia</c>.
           See <seealso marker="kernel:code">code(3)</seealso> about archive files.</p>
       </desc>
     </func>
     <func>
-      <name>set_path(Path) -> ok</name>
+      <name name="set_path" arity="1"/>
       <fsummary>Set the path of the loader</fsummary>
-      <type>
-        <v>Path = [Dir]</v>
-        <v>Dir = string()</v>
-      </type>
       <desc>
         <p>This function sets the path of the loader if <c>init</c>
           interprets a <c>path</c> command in the start script.</p>
diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml
index 46f8df4683..e69f223897 100644
--- a/erts/doc/src/erlang.xml
+++ b/erts/doc/src/erlang.xml
@@ -4,7 +4,7 @@
 <erlref>
   <header>
     <copyright>
-      <year>1996</year><year>2010</year>
+      <year>1996</year><year>2012</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -48,22 +48,24 @@
       "Allowed in guard tests".</p>
   </description>
 
-  <section>
-    <title>DATA TYPES</title>
-    <marker id="iolist_definition"></marker>
-    <code type="none">
-ext_binary()
-  a binary data object,
-  structured according to the Erlang external term format
-
-iodata() = iolist() | binary()
+  <datatypes>
+    <datatype>
+      <name><marker id="type-ext_binary">ext_binary()</marker></name>
+      <desc>
+        <p>A binary data object, structured according to
+           the Erlang external term format.</p>
+      </desc>
+    </datatype>
+    <datatype>
+      <name name="timestamp"></name>
+      <desc><p>See <seealso marker="#now/0">now/0</seealso>.</p>
+      </desc>
+    </datatype>
+  </datatypes>
 
-iolist() = [char() | binary() | iolist()]
-  a binary is allowed as the tail of the list</code>
-  </section>
   <funcs>
     <func>
-      <name>abs(Number) ->  int() | float()</name>
+      <name>abs(Number) -> integer() | float()</name>
       <fsummary>Arithmetical absolute value</fsummary>
       <type>
         <v>Number = number()</v>
@@ -80,7 +82,7 @@ iolist() = [char() | binary() | iolist()]
       </desc>
     </func>
     <func>
-      <name>adler32(Data) -> int()</name>
+      <name>erlang:adler32(Data) -> integer()</name>
       <fsummary>Compute adler32 checksum</fsummary>
       <type>
         <v>Data = iodata()</v>
@@ -90,10 +92,10 @@ iolist() = [char() | binary() | iolist()]
       </desc>
     </func>
     <func>
-      <name>adler32(OldAdler, Data) -> int()</name>
+      <name>erlang:adler32(OldAdler, Data) -> integer()</name>
       <fsummary>Compute adler32 checksum</fsummary>
       <type>
-        <v>OldAdler = int()</v>
+        <v>OldAdler = integer()</v>
         <v>Data = iodata()</v>
       </type>
       <desc>
@@ -102,21 +104,21 @@ iolist() = [char() | binary() | iolist()]
 	<c>Data</c>.</p>
 	<p>The following code:</p>
 	<code>
-	X = adler32(Data1),
-	Y = adler32(X,Data2).
+	X = erlang:adler32(Data1),
+	Y = erlang:adler32(X,Data2).
 	</code>
 	<p>- would assign the same value to <c>Y</c> as this would:</p>
 	<code>
-	Y = adler32([Data1,Data2]).
+	Y = erlang:adler32([Data1,Data2]).
 	</code>
       </desc>
     </func>
     <func>
-      <name>adler32_combine(FirstAdler, SecondAdler, SecondSize) -> int()</name>
+      <name>erlang:adler32_combine(FirstAdler, SecondAdler, SecondSize) -> integer()</name>
       <fsummary>Combine two adler32 checksums</fsummary>
       <type>
-        <v>FirstAdler = SecondAdler = int()</v>
-        <v>SecondSize = int()</v>
+        <v>FirstAdler = SecondAdler = integer()</v>
+        <v>SecondSize = integer()</v>
       </type>
       <desc>
         <p>Combines two previously computed adler32 checksums. 
@@ -124,14 +126,14 @@ iolist() = [char() | binary() | iolist()]
 	the second checksum to be known.</p>
 	<p>The following code:</p>
 	<code>
-	Y = adler32(Data1),
-	Z = adler32(Y,Data2).
+	Y = erlang:adler32(Data1),
+	Z = erlang:adler32(Y,Data2).
 	</code>
 	<p>- would assign the same value to <c>Z</c> as this would:</p>
 	<code>
-	X = adler32(Data1),
-	Y = adler32(Data2),
-	Z = adler32_combine(X,Y,iolist_size(Data2)).
+	X = erlang:adler32(Data1),
+	Y = erlang:adler32(Data2),
+	Z = erlang:adler32_combine(X,Y,iolist_size(Data2)).
 	</code>
       </desc>
     </func>
@@ -147,7 +149,7 @@ iolist() = [char() | binary() | iolist()]
           <c>Tuple1</c>, and contains the elements in <c>Tuple1</c>
           followed by <c>Term</c> as the last element. Semantically
           equivalent to
-          <c>list_to_tuple(tuple_to_list(Tuple ++ [Term])</c>, but much
+          <c>list_to_tuple(tuple_to_list(Tuple) ++ [Term])</c>, but much
           faster.</p>
         <pre>
 > <input>erlang:append_element({one, two}, three).</input>
@@ -155,20 +157,16 @@ iolist() = [char() | binary() | iolist()]
       </desc>
     </func>
     <func>
-      <name>apply(Fun, Args) -> term() | empty()</name>
+      <name name="apply" arity="2"/>
       <fsummary>Apply a function to an argument list</fsummary>
-      <type>
-        <v>Fun = fun()</v>
-        <v>Args = [term()]</v>
-      </type>
       <desc>
-        <p>Call a fun, passing the elements in <c>Args</c> as
+        <p>Call a fun, passing the elements in <c><anno>Args</anno></c> as
           arguments.</p>
         <p>Note: If the number of elements in the arguments are known at
           compile-time, the call is better written as
-          <c>Fun(Arg1, Arg2, ... ArgN)</c>.</p>
+          <c><anno>Fun</anno>(Arg1, Arg2, ... ArgN)</c>.</p>
         <warning>
-          <p>Earlier, <c>Fun</c> could also be given as
+          <p>Earlier, <c><anno>Fun</anno></c> could also be given as
             <c>{Module, Function}</c>, equivalent to
             <c>apply(Module, Function, Args)</c>. This usage is
             deprecated and will stop working in a future release of
@@ -177,15 +175,11 @@ iolist() = [char() | binary() | iolist()]
       </desc>
     </func>
     <func>
-      <name>apply(Module, Function, Args) -> term() | empty()</name>
+      <name name="apply" arity="3"/>
       <fsummary>Apply a function to an argument list</fsummary>
-      <type>
-        <v>Module = Function = atom()</v>
-        <v>Args = [term()]</v>
-      </type>
       <desc>
         <p>Returns the result of applying <c>Function</c> in
-          <c>Module</c> to <c>Args</c>. The applied function must
+          <c><anno>Module</anno></c> to <c><anno>Args</anno></c>. The applied function must
           be exported from <c>Module</c>. The arity of the function is
           the length of <c>Args</c>.</p>
         <pre>
@@ -198,7 +192,7 @@ iolist() = [char() | binary() | iolist()]
 "Erlang"</pre>
         <p>Note: If the number of arguments are known at compile-time,
           the call is better written as
-          <c>Module:Function(Arg1, Arg2, ..., ArgN)</c>.</p>
+          <c><anno>Module</anno>:<anno>Function</anno>(Arg1, Arg2, ..., ArgN)</c>.</p>
         <p>Failure: <c>error_handler:undefined_function/3</c> is called
           if the applied function is not exported. The error handler
           can be redefined (see
@@ -221,9 +215,9 @@ iolist() = [char() | binary() | iolist()]
           representation of <c>Atom</c>. If <c>Encoding</c>
 	  is <c>latin1</c>, there will be one byte for each character
 	  in the text representation. If <c>Encoding</c> is <c>utf8</c> or
-	  <c>unicode</c>, the characters will encoded using UTF-8
+	  <c>unicode</c>, the characters will be encoded using UTF-8
 	  (meaning that characters from 16#80 up to 0xFF will be
-	  encode in two bytes).</p>
+	  encoded in two bytes).</p>
 
 	<note><p>Currently, <c>atom_to_binary(Atom, latin1)</c> can
 	never fail because the text representation of an atom can only contain
@@ -253,6 +247,54 @@ iolist() = [char() | binary() | iolist()]
       </desc>
     </func>
     <func>
+      <name>binary_part(Subject, PosLen) -> binary()</name>
+      <fsummary>Extracts a part of a binary</fsummary>
+      <type>
+        <v>Subject = binary()</v>
+	<v>PosLen = {Start,Length}</v>
+	<v>Start = integer() >= 0</v>
+	<v>Length = integer() >= 0</v>
+      </type>
+      <desc>
+      <p>Extracts the part of the binary described by <c>PosLen</c>.</p>
+
+      <p>Negative length can be used to extract bytes at the end of a binary:</p>
+
+<code>
+1> Bin = &lt;&lt;1,2,3,4,5,6,7,8,9,10&gt;&gt;.
+2> binary_part(Bin,{byte_size(Bin), -5)).
+&lt;&lt;6,7,8,9,10&gt;&gt;
+</code>
+
+      <p>If <c>PosLen</c> in any way references outside the binary, a <c>badarg</c> exception is raised.</p>
+
+      <p><c>Start</c> is zero-based, i.e.:</p>
+<code>
+1> Bin = &lt;&lt;1,2,3&gt;&gt;
+2> binary_part(Bin,{0,2}).
+&lt;&lt;1,2&gt;&gt;
+</code>
+
+      <p>See the STDLIB module <c>binary</c> for details about the <c>PosLen</c> semantics.</p>
+
+      <p>Allowed in guard tests.</p>
+      </desc>
+    </func>
+    <func>
+      <name>binary_part(Subject, Start, Length) -> binary()</name>
+      <fsummary>Extracts a part of a binary</fsummary>
+      <type>
+        <v>Subject = binary()</v>
+	<v>Start = integer() >= 0</v>
+	<v>Length = integer() >= 0</v>
+      </type>
+      <desc>
+      <p>The same as <c>binary_part(Subject, {Pos, Len})</c>.</p>
+
+      <p>Allowed in guard tests.</p>
+      </desc>
+    </func>
+    <func>
       <name>binary_to_atom(Binary, Encoding) -> atom()</name>
       <fsummary>Convert from text representation to an atom</fsummary>
       <type>
@@ -318,6 +360,11 @@ iolist() = [char() | binary() | iolist()]
           corresponding to the bytes from position <c>Start</c> to
           position <c>Stop</c> in <c>Binary</c>. Positions in the
           binary are numbered starting from 1.</p>
+
+	  <note><p>This function's indexing style of using one-based indices for
+	  binaries is deprecated. New code should use the functions in
+	  the STDLIB module <c>binary</c> instead. They consequently
+	  use the same (zero-based) style of indexing.</p></note>
       </desc>
     </func>
     <func>
@@ -337,7 +384,7 @@ iolist() = [char() | binary() | iolist()]
       <name>binary_to_term(Binary) -> term()</name>
       <fsummary>Decode an Erlang external term format binary</fsummary>
       <type>
-        <v>Binary = ext_binary()</v>
+        <v>Binary = <seealso marker="#type-ext_binary">ext_binary()</seealso></v>
       </type>
       <desc>
         <p>Returns an Erlang term which is the result of decoding
@@ -354,11 +401,11 @@ iolist() = [char() | binary() | iolist()]
       </desc>
     </func>
     <func>
-      <name>erlang:binary_to_term(Binary, Opts) -> term()</name>
+      <name>binary_to_term(Binary, Opts) -> term()</name>
       <fsummary>Decode an Erlang external term format binary</fsummary>
       <type>
         <v>Opts = [safe]</v>
-        <v>Binary = ext_binary()</v>
+        <v>Binary = <seealso marker="#type-ext_binary">ext_binary()</seealso></v>
       </type>
       <desc>
         <p>As <c>binary_to_term/1</c>, but takes options that affect decoding
@@ -389,7 +436,7 @@ iolist() = [char() | binary() | iolist()]
       </desc>
     </func>
     <func>
-      <name>bit_size(Bitstring) -> int()</name>
+      <name>bit_size(Bitstring) -> integer() >= 0</name>
       <fsummary>Return the size of a bitstring</fsummary>
       <type>
         <v>Bitstring = bitstring()</v>
@@ -408,7 +455,7 @@ iolist() = [char() | binary() | iolist()]
       <name>erlang:bump_reductions(Reductions) -> void()</name>
       <fsummary>Increment the reduction counter</fsummary>
       <type>
-        <v>Reductions = int()</v>
+        <v>Reductions = integer() >= 0</v>
       </type>
       <desc>
         <p>This implementation-dependent function increments
@@ -425,7 +472,7 @@ iolist() = [char() | binary() | iolist()]
       </desc>
     </func>
     <func>
-      <name>byte_size(Bitstring) -> int()</name>
+      <name>byte_size(Bitstring) -> integer() >= 0</name>
       <fsummary>Return the size of a bitstring (or binary)</fsummary>
       <type>
         <v>Bitstring = bitstring()</v>
@@ -446,8 +493,8 @@ iolist() = [char() | binary() | iolist()]
       <name>erlang:cancel_timer(TimerRef) -> Time | false</name>
       <fsummary>Cancel a timer</fsummary>
       <type>
-        <v>TimerRef = ref()</v>
-        <v>Time = int()</v>
+        <v>TimerRef = reference()</v>
+        <v>Time = integer() >= 0</v>
       </type>
       <desc>
         <p>Cancels a timer, where <c>TimerRef</c> was returned by
@@ -471,7 +518,19 @@ iolist() = [char() | binary() | iolist()]
     </func>
 
     <func>
-      <name>check_process_code(Pid, Module) -> bool()</name>
+      <name>check_old_code(Module) -> boolean()</name>
+      <fsummary>Check if a module has old code</fsummary>
+      <type>
+        <v>Module = atom()</v>
+      </type>
+      <desc>
+        <p>Returns <c>true</c> if the <c>Module</c> has old code,
+	and <c>false</c> otherwise.</p>
+        <p>See also <seealso marker="kernel:code">code(3)</seealso>.</p>
+      </desc>
+    </func>
+    <func>
+      <name>check_process_code(Pid, Module) -> boolean()</name>
       <fsummary>Check if a process is executing old code for a module</fsummary>
       <type>
         <v>Pid = pid()</v>
@@ -491,16 +550,7 @@ false</pre>
       </desc>
     </func>
     <func>
-      <name>concat_binary(ListOfBinaries)</name>
-      <fsummary>Concatenate a list of binaries (deprecated)</fsummary>
-      <desc>
-        <p>Do not use; use
-          <seealso marker="#list_to_binary/1">list_to_binary/1</seealso>
-          instead.</p>
-      </desc>
-    </func>
-    <func>
-      <name>crc32(Data) -> int()</name>
+      <name>erlang:crc32(Data) -> integer() >= 0</name>
       <fsummary>Compute crc32 (IEEE 802.3) checksum</fsummary>
       <type>
         <v>Data = iodata()</v>
@@ -510,10 +560,10 @@ false</pre>
       </desc>
     </func>
     <func>
-      <name>crc32(OldCrc, Data) -> int()</name>
+      <name>erlang:crc32(OldCrc, Data) -> integer() >= 0</name>
       <fsummary>Compute crc32 (IEEE 802.3) checksum</fsummary>
       <type>
-        <v>OldCrc = int()</v>
+        <v>OldCrc = integer() >= 0</v>
         <v>Data = iodata()</v>
       </type>
       <desc>
@@ -522,21 +572,21 @@ false</pre>
 	<c>Data</c>.</p>
 	<p>The following code:</p>
 	<code>
-	X = crc32(Data1),
-	Y = crc32(X,Data2).
+	X = erlang:crc32(Data1),
+	Y = erlang:crc32(X,Data2).
 	</code>
 	<p>- would assign the same value to <c>Y</c> as this would:</p>
 	<code>
-	Y = crc32([Data1,Data2]).
+	Y = erlang:crc32([Data1,Data2]).
 	</code>
       </desc>
     </func>
     <func>
-      <name>crc32_combine(FirstCrc, SecondCrc, SecondSize) -> int()</name>
+      <name>erlang:crc32_combine(FirstCrc, SecondCrc, SecondSize) -> integer() >= 0</name>
       <fsummary>Combine two crc32 (IEEE 802.3) checksums</fsummary>
       <type>
-        <v>FirstCrc = SecondCrc = int()</v>
-        <v>SecondSize = int()</v>
+        <v>FirstCrc = SecondCrc = integer() >= 0</v>
+        <v>SecondSize = integer() >= 0</v>
       </type>
       <desc>
         <p>Combines two previously computed crc32 checksums. 
@@ -544,22 +594,22 @@ false</pre>
 	the second checksum to be known.</p>
 	<p>The following code:</p>
 	<code>
-	Y = crc32(Data1),
-	Z = crc32(Y,Data2).
+	Y = erlang:crc32(Data1),
+	Z = erlang:crc32(Y,Data2).
 	</code>
 	<p>- would assign the same value to <c>Z</c> as this would:</p>
 	<code>
-	X = crc32(Data1),
-	Y = crc32(Data2),
-	Z = crc32_combine(X,Y,iolist_size(Data2)).
+	X = erlang:crc32(Data1),
+	Y = erlang:crc32(Data2),
+	Z = erlang:crc32_combine(X,Y,iolist_size(Data2)).
 	</code>
       </desc>
     </func>
     <func>
-      <name>date() -> {Year, Month, Day}</name>
+      <name>date() -> Date</name>
       <fsummary>Current date</fsummary>
       <type>
-        <v>Year = Month = Day = int()</v>
+        <v>Date = <seealso marker="calendar#type-date">calendar:date()</seealso></v>
       </type>
       <desc>
         <p>Returns the current date as <c>{Year, Month, Day}</c>.</p>
@@ -571,27 +621,27 @@ false</pre>
       </desc>
     </func>
     <func>
-      <name>decode_packet(Type,Bin,Options) -> {ok,Packet,Rest} | {more,Length} | {error,Reason}</name>
+      <name>erlang:decode_packet(Type,Bin,Options) -> {ok,Packet,Rest} | {more,Length} | {error,Reason}</name>
       <fsummary>Extracts a protocol packet from a binary</fsummary>
       <type>
         <v>Bin = binary()</v>
         <v>Options = [Opt]</v>
         <v>Packet = binary() | HttpPacket</v>
         <v>Rest = binary()</v>
-        <v>Length = int() | undefined</v>
+        <v>Length = integer() > 0 | undefined</v>
         <v>Reason = term()</v>
         <v>&nbsp;Type, Opt -- see below</v>
         <v></v>
         <v>HttpPacket = HttpRequest | HttpResponse | HttpHeader | http_eoh | HttpError</v>
         <v>HttpRequest = {http_request, HttpMethod, HttpUri, HttpVersion}</v>
         <v>HttpResponse = {http_response, HttpVersion, integer(), HttpString}</v>
-        <v>HttpHeader = {http_header, int(), HttpField, Reserved=term(), Value=HttpString}</v>
+        <v>HttpHeader = {http_header, integer(), HttpField, Reserved=term(), Value=HttpString}</v>
         <v>HttpError = {http_error, HttpString}</v>
         <v>HttpMethod = HttpMethodAtom | HttpString</v>
         <v>HttpMethodAtom = 'OPTIONS' | 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'TRACE'</v>
-        <v>HttpUri = '*' | {absoluteURI, http|https, Host=HttpString, Port=int()|undefined, Path=HttpString} | 
+        <v>HttpUri = '*' | {absoluteURI, http|https, Host=HttpString, Port=integer()|undefined, Path=HttpString} |
                      {scheme, Scheme=HttpString, HttpString} | {abs_path, HttpString} | HttpString</v>
-        <v>HttpVersion = {Major=int(), Minor=int()}</v>
+        <v>HttpVersion = {Major=integer(), Minor=integer()}</v>
         <v>HttpString = string() | binary()</v>
         <v>HttpField = HttpFieldAtom | HttpString</v>
         <v>HttpFieldAtom = 'Cache-Control' | 'Connection' | 'Date' | 'Pragma' | 'Transfer-Encoding' | 'Upgrade' | 'Via' | 'Accept' | 'Accept-Charset' | 'Accept-Encoding' | 'Accept-Language' | 'Authorization' | 'From' | 'Host' | 'If-Modified-Since' | 'If-Match' | 'If-None-Match' | 'If-Range' | 'If-Unmodified-Since' | 'Max-Forwards' | 'Proxy-Authorization' | 'Range' | 'Referer' | 'User-Agent' | 'Age' | 'Location' | 'Proxy-Authenticate' | 'Public' | 'Retry-After' | 'Server' | 'Vary' | 'Warning' | 'Www-Authenticate' | 'Allow' | 'Content-Base' | 'Content-Encoding' | 'Content-Language' | 'Content-Length' | 'Content-Location' | 'Content-Md5' | 'Content-Range' | 'Content-Type' | 'Etag' | 'Expires' | 'Last-Modified' | 'Accept-Ranges' | 'Set-Cookie' | 'Set-Cookie2' | 'X-Forwarded-For' | 'Cookie' | 'Keep-Alive' | 'Proxy-Connection'</v>
@@ -666,17 +716,20 @@ false</pre>
         </taglist>
         <p>The following options are available:</p>
             <taglist>
-              <tag><c>{packet_size, int()}</c></tag>
+              <tag><c>{packet_size, integer()}</c></tag>
               <item><p>Sets the max allowed size of the packet body. If
                 the packet header indicates that the length of the
                 packet is longer than the max allowed length, the packet
                 is considered invalid. Default is 0 which means no
                 size limit.</p>
               </item>
-              <tag><c>{line_length, int()}</c></tag>
-              <item><p>Applies only to line oriented protocols
-                (<c>line</c>, <c>http</c>). Lines longer than this
-                will be truncated.</p>
+              <tag><c>{line_length, integer()}</c></tag>
+              <item><p>For packet type <c>line</c>, truncate lines longer
+	      than the indicated length.</p>
+	      <p>Option <c>line_length</c> also applies to <c>http*</c> 
+	      packet types as an alias for option <c>packet_size</c> in the
+	      case when <c>packet_size</c> itself is not set. This usage is
+	      only intended for backward compatibility.</p>
               </item>
             </taglist>
         <pre>
@@ -707,29 +760,29 @@ false</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:demonitor(MonitorRef) -> true</name>
+      <name>demonitor(MonitorRef) -> true</name>
       <fsummary>Stop monitoring</fsummary>
       <type>
-        <v>MonitorRef = ref()</v>
+        <v>MonitorRef = reference()</v>
       </type>
       <desc>
         <p>If <c>MonitorRef</c> is a reference which the calling process
           obtained by calling
-          <seealso marker="#monitor/2">erlang:monitor/2</seealso>,
+          <seealso marker="#monitor/2">monitor/2</seealso>,
           this monitoring is turned off. If the monitoring is already
           turned off, nothing happens.</p>
-        <p>Once <c>erlang:demonitor(MonitorRef)</c> has returned it is
+        <p>Once <c>demonitor(MonitorRef)</c> has returned it is
           guaranteed that no <c>{'DOWN', MonitorRef, _, _, _}</c> message
-          due to the monitor will be placed in the callers message queue
+          due to the monitor will be placed in the caller's message queue
           in the future. A <c>{'DOWN', MonitorRef, _, _, _}</c> message
-          might have been placed in the callers message queue prior to
+          might have been placed in the caller's message queue prior to
           the call, though. Therefore, in most cases, it is advisable
           to remove such a <c>'DOWN'</c> message from the message queue
           after monitoring has been stopped. 
-          <seealso marker="#demonitor/2">erlang:demonitor(MonitorRef, [flush])</seealso> can be used instead of
-          <c>erlang:demonitor(MonitorRef)</c> if this cleanup is wanted.</p>
+          <seealso marker="#demonitor/2">demonitor(MonitorRef, [flush])</seealso> can be used instead of
+          <c>demonitor(MonitorRef)</c> if this cleanup is wanted.</p>
         <note>
-          <p>Prior to OTP release R11B (erts version 5.5) <c>erlang:demonitor/1</c>
+          <p>Prior to OTP release R11B (erts version 5.5) <c>demonitor/1</c>
             behaved completely asynchronous, i.e., the monitor was active
             until the "demonitor signal" reached the monitored entity. This
             had one undesirable effect, though. You could never know when
@@ -747,32 +800,31 @@ false</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:demonitor(MonitorRef, OptionList) -> true|false</name>
+      <name>demonitor(MonitorRef, OptionList) -> boolean()</name>
       <fsummary>Stop monitoring</fsummary>
       <type>
-        <v>MonitorRef = ref()</v>
+        <v>MonitorRef = reference()</v>
         <v>OptionList = [Option]</v>
-        <v>Option = flush</v>
-        <v>Option = info</v>
+        <v>&nbsp;Option = flush | info</v>
       </type>
       <desc>
         <p>The returned value is <c>true</c> unless <c>info</c> is part
 	   of <c>OptionList</c>.
 	</p>
-        <p><c>erlang:demonitor(MonitorRef, [])</c> is equivalent to
-          <seealso marker="#demonitor/1">erlang:demonitor(MonitorRef)</seealso>.</p>
+        <p><c>demonitor(MonitorRef, [])</c> is equivalent to
+          <seealso marker="#demonitor/1">demonitor(MonitorRef)</seealso>.</p>
         <p>Currently the following <c>Option</c>s are valid:</p>
         <taglist>
           <tag><c>flush</c></tag>
           <item>
             <p>Remove (one) <c>{_, MonitorRef, _, _, _}</c> message,
-              if there is one, from the callers message queue after
+              if there is one, from the caller's message queue after
               monitoring has been stopped.</p>
-            <p>Calling <c>erlang:demonitor(MonitorRef, [flush])</c>
+            <p>Calling <c>demonitor(MonitorRef, [flush])</c>
               is equivalent to the following, but more efficient:</p>
             <code type="none">
 
-    erlang:demonitor(MonitorRef),
+    demonitor(MonitorRef),
     receive
         {_, MonitorRef, _, _, _} ->
             true
@@ -795,7 +847,7 @@ false</pre>
 	      <item><p>The monitor was not found and could not be removed.
 	      	       This probably because someone already has placed a
 		       <c>'DOWN'</c> message corresponding to this monitor
-		       in the callers message queue.
+		       in the caller's message queue.
 		    </p>
 	      </item>
 	    </taglist>
@@ -810,18 +862,15 @@ false</pre>
         </note>
         <p>Failure: <c>badarg</c> if <c>OptionList</c> is not a list, or
           if <c>Option</c> is not a valid option, or the same failure as for
-          <seealso marker="#demonitor/1">erlang:demonitor/1</seealso></p>
+          <seealso marker="#demonitor/1">demonitor/1</seealso></p>
       </desc>
     </func>
     <func>
-      <name>disconnect_node(Node) -> bool() | ignored</name>
+      <name name="disconnect_node" arity="1"/>
       <fsummary>Force the disconnection of a node</fsummary>
-      <type>
-        <v>Node = atom()</v>
-      </type>
       <desc>
         <p>Forces the disconnection of a node. This will appear to
-          the node <c>Node</c> as if the local node has crashed. This
+          the node <c><anno>Node</anno></c> as if the local node has crashed. This
           BIF is mainly used in the Erlang network authentication
           protocols. Returns <c>true</c> if disconnection succeeds,
           otherwise <c>false</c>. If the local node is not alive,
@@ -891,7 +940,7 @@ b</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:error(Reason)</name>
+      <name>error(Reason)</name>
       <fsummary>Stop execution with a given reason</fsummary>
       <type>
         <v>Reason = term()</v>
@@ -904,7 +953,7 @@ b</pre>
           function first). Since evaluating this function causes
           the process to terminate, it has no return value.</p>
         <pre>
-> <input>catch erlang:error(foobar).</input>
+> <input>catch error(foobar).</input>
 {'EXIT',{foobar,[{erl_eval,do_apply,5},
                  {erl_eval,expr,5},
                  {shell,exprs,6},
@@ -913,7 +962,7 @@ b</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:error(Reason, Args)</name>
+      <name>error(Reason, Args)</name>
       <fsummary>Stop execution with a given reason</fsummary>
       <type>
         <v>Reason = term()</v>
@@ -979,6 +1028,56 @@ b</pre>
       </desc>
     </func>
     <func>
+      <name>erlang:external_size(Term) -> integer() >= 0</name>
+      <fsummary>Calculate the maximum size for a term encoded in the Erlang
+        external term format</fsummary>
+      <type>
+        <v>Term = term()</v>
+      </type>
+      <desc>
+        <p>Calculates, without doing the encoding, the maximum byte size for
+          a term encoded in the Erlang external term format. The following
+          condition applies always:</p>
+        <p>
+          <pre>
+> <input>Size1 = byte_size(term_to_binary(Term)),</input>
+> <input>Size2 = erlang:external_size(Term),</input>
+> <input>true = Size1 =&lt; Size2.</input>
+true
+          </pre>
+        </p>
+        <p>This is equivalent to a call to: <code>erlang:external_size(Term, [])
+          </code></p>
+      </desc>
+    </func>
+    <func>
+      <name>erlang:external_size(Term, [Option]) -> integer() >= 0</name>
+      <fsummary>Calculate the maximum size for a term encoded in the Erlang
+        external term format</fsummary>
+      <type>
+        <v>Term = term()</v>
+        <v>Option = {minor_version, Version}</v>
+      </type>
+      <desc>
+        <p>Calculates, without doing the encoding, the maximum byte size for
+          a term encoded in the Erlang external term format. The following
+          condition applies always:</p>
+        <p>
+          <pre>
+> <input>Size1 = byte_size(term_to_binary(Term, Options)),</input>
+> <input>Size2 = erlang:external_size(Term, Options),</input>
+> <input>true = Size1 =&lt; Size2.</input>
+true
+          </pre>
+        </p>
+        <p>The option <c>{minor_version, Version}</c> specifies how floats
+          are encoded. See
+          <seealso marker="#term_to_binary/2">term_to_binary/2</seealso> for
+          a more detailed description.
+        </p>
+      </desc>
+    </func>
+    <func>
       <name>float(Number) -> float()</name>
       <fsummary>Convert a number to a float</fsummary>
       <type>
@@ -1016,15 +1115,11 @@ b</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:fun_info(Fun) -> [{Item, Info}]</name>
+      <name name="fun_info" arity="1"/>
       <fsummary>Information about a fun</fsummary>
-      <type>
-        <v>Fun = fun()</v>
-        <v>Item, Info -- see below</v>
-      </type>
       <desc>
         <p>Returns a list containing information about the fun
-          <c>Fun</c>. Each element of the list is a tuple. The order of
+          <c><anno>Fun</anno></c>. Each element of the list is a tuple. The order of
           the tuples is not defined, and more tuples may be added in a
           future release.</p>
         <warning>
@@ -1103,11 +1198,16 @@ b</pre>
           </item>
           <tag><c>{new_uniq, Uniq}</c></tag>
           <item>
-            <p><c>Uniq</c> (a binary) is a unique value for this fun.</p>
+            <p><c>Uniq</c> (a binary) is a unique value for this fun.
+	    It is calculated from the compiled code for the entire module.</p>
           </item>
           <tag><c>{uniq, Uniq}</c></tag>
           <item>
-            <p><c>Uniq</c> (an integer) is a unique value for this fun.</p>
+            <p><c>Uniq</c> (an integer) is a unique value for this fun.
+	    Starting in the R15 release, this integer is calculated from
+	    the compiled code for the entire module. Before R15, this
+	    integer was based on only the body of the fun.
+	    </p>
           </item>
         </taglist>
       </desc>
@@ -1123,7 +1223,7 @@ b</pre>
         <p>Returns information about <c>Fun</c> as specified by
           <c>Item</c>, in the form <c>{Item,Info}</c>.</p>
         <p>For any fun, <c>Item</c> can be any of the atoms
-          <c>module</c>, <c>name</c>, <c>arity</c>, or <c>env</c>.</p>
+          <c>module</c>, <c>name</c>, <c>arity</c>, <c>env</c>, or <c>type</c>.</p>
         <p>For a local fun, <c>Item</c> can also be any of the atoms
           <c>index</c>, <c>new_index</c>, <c>new_uniq</c>,
           <c>uniq</c>, and <c>pid</c>. For an external fun, the value
@@ -1144,11 +1244,11 @@ b</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:function_exported(Module, Function, Arity) -> bool()</name>
+      <name>erlang:function_exported(Module, Function, Arity) -> boolean()</name>
       <fsummary>Check if a function is exported and loaded</fsummary>
       <type>
         <v>Module = Function = atom()</v>
-        <v>Arity = int()</v>
+        <v>Arity = arity()</v>
       </type>
       <desc>
         <p>Returns <c>true</c> if the module <c>Module</c> is loaded
@@ -1176,7 +1276,7 @@ b</pre>
       </desc>
     </func>
     <func>
-      <name>garbage_collect(Pid) -> bool()</name>
+      <name>garbage_collect(Pid) -> boolean()</name>
       <fsummary>Force an immediate garbage collection of a process</fsummary>
       <type>
         <v>Pid = pid()</v>
@@ -1223,11 +1323,8 @@ b</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:get_cookie() -> Cookie | nocookie</name>
+      <name name="get_cookie" arity="0"/>
       <fsummary>Get the magic cookie of the local node</fsummary>
-      <type>
-        <v>Cookie = atom()</v>
-      </type>
       <desc>
         <p>Returns the magic cookie of the local node, if the node is
           alive; otherwise the atom <c>nocookie</c>.</p>
@@ -1254,17 +1351,18 @@ b</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:get_stacktrace() -> [{Module, Function, Arity | Args}]</name>
+      <name>erlang:get_stacktrace() -> [{Module, Function, Arity | Args, Location}]</name>
       <fsummary>Get the call stack back-trace of the last exception</fsummary>
       <type>
         <v>Module = Function = atom()</v>
-        <v>Arity = int()</v>
+        <v>Arity = arity()</v>
         <v>Args = [term()]</v>
+	<v>Location = [{atom(),term()}]</v>
       </type>
       <desc>
         <p>Get the call stack back-trace (<em>stacktrace</em>) of the last
           exception in the calling process as a list of 
-          <c>{Module,Function,Arity}</c> tuples.
+          <c>{Module,Function,Arity,Location}</c> tuples.
           The <c>Arity</c> field in the first tuple may be the argument
           list of that function call instead of an arity integer,
           depending on the exception.</p>
@@ -1274,6 +1372,25 @@ b</pre>
         <p>The stacktrace is the same data as the <c>catch</c> operator
           returns, for example:</p>
         <p><c>{'EXIT',{badarg,Stacktrace}} = catch abs(x)</c></p>
+	<p><c>Location</c> is a (possibly empty) list of two-tuples that
+	  may indicate the location in the source code of the function.
+	  The first element is an atom that describes the type of
+	  information in the second element. Currently the following
+	  items may occur:</p>
+	<taglist>
+	  <tag><c>file</c></tag>
+	  <item>
+	    <p>The second element of the tuple is a string (list of
+	      characters) representing the filename of the source file
+	      of the function.</p>
+	  </item>
+	  <tag><c>line</c></tag>
+	  <item>
+	    <p>The second element of the tuple is the line number
+	      (an integer greater than zero) in the source file
+	      where the exception occurred or the function was called.</p>
+	  </item>
+	</taglist>
         <p>See also
           <seealso marker="#error/1">erlang:error/1</seealso> and
           <seealso marker="#error/2">erlang:error/2</seealso>.</p>
@@ -1315,29 +1432,69 @@ b</pre>
       <name>halt()</name>
       <fsummary>Halt the Erlang runtime system and indicate normal exit to the calling environment</fsummary>
       <desc>
-        <p>Halts the Erlang runtime system and indicates normal exit to
-          the calling environment. Has no return value.</p>
+	<p>The same as
+	  <seealso marker="#halt/2"><c>halt(0, [])</c></seealso>.</p>
         <pre>
 > <input>halt().</input>
-os_prompt%</pre>
+os_prompt% </pre>
       </desc>
     </func>
     <func>
       <name>halt(Status)</name>
       <fsummary>Halt the Erlang runtime system</fsummary>
       <type>
-        <v>Status = int()>=0 | string()</v>
+        <v>Status = integer() >= 0 | string() | abort</v>
       </type>
       <desc>
-        <p><c>Status</c> must be a non-negative integer, or a string.
-          Halts the Erlang runtime system. Has no return value.
-          If <c>Status</c> is an integer, it is returned as an exit
-          status of Erlang to the calling environment.
-          If <c>Status</c> is a string, produces an Erlang crash dump
-          with <c>String</c> as slogan, and then exits with a non-zero
-          status code.</p>
-        <p>Note that on many platforms, only the status codes 0-255 are
-          supported by the operating system.</p>
+	<p>The same as
+	  <seealso marker="#halt/2"><c>halt(Status, [])</c></seealso>.</p>
+        <pre>
+> <input>halt(17).</input>
+os_prompt% <input>echo $?</input>
+17
+os_prompt% </pre>
+      </desc>
+    </func>
+    <func>
+      <name>halt(Status, Options)</name>
+      <fsummary>Halt the Erlang runtime system</fsummary>
+      <type>
+        <v>Status = integer() >= 0 | string() | abort</v>
+	<v>Options = [Option]</v>
+	<v>Option = {flush,boolean()} | term()</v>
+      </type>
+      <desc>
+        <p><c>Status</c> must be a non-negative integer, a string,
+	  or the atom <c>abort</c>.
+	  Halts the Erlang runtime system. Has no return value.
+	  Depending on <c>Status</c>:
+	</p>
+	<taglist>
+	  <tag>integer()</tag>
+	  <item>The runtime system exits with the integer value <c>Status</c>
+	  as status code to the calling environment (operating system).
+	  </item>
+	  <tag>string()</tag>
+	  <item>An erlang crash dump is produced with <c>Status</c> as slogan,
+	  and then the runtime system exits with status code <c>1</c>.
+	  </item>
+	  <tag><c>abort</c></tag>
+	  <item>
+	    The runtime system aborts producing a core dump, if that is
+	    enabled in the operating system.
+	  </item>
+	</taglist>
+	<p>Note that on many platforms, only the status codes 0-255 are
+	  supported by the operating system.
+	</p>
+	<p>For integer <c>Status</c> the Erlang runtime system closes all ports
+	  and allows async threads to finish their operations before exiting.
+	  To exit without such flushing use
+	  <c>Option</c> as <c>{flush,false}</c>.
+	</p>
+	<p>For statuses <c>string()</c> and <c>abort</c> the <c>flush</c>
+	  option is ignored and flushing is <em>not</em> done.
+	</p>
       </desc>
     </func>
     <func>
@@ -1419,7 +1576,7 @@ os_prompt%</pre>
       <name>integer_to_list(Integer) -> string()</name>
       <fsummary>Text representation of an integer</fsummary>
       <type>
-        <v>Integer = int()</v>
+        <v>Integer = integer()</v>
       </type>
       <desc>
         <p>Returns a string which corresponds to the text
@@ -1430,17 +1587,13 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:integer_to_list(Integer, Base) -> string()</name>
+      <name name="integer_to_list" arity="2"/>
       <fsummary>Text representation of an integer</fsummary>
-      <type>
-        <v>Integer = int()</v>
-        <v>Base = 2..36</v>
-      </type>
       <desc>
         <p>Returns a string which corresponds to the text
-          representation of <c>Integer</c> in base <c>Base</c>.</p>
+          representation of <c><anno>Integer</anno></c> in base <c><anno>Base</anno></c>.</p>
         <pre>
-> <input>erlang:integer_to_list(1023, 16).</input>
+> <input>integer_to_list(1023, 16).</input>
 "3FF"</pre>
       </desc>
     </func>
@@ -1465,7 +1618,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>iolist_size(Item) -> int()</name>
+      <name>iolist_size(Item) -> integer() >= 0</name>
       <fsummary>Size of an iolist</fsummary>
       <type>
         <v>Item = iolist() | binary()</v>
@@ -1480,7 +1633,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>is_alive() -> bool()</name>
+      <name>is_alive() -> boolean()</name>
       <fsummary>Check whether the local node is alive</fsummary>
       <desc>
         <p>Returns <c>true</c> if the local node is alive; that is, if
@@ -1489,7 +1642,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>is_atom(Term) -> bool()</name>
+      <name>is_atom(Term) -> boolean()</name>
       <fsummary>Check whether a term is an atom</fsummary>
       <type>
         <v>Term = term()</v>
@@ -1501,7 +1654,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>is_binary(Term) -> bool()</name>
+      <name>is_binary(Term) -> boolean()</name>
       <fsummary>Check whether a term is a binary</fsummary>
       <type>
         <v>Term = term()</v>
@@ -1516,7 +1669,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>is_bitstring(Term) -> bool()</name>
+      <name>is_bitstring(Term) -> boolean()</name>
       <fsummary>Check whether a term is a bitstring</fsummary>
       <type>
         <v>Term = term()</v>
@@ -1529,7 +1682,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>is_boolean(Term) -> bool()</name>
+      <name>is_boolean(Term) -> boolean()</name>
       <fsummary>Check whether a term is a boolean</fsummary>
       <type>
         <v>Term = term()</v>
@@ -1542,11 +1695,11 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:is_builtin(Module, Function, Arity) -> bool()</name>
+      <name>erlang:is_builtin(Module, Function, Arity) -> boolean()</name>
       <fsummary>Check if a function is a BIF implemented in C</fsummary>
       <type>
         <v>Module = Function = atom()</v>
-        <v>Arity = int()</v>
+        <v>Arity = arity()</v>
       </type>
       <desc>
         <p>Returns <c>true</c> if <c>Module:Function/Arity</c> is
@@ -1555,7 +1708,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>is_float(Term) -> bool()</name>
+      <name>is_float(Term) -> boolean()</name>
       <fsummary>Check whether a term is a float</fsummary>
       <type>
         <v>Term = term()</v>
@@ -1567,7 +1720,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>is_function(Term) -> bool()</name>
+      <name>is_function(Term) -> boolean()</name>
       <fsummary>Check whether a term is a fun</fsummary>
       <type>
         <v>Term = term()</v>
@@ -1579,11 +1732,11 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>is_function(Term, Arity) -> bool()</name>
+      <name>is_function(Term, Arity) -> boolean()</name>
       <fsummary>Check whether a term is a fun with a given arity</fsummary>
       <type>
         <v>Term = term()</v>
-        <v>Arity = int()</v>
+        <v>Arity = arity()</v>
       </type>
       <desc>
         <p>Returns <c>true</c> if <c>Term</c> is a fun that can be
@@ -1600,7 +1753,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>is_integer(Term) -> bool()</name>
+      <name>is_integer(Term) -> boolean()</name>
       <fsummary>Check whether a term is an integer</fsummary>
       <type>
         <v>Term = term()</v>
@@ -1612,7 +1765,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>is_list(Term) -> bool()</name>
+      <name>is_list(Term) -> boolean()</name>
       <fsummary>Check whether a term is a list</fsummary>
       <type>
         <v>Term = term()</v>
@@ -1624,7 +1777,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>is_number(Term) -> bool()</name>
+      <name>is_number(Term) -> boolean()</name>
       <fsummary>Check whether a term is a number</fsummary>
       <type>
         <v>Term = term()</v>
@@ -1636,7 +1789,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>is_pid(Term) -> bool()</name>
+      <name>is_pid(Term) -> boolean()</name>
       <fsummary>Check whether a term is a pid</fsummary>
       <type>
         <v>Term = term()</v>
@@ -1648,7 +1801,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>is_port(Term) -> bool()</name>
+      <name>is_port(Term) -> boolean()</name>
       <fsummary>Check whether a term is a port</fsummary>
       <type>
         <v>Term = term()</v>
@@ -1660,7 +1813,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>is_process_alive(Pid) -> bool()</name>
+      <name>is_process_alive(Pid) -> boolean()</name>
       <fsummary>Check whether a process is alive</fsummary>
       <type>
         <v>Pid = pid()</v>
@@ -1675,7 +1828,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>is_record(Term, RecordTag) -> bool()</name>
+      <name>is_record(Term, RecordTag) -> boolean()</name>
       <fsummary>Check whether a term appears to be a record</fsummary>
       <type>
         <v>Term = term()</v>
@@ -1698,12 +1851,12 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>is_record(Term, RecordTag, Size) -> bool()</name>
+      <name>is_record(Term, RecordTag, Size) -> boolean()</name>
       <fsummary>Check whether a term appears to be a record</fsummary>
       <type>
         <v>Term = term()</v>
         <v>RecordTag = atom()</v>
-        <v>Size = int()</v>
+        <v>Size = integer()</v>
       </type>
       <desc>
         <p><c>RecordTag</c> must be an atom. Returns <c>true</c> if
@@ -1718,7 +1871,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>is_reference(Term) -> bool()</name>
+      <name>is_reference(Term) -> boolean()</name>
       <fsummary>Check whether a term is a reference</fsummary>
       <type>
         <v>Term = term()</v>
@@ -1730,7 +1883,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>is_tuple(Term) -> bool()</name>
+      <name>is_tuple(Term) -> boolean()</name>
       <fsummary>Check whether a term is a tuple</fsummary>
       <type>
         <v>Term = term()</v>
@@ -1742,7 +1895,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>length(List) -> int()</name>
+      <name>length(List) -> integer() >= 0</name>
       <fsummary>Length of a list</fsummary>
       <type>
         <v>List = [term()]</v>
@@ -1863,7 +2016,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>list_to_integer(String) -> int()</name>
+      <name>list_to_integer(String) -> integer()</name>
       <fsummary>Convert from text representation to an integer</fsummary>
       <type>
         <v>String = string()</v>
@@ -1879,19 +2032,15 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:list_to_integer(String, Base) -> int()</name>
+      <name name="list_to_integer" arity="2"/>
       <fsummary>Convert from text representation to an integer</fsummary>
-      <type>
-        <v>String = string()</v>
-        <v>Base = 2..36</v>
-      </type>
       <desc>
         <p>Returns an integer whose text representation in base
-          <c>Base</c> is <c>String</c>.</p>
+          <c><anno>Base</anno></c> is <c><anno>String</anno></c>.</p>
         <pre>
-> <input>erlang:list_to_integer("3FF", 16).</input>
+> <input>list_to_integer("3FF", 16).</input>
 1023</pre>
-        <p>Failure: <c>badarg</c> if <c>String</c> contains a bad
+        <p>Failure: <c>badarg</c> if <c><anno>String</anno></c> contains a bad
           representation of an integer.</p>
       </desc>
     </func>
@@ -1981,16 +2130,18 @@ os_prompt%</pre>
         <v>Text = string()</v>
       </type>
       <desc>
-        <warning>
-          <p>This BIF is still an experimental feature. The interface
-          may be changed in any way in future releases.</p><p>In
-          R13B03 the return value on failure was
+        <note>
+          <p>In releases older than OTP R14B, NIFs were an
+          experimental feature. Versions of OTP older than R14B might
+          have different and possibly incompatible NIF semantics and
+          interfaces. For example, in R13B03 the return value on
+          failure was
           <c>{error,Reason,Text}</c>.</p>
-        </warning>
+        </note>
         <p>Loads and links a dynamic library containing native
         implemented functions (NIFs) for a module. <c>Path</c> is a
         file path to the sharable object/dynamic library file minus
-        the OS-dependant file extension (.so for Unix and .ddl for
+        the OS-dependent file extension (.so for Unix and .dll for
         Windows). See <seealso marker="erl_nif">erl_nif</seealso>
         on how to implement a NIF library.</p> 
         <p><c>LoadInfo</c> can be any term. It will be passed on to
@@ -2040,12 +2191,10 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:localtime() -> {Date, Time}</name>
+      <name>erlang:localtime() -> DateTime</name>
       <fsummary>Current local date and time</fsummary>
       <type>
-        <v>Date = {Year, Month, Day}</v>
-        <v>Time = {Hour, Minute, Second}</v>
-        <v>&nbsp;Year = Month = Day = Hour = Minute = Second = int()</v>
+        <v>DateTime = <seealso marker="calendar#type-datetime">calendar:datetime()</seealso></v>
       </type>
       <desc>
         <p>Returns the current local date and time
@@ -2058,17 +2207,12 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:localtime_to_universaltime({Date1, Time1}) -> {Date2, Time2}</name>
+      <name name="localtime_to_universaltime" arity="1"/>
       <fsummary>Convert from local to Universal Time Coordinated (UTC) date and time</fsummary>
-      <type>
-        <v>Date1 = Date2 = {Year, Month, Day}</v>
-        <v>Time1 = Time2 = {Hour, Minute, Second}</v>
-        <v>&nbsp;Year = Month = Day = Hour = Minute = Second = int()</v>
-      </type>
       <desc>
         <p>Converts local date and time to Universal Time Coordinated
           (UTC), if this is supported by the underlying OS. Otherwise,
-          no conversion is done and <c>{Date1, Time1}</c> is returned.</p>
+          no conversion is done and <c>{<anno>Date1</anno>, <anno>Time1</anno>}</c> is returned.</p>
         <pre>
 > <input>erlang:localtime_to_universaltime({{1996,11,6},{14,45,17}}).</input>
 {{1996,11,6},{13,45,17}}</pre>
@@ -2080,9 +2224,8 @@ os_prompt%</pre>
       <name>erlang:localtime_to_universaltime({Date1, Time1}, IsDst) -> {Date2, Time2}</name>
       <fsummary>Convert from local to Universal Time Coordinated (UTC) date and time</fsummary>
       <type>
-        <v>Date1 = Date2 = {Year, Month, Day}</v>
-        <v>Time1 = Time2 = {Hour, Minute, Second}</v>
-        <v>&nbsp;Year = Month = Day = Hour = Minute = Second = int()</v>
+        <v>Date1 = Date2 = <seealso marker="calendar#type-date">calendar:date()</seealso></v>
+        <v>Time1 = Time2 = <seealso marker="calendar#type-time">calendar:time()</seealso></v>
         <v>IsDst = true | false | undefined</v>
       </type>
       <desc>
@@ -2107,7 +2250,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>make_ref() -> ref()</name>
+      <name>make_ref() -> reference()</name>
       <fsummary>Return an almost unique reference</fsummary>
       <desc>
         <p>Returns an almost unique reference.</p>
@@ -2122,7 +2265,7 @@ os_prompt%</pre>
       <name>erlang:make_tuple(Arity, InitialValue) -> tuple()</name>
       <fsummary>Create a new tuple of a given arity</fsummary>
       <type>
-        <v>Arity = int()</v>
+        <v>Arity = arity()</v>
         <v>InitialValue = term()</v>
       </type>
       <desc>
@@ -2137,7 +2280,7 @@ os_prompt%</pre>
       <name>erlang:make_tuple(Arity, Default, InitList) -> tuple()</name>
       <fsummary>Create a new tuple with given arity and contents</fsummary>
       <type>
-        <v>Arity = int()</v>
+        <v>Arity = arity()</v>
         <v>Default = term()</v>
         <v>InitList = [{Position,term()}]</v>
         <v>Position = integer()</v>
@@ -2156,14 +2299,11 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:max(Term1, Term2) -> Maximum</name>
+      <name name="max" arity="2"/>
       <fsummary>Return the largest of two term</fsummary>
-      <type>
-        <v>Term1 = Term2 = Maximum = term()</v>
-      </type>
       <desc>
-        <p>Return the largest of <c>Term1</c> and <c>Term2</c>;
-	if the terms compares equal, <c>Term1</c> will be returned.</p>
+        <p>Return the largest of <c><anno>Term1</anno></c> and <c><anno>Term2</anno></c>;
+	if the terms compare equal, <c><anno>Term1</anno></c> will be returned.</p>
       </desc>
     </func>
     <func>
@@ -2301,6 +2441,14 @@ os_prompt%</pre>
               <seealso marker="tools:instrument">instrument(3)</seealso>
               and/or <seealso marker="erts:erl">erl(1)</seealso>.</p>
           </item>
+          <tag><c>low</c></tag>
+          <item>
+            <p>Only on 64-bit halfword emulator.</p>
+            <p>The total amount of memory allocated in low memory areas
+	      that are restricted to less than 4 Gb even though
+	      the system may have more physical memory.</p>
+	    <p>May be removed in future releases of halfword emulator.</p>
+          </item>
         </taglist>
         <note>
           <p>The <c>system</c> value is not complete. Some allocated
@@ -2405,18 +2553,15 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:min(Term1, Term2) -> Minimum</name>
+      <name name="min" arity="2"/>
       <fsummary>Return the smallest of two term</fsummary>
-      <type>
-        <v>Term1 = Term2 = Minimum = term()</v>
-      </type>
       <desc>
-        <p>Return the smallest of <c>Term1</c> and <c>Term2</c>;
-	if the terms compare equal, <c>Term1</c> will be returned.</p>
+        <p>Return the smallest of <c><anno>Term1</anno></c> and <c><anno>Term2</anno></c>;
+	if the terms compare equal, <c><anno>Term1</anno></c> will be returned.</p>
       </desc>
     </func>
     <func>
-      <name>module_loaded(Module) -> bool()</name>
+      <name>module_loaded(Module) -> boolean()</name>
       <fsummary>Check if a module is loaded</fsummary>
       <type>
         <v>Module = atom()</v>
@@ -2433,7 +2578,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:monitor(Type, Item) -> MonitorRef</name>
+      <name>monitor(Type, Item) -> MonitorRef</name>
       <fsummary>Start monitoring</fsummary>
       <type>
         <v>Type = process</v>
@@ -2469,7 +2614,7 @@ os_prompt%</pre>
         <note>
           <p>When a process is monitored by registered name, the process
             that has the registered name at the time when
-            <c>erlang:monitor/2</c> is called will be monitored.
+            <c>monitor/2</c> is called will be monitored.
             The monitor will not be effected, if the registered name is
             unregistered.</p>
         </note>
@@ -2503,20 +2648,20 @@ os_prompt%</pre>
           </item>
         </taglist>
         <note>
-          <p>If/when <c>erlang:monitor/2</c> is extended (e.g. to
+          <p>If/when <c>monitor/2</c> is extended (e.g. to
             handle other item types than <c>process</c>), other
             possible values for <c>Object</c>, and <c>Info</c> in the
             <c>'DOWN'</c> message will be introduced.</p>
         </note>
         <p>The monitoring is turned off either when the <c>'DOWN'</c>
           message is sent, or when
-          <seealso marker="#demonitor/1">erlang:demonitor/1</seealso>
+          <seealso marker="#demonitor/1">demonitor/1</seealso>
           is called.</p>
         <p>If an attempt is made to monitor a process on an older node
           (where remote process monitoring is not implemented or one
           where remote process monitoring by registered name is not
           implemented), the call fails with <c>badarg</c>.</p>
-        <p>Making several calls to <c>erlang:monitor/2</c> for the same
+        <p>Making several calls to <c>monitor/2</c> for the same
           <c>Item</c> is not an error; it results in as many, completely
           independent, monitorings.</p>
         <note>
@@ -2539,7 +2684,7 @@ os_prompt%</pre>
       <fsummary>Monitor the status of a node</fsummary>
       <type>
         <v>Node = node()</v>
-        <v>Flag = bool()</v>
+        <v>Flag = boolean()</v>
       </type>
       <desc>
         <p>Monitors the status of the node <c>Node</c>. If <c>Flag</c>
@@ -2565,7 +2710,7 @@ os_prompt%</pre>
       <fsummary>Monitor the status of a node</fsummary>
       <type>
         <v>Node = node()</v>
-        <v>Flag = bool()</v>
+        <v>Flag = boolean()</v>
         <v>Options = [Option]</v>
         <v>Option = allow_passive_connect</v>
       </type>
@@ -2591,6 +2736,37 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
+      <name>erlang:nif_error(Reason)</name>
+      <fsummary>Stop execution with a given reason</fsummary>
+      <type>
+        <v>Reason = term()</v>
+      </type>
+      <desc>
+        <p>Works exactly like
+	 <seealso marker="#error/1">erlang:error/1</seealso>,
+	 but Dialyzer thinks that this BIF will return an arbitrary term.
+	 When used in a stub function for a NIF to generate an
+	 exception when the NIF library is not loaded, Dialyzer
+	 will not generate false warnings.</p>
+      </desc>
+    </func>
+    <func>
+      <name>erlang:nif_error(Reason, Args)</name>
+      <fsummary>Stop execution with a given reason</fsummary>
+      <type>
+        <v>Reason = term()</v>
+        <v>Args = [term()]</v>
+      </type>
+      <desc>
+        <p>Works exactly like
+	 <seealso marker="#error/2">erlang:error/2</seealso>,
+	 but Dialyzer thinks that this BIF will return an arbitrary term.
+	 When used in a stub function for a NIF to generate an
+	 exception when the NIF library is not loaded, Dialyzer
+	 will not generate false warnings.</p>
+      </desc>
+    </func>
+    <func>
       <name>node() -> Node</name>
       <fsummary>Name of the local node</fsummary>
       <type>
@@ -2606,7 +2782,7 @@ os_prompt%</pre>
       <name>node(Arg) -> Node</name>
       <fsummary>At which node is a pid, port or reference located</fsummary>
       <type>
-        <v>Arg = pid() | port() | ref()</v>
+        <v>Arg = pid() | port() | reference()</v>
         <v>Node = node()</v>
       </type>
       <desc>
@@ -2617,11 +2793,8 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>nodes() -> Nodes</name>
+      <name name="nodes" arity="0"/>
       <fsummary>All visible nodes in the system</fsummary>
-      <type>
-        <v>Nodes = [node()]</v>
-      </type>
       <desc>
         <p>Returns a list of all visible nodes in the system, excluding
           the local node. Same as <c>nodes(visible)</c>.</p>
@@ -2671,11 +2844,12 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>now() -> {MegaSecs, Secs, MicroSecs}</name>
-      <fsummary>Elapsed time since 00:00 GMT</fsummary>
+      <name>now() -> timestamp()</name>
       <type>
-        <v>MegaSecs = Secs = MicroSecs = int()</v>
+        <v>timestamp() = {MegaSecs, Secs, MicroSecs}</v>
+        <v>MegaSecs = Secs = MicroSecs = integer() >= 0</v>
       </type>
+      <fsummary>Elapsed time since 00:00 GMT</fsummary>
       <desc>
         <p>Returns the tuple <c>{MegaSecs, Secs, MicroSecs}</c> which is
           the elapsed time since 00:00 GMT, January 1, 1970 (zero hour)
@@ -2683,24 +2857,33 @@ os_prompt%</pre>
           Otherwise, some other point in time is chosen. It is also
           guaranteed that subsequent calls to this BIF returns
           continuously increasing values. Hence, the return value from
-          <c>now()</c> can be used to generate unique time-stamps. It
-          can only be used to check the local time of day if
+          <c>now()</c> can be used to generate unique time-stamps,
+          and if it is called in a tight loop on a fast machine
+          the time of the node can become skewed.</p>
+        <p>It can only be used to check the local time of day if
           the time-zone info of the underlying operating system is
           properly configured.</p>
+        <p>If you do not need the return value to be unique and
+          monotonically increasing, use
+          <seealso marker="kernel:os#timestamp/0">os:timestamp/0</seealso>
+          instead to avoid some overhead.</p>
       </desc>
     </func>
     <func>
       <name>open_port(PortName, PortSettings) -> port()</name>
       <fsummary>Open a port</fsummary>
       <type>
-        <v>PortName = {spawn, Command} | {spawn_driver, Command} | {spawn_executable, Command} | {fd, In, Out}</v>
+        <v>PortName = {spawn, Command} | {spawn_driver, Command} | {spawn_executable, FileName} | {fd, In, Out}</v>
         <v>&nbsp;Command = string()</v>
-        <v>&nbsp;In = Out = int()</v>
+	<v>&nbsp;FileName = [ FileNameChar ] | binary()</v>
+	<v>&nbsp;FileNameChar = integer() (1..255 or any Unicode codepoint, see description)</v>
+        <v>&nbsp;In = Out = integer()</v>
         <v>PortSettings = [Opt]</v>
-        <v>&nbsp;Opt = {packet, N} | stream | {line, L} | {cd, Dir} | {env, Env} | {args, [ string() ]} | {arg0, string()} | exit_status | use_stdio | nouse_stdio | stderr_to_stdout | in | out | binary | eof</v>
+        <v>&nbsp;Opt = {packet, N} | stream | {line, L} | {cd, Dir} | {env, Env} | {args, [ ArgString ]} | {arg0, ArgString} | exit_status | use_stdio | nouse_stdio | stderr_to_stdout | in | out | binary | eof</v>
         <v>&nbsp;&nbsp;N = 1 | 2 | 4</v>
-        <v>&nbsp;&nbsp;L = int()</v>
+        <v>&nbsp;&nbsp;L = integer()</v>
         <v>&nbsp;&nbsp;Dir = string()</v>
+	<v>&nbsp;&nbsp;ArgString = [ FileNameChar ] | binary()</v>
         <v>&nbsp;&nbsp;Env = [{Name, Val}]</v>
         <v>&nbsp;&nbsp;&nbsp;Name = string()</v>
         <v>&nbsp;&nbsp;&nbsp;Val = string() | false</v>
@@ -2730,7 +2913,7 @@ os_prompt%</pre>
 	      <p>For external programs, the <c>PATH</c> is searched
 	      (or an equivalent method is used to find programs,
 	      depending on operating system). This is done by invoking
-	      the shell och certain platforms. The first space
+	      the shell on certain platforms. The first space
 	      separated token of the command will be considered as the
 	      name of the executable (or driver). This (among other
 	      things) makes this option unsuitable for running
@@ -2749,7 +2932,7 @@ os_prompt%</pre>
           <item>
 
 	      <p>Works like <c>{spawn, Command}</c>, but only runs
-	      external executables. The <c>Command</c> in it's whole
+	      external executables. The <c>Command</c> in its whole
 	      is used as the name of the executable, including any
 	      spaces. If arguments are to be passed, the
 	      <c>args</c> and <c>arg0</c> <c>PortSettings</c> can be used.</p>
@@ -2763,7 +2946,26 @@ os_prompt%</pre>
 	      executed, the appropriate command interpreter will
 	      implicitly be invoked, but there will still be no
 	      command argument expansion or implicit PATH search.</p>
-	      
+
+	      <p>The name of the executable as well as the arguments
+	      given in <c>args</c> and <c>arg0</c> is subject to
+	      Unicode file name translation if the system is running
+	      in Unicode file name mode. To avoid
+	      translation or force i.e. UTF-8, supply the executable
+	      and/or arguments as a binary in the correct
+	      encoding. See the <seealso
+	      marker="kernel:file">file</seealso> module, the
+	      <seealso marker="kernel:file#native_name_encoding/0">
+	      file:native_name_encoding/0</seealso> function and the 
+	      <seealso marker="stdlib:unicode_usage">stdlib users guide
+	      </seealso> for details.</p>
+
+	      <note><p>The characters in the name (if given as a list)
+	      can only be &gt; 255 if the Erlang VM is started in
+	      Unicode file name translation mode, otherwise the name
+	      of the executable is limited to the ISO-latin-1
+	      character set.</p></note>
+  
 	      <p>If the <c>Command</c> cannot be run, an error
 	      exception, with the posix error code as the reason, is
 	      raised. The error reason may differ between operating
@@ -2866,6 +3068,21 @@ os_prompt%</pre>
 	    should not be given in this list. The proper executable name will
 	    automatically be used as argv[0] where applicable.</p>
 
+	    <p>When the Erlang VM is running in Unicode file name
+	    mode, the arguments can contain any Unicode characters and
+	    will be translated into whatever is appropriate on the
+	    underlying OS, which means UTF-8 for all platforms except
+	    Windows, which has other (more transparent) ways of
+	    dealing with Unicode arguments to programs. To avoid
+	    Unicode translation of arguments, they can be supplied as
+	    binaries in whatever encoding is deemed appropriate.</p>
+
+	      <note><p>The characters in the arguments (if given as a
+	      list of characters) can only be &gt; 255 if the Erlang
+	      VM is started in Unicode file name mode,
+	      otherwise the arguments are limited to the
+	      ISO-latin-1 character set.</p></note>
+
 	    <p>If one, for any reason, wants to explicitly set the
 	    program name in the argument vector, the <c>arg0</c>
 	    option can be used.</p>
@@ -2881,6 +3098,9 @@ os_prompt%</pre>
 	    responds to this is highly system dependent and no specific 
 	    effect is guaranteed.</p>
 
+	    <p>The unicode file name translation rules of the
+	    <c>args</c> option apply to this option as well.</p>
+
 	  </item> 
 
           <tag><c>exit_status</c></tag>
@@ -2926,7 +3146,7 @@ os_prompt%</pre>
 	    The standard input and standard output handles of the port program
 	    will, if this option is supplied, be opened with the flag
 	    FILE_FLAG_OVERLAPPED, so that the port program can (and has to) do
-	    overlapped I/O on it's standard handles. This is not normally
+	    overlapped I/O on its standard handles. This is not normally
 	    the case for simple port programs, but an option of value for the
 	    experienced Windows programmer. <em>On all other platforms, this 
 	    option is silently discarded</em>.</p> 
@@ -3147,7 +3367,7 @@ os_prompt%</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:port_command(Port, Data, OptionList) -> true|false</name>
+      <name>port_command(Port, Data, OptionList) -> boolean()</name>
       <fsummary>Send data to a port</fsummary>
       <type>
         <v>Port = port() | atom()</v>
@@ -3183,10 +3403,6 @@ os_prompt%</pre>
         <note>
           <p>More options may be added in the future.</p>
         </note>
-        <note>
-          <p><c>erlang:port_command/3</c> is currently not auto imported, but
-	     it is planned to be auto imported in OTP R14.</p>
-        </note>
         <p>Failures:</p>
         <taglist>
           <tag><c>badarg</c></tag>
@@ -3267,7 +3483,7 @@ os_prompt%</pre>
       <fsummary>Perform a synchronous control operation on a port</fsummary>
       <type>
         <v>Port = port() | atom()</v>
-        <v>Operation = int()</v>
+        <v>Operation = integer()</v>
         <v>Data = Res = iodata()</v>
       </type>
       <desc>
@@ -3291,7 +3507,7 @@ os_prompt%</pre>
       <fsummary>Synchronous call to a port with term data</fsummary>
       <type>
         <v>Port = port() | atom()</v>
-        <v>Operation = int()</v>
+        <v>Operation = integer()</v>
         <v>Data = term()</v>
       </type>
       <desc>
@@ -3361,6 +3577,10 @@ os_prompt%</pre>
             <p><c>Bytes</c> is the total number of bytes written to
               the port.</p>
           </item>
+          <tag><c>{os_pid, Integer | undefined}</c></tag>
+          <item>
+            <p><c>Integer</c> is the process identifier (or equivalent) of an OS process created with <c>open_port({spawn | spawn_executable, Command}, Options)</c>. If the port is not the result of spawning an OS process, the value is <c>undefined</c>.</p>
+          </item>
         </taglist>
         <p>Failure: <c>badarg</c> if <c>Port</c> is not a local port.</p>
       </desc>
@@ -3550,12 +3770,6 @@ os_prompt%</pre>
 
           <tag><c>process_flag(save_calls, N)</c></tag>
           <item>
-	    <p>When there are runnable processes on priority <c>max</c>
-	      no processes on priority <c>low</c>, <c>normal</c>, or
-	      <c>high</c> will be selected for execution. As with the
-	      <c>high</c> priority, processes on lower priorities might
-	      execute in parallel with processes on priority <c>max</c>.
-	    </p>
             <p><c>N</c> must be an integer in the interval 0..10000.
               If <c>N</c> &gt; 0, call saving is made active for the
               process, which means that information about the <c>N</c>
@@ -3726,11 +3940,26 @@ os_prompt%</pre>
 	    catches in this process. This <c>InfoTuple</c> may be
 	    changed or removed without prior notice.</p>
           </item>
-          <tag><c>{current_function, {Module, Function, Args}}</c></tag>
+          <tag><c>{current_function, {Module, Function, Arity}}</c></tag>
           <item>
-            <p><c>Module</c>, <c>Function</c>, <c>Args</c> is
+            <p><c>Module</c>, <c>Function</c>, <c>Arity</c> is
               the current function call of the process.</p>
           </item>
+          <tag><c>{current_location, {Module, Function, Arity, Location}}</c></tag>
+          <item>
+            <p><c>Module</c>, <c>Function</c>, <c>Arity</c> is
+              the current function call of the process.
+	      <c>Location</c> is a list of two-tuples that describes the
+	      location in the source code.
+	    </p>
+          </item>
+          <tag><c>{current_stacktrace, Stack}</c></tag>
+          <item>
+            <p>Return the current call stack back-trace (<em>stacktrace</em>)
+              of the process. The stack has the same format as returned by
+	      <seealso marker="#get_stacktrace/0">erlang:get_stacktrace/0</seealso>.
+	    </p>
+          </item>
           <tag><c>{dictionary, Dictionary}</c></tag>
           <item>
             <p><c>Dictionary</c> is the dictionary of the process.</p>
@@ -3784,14 +4013,6 @@ os_prompt%</pre>
             <p><c>Size</c> is the size in bytes of the process. This
               includes call stack, heap and internal structures.</p>
           </item>
-          <tag><c>{message_binary, BinInfo}</c></tag>
-          <item>
-            <p><c>BinInfo</c> is a list containing miscellaneous information
-	      about binaries currently being referred to by the message
-	      area. This <c>InfoTuple</c> is only valid on an emulator
-	      using the hybrid heap type. This <c>InfoTuple</c> may be
-	      changed or removed without prior notice.</p>
-          </item>
           <tag><c>{message_queue_len, MessageQueueLen}</c></tag>
           <item>
             <p><c>MessageQueueLen</c> is the number of messages
@@ -3815,11 +4036,11 @@ os_prompt%</pre>
           <tag><c>{monitored_by, Pids}</c></tag>
           <item>
             <p>A list of pids that are monitoring the process (with
-              <c>erlang:monitor/2</c>).</p>
+              <c>monitor/2</c>).</p>
           </item>
           <tag><c>{monitors, Monitors}</c></tag>
           <item>
-            <p>A list of monitors (started by <c>erlang:monitor/2</c>)
+            <p>A list of monitors (started by <c>monitor/2</c>)
               that are active for the process. For a local process
               monitor or a remote process monitor by pid, the list item
               is <c>{process, Pid}</c>, and for a remote process
@@ -3856,7 +4077,8 @@ os_prompt%</pre>
           <tag><c>{status, Status}</c></tag>
           <item>
             <p><c>Status</c> is the status of the process. <c>Status</c>
-              is <c>waiting</c> (waiting for a message), <c>running</c>,
+              is <c>exiting</c>, <c>garbage_collecting</c>,
+              <c>waiting</c> (for a message), <c>running</c>,
               <c>runnable</c> (ready to run, but another process is
               running), or <c>suspended</c> (suspended on a "busy" port
               or by the <c>erlang:suspend_process/[1,2]</c> BIF).</p>
@@ -3976,7 +4198,7 @@ os_prompt%</pre>
         <v>Reason = term()</v>
         <v>Stacktrace = [{Module, Function, Arity | Args} | {Fun, Args}]</v>
         <v>&nbsp;Module = Function = atom()</v>
-        <v>&nbsp;Arity = int()</v>
+        <v>&nbsp;Arity = arity()</v>
         <v>&nbsp;Args = [term()]</v>
         <v>&nbsp;Fun = [fun()]</v>
       </type>
@@ -3996,11 +4218,14 @@ os_prompt%</pre>
           equivalent to <c>erlang:Class(Reason)</c>.
           <c>Reason</c> is any term and <c>Stacktrace</c> is a list as
           returned from <c>get_stacktrace()</c>, that is a list of
-          3-tuples <c>{Module, Function, Arity | Args}</c> where
-          <c>Module</c> and <c>Function</c> are atoms and the third
-          element is an integer arity or an argument list. The
-          stacktrace may also contain <c>{Fun, Args}</c> tuples where
+          4-tuples <c>{Module, Function, Arity | Args,
+          Location}</c> where <c>Module</c> and <c>Function</c>
+          are atoms and the third element is an integer arity or an
+          argument list. The stacktrace may also contain <c>{Fun,
+          Args, Location}</c> tuples where
           <c>Fun</c> is a local fun and <c>Args</c> is an argument list.</p>
+	<p>The <c>Location</c> element at the end is optional.
+	  Omitting it is equivalent to specifying an empty list.</p>
         <p>The stacktrace is used as the exception stacktrace for the
           calling process; it will be truncated to the current
           maximum stacktrace depth.</p>
@@ -4008,15 +4233,15 @@ os_prompt%</pre>
           terminate, it has no return value - unless the arguments are
           invalid, in which case the function <em>returns the error reason</em>, that is <c>badarg</c>. If you want to be
           really sure not to return you can call
-          <c>erlang:error(erlang:raise(Class, Reason, Stacktrace))</c>
+          <c>error(erlang:raise(Class, Reason, Stacktrace))</c>
           and hope to distinguish exceptions later.</p>
       </desc>
     </func>
     <func>
-      <name>erlang:read_timer(TimerRef) -> int() | false</name>
+      <name>erlang:read_timer(TimerRef) -> integer() >= 0 | false</name>
       <fsummary>Number of milliseconds remaining for a timer</fsummary>
       <type>
-        <v>TimerRef = ref()</v>
+        <v>TimerRef = reference()</v>
       </type>
       <desc>
         <p><c>TimerRef</c> is a timer reference returned by
@@ -4039,7 +4264,7 @@ os_prompt%</pre>
       <name>erlang:ref_to_list(Ref) -> string()</name>
       <fsummary>Text representation of a reference</fsummary>
       <type>
-        <v>Ref = ref()</v>
+        <v>Ref = reference()</v>
       </type>
       <desc>
         <p>Returns a string which corresponds to the text
@@ -4129,7 +4354,7 @@ true</pre>
       </desc>
     </func>
     <func>
-      <name>round(Number) -> int()</name>
+      <name>round(Number) -> integer()</name>
       <fsummary>Return an integer by rounding a number</fsummary>
       <type>
         <v>Number = number()</v>
@@ -4213,12 +4438,12 @@ true</pre>
       <name>erlang:send_after(Time, Dest, Msg) -> TimerRef</name>
       <fsummary>Start a timer</fsummary>
       <type>
-        <v>Time = int()</v>
+        <v>Time = integer() >= 0</v>
         <v>&nbsp;0 &lt;= Time &lt;= 4294967295</v>
         <v>Dest = pid() | RegName </v>
         <v>&nbsp;LocalPid = pid() (of a process, alive or dead, on the local node)</v>
         <v>Msg = term()</v>
-        <v>TimerRef = ref()</v>
+        <v>TimerRef = reference()</v>
       </type>
       <desc>
         <p>Starts a timer which will send the message <c>Msg</c>
@@ -4242,17 +4467,12 @@ true</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:send_nosuspend(Dest, Msg) -> bool()</name>
+      <name name="send_nosuspend" arity="2"/>
       <fsummary>Try to send a message without ever blocking</fsummary>
-      <type>
-        <v>Dest = pid() | port() | RegName | {RegName, Node}</v>
-        <v>&nbsp;RegName = atom()</v>
-        <v>&nbsp;Node = node()</v>
-        <v>Msg = term()</v>
-      </type>
+      <type name="dst"/>
       <desc>
         <p>The same as
-          <seealso marker="#send/3">erlang:send(Dest, Msg, [nosuspend])</seealso>, but returns <c>true</c> if
+          <seealso marker="#send/3">erlang:send(<anno>Dest</anno>, <anno>Msg</anno>, [nosuspend])</seealso>, but returns <c>true</c> if
           the message was sent and <c>false</c> if the message was not
           sent because the sender would have had to be suspended.</p>
         <p>This function is intended for send operations towards an
@@ -4260,7 +4480,7 @@ true</pre>
           (Erlang) process. If the connection to the remote node
           (usually not a real Erlang node, but a node written in C or
           Java) is overloaded, this function <em>will not send the message</em> but return <c>false</c> instead.</p>
-        <p>The same happens, if <c>Dest</c> refers to a local port that
+        <p>The same happens, if <c><anno>Dest</anno></c> refers to a local port that
           is busy. For all other destinations (allowed for the ordinary
           send operator <c>'!'</c>) this function sends the message and
           returns <c>true</c>.</p>
@@ -4293,18 +4513,12 @@ true</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:send_nosuspend(Dest, Msg, Options) -> bool()</name>
+      <name name="send_nosuspend" arity="3"/>
       <fsummary>Try to send a message without ever blocking</fsummary>
-      <type>
-        <v>Dest = pid() | port() | RegName | {RegName, Node}</v>
-        <v>&nbsp;RegName = atom()</v>
-        <v>&nbsp;Node = node()</v>
-        <v>Msg = term()</v>
-        <v>Option = noconnect</v>
-      </type>
+      <type name="dst"/>
       <desc>
         <p>The same as
-          <seealso marker="#send/3">erlang:send(Dest, Msg, [nosuspend | Options])</seealso>,
+          <seealso marker="#send/3">erlang:send(<anno>Dest</anno>, <anno>Msg</anno>, [nosuspend | <anno>Options</anno>])</seealso>,
           but with boolean return value.</p>
         <p>This function behaves like
           <seealso marker="#send_nosuspend/2">erlang:send_nosuspend/2)</seealso>,
@@ -4329,17 +4543,13 @@ true</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:set_cookie(Node, Cookie) -> true</name>
+      <name name="set_cookie" arity="2"/>
       <fsummary>Set the magic cookie of a node</fsummary>
-      <type>
-        <v>Node = node()</v>
-        <v>Cookie = atom()</v>
-      </type>
       <desc>
-        <p>Sets the magic cookie of <c>Node</c> to the atom
-          <c>Cookie</c>. If <c>Node</c> is the local node, the function
+        <p>Sets the magic cookie of <c><anno>Node</anno></c> to the atom
+          <c><anno>Cookie</anno></c>. If <c><anno>Node</anno></c> is the local node, the function
           also sets the cookie of all other unknown nodes to
-          <c>Cookie</c> (see
+          <c><anno>Cookie</anno></c> (see
           <seealso marker="doc/reference_manual:distributed">Distributed Erlang</seealso> in the Erlang Reference Manual).</p>
         <p>Failure: <c>function_clause</c> if the local node is not
           alive.</p>
@@ -4364,7 +4574,7 @@ true</pre>
       </desc>
     </func>
     <func>
-      <name>size(Item) -> int()</name>
+      <name>size(Item) -> integer() >= 0</name>
       <fsummary>Size of a tuple or binary</fsummary>
       <type>
         <v>Item = tuple() | binary()</v>
@@ -4379,28 +4589,21 @@ true</pre>
       </desc>
     </func>
     <func>
-      <name>spawn(Fun) -> pid()</name>
+      <name name="spawn" arity="1"/>
       <fsummary>Create a new process with a fun as entry point</fsummary>
-      <type>
-        <v>Fun = fun()</v>
-      </type>
       <desc>
         <p>Returns the pid of a new process started by the application
-          of <c>Fun</c> to the empty list <c>[]</c>. Otherwise works
+          of <c><anno>Fun</anno></c> to the empty list <c>[]</c>. Otherwise works
           like <seealso marker="#spawn/3">spawn/3</seealso>.</p>
       </desc>
     </func>
     <func>
-      <name>spawn(Node, Fun) -> pid()</name>
+      <name name="spawn" arity="2"/>
       <fsummary>Create a new process with a fun as entry point on a given node</fsummary>
-      <type>
-        <v>Node = node()</v>
-        <v>Fun = fun()</v>
-      </type>
       <desc>
         <p>Returns the pid of a new process started by the application
-          of <c>Fun</c> to the empty list <c>[]</c> on <c>Node</c>. If
-          <c>Node</c> does not exist, a useless pid is returned.
+          of <c><anno>Fun</anno></c> to the empty list <c>[]</c> on <c><anno>Node</anno></c>. If
+          <c><anno>Node</anno></c> does not exist, a useless pid is returned.
           Otherwise works like
           <seealso marker="#spawn/3">spawn/3</seealso>.</p>
       </desc>
@@ -4431,47 +4634,35 @@ true</pre>
       </desc>
     </func>
     <func>
-      <name>spawn(Node, Module, Function, ArgumentList) -> pid()</name>
+      <name name="spawn" arity="4"/>
       <fsummary>Create a new process with a function as entry point on a given node</fsummary>
-      <type>
-        <v>Node = node()</v>
-        <v>Module = Function = atom()</v>
-        <v>Args = [term()]</v>
-      </type>
       <desc>
         <p>Returns the pid of a new process started by the application
-          of <c>Module:Function</c> to <c>Args</c> on <c>Node</c>. If
-          <c>Node</c> does not exists, a useless pid is returned.
+          of <c><anno>Module</anno>:<anno>Function</anno></c> to <c><anno>Args</anno></c> on <c>Node</c>. If
+          <c><anno>Node</anno></c> does not exists, a useless pid is returned.
           Otherwise works like
           <seealso marker="#spawn/3">spawn/3</seealso>.</p>
       </desc>
     </func>
     <func>
-      <name>spawn_link(Fun) -> pid()</name>
+      <name name="spawn_link" arity="1"/>
       <fsummary>Create and link to a new process with a fun as entry point</fsummary>
-      <type>
-        <v>Fun = fun()</v>
-      </type>
       <desc>
         <p>Returns the pid of a new process started by the application
-          of <c>Fun</c> to the empty list []. A link is created between
+          of <c><anno>Fun</anno></c> to the empty list []. A link is created between
           the calling process and the new process, atomically.
           Otherwise works like
           <seealso marker="#spawn/3">spawn/3</seealso>.</p>
       </desc>
     </func>
     <func>
-      <name>spawn_link(Node, Fun) -> pid()</name>
+      <name name="spawn_link" arity="2"/>
       <fsummary>Create and link to a new process with a fun as entry point on a specified node</fsummary>
-      <type>
-        <v>Node = node()</v>
-        <v>Fun = fun()</v>
-      </type>
       <desc>
         <p>Returns the pid of a new process started by the application
-          of <c>Fun</c> to the empty list [] on <c>Node</c>. A link is
+          of <c><anno>Fun</anno></c> to the empty list [] on <c><anno>Node</anno></c>. A link is
           created between the calling process and the new process,
-          atomically. If <c>Node</c> does not exist, a useless pid is
+          atomically. If <c><anno>Node</anno></c> does not exist, a useless pid is
           returned (and due to the link, an exit signal with exit
           reason <c>noconnection</c> will be received). Otherwise works
           like <seealso marker="#spawn/3">spawn/3</seealso>.</p>
@@ -4493,47 +4684,35 @@ true</pre>
       </desc>
     </func>
     <func>
-      <name>spawn_link(Node, Module, Function, Args) -> pid()</name>
+      <name name="spawn_link" arity="4"/>
       <fsummary>Create and link to a new process with a function as entry point on a given node</fsummary>
-      <type>
-        <v>Node = node()</v>
-        <v>Module = Function = atom()</v>
-        <v>Args = [term()]</v>
-      </type>
       <desc>
         <p>Returns the pid of a new process started by the application
-          of <c>Module:Function</c> to <c>Args</c> on <c>Node</c>. A
+          of <c><anno>Module</anno>:<anno>Function</anno></c> to <c><anno>Args</anno></c> on <c>Node</c>. A
           link is created between the calling process and the new
-          process, atomically. If <c>Node</c> does not exist, a useless
+          process, atomically. If <c><anno>Node</anno></c> does not exist, a useless
           pid is returned (and due to the link, an exit signal with exit
           reason <c>noconnection</c> will be received). Otherwise works
           like <seealso marker="#spawn/3">spawn/3</seealso>.</p>
       </desc>
     </func>
     <func>
-      <name>spawn_monitor(Fun) -> {pid(),reference()}</name>
+      <name name="spawn_monitor" arity="1"/>
       <fsummary>Create and monitor a new process with a fun as entry point</fsummary>
-      <type>
-        <v>Fun = fun()</v>
-      </type>
       <desc>
         <p>Returns the pid of a new process started by the application
-          of <c>Fun</c> to the empty list [] and reference for a monitor
+          of <c><anno>Fun</anno></c> to the empty list [] and reference for a monitor
           created to the new process.
           Otherwise works like
           <seealso marker="#spawn/3">spawn/3</seealso>.</p>
       </desc>
     </func>
     <func>
-      <name>spawn_monitor(Module, Function, Args) -> {pid(),reference()}</name>
+      <name name="spawn_monitor" arity="3"/>
       <fsummary>Create and monitor a new process with a function as entry point</fsummary>
-      <type>
-        <v>Module = Function = atom()</v>
-        <v>Args = [term()]</v>
-      </type>
       <desc>
         <p>A new process is started by the application
-          of <c>Module:Function</c> to <c>Args</c>, and the process is
+          of <c><anno>Module</anno>:<anno>Function</anno></c> to <c><anno>Args</anno></c>, and the process is
           monitored at the same time. Returns the pid and a reference
           for the monitor.
           Otherwise works like
@@ -4541,19 +4720,11 @@ true</pre>
       </desc>
     </func>
     <func>
-      <name>spawn_opt(Fun, [Option]) -> pid() | {pid(),reference()}</name>
+      <name name="spawn_opt" arity="2"/>
       <fsummary>Create a new process with a fun as entry point</fsummary>
-      <type>
-        <v>Fun = fun()</v>
-        <v>Option = link | monitor | {priority, Level} | {fullsweep_after, Number} | {min_heap_size, Size} | {min_bin_vheap_size, VSize}</v>
-        <v>&nbsp;Level = low | normal | high</v>
-        <v>&nbsp;Number = int()</v>
-        <v>&nbsp;Size = int()</v>
-        <v>&nbsp;VSize = int()</v>
-      </type>
       <desc>
         <p>Returns the pid of a new process started by the application
-          of <c>Fun</c> to the empty list <c>[]</c>. Otherwise
+          of <c><anno>Fun</anno></c> to the empty list <c>[]</c>. Otherwise
           works like
           <seealso marker="#spawn_opt/4">spawn_opt/4</seealso>.</p>
         <p>If the option <c>monitor</c> is given, the newly created
@@ -4562,37 +4733,19 @@ true</pre>
       </desc>
     </func>
     <func>
-      <name>spawn_opt(Node, Fun, [Option]) -> pid()</name>
+      <name name="spawn_opt" arity="3"/>
       <fsummary>Create a new process with a fun as entry point on a given node</fsummary>
-      <type>
-        <v>Node = node()</v>
-        <v>Fun = fun()</v>
-        <v>Option = link | {priority, Level} | {fullsweep_after, Number} | {min_heap_size, Size} | {min_bin_vheap_size, VSize}</v>
-        <v>&nbsp;Level = low | normal | high</v>
-        <v>&nbsp;Number = int()</v>
-        <v>&nbsp;Size = int()</v>
-        <v>&nbsp;VSize = int()</v>
-      </type>
       <desc>
         <p>Returns the pid of a new process started by the application
-          of <c>Fun</c> to the empty list <c>[]</c> on <c>Node</c>. If
-          <c>Node</c> does not exist, a useless pid is returned.
+          of <c><anno>Fun</anno></c> to the empty list <c>[]</c> on <c><anno>Node</anno></c>. If
+          <c><anno>Node</anno></c> does not exist, a useless pid is returned.
           Otherwise works like
           <seealso marker="#spawn_opt/4">spawn_opt/4</seealso>.</p>
       </desc>
     </func>
     <func>
-      <name>spawn_opt(Module, Function, Args, [Option]) -> pid() | {pid(),reference()}</name>
+      <name name="spawn_opt" arity="4"/>
       <fsummary>Create a new process with a function as entry point</fsummary>
-      <type>
-        <v>Module = Function = atom()</v>
-        <v>Args = [term()]</v>
-        <v>Option = link | monitor | {priority, Level} | {fullsweep_after, Number} | {min_heap_size, Size} | {min_bin_vheap_size, VSize}</v>
-        <v>&nbsp;Level = low | normal | high</v>
-        <v>&nbsp;Number = int()</v>
-        <v>&nbsp;Size = int()</v>
-        <v>&nbsp;VSize = int()</v>
-      </type>
       <desc>
         <p>Works exactly like
           <seealso marker="#spawn/3">spawn/3</seealso>, except that an
@@ -4609,19 +4762,19 @@ true</pre>
           <tag><c>monitor</c></tag>
           <item>
             <p>Monitor the new process (just like
-              <seealso marker="#monitor/2">erlang:monitor/2</seealso> does).</p>
+              <seealso marker="#monitor/2">monitor/2</seealso> does).</p>
           </item>
-          <tag><c>{priority, Level}</c></tag>
+          <tag><c>{priority, <anno>Level</anno>}</c></tag>
           <item>
             <p>Sets the priority of the new process. Equivalent to
               executing
-              <seealso marker="#process_flag_priority">process_flag(priority, Level)</seealso> in the start function of the new process,
+              <seealso marker="#process_flag_priority">process_flag(priority, <anno>Level</anno>)</seealso> in the start function of the new process,
               except that the priority will be set before the process is
               selected for execution for the first time. For more information
               on priorities see
               <seealso marker="#process_flag_priority">process_flag(priority, Level)</seealso>.</p>
           </item>
-          <tag><c>{fullsweep_after, Number}</c></tag>
+          <tag><c>{fullsweep_after, <anno>Number</anno>}</c></tag>
           <item>
             <p>This option is only useful for performance tuning.
               In general, you should not use this option unless you
@@ -4643,18 +4796,18 @@ true</pre>
             <p>Here are a few cases when it could be useful to change
               <c>fullsweep_after</c>. Firstly, if binaries that are no
               longer used should be thrown away as soon as possible.
-              (Set <c>Number</c> to zero.) Secondly, a process that
+              (Set <c><anno>Number</anno></c> to zero.) Secondly, a process that
               mostly have short-lived data will be fullsweeped seldom
               or never, meaning that the old heap will contain mostly
               garbage. To ensure a fullsweep once in a while, set
-              <c>Number</c> to a suitable value such as 10 or 20.
+              <c><anno>Number</anno></c> to a suitable value such as 10 or 20.
               Thirdly, in embedded systems with limited amount of RAM
               and no virtual memory, one might want to preserve memory
-              by setting <c>Number</c> to zero. (The value may be set
+              by setting <c><anno>Number</anno></c> to zero. (The value may be set
               globally, see
               <seealso marker="#system_flag/2">erlang:system_flag/2</seealso>.)</p>
           </item>
-          <tag><c>{min_heap_size, Size}</c></tag>
+          <tag><c>{min_heap_size, <anno>Size</anno>}</c></tag>
           <item>
             <p>This option is only useful for performance tuning.
               In general, you should not use this option unless you
@@ -4669,9 +4822,9 @@ true</pre>
               slow down the system due to worse data locality.
               Therefore, it is recommended to use this option only for
               fine-tuning an application and to measure the execution
-              time with various <c>Size</c> values.</p>
+              time with various <c><anno>Size</anno></c> values.</p>
           </item>
-          <tag><c>{min_bin_vheap_size, VSize}</c></tag>
+          <tag><c>{min_bin_vheap_size, <anno>VSize</anno>}</c></tag>
           <item>
             <p>This option is only useful for performance tuning.
               In general, you should not use this option unless you
@@ -4685,29 +4838,19 @@ true</pre>
               Setting too high value, however, might waste memory.
               Therefore, it is recommended to use this option only for
               fine-tuning an application and to measure the execution
-              time with various <c>VSize</c> values.</p>
+              time with various <c><anno>VSize</anno></c> values.</p>
           </item>
 
         </taglist>
       </desc>
     </func>
     <func>
-      <name>spawn_opt(Node, Module, Function, Args, [Option]) -> pid()</name>
+      <name name="spawn_opt" arity="5"/>
       <fsummary>Create a new process with a function as entry point on a given node</fsummary>
-      <type>
-        <v>Node = node()</v>
-        <v>Module = Function = atom()</v>
-        <v>Args = [term()]</v>
-        <v>Option = link | {priority, Level} | {fullsweep_after, Number} | {min_heap_size, Size} | {min_bin_vheap_size, VSize}</v>
-        <v>&nbsp;Level = low | normal | high</v>
-        <v>&nbsp;Number = int()</v>
-        <v>&nbsp;Size = int()</v>
-        <v>&nbsp;VSize = int()</v>
-      </type>
       <desc>
         <p>Returns the pid of a new process started by the application
-          of <c>Module:Function</c> to <c>Args</c> on <c>Node</c>. If
-          <c>Node</c> does not exist, a useless pid is returned.
+          of <c><anno>Module</anno>:<anno>Function</anno></c> to <c><anno>Args</anno></c> on <c>Node</c>. If
+          <c><anno>Node</anno></c> does not exist, a useless pid is returned.
           Otherwise works like
           <seealso marker="#spawn_opt/4">spawn_opt/4</seealso>.</p>
       </desc>
@@ -4741,13 +4884,13 @@ true</pre>
       <name>erlang:start_timer(Time, Dest, Msg) -> TimerRef</name>
       <fsummary>Start a timer</fsummary>
       <type>
-        <v>Time = int()</v>
+        <v>Time = integer() >= 0</v>
         <v>&nbsp;0 &lt;= Time &lt;= 4294967295</v>
         <v>Dest = LocalPid | RegName </v>
         <v>&nbsp;LocalPid = pid() (of a process, alive or dead, on the local node)</v>
         <v>&nbsp;RegName = atom()</v>
         <v>Msg = term()</v>
-        <v>TimerRef = ref()</v>
+        <v>TimerRef = reference()</v>
       </type>
       <desc>
         <p>Starts a timer which will send the message
@@ -4778,6 +4921,7 @@ true</pre>
         <v>Type, Res -- see below</v>
       </type>
       <desc>
+	        <p>All times are in milliseconds unless otherwise specified.</p>
         <p>Returns information about the system as specified by
           <c>Type</c>:</p>
         <taglist>
@@ -4791,15 +4935,20 @@ true</pre>
           <item>
             <p>Returns
               <c>{Total_Exact_Reductions, Exact_Reductions_Since_Last_Call}</c>.</p>
-            <p><em>NOTE:</em><c>statistics(exact_reductions)</c> is
-              a more expensive operation than
-              <seealso marker="#statistics_reductions">statistics(reductions)</seealso>
-              especially on an Erlang machine with SMP support.</p>
+		  <note><p><c>statistics(exact_reductions)</c> is
+				  a more expensive operation than
+				  <seealso marker="#statistics_reductions">statistics(reductions)</seealso>
+				  especially on an Erlang machine with SMP support.</p>
+		  </note>
           </item>
           <tag><c>garbage_collection</c></tag>
           <item>
             <p>Returns <c>{Number_of_GCs, Words_Reclaimed, 0}</c>. This
               information may not be valid for all implementations.</p>
+			<pre>
+> <input>statistics(garbage_collection).</input>
+{85,23961,0}
+</pre>
           </item>
           <tag><c>io</c></tag>
           <item>
@@ -4811,12 +4960,18 @@ true</pre>
           <tag><marker id="statistics_reductions"><c>reductions</c></marker></tag>
           <item>
             <p>Returns
-              <c>{Total_Reductions, Reductions_Since_Last_Call}</c>.</p>
-            <p><em>NOTE:</em> From erts version 5.5 (OTP release R11B)
-              this value does not include reductions performed in current
-              time slices of currently scheduled processes. If an
-              exact value is wanted, use
-              <seealso marker="#statistics_exact_reductions">statistics(exact_reductions)</seealso>.</p>
+				<c>{Total_Reductions, Reductions_Since_Last_Call}</c>.</p>
+			<note>
+				<p>From erts version 5.5 (OTP release R11B)
+					this value does not include reductions performed in current
+					time slices of currently scheduled processes. If an
+					exact value is wanted, use
+					<seealso marker="#statistics_exact_reductions">statistics(exact_reductions)</seealso>.</p>
+			</note>
+        <pre>
+> <input>statistics(reductions).</input>
+{2046,11}
+</pre>
           </item>
           <tag><c>run_queue</c></tag>
           <item>
@@ -4829,7 +4984,74 @@ true</pre>
 	    Note that the run-time is the sum of the run-time for all
 	    threads in the Erlang run-time system and may therefore be greater
 	    than the wall-clock time.</p>
+        <pre>
+> <input>statistics(runtime).</input>
+{1690,1620}
+</pre>
+          </item>
+          <tag><marker id="statistics_scheduler_wall_time"><c>scheduler_wall_time</c></marker></tag>
+          <item>
+            <p>Returns a list of tuples with 
+	    <c>{SchedulerId, ActiveTime, TotalTime}</c>, where <c>SchedulerId</c> is an integer id of the scheduler, <c>ActiveTime</c> is
+		 the duration the scheduler has been busy, <c>TotalTime</c> is the total time duration since 
+	    <seealso marker="#system_flag_scheduler_wall_time">scheduler_wall_time</seealso>
+		activation. The time unit is not defined and may be subject to change
+		between releases, operating systems and system restarts.
+		<c>scheduler_wall_time</c> should only be used to calculate relative
+		values for scheduler-utilization. <c>ActiveTime</c> can never exceed <c>TotalTime</c>.
+		</p>
+
+		<p>The definition of a busy scheduler is when it is not idle or not
+			scheduling (selecting) a process or port, meaning; executing process
+			code, executing linked-in-driver or NIF code, executing
+			built-in-functions or any other runtime handling, garbage collecting
+			or handling any other memory management. Note, a scheduler may also be
+			busy even if the operating system has scheduled out the scheduler
+			thread.
+		</p>
+
+		<p>
+	    Returns <c>undefined</c> if the system flag <seealso marker="#system_flag_scheduler_wall_time">
+	    scheduler_wall_time</seealso> is turned off.
+	    </p>
+
+	    <p>The list of scheduler information is unsorted and may appear in different order
+	    between calls. 
+	    </p>
+		<p>Using <c>scheduler_wall_time</c> to calculate scheduler utilization.</p>
+<pre>
+> <input>erlang:system_flag(scheduler_wall_time, true).</input>
+false
+> <input>Ts0 = lists:sort(erlang:statistics(scheduler_wall_time)), ok.</input>
+ok
+</pre>
+		<p>Some time later we will take another snapshot and calculate scheduler-utilization per scheduler.</p>
+<pre>
+> <input>Ts1 = lists:sort(erlang:statistics(scheduler_wall_time)), ok.</input>
+ok
+> <input>lists:map(fun({{I, A0, T0}, {I, A1, T1}}) ->
+	{I, (A1 - A0)/(T1 - T0)} end, lists:zip(Ts0,Ts1)).</input>
+[{1,0.9743474730177548},
+ {2,0.9744843782751444},
+ {3,0.9995902361669045},
+ {4,0.9738012596572161},
+ {5,0.9717956667018103},
+ {6,0.9739235846420741},
+ {7,0.973237033077876},
+ {8,0.9741297293248656}]
+</pre>
+		<p>Using the same snapshots to calculate a total scheduler-utilization.</p>
+<pre>
+> <input>{A, T} = lists:foldl(fun({{_, A0, T0}, {_, A1, T1}}, {Ai,Ti}) ->
+	{Ai + (A1 - A0), Ti + (T1 - T0)} end, {0, 0}, lists:zip(Ts0,Ts1)), A/T.</input>
+0.9769136803764825
+</pre>
+ 
+        <note>
+          <p><c>scheduler_wall_time</c> is by default disabled. Use <c>erlang:system_flag(scheduler_wall_time, true)</c> to enable it. </p>
+        </note>
           </item>
+
           <tag><c>wall_clock</c></tag>
           <item>
             <p>Returns
@@ -4839,18 +5061,10 @@ true</pre>
               opposed to runtime or CPU time.</p>
           </item>
         </taglist>
-        <p>All times are in milliseconds.</p>
-        <pre>
-> <input>statistics(runtime).</input>
-{1690,1620}
-> <input>statistics(reductions).</input>
-{2046,11}
-> <input>statistics(garbage_collection).</input>
-{85,23961,0}</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:suspend_process(Suspendee, OptList) -> true | false</name>
+      <name>erlang:suspend_process(Suspendee, OptList) -> boolean()</name>
       <fsummary>Suspend a process</fsummary>
       <type>
         <v>Suspendee = pid()</v>
@@ -4950,15 +5164,12 @@ true</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:suspend_process(Suspendee) -> true</name>
+      <name name="suspend_process" arity="1"/>
       <fsummary>Suspend a process</fsummary>
-      <type>
-        <v>Suspendee = pid()</v>
-      </type>
       <desc>
-        <p>Suspends the process identified by <c>Suspendee</c>. The
+        <p>Suspends the process identified by <c><anno>Suspendee</anno></c>. The
 	same as calling
-	<seealso marker="#suspend_process/2">erlang:suspend_process(Suspendee, [])</seealso>. For more information see the documentation of <seealso marker="#suspend_process/2">erlang:suspend_process/2</seealso>.
+	<seealso marker="#suspend_process/2">erlang:suspend_process(<anno>Suspendee</anno>, [])</seealso>. For more information see the documentation of <seealso marker="#suspend_process/2">erlang:suspend_process/2</seealso>.
 	</p>
         <warning>
           <p>This BIF is intended for debugging only.</p>
@@ -4972,6 +5183,14 @@ true</pre>
         <v>Flag, Value, OldValue -- see below</v>
       </type>
       <desc>
+        <warning>
+          <p>The
+	  <seealso marker="#system_flag_cpu_topology">cpu_topology</seealso>,
+	  and
+	  <seealso marker="#system_flag_scheduler_bind_type">scheduler_bind_type</seealso>
+	  <c>Flag</c>s are <em>deprecated</em> and have been scheduled for
+	  removal in erts-5.10/OTP-R16.</p>
+        </warning>
         <p>Sets various system properties of the Erlang node. Returns
           the old value of the flag.</p>
         <taglist>
@@ -4982,6 +5201,12 @@ true</pre>
           </item>
           <tag><marker id="system_flag_cpu_topology"><c>erlang:system_flag(cpu_topology, CpuTopology)</c></marker></tag>
           <item>
+	    <p><em>NOTE:</em> This argument is <em>deprecated</em> and
+	    scheduled for removal in erts-5.10/OTP-R16. Instead of using
+	    this argument you are advised to use the <c>erl</c> command
+	    line argument <seealso marker="erts:erl#+sct">+sct</seealso>.
+	    When this argument has been removed a final CPU topology to use
+	    will be determined at emulator boot time.</p>
             <p>Sets the user defined <c>CpuTopology</c>. The user defined
 	       CPU topology will override any automatically detected
 	       CPU topology. By passing <c>undefined</c> as <c>CpuTopology</c>
@@ -4996,15 +5221,15 @@ true</pre>
 	       to rebind according to the new CPU topology.
 	    </p>
 	    <p>The user defined CPU topology can also be set by passing
-	       the <seealso marker="erl#+sct">+sct</seealso> command
+	       the <seealso marker="erts:erl#+sct">+sct</seealso> command
 	       line argument to <c>erl</c>.
 	    </p>
 	    <p>For information on the <c>CpuTopology</c> type
 	       and more, see the documentation of
 	       <seealso marker="#system_info_cpu_topology">erlang:system_info(cpu_topology)</seealso>,
-	       the <c>erl</c> <seealso marker="erl#+sct">+sct</seealso>
-	       emulator flag, and
-	       <seealso marker="#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, How)</seealso>.
+	       and the <c>erl</c> <seealso marker="erts:erl#+sct">+sct</seealso>
+	       and <seealso marker="erts:erl#+sbt">+sbt</seealso>
+	       command line flags.
 	    </p>
           </item>
           <tag><c>erlang:system_flag(fullsweep_after, Number)</c></tag>
@@ -5080,6 +5305,12 @@ true</pre>
           </item>
           <tag><marker id="system_flag_scheduler_bind_type"><c>erlang:system_flag(scheduler_bind_type, How)</c></marker></tag>
           <item>
+	    <p><em>NOTE:</em> This argument is <em>deprecated</em> and
+	    scheduled for removal in erts-5.10/OTP-R16. Instead of using
+	    this argument you are advised to use the <c>erl</c> command
+	    line argument <seealso marker="erts:erl#+sbt">+sbt</seealso>.
+	    When this argument has been removed a final scheduler bind type
+	    to use will be determined at emulator boot time.</p>
 	    <p>Controls if and how schedulers are bound to logical
 	       processors.</p>
 	    <p>When <c>erlang:system_flag(scheduler_bind_type, How)</c> is
@@ -5093,90 +5324,69 @@ true</pre>
 	       schedulers actually have bound as requested, call
 	       <seealso marker="#system_info_scheduler_bindings">erlang:system_info(scheduler_bindings)</seealso>.
 	    </p>
-            <p>Schedulers can currently only be bound on newer Linux
-	       and Solaris systems, but more systems will be supported
-	       in the future.
+            <p>Schedulers can currently only be bound on newer Linux,
+	       Solaris, FreeBSD, and Windows systems, but more systems will be
+	       supported in the future.
 	    </p>
             <p>In order for the runtime system to be able to bind schedulers,
 	       the CPU topology needs to be known. If the runtime system fails
 	       to automatically detect the CPU topology, it can be defined.
 	       For more information on how to define the CPU topology, see 
-	       <seealso marker="#system_flag_cpu_topology">erlang:system_flag(cpu_topology, CpuTopology)</seealso>.
+	       the <c>erl</c> <seealso marker="erts:erl#+sct">+sct</seealso> command
+	       line flag.
+	    </p>
+            <p>The runtime system will by default <em>not</em> bind schedulers
+	       to logical processors.
 	    </p>
-	    <p><em>NOTE:</em> If other programs on the system have bound
-	       to processors, e.g. another Erlang runtime system, you
-	       may loose performance when binding schedulers. Therefore,
-	       schedulers are by default not bound.</p>
+	    <p><em>NOTE:</em> If the Erlang runtime system is the only
+	       operating system process that binds threads to logical processors,
+	       this improves the performance of the runtime system. However,
+	       if other operating system processes (as for example another Erlang
+	       runtime system) also bind threads to logical processors, there
+	       might be a performance penalty instead. In some cases this
+	       performance penalty might be severe. If this is the case, you
+	       are advised to not bind the schedulers.</p>
 	    <p>Schedulers can be bound in different ways. The <c>How</c>
 	       argument determines how schedulers are bound. <c>How</c> can
 	       currently be one of:</p>
             <taglist>
               <tag><c>unbound</c></tag>
-              <item>
-                <p>Schedulers will not be bound to logical processors, i.e.,
-		   the operating system decides where the scheduler threads
-		   execute, and when to migrate them. This is the default.</p>
-              </item>
+              <item><p>Same as the <c>erl</c> command line argument
+	      <seealso marker="erts:erl#+sbt">+sbt u</seealso>.
+	      </p></item>
               <tag><c>no_spread</c></tag>
-              <item>
-                <p>Schedulers with close scheduler identifiers will be bound
-		   as close as possible in hardware.</p>
-              </item>
+              <item><p>Same as the <c>erl</c> command line argument
+	      <seealso marker="erts:erl#+sbt">+sbt ns</seealso>.
+	      </p></item>
               <tag><c>thread_spread</c></tag>
-              <item>
-                <p>Thread refers to hardware threads (e.g. Intels
-		   hyper-threads). Schedulers with low scheduler identifiers,
-		   will be bound to the first hardware thread of each core,
-		   then schedulers with higher scheduler identifiers will be
-		   bound to the second hardware thread of each core, etc.</p>
-              </item>
+              <item><p>Same as the <c>erl</c> command line argument
+	      <seealso marker="erts:erl#+sbt">+sbt ts</seealso>.
+	      </p></item>
               <tag><c>processor_spread</c></tag>
-              <item>
-                <p>Schedulers will be spread like <c>thread_spread</c>, but
-		   also over physical processor chips.</p>
-              </item>
+              <item><p>Same as the <c>erl</c> command line argument
+	      <seealso marker="erts:erl#+sbt">+sbt ps</seealso>.
+	      </p></item>
               <tag><c>spread</c></tag>
-              <item>
-                <p>Schedulers will be spread as much as possible.</p>
-              </item>
+              <item><p>Same as the <c>erl</c> command line argument
+	      <seealso marker="erts:erl#+sbt">+sbt s</seealso>.
+	      </p></item>
               <tag><c>no_node_thread_spread</c></tag>
-              <item>
-                <p>Like <c>thread_spread</c>, but if multiple NUMA
-		   (Non-Uniform Memory Access) nodes exists,
-		   schedulers will be spread over one NUMA node at a time,
-		   i.e., all logical processors of one NUMA node will
-		   be bound to schedulers in sequence.</p>
-              </item>
+              <item><p>Same as the <c>erl</c> command line argument
+	      <seealso marker="erts:erl#+sbt">+sbt nnts</seealso>.
+	      </p></item>
               <tag><c>no_node_processor_spread</c></tag>
-              <item>
-                <p>Like <c>processor_spread</c>, but if multiple NUMA
-		   nodes exists, schedulers will be spread over one
-		   NUMA node at a time, i.e., all logical processors of
-		   one NUMA node will be bound to schedulers in sequence.</p>
-              </item>
+              <item><p>Same as the <c>erl</c> command line argument
+	      <seealso marker="erts:erl#+sbt">+sbt nnps</seealso>.
+	      </p></item>
               <tag><c>thread_no_node_processor_spread</c></tag>
-              <item>
-                <p>A combination of <c>thread_spread</c>, and
-		<c>no_node_processor_spread</c>. Schedulers will be
-		spread over hardware threads across NUMA nodes, but
-		schedulers will only be spread over processors internally
-		in one NUMA node at a time.</p>
-              </item>
+              <item><p>Same as the <c>erl</c> command line argument
+	      <seealso marker="erts:erl#+sbt">+sbt tnnps</seealso>.
+	      </p></item>
               <tag><c>default_bind</c></tag>
-              <item>
-                <p>Binds schedulers the default way. Currently the default
-		is <c>thread_no_node_processor_spread</c> (which might change
-		in the future).</p>
-              </item>
+              <item><p>Same as the <c>erl</c> command line argument
+	      <seealso marker="erts:erl#+sbt">+sbt db</seealso>.
+	      </p></item>
 	    </taglist>
-            <p>How schedulers are bound matters. For example, in
-	       situations when there are fewer running processes than
-	       schedulers online, the runtime system tries to migrate
-	       processes to schedulers with low scheduler identifiers.
-	       The more the schedulers are spread over the hardware,
-	       the more resources will be available to the runtime
-	       system in such situations.
-	    </p>
 	    <p>The value returned equals <c>How</c> before the
 	       <c>scheduler_bind_type</c> flag was changed.</p>
 	    <p>Failure:</p>
@@ -5195,17 +5405,25 @@ true</pre>
               </item>
             </taglist>
 	    <p>The scheduler bind type can also be set by passing
-	       the <seealso marker="erl#+sbt">+sbt</seealso> command
+	       the <seealso marker="erts:erl#+sbt">+sbt</seealso> command
 	       line argument to <c>erl</c>.
 	    </p>
 	    <p>For more information, see
 	       <seealso marker="#system_info_scheduler_bind_type">erlang:system_info(scheduler_bind_type)</seealso>,
 	       <seealso marker="#system_info_scheduler_bindings">erlang:system_info(scheduler_bindings)</seealso>,
-	       the <c>erl</c> <seealso marker="erl#+sbt">+sbt</seealso>
-	       emulator flag, and
-	       <seealso marker="#system_flag_cpu_topology">erlang:system_flag(cpu_topology, CpuTopology)</seealso>.
+	       the <c>erl</c> <seealso marker="erts:erl#+sbt">+sbt</seealso>
+	       and <seealso marker="erts:erl#+sct">+sct</seealso> command line
+	       flags.
+	    </p>
+          </item>
+	  <tag><marker id="system_flag_scheduler_wall_time"><c>erlang:system_flag(scheduler_wall_time, Boolean)</c></marker></tag>
+          <item>
+            <p>Turns on/off scheduler wall time measurements. </p>
+	    <p>For more information see,
+	    <seealso marker="#statistics_scheduler_wall_time">erlang:statistics(scheduler_wall_time)</seealso>.
 	    </p>
           </item>
+
           <tag><marker id="system_flag_schedulers_online"><c>erlang:system_flag(schedulers_online, SchedulersOnline)</c></marker></tag>
           <item>
             <p>Sets the amount of schedulers online. Valid range is
@@ -5217,6 +5435,7 @@ true</pre>
 	    <seealso marker="#system_info_schedulers_online">erlang:system_info(schedulers_online)</seealso>.
 	    </p>
           </item>
+
           <tag><c>erlang:system_flag(trace_control_word, TCW)</c></tag>
           <item>
             <p>Sets the value of the node's trace control word to
@@ -5271,8 +5490,8 @@ true</pre>
             <p>Returns <c>{Allocator, Version, Features, Settings}.</c></p>
             <p>Types:</p>
             <list type="bulleted">
-              <item><c>Allocator = undefined | elib_malloc | glibc</c></item>
-              <item><c>Version = [int()]</c></item>
+              <item><c>Allocator = undefined | glibc</c></item>
+              <item><c>Version = [integer()]</c></item>
               <item><c>Features = [atom()]</c></item>
               <item><c>Settings = [{Subsystem, [{Parameter, Value}]}]</c></item>
               <item><c>Subsystem = atom()</c></item>
@@ -5286,7 +5505,7 @@ true</pre>
                   implementation used. If <c>Allocator</c> equals
                   <c>undefined</c>, the <c>malloc()</c> implementation
                   used could not be identified. Currently
-                  <c>elib_malloc</c> and <c>glibc</c> can be identified.</p>
+                  <c>glibc</c> can be identified.</p>
               </item>
               <item>
                 <p><c>Version</c> is a list of integers (but not a
@@ -5363,6 +5582,16 @@ true</pre>
 	    <seealso marker="#system_info_allocator_tuple">erlang:system_info({allocator, Alloc})</seealso>.
 	    </p>
           </item>
+          <tag><c>build_type</c></tag>
+          <item>
+            <p>Returns an atom describing the build type of the runtime
+	       system. This is normally the atom <c>opt</c> for optimized.
+               Other possible return values are <c>debug</c>, <c>purify</c>,
+	       <c>quantify</c>, <c>purecov</c>, <c>gcov</c>, <c>valgrind</c>,
+	       <c>gprof</c>, and <c>lcnt</c>. Possible return values
+	       may be added and/or removed at any time without prior notice.
+	     </p>
+          </item>
           <tag><c>c_compiler_used</c></tag>
           <item>
             <p>Returns a two-tuple describing the C compiler used when
@@ -5394,10 +5623,12 @@ true</pre>
           <item>
 	    <p>Returns the <c>CpuTopology</c> which currently is used by the
 	       emulator. The CPU topology is used when binding schedulers
-	       to logical processors. The CPU topology used is the user defined
-	       CPU topology if such exist; otherwise, the automatically
-	       detected CPU topology if such exist. If no CPU topology
-	       exist <c>undefined</c> is returned.</p>
+	       to logical processors. The CPU topology used is the
+	       <seealso marker="erlang#system_info_cpu_topology_defined">user
+	       defined CPU topology</seealso> if such exists; otherwise, the
+	       <seealso marker="erlang#system_info_cpu_topology_detected">automatically
+	       detected CPU topology</seealso> if such exists. If no CPU topology
+	       exists, <c>undefined</c> is returned.</p>
             <p>Types:</p>
             <list type="bulleted">
               <item><c>CpuTopology = LevelEntryList | undefined</c></item>
@@ -5440,22 +5671,24 @@ true</pre>
 	       <c>CpuTopology</c> type to change.
 	    </p>
           </item>
-          <tag><c>{cpu_topology, defined}</c></tag>
+          <tag><marker id="system_info_cpu_topology_defined"><c>{cpu_topology, defined}</c></marker></tag>
           <item>
             <p>Returns the user defined <c>CpuTopology</c>. For more
 	       information see the documentation of
-	       <seealso marker="#system_flag_cpu_topology">erlang:system_flag(cpu_topology, CpuTopology)</seealso>
-               and the documentation of the
+	       the <c>erl</c> <seealso marker="erts:erl#+sct">+sct</seealso> command
+	       line flag, and the documentation of the
 	       <seealso marker="#system_info_cpu_topology">cpu_topology</seealso>
 	       argument.
 	    </p>
           </item>
-          <tag><c>{cpu_topology, detected}</c></tag>
+          <tag><marker id="system_info_cpu_topology_detected"><c>{cpu_topology, detected}</c></marker></tag>
           <item>
             <p>Returns the automatically detected <c>CpuTopology</c>. The
 	       emulator currently only detects the CPU topology on some newer
-	       linux and solaris systems. For more information see the 
-               documentation of the
+	       Linux, Solaris, FreeBSD, and Windows systems. On Windows system with
+	       more than 32 logical processors the CPU topology is not detected.
+	    </p>
+            <p>For more information see the documentation of the
 	       <seealso marker="#system_info_cpu_topology">cpu_topology</seealso>
 	       argument.
 	    </p>
@@ -5511,58 +5744,45 @@ true</pre>
               used by the runtime system. It will be on the form
               <seealso marker="erts:erl_driver#version_management">"&lt;major ver&gt;.&lt;minor ver&gt;"</seealso>.</p>
           </item>
+          <tag><c>dynamic_trace</c></tag>
+          <item>
+            <p>Returns an atom describing the dynamic trace framework
+            compiled into the virtual machine. It can currently be either
+            <c>dtrace</c>, <c>systemtap</c> or <c>none</c>. For a
+            commercial or standard build, this is always <c>none</c>,
+            the other return values indicate a custom configuration
+            (e.g. <c>./configure --with-dynamic-trace=dtrace</c>). See
+            the <seealso marker="runtime_tools:dyntrace">dyntrace
+            </seealso> manual page and the
+            <c>README.dtrace</c>/<c>README.systemtap</c> files in the
+            Erlang source code top directory for more information
+            about dynamic tracing.</p>
+          </item>
+          <tag><c>dynamic_trace_probes</c></tag>
+          <item>
+            <p>Returns a <c>boolean()</c> indicating if dynamic trace probes
+            (either dtrace or systemtap) are built into the
+            emulator. This can only be <c>true</c> if the virtual
+            machine was built for dynamic tracing
+            (i.e. <c>system_info(dynamic_trace)</c> returns
+            <c>dtrace</c> or <c>systemtap</c>).</p>
+          </item>
           <tag><c>elib_malloc</c></tag>
           <item>
-            <p>If the emulator uses the <c>elib_malloc</c> memory
-              allocator, a list of two-element tuples containing status
-              information is returned; otherwise, <c>false</c> is
-              returned. The list currently contains the following
-              two-element tuples (all sizes are presented in bytes):</p>
-            <taglist>
-              <tag><c>{heap_size, Size}</c></tag>
-              <item>
-                <p>Where <c>Size</c> is the current heap size.</p>
-              </item>
-              <tag><c>{max_alloced_size, Size}</c></tag>
-              <item>
-                <p>Where <c>Size</c> is the maximum amount of memory
-                  allocated on the heap since the emulator started.</p>
-              </item>
-              <tag><c>{alloced_size, Size}</c></tag>
-              <item>
-                <p>Where <c>Size</c> is the current amount of memory
-                  allocated on the heap.</p>
-              </item>
-              <tag><c>{free_size, Size}</c></tag>
-              <item>
-                <p>Where <c>Size</c> is the current amount of free
-                  memory on the heap.</p>
-              </item>
-              <tag><c>{no_alloced_blocks, No}</c></tag>
-              <item>
-                <p>Where <c>No</c> is the current number of allocated
-                  blocks on the heap.</p>
-              </item>
-              <tag><c>{no_free_blocks, No}</c></tag>
-              <item>
-                <p>Where <c>No</c> is the current number of free blocks
-                  on the heap.</p>
-              </item>
-              <tag><c>{smallest_alloced_block, Size}</c></tag>
-              <item>
-                <p>Where <c>Size</c> is the size of the smallest
-                  allocated block on the heap.</p>
-              </item>
-              <tag><c>{largest_free_block, Size}</c></tag>
-              <item>
-                <p>Where <c>Size</c> is the size of the largest free
-                  block on the heap.</p>
-              </item>
-            </taglist>
+            <p>This option will be removed in a future release.
+	      The return value will always be <c>false</c> since
+	      the elib_malloc allocator has been removed.</p>
+          </item>
+          <tag><marker id="system_info_dist_buf_busy_limit"><c>dist_buf_busy_limit</c></marker></tag>
+          <item>
+            <p>Returns the value of the distribution buffer busy limit
+	    in bytes. This limit can be set on startup by passing the
+	    <seealso marker="erts:erl#+zdbbl">+zdbbl</seealso> command line
+	    flag to <c>erl</c>.</p>
           </item>
           <tag><c>fullsweep_after</c></tag>
           <item>
-            <p>Returns <c>{fullsweep_after, int()}</c> which is the
+            <p>Returns <c>{fullsweep_after, integer()}</c> which is the
               <c>fullsweep_after</c> garbage collection setting used
               by default. For more information see
               <c>garbage_collection</c> described below.</p>
@@ -5579,10 +5799,6 @@ true</pre>
               can spawn a process that does not use the default
               settings.</p>
           </item>
-          <tag><c>global_heaps_size</c></tag>
-          <item>
-            <p>Returns the current size of the shared (global) heap.</p>
-          </item>
           <tag><c>heap_sizes</c></tag>
           <item>
             <p>Returns a list of integers representing valid heap sizes 
@@ -5592,7 +5808,7 @@ true</pre>
           <tag><c>heap_type</c></tag>
           <item>
             <p>Returns the heap type used by the current emulator.
-              Currently the following heap types exist:</p>
+              Currently only the following heap type exists:</p>
             <taglist>
               <tag><c>private</c></tag>
               <item>
@@ -5601,17 +5817,6 @@ true</pre>
                   allowed. Messages passed between processes are copied
                   between heaps.</p>
               </item>
-              <tag><c>shared</c></tag>
-              <item>
-                <p>One heap for use by all processes. Messages passed
-                  between processes are passed by reference.</p>
-              </item>
-              <tag><c>hybrid</c></tag>
-              <item>
-                <p>A hybrid of the <c>private</c> and <c>shared</c> heap
-                  types. A shared heap as well as private heaps are
-                  used.</p>
-              </item>
             </taglist>
           </item>
           <tag><c>info</c></tag>
@@ -5634,11 +5839,34 @@ true</pre>
               information see the <seealso marker="erts:crash_dump">"How to interpret the Erlang crash dumps"</seealso> chapter
               in the ERTS User's Guide.</p>
           </item>
-	  <tag><c>logical_processors</c></tag>
+	  <tag><marker id="logical_processors"><c>logical_processors</c></marker></tag>
+          <item>
+            <p>Returns the detected number of logical processors configured
+	       on the system. The return value is either an integer, or
+	       the atom <c>unknown</c> if the emulator wasn't able to
+	       detect logical processors configured.
+	    </p>
+          </item>
+	  <tag><marker id="logical_processors_available"><c>logical_processors_available</c></marker></tag>
           <item>
-            <p>Returns the number of logical processors detected on the
-	       system as an integer or the atom <c>unknown</c> if the
-	       emulator wasn't able to detect any.
+            <p>Returns the detected number of logical processors available to
+	       the Erlang runtime system. The return value is either an
+	       integer, or the atom <c>unknown</c> if the emulator wasn't
+	       able to detect logical processors available. The number
+	       of logical processors available is less than or equal to
+	       the number of <seealso marker="#logical_processors_online">logical
+	       processors online</seealso>.
+	    </p>
+          </item>
+	  <tag><marker id="logical_processors_online"><c>logical_processors_online</c></marker></tag>
+          <item>
+            <p>Returns the detected number of logical processors online on
+	       the system. The return value is either an integer,
+	       or the atom <c>unknown</c> if the emulator wasn't able to
+	       detect logical processors online. The number of logical
+	       processors online is less than or equal to the number of
+	       <seealso marker="#logical_processors">logical processors
+	       configured</seealso>.
 	    </p>
           </item>
           <tag><c>machine</c></tag>
@@ -5736,14 +5964,13 @@ true</pre>
 	    <p>Returns information on how user has requested
 	       schedulers to be bound or not bound.</p>
 	    <p><em>NOTE:</em> Even though user has requested
-	       schedulers to be bound via
-	       <seealso marker="#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, How)</seealso>,
-	       they might have silently failed to bind. In order to
-	       inspect actual scheduler bindings call
+	       schedulers to be bound, they might have silently failed
+	       to bind. In order to inspect actual scheduler bindings call
 	       <seealso marker="#system_info_scheduler_bindings">erlang:system_info(scheduler_bindings)</seealso>.
 	    </p>
 	    <p>For more information, see
-	       <seealso marker="#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, How)</seealso>, and
+	       the <c>erl</c> <seealso marker="erts:erl#+sbt">+sbt</seealso>
+	       command line argument, and
 	       <seealso marker="#system_info_scheduler_bindings">erlang:system_info(scheduler_bindings)</seealso>.
 	    </p>
           </item>
@@ -5766,7 +5993,8 @@ true</pre>
 	    <p>Note that only schedulers online can be bound to logical
 	       processors.</p>
 	    <p>For more information, see
-	       <seealso marker="#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, How)</seealso>,
+	       the <c>erl</c> <seealso marker="erts:erl#+sbt">+sbt</seealso>
+	       command line argument,
 	       <seealso marker="#system_info_schedulers_online">erlang:system_info(schedulers_online)</seealso>.
 	    </p>
           </item>
@@ -5843,6 +6071,26 @@ true</pre>
               <c>get_tcw</c> in "Match Specifications in Erlang",
               <seealso marker="erts:match_spec#get_tcw">ERTS User's Guide</seealso>.</p>
           </item>
+          <tag><marker id="update_cpu_info"><c>update_cpu_info</c></marker></tag>
+          <item>
+            <p>The runtime system rereads the CPU information available and
+	       updates its internally stored information about the
+	       <seealso marker="#system_info_cpu_topology_detected">detected CPU
+	       topology</seealso> and the amount of logical processors
+	       <seealso marker="#logical_processors">configured</seealso>,
+	       <seealso marker="#logical_processors_online">online</seealso>, and
+	       <seealso marker="#logical_processors_available">available</seealso>.
+	       If the CPU information has changed since the last time it was read,
+	       the atom <c>changed</c> is returned; otherwise, the atom
+	       <c>unchanged</c> is returned. If the CPU information has changed
+	       you probably want to
+	       <seealso marker="#system_flag_schedulers_online">adjust the amount
+	       of schedulers online</seealso>. You typically want to have as
+	       many schedulers online as
+	       <seealso marker="#logical_processors_available">logical processors
+	       available</seealso>.
+	    </p>
+          </item>
           <tag><marker id="system_info_version"><c>version</c></marker></tag>
           <item>
             <p>Returns a string containing the version number of the
@@ -5850,9 +6098,23 @@ true</pre>
           </item>
           <tag><c>wordsize</c></tag>
           <item>
-            <p>Returns the word size in bytes as an integer, i.e. on a
-              32-bit architecture 4 is returned, and on a 64-bit
-              architecture 8 is returned.</p>
+            <p>Same as <c>{wordsize, internal}.</c></p>
+          </item>
+          <tag><c>{wordsize, internal}</c></tag>
+          <item>
+            <p>Returns the size of Erlang term words in bytes as an
+              integer, i.e. on a 32-bit architecture 4 is returned,
+              and on a pure 64-bit architecture 8 is returned. On a
+              halfword 64-bit emulator, 4 is returned, as the Erlang
+              terms are stored using a virtual wordsize of half the
+              system's wordsize.</p>
+          </item>
+          <tag><c>{wordsize, external}</c></tag>
+          <item>
+	    <p>Returns the true wordsize of the emulator, i.e. the size
+	       of a pointer, in bytes as an integer. On a pure 32-bit
+	       architecture 4 is returned, on both a halfword and pure
+	       64-bit architecture, 8 is returned.</p>
           </item>
         </taglist>
         <note>
@@ -5873,7 +6135,7 @@ true</pre>
         <v>&nbsp;MonitorPid = pid()</v>
         <v>&nbsp;Options = [Option]</v>
         <v>&nbsp;&nbsp;Option = {long_gc, Time} | {large_heap, Size} | busy_port | busy_dist_port</v>
-        <v>&nbsp;&nbsp;&nbsp;Time = Size = int()</v>
+        <v>&nbsp;&nbsp;&nbsp;Time = Size = integer()</v>
       </type>
       <desc>
         <p>Returns the current system monitoring settings set by
@@ -5907,7 +6169,7 @@ true</pre>
       <type>
         <v>MonitorPid = pid()</v>
         <v>Option = {long_gc, Time} | {large_heap, Size} | busy_port | busy_dist_port</v>
-        <v>&nbsp;Time = Size = int()</v>
+        <v>&nbsp;Time = Size = integer()</v>
         <v>MonSettings = {OldMonitorPid, [Option]}</v>
         <v>&nbsp;OldMonitorPid = pid()</v>
       </type>
@@ -6137,7 +6399,7 @@ true</pre>
       <name>time() -> {Hour, Minute, Second}</name>
       <fsummary>Current time</fsummary>
       <type>
-        <v>Hour = Minute = Second = int()</v>
+        <v>Hour = Minute = Second = integer() >= 0</v>
       </type>
       <desc>
         <p>Returns the current time as <c>{Hour, Minute, Second}</c>.</p>
@@ -6165,11 +6427,11 @@ true</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:trace(PidSpec, How, FlagList) -> int()</name>
+      <name>erlang:trace(PidSpec, How, FlagList) -> integer() >= 0</name>
       <fsummary>Set trace flags for a process or processes</fsummary>
       <type>
         <v>PidSpec = pid() | existing | new | all</v>
-        <v>How = bool()</v>
+        <v>How = boolean()</v>
         <v>FlagList = [Flag]</v>
         <v>&nbsp;Flag -- see below</v>
       </type>
@@ -6570,7 +6832,7 @@ true</pre>
       <type>
         <v>PidOrFunc = pid() | new | {Module, Function, Arity} | on_load</v>
         <v>&nbsp;Module = Function = atom()</v>
-        <v>&nbsp;Arity = int()</v>
+        <v>&nbsp;Arity = arity()</v>
         <v>Item, Res -- see below</v>
       </type>
       <desc>
@@ -6645,6 +6907,17 @@ true</pre>
               See also
               <seealso marker="#trace_pattern/3">erlang:trace_pattern/3</seealso>.</p>
           </item>
+          <tag><c>call_time</c></tag>
+          <item>
+	      <p>Return the call time values for this function or
+              <c>true</c> for the pseudo function <c>on_load</c> if call
+	      time tracing is active. Returns <c>false</c> otherwise.
+	      The call time values returned, <c>[{Pid, Count, S, Us}]</c>,
+	      is a list of each process that has executed the function and its specific counters.
+              See also
+              <seealso marker="#trace_pattern/3">erlang:trace_pattern/3</seealso>.</p>
+          </item>
+
           <tag><c>all</c></tag>
           <item>
             <p>Return a list containing the <c>{Item, Value}</c> tuples
@@ -6662,7 +6935,7 @@ true</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:trace_pattern(MFA, MatchSpec) -> int()</name>
+      <name>erlang:trace_pattern(MFA, MatchSpec) -> integer() >= 0</name>
       <fsummary>Set trace patterns for global call tracing</fsummary>
       <desc>
         <p>The same as
@@ -6671,7 +6944,7 @@ true</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:trace_pattern(MFA, MatchSpec, FlagList) -> int()</name>
+      <name>erlang:trace_pattern(MFA, MatchSpec, FlagList) -> integer() >= 0</name>
       <fsummary>Set trace patterns for tracing of function calls</fsummary>
       <type>
         <v>MFA, MatchSpec, FlagList -- see below</v>
@@ -6747,13 +7020,13 @@ true</pre>
           </item>
           <tag><c>restart</c></tag>
           <item>
-            <p>For the <c>FlagList</c> option <c>call_count</c>:
+	      <p>For the <c>FlagList</c> option <c>call_count</c> and <c>call_time</c>:
               restart the existing counters. The behaviour is undefined
               for other <c>FlagList</c> options.</p>
           </item>
           <tag><c>pause</c></tag>
           <item>
-            <p>For the <c>FlagList</c> option <c>call_count</c>: pause
+	      <p>For the <c>FlagList</c> option <c>call_count</c> and <c>call_time</c>: pause
               the existing counters. The behaviour is undefined for
               other <c>FlagList</c> options.</p>
           </item>
@@ -6808,6 +7081,23 @@ true</pre>
             <p>The counter value can be read with
               <seealso marker="#trace_info/2">erlang:trace_info/2</seealso>.</p>
           </item>
+          <tag><c>call_time</c></tag>
+          <item>
+            <p>Starts (<c>MatchSpec == true</c>) or stops
+              (<c>MatchSpec == false</c>) call time tracing for all
+              types of function calls. For every function a counter is
+	      incremented when the function is called. Time spent in the function
+	      is accumulated in two other counters, seconds and micro-seconds.
+	      The counters are stored for each call traced process.</p>
+            <p>If call time tracing is started while already running,
+              the count and time is restarted from zero. Running counters can be
+              paused with <c>MatchSpec == pause</c>. Paused and running
+              counters can be restarted from zero with
+              <c>MatchSpec == restart</c>.</p>
+            <p>The counter value can be read with
+              <seealso marker="#trace_info/2">erlang:trace_info/2</seealso>.</p>
+          </item>
+
         </taglist>
         <p>The <c>global</c> and <c>local</c> options are mutually
           exclusive and <c>global</c> is the default (if no options are
@@ -6815,7 +7105,7 @@ true</pre>
           perform a kind of local tracing, and can also not be combined
           with <c>global</c>. A function can be either globally or
           locally traced. If global tracing is specified for a
-          specified set of functions; local, meta and call count
+          specified set of functions; local, meta, call time and call count
           tracing for the matching set of local functions will be
           disabled, and vice versa.</p>
         <p>When disabling trace, the option must match the type of trace
@@ -6834,7 +7124,7 @@ true</pre>
       </desc>
     </func>
     <func>
-      <name>trunc(Number) -> int()</name>
+      <name>trunc(Number) -> integer()</name>
       <fsummary>Return an integer by the truncating a number</fsummary>
       <type>
         <v>Number = number()</v>
@@ -6848,7 +7138,7 @@ true</pre>
       </desc>
     </func>
     <func>
-      <name>tuple_size(Tuple) -> int()</name>
+      <name>tuple_size(Tuple) -> integer() >= 0</name>
       <fsummary>Return the size of a tuple</fsummary>
       <type>
         <v>Tuple = tuple()</v>
@@ -6876,12 +7166,10 @@ true</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:universaltime() -> {Date, Time}</name>
+      <name>erlang:universaltime() -> DateTime</name>
       <fsummary>Current date and time according to Universal Time Coordinated (UTC)</fsummary>
       <type>
-        <v>Date = {Year, Month, Day}</v>
-        <v>Time = {Hour, Minute, Second}</v>
-        <v>&nbsp;Year = Month = Day = Hour = Minute = Second = int()</v>
+        <v>DateTime = <seealso marker="calendar#type-datetime">calendar:datetime()</seealso></v>
       </type>
       <desc>
         <p>Returns the current date and time according to Universal
@@ -6899,9 +7187,8 @@ true</pre>
       <name>erlang:universaltime_to_localtime({Date1, Time1}) -> {Date2, Time2}</name>
       <fsummary>Convert from Universal Time Coordinated (UTC) to local date and time</fsummary>
       <type>
-        <v>Date1 = Date2 = {Year, Month, Day}</v>
-        <v>Time1 = Time2 = {Hour, Minute, Second}</v>
-        <v>&nbsp;Year = Month = Day = Hour = Minute = Second = int()</v>
+        <v>Date1 = Date2 = <seealso marker="calendar#type-date">calendar:date()</seealso></v>
+        <v>Time1 = Time2 = <seealso marker="calendar#type-time">calendar:time()</seealso></v>
       </type>
       <desc>
         <p>Converts Universal Time Coordinated (UTC) date and time to
@@ -6931,7 +7218,7 @@ true</pre>
           <c>Id</c> has no effect on the caller in the future (unless
           the link is setup again). If caller is trapping exits, an
           <c>{'EXIT', Id, _}</c> message due to the link might have
-          been placed in the callers message queue prior to the call,
+          been placed in the caller's message queue prior to the call,
           though. Note, the <c>{'EXIT', Id, _}</c> message can be the
           result of the link, but can also be the result of <c>Id</c>
           calling <c>exit/2</c>. Therefore, it <em>may</em> be
@@ -6988,7 +7275,7 @@ true</pre>
       </desc>
     </func>
     <func>
-      <name>erlang:yield() -> true</name>
+      <name name="yield" arity="0"/>
       <fsummary>Let other processes get a chance to execute</fsummary>
       <desc>
         <p>Voluntarily let other processes (if any) get a chance to
diff --git a/erts/doc/src/erlc.xml b/erts/doc/src/erlc.xml
index 1e8960c22c..3358b8f115 100644
--- a/erts/doc/src/erlc.xml
+++ b/erts/doc/src/erlc.xml
@@ -4,7 +4,7 @@
 <comref>
   <header>
     <copyright>
-      <year>1997</year><year>2010</year>
+      <year>1997</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -129,18 +129,56 @@
           This option will be ignored by compilers that have a
           a single output format.</p>
       </item>
-      <tag>-hybrid</tag>
-      <item>
-        <p>Compile using the hybrid-heap emulator. This is mainly useful
-          for compiling native code, which needs to be compiled with the same
-          run-time system that it should be run on.</p>
-      </item>
       <tag>-smp</tag>
       <item>
         <p>Compile using the SMP emulator. This is mainly useful
           for compiling native code, which needs to be compiled with the same
           run-time system that it should be run on.</p>
       </item>
+      <tag>-M</tag>
+      <item>
+        <p>Produces a Makefile rule to track headers dependencies. The
+          rule is sent to stdout. No object file is produced.
+        </p>
+      </item>
+      <tag>-MF <em>Makefile</em></tag>
+      <item>
+        <p>Like the <c><![CDATA[-M]]></c> option above, except that the
+          Makefile is written to <em>Makefile</em>. No object
+          file is produced.
+        </p>
+      </item>
+      <tag>-MD</tag>
+      <item>
+        <p>Same as <c><![CDATA[-M -MF <File>.Pbeam]]></c>.
+        </p>
+      </item>
+      <tag>-MT <em>Target</em></tag>
+      <item>
+        <p>In conjunction with <c><![CDATA[-M]]></c> or
+          <c><![CDATA[-MF]]></c>, change the name of the rule emitted
+          to <em>Target</em>.
+        </p>
+      </item>
+      <tag>-MQ <em>Target</em></tag>
+      <item>
+        <p>Like the <c><![CDATA[-MT]]></c> option above, except that
+          characters special to make(1) are quoted.
+        </p>
+      </item>
+      <tag>-MP</tag>
+      <item>
+        <p>In conjunction with <c><![CDATA[-M]]></c> or
+          <c><![CDATA[-MF]]></c>, add a phony target for each dependency.
+        </p>
+      </item>
+      <tag>-MG</tag>
+      <item>
+        <p>In conjunction with <c><![CDATA[-M]]></c> or
+          <c><![CDATA[-MF]]></c>, consider missing headers as generated
+          files and add them to the dependencies.
+        </p>
+      </item>
       <tag>--</tag>
       <item>
         <p>Signals that no more options will follow.
diff --git a/erts/doc/src/erlsrv.xml b/erts/doc/src/erlsrv.xml
index 0dfad2a112..c1ecbc7b77 100644
--- a/erts/doc/src/erlsrv.xml
+++ b/erts/doc/src/erlsrv.xml
@@ -4,7 +4,7 @@
 <comref>
   <header>
     <copyright>
-      <year>1998</year><year>2010</year>
+      <year>1998</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -273,7 +273,7 @@
       </desc>
     </func>
     <func>
-      <name>erlsrv {start | stop | disable | enable} &lt;service-name></name>
+      <name>erlsrv {start | start_disabled | stop | disable | enable} &lt;service-name></name>
       <fsummary>Manipulate the current service status.</fsummary>
       <desc>
         <p>These commands are only added for convenience, the normal
@@ -287,6 +287,21 @@
           service actually is stopped. Enabling a service sets it in
           automatic mode, that is started at boot. This command cannot
           set the service to manual.  </p>
+
+	<p>The <c>start_disabled</c> command operates on a service
+	regardless of if it's enabled/disabled or started/stopped.  It
+	does this by first enabling it (regardless of if it's enabled
+	or not), then starting it (if it's not already started) and
+	then disabling it. The result will be a disabled but started
+	service, regardless of its earlier state. This is useful for
+	starting services temporarily during a release upgrade. The
+	difference between using <c>start_disabled</c> and the
+	sequence <c>enable</c>, <c>start</c> and <c>disable</c> is
+	that all other <c>erlsrv</c> commands are locked out during
+	the sequence of operations in <c>start_disable</c>, making the
+	operation atomic from an <c>erlsrv</c> user's point of
+	view.</p>
+
       </desc>
     </func>
     <func>
diff --git a/erts/doc/src/erts_alloc.xml b/erts/doc/src/erts_alloc.xml
index 51a4a2bca0..ec5e7d9b74 100644
--- a/erts/doc/src/erts_alloc.xml
+++ b/erts/doc/src/erts_alloc.xml
@@ -4,7 +4,7 @@
 <cref>
   <header>
     <copyright>
-      <year>2002</year><year>2010</year>
+      <year>2002</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -58,11 +58,8 @@
       <item>Allocator used for memory blocks that are expected to be
        long-lived, for example Erlang code.</item>
       <tag><c>fix_alloc</c></tag>
-      <item>A very fast allocator used for some fix-sized
-       data. <c>fix_alloc</c> manages a set of memory pools from
-       which memory blocks are handed out. <c>fix_alloc</c>
-       allocates memory pools from <c>ll_alloc</c>. Memory pools
-       that have been allocated are never deallocated.</item>
+      <item>A fast allocator used for some frequently used
+       fixed size data types.</item>
       <tag><c>std_alloc</c></tag>
       <item>Allocator used for most memory blocks not allocated via any of
        the other allocators described above.</item>
@@ -78,14 +75,20 @@
        segments are allocated, cached segments are used if possible
        instead of creating new segments.  This in order to reduce
        the number of system calls made.</item>
+      <tag><c>sbmbc_alloc</c></tag>
+      <item>Allocator used by other allocators for allocation of carriers
+      where only small blocks are placed. Currently this allocator is
+      disabled by default.</item>
     </taglist>
-    <p><c>sys_alloc</c> and <c>fix_alloc</c> are always enabled and
+    <p><c>sys_alloc</c> is always enabled and
       cannot be disabled. <c>mseg_alloc</c> is always enabled if it is
       available and an allocator that uses it is enabled. All other
       allocators can be <seealso marker="#M_e">enabled or disabled</seealso>.
       By default all allocators are enabled.
-      When an allocator is disabled, <c>sys_alloc</c>
-      is used instead of the disabled allocator.</p>
+      When an allocator is disabled, <c>sys_alloc</c> is used instead of
+      the disabled allocator. <c>sbmbc_alloc</c> is an exception. If
+      <c>sbmbc_alloc</c> is disabled, other allocators will not handle
+      small blocks in separate carriers.</p>
     <p>The main idea with the <c>erts_alloc</c> library is to separate
       memory blocks that are used differently into different memory
       areas, and by this achieving less memory fragmentation. By
@@ -98,26 +101,30 @@
     <marker id="alloc_util"></marker>
     <title>The alloc_util framework</title>
     <p>Internally a framework called <c>alloc_util</c> is used for
-      implementing allocators. <c>sys_alloc</c>, <c>fix_alloc</c>, and
+      implementing allocators. <c>sys_alloc</c>, and
       <c>mseg_alloc</c> do not use this framework; hence, the
       following does <em>not</em> apply to them.</p>
     <p>An allocator manages multiple areas, called carriers, in which
       memory blocks are placed. A carrier is either placed in a
-      separate memory segment (allocated via <c>mseg_alloc</c>) or in
-      the heap segment (allocated via <c>sys_alloc</c>). Multiblock
+      separate memory segment (allocated via <c>mseg_alloc</c>), in
+      the heap segment (allocated via <c>sys_alloc</c>), or inside
+      another carrier (in case it is a carrier created by
+      <c>sbmbc_alloc</c>). Multiblock
       carriers are used for storage of several blocks. Singleblock
       carriers are used for storage of one block. Blocks that are
       larger than the value of the singleblock carrier threshold
       (<seealso marker="#M_sbct">sbct</seealso>) parameter are placed
-      in singleblock carriers. Blocks smaller than the value of the
-      <c>sbct</c> parameter are placed in multiblock
-      carriers. Normally an allocator creates a "main multiblock
+      in singleblock carriers. Blocks that are smaller than the value
+      of the <c>sbct</c> parameter are placed in multiblock
+      carriers. Blocks that are smaller than the small block multiblock
+      carrier threshold (<seealso marker="#M_sbmbct">sbmbct</seealso>)
+      will be placed in multiblock carriers only used for small blocks.
+      Normally an allocator creates a "main multiblock
       carrier". Main multiblock carriers are never deallocated. The
       size of the main multiblock carrier is determined by the value
       of the <seealso marker="#M_mmbcs">mmbcs</seealso> parameter.</p>
-    <p>      <marker id="mseg_mbc_sizes"></marker>
-
-      Sizes of multiblock carriers allocated via <c>mseg_alloc</c> are
+    <p><marker id="mseg_mbc_sizes"></marker>Sizes of multiblock carriers
+      allocated via <c>mseg_alloc</c> are
       decided based on the values of the largest multiblock carrier
       size (<seealso marker="#M_lmbcs">lmbcs</seealso>), the smallest
       multiblock carrier size (<seealso marker="#M_smbcs">smbcs</seealso>),
@@ -133,8 +140,11 @@
       <c>sbct</c> parameter should be larger than the value of the
       <c>lmbcs</c> parameter, the allocator may have to create
       multiblock carriers that are larger than the value of the
-      <c>lmbcs</c> parameter, though. Singleblock carriers allocated
-      via <c>mseg_alloc</c> are sized to whole pages.</p>
+      <c>lmbcs</c> parameter, though. The size of multiblock carriers
+      for small blocks is determined by the small block multiblock
+      carrier size (<seealso marker="#M_sbmbcs">sbmbcs</seealso>).
+      Singleblock carriers allocated via <c>mseg_alloc</c> are sized
+      to whole pages.</p>
     <p>Sizes of carriers allocated via <c>sys_alloc</c> are
       decided based on the value of the <c>sys_alloc</c> carrier size
       (<seealso marker="#Muycs">ycs</seealso>) parameter. The size of
@@ -143,9 +153,8 @@
     <p>Coalescing of free blocks are always performed immediately.
       Boundary tags (headers and footers) in free blocks are used
       which makes the time complexity for coalescing constant.</p>
-    <p>      <marker id="strategy"></marker>
-
-      The memory allocation strategy used for multiblock carriers by an
+    <p><marker id="strategy"></marker>The memory allocation strategy
+      used for multiblock carriers by an
       allocator is configurable via the <seealso marker="#M_as">as</seealso>
       parameter. Currently the following strategies are available:</p>
     <taglist>
@@ -166,6 +175,14 @@
           used. The time complexity is proportional to log N, where
           N is the number of free blocks.</p>
       </item>
+      <tag>Address order first fit</tag>
+      <item>
+        <p>Strategy: Find the block with the lowest address that satisfies the
+          requested block size.</p>
+        <p>Implementation: A balanced binary search tree is
+          used. The time complexity is proportional to log N, where
+          N is the number of free blocks.</p>
+      </item>
       <tag>Good fit</tag>
       <item>
         <p>Strategy: Try to find the best fit, but settle for the best fit
@@ -192,8 +209,21 @@
 	  This since it will only cause problems for other allocators.</p>
       </item>
     </taglist>
+    <p>Apart from the ordinary allocators described above a number of
+       pre-allocators are used for some specific data types. These
+       pre-allocators pre-allocate a fixed amount of memory for certain data
+       types when the run-time system starts. As long as pre-allocated memory
+       is available, it will be used. When no pre-allocated memory is
+       available, memory will be allocated in ordinary allocators. These
+       pre-allocators are typically much faster than the ordinary allocators,
+       but can only satisfy a limited amount of requests.</p>
   </section>
 
+  <note><p>
+    Currently only allocators using the best fit and the address order
+    best fit strategies are able to use "small block multi block carriers".
+  </p></note>
+
   <section>
     <marker id="flags"></marker>
     <title>System Flags Effecting erts_alloc</title>
@@ -215,6 +245,7 @@
       the currently present allocators:</p>
     <list type="bulleted">
       <item><c>B: binary_alloc</c></item>
+      <item><c>C: sbmbc_alloc</c></item>
       <item><c>D: std_alloc</c></item>
       <item><c>E: ets_alloc</c></item>
       <item><c>F: fix_alloc</c></item>
@@ -246,18 +277,6 @@
        Max cached segments. The maximum number of memory segments
        stored in the memory segment cache. Valid range is
        0-30. Default value is 5.</item>
-      <tag><marker id="MMcci"><c><![CDATA[+MMcci <time>]]></c></marker></tag>
-      <item>
-       Cache check interval (in milliseconds). The memory segment
-       cache is checked for segments to destroy at an interval
-       determined by this parameter. Default value is 1000.</item>
-    </taglist>
-    <p>The following flags are available for configuration of
-      <c>fix_alloc</c>:</p>
-    <taglist>
-      <tag><marker id="MFe"><c>+MFe true</c></marker></tag>
-      <item>
-       Enable <c>fix_alloc</c>. Note: <c>fix_alloc</c> cannot be disabled.</item>
     </taglist>
     <p>The following flags are available for configuration of
       <c>sys_alloc</c>:</p>
@@ -296,15 +315,15 @@
        based on <c>alloc_util</c>. If <c>u</c> is used as subsystem
        identifier (i.e., <c><![CDATA[<S> = u]]></c>) all allocators based on
        <c>alloc_util</c> will be effected. If <c>B</c>, <c>D</c>, <c>E</c>,
-       <c>H</c>, <c>L</c>, <c>R</c>, <c>S</c>, or <c>T</c> is used as
+        <c>F</c>, <c>H</c>, <c>L</c>, <c>R</c>, <c>S</c>, or <c>T</c> is used as
        subsystem identifier, only the specific allocator identified will be
        effected:</p>
     <taglist>
-      <tag><marker id="M_as"><c><![CDATA[+M<S>as bf|aobf|gf|af]]></c></marker></tag>
+      <tag><marker id="M_as"><c><![CDATA[+M<S>as bf|aobf|aoff|gf|af]]></c></marker></tag>
       <item>
        Allocation strategy. Valid strategies are <c>bf</c> (best fit),
-      <c>aobf</c> (address order best fit), <c>gf</c> (good fit),
-       and <c>af</c> (a fit). See 
+      <c>aobf</c> (address order best fit), <c>aoff</c> (address order first fit),
+      <c>gf</c> (good fit), and <c>af</c> (a fit). See 
       <seealso marker="#strategy">the description of allocation strategies</seealso> in "the <c>alloc_util</c> framework" section.</item>
       <tag><marker id="M_asbcst"><c><![CDATA[+M<S>asbcst <size>]]></c></marker></tag>
       <item>
@@ -395,32 +414,43 @@
        threshold will be placed in singleblock carriers. Blocks
        smaller than this threshold will be placed in multiblock
        carriers.</item>
+      <tag><marker id="M_sbmbcs"><c><![CDATA[+M<S>sbmbcs <size>]]></c></marker></tag>
+      <item>
+	Small block multiblock carrier size (in bytes). Memory blocks smaller
+	than the small block multiblock carrier threshold
+	(<seealso marker="#M_sbmbct">sbmbct</seealso>) will be placed in
+	multiblock carriers used for small blocks only. This parameter
+	determines the size of such carriers.
+      </item>
+      <tag><marker id="M_sbmbct"><c><![CDATA[+M<S>sbmbct <size>]]></c></marker></tag>
+      <item>
+	Small block multiblock carrier threshold (in bytes). Memory blocks
+	smaller than this threshold will be placed in multiblock carriers
+	used for small blocks only.
+      </item>
       <tag><marker id="M_smbcs"><c><![CDATA[+M<S>smbcs <size>]]></c></marker></tag>
       <item>
        Smallest (<c>mseg_alloc</c>) multiblock carrier size (in
        kilobytes). See <seealso marker="#mseg_mbc_sizes">the description
        on how sizes for mseg_alloc multiblock carriers are decided</seealso>
        in "the <c>alloc_util</c> framework" section.</item>
-      <tag><marker id="M_t"><c><![CDATA[+M<S>t true|false|<amount>]]></c></marker></tag>
+      <tag><marker id="M_t"><c><![CDATA[+M<S>t true|false]]></c></marker></tag>
       <item>
-       Multiple, thread specific instances of the allocator.
+       <p>Multiple, thread specific instances of the allocator.
        This option will only have any effect on the runtime system
        with SMP support. Default behaviour on the runtime system with
-       SMP support (<c>N</c> equals the number of scheduler threads):
+       SMP support:</p>
        <taglist>
-         <tag><c>temp_alloc</c></tag>
-	 <item><c>N + 1</c> instances.</item>
          <tag><c>ll_alloc</c></tag>
 	 <item><c>1</c> instance.</item>
          <tag>Other allocators</tag>
-	 <item><c>N</c> instances when <c>N</c> is less than or equal to
-	 <c>16</c>. <c>16</c> instances when <c>N</c> is greater than
-	 <c>16</c>.</item>
+	 <item><c>NoSchedulers+1</c> instances. Each scheduler will use
+	 a lock-free instance of its own and other threads will use
+	 a common instance.</item>
        </taglist>
-       <c>temp_alloc</c> will always use <c>N + 1</c> instances when
-       this option has been enabled regardless of the amount passed.
-       Other allocators will use the same amount of instances as the
-       amount passed as long as it isn't greater than <c>N</c>.       
+       <p>It was previously (before ERTS version 5.9) possible to configure
+       a smaller amount of thread specific instances than schedulers.
+       This is, however, not possible any more.</p>
       </item>
     </taglist>
     <p>Currently the following flags are available for configuration of
diff --git a/erts/doc/src/escript.xml b/erts/doc/src/escript.xml
index a89449df23..66e904f64f 100644
--- a/erts/doc/src/escript.xml
+++ b/erts/doc/src/escript.xml
@@ -4,7 +4,7 @@
 <comref>
   <header>
     <copyright>
-      <year>2007</year><year>2010</year>
+      <year>2007</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -31,7 +31,7 @@
   <com>escript</com>
   <comsummary>Erlang scripting support</comsummary>
   <description>
-    <p><c><![CDATA[escript]]></c> provides support for running short Erlang programs
+    <p><c>escript</c> provides support for running short Erlang programs
       without having to compile them first and an easy way to retrieve the
       command line arguments.</p>
   </description>
@@ -41,10 +41,10 @@
       <name>escript escript-flags script-name script-arg1 script-arg2...</name>
       <fsummary>Run a script written in Erlang</fsummary>
       <desc>
-        <p><c><![CDATA[escript]]></c> runs a script written in Erlang.</p>
+        <p><c>escript</c> runs a script written in Erlang.</p>
         <p>Here follows an example.</p>
         <pre>
-$ <input>cat factorial</input>        
+$ <input>cat factorial</input>
 #!/usr/bin/env escript
 %% -*- erlang -*-
 %%! -smp enable -sname factorial -mnesia debug verbose
@@ -59,11 +59,11 @@ main([String]) ->
     end;
 main(_) ->
     usage().
-        
+
 usage() ->
     io:format("usage: factorial integer\n"),
     halt(1).
-        
+
 fac(0) -> 1;
 fac(N) -> N * fac(N-1).
 $ <input>factorial 5</input>
@@ -74,9 +74,8 @@ $ <input>factorial five</input>
 usage: factorial integer        </pre>
         <p>The header of the Erlang script in the example differs from
         a normal Erlang module.  The first line is intended to be the
-        interpreter line, which invokes
-        <c><![CDATA[escript]]></c>. However if you invoke the
-        <c><![CDATA[escript]]></c> like this</p>
+        interpreter line, which invokes <c>escript</c>. However if you
+        invoke the <c>escript</c> like this</p>
         <pre>
 $ <input>escript factorial 5</input>        </pre>
         <p>the contents of the first line does not matter, but it
@@ -93,13 +92,13 @@ $ <input>escript factorial 5</input>        </pre>
 %%! -smp enable -sname factorial -mnesia debug verbose</pre>
         <p>Such an argument line must start with <c>%%!</c> and the
           rest of the line will interpreted as arguments to the emulator.</p>
-        <p>If you know the location of the <c><![CDATA[escript]]></c> executable, the first
-          line can directly give the path to <c><![CDATA[escript]]></c>. For instance:</p>
+        <p>If you know the location of the <c>escript</c> executable, the first
+          line can directly give the path to <c>escript</c>. For instance:</p>
         <pre>
 #!/usr/local/bin/escript        </pre>
         <p>As any other kind of scripts, Erlang scripts will not work on
           Unix platforms if the execution bit for the script file is not set.
-          (Use <c><![CDATA[chmod +x script-name]]></c> to turn on the execution bit.)
+          (Use <c>chmod +x script-name</c> to turn on the execution bit.)
           </p>
 
         <p>The rest of the Erlang script file may either contain
@@ -108,33 +107,33 @@ $ <input>escript factorial 5</input>        </pre>
 
         <p>An Erlang script file must always contain the function
         <em>main/1</em>.  When the script is run, the
-        <c><![CDATA[main/1]]></c> function will be called with a list
+        <c>main/1</c> function will be called with a list
         of strings representing the arguments given to the script (not
         changed or interpreted in any way).</p>
 
-        <p>If the <c><![CDATA[main/1]]></c> function in the script returns successfully,
+        <p>If the <c>main/1</c> function in the script returns successfully,
           the exit status for the script will be 0. If an exception is generated
           during execution, a short message will be printed and the script terminated
           with exit status 127.</p>
 
-        <p>To return your own non-zero exit code, call <c><![CDATA[halt(ExitCode)]]></c>;
+        <p>To return your own non-zero exit code, call <c>halt(ExitCode)</c>;
           for instance:</p>
         <pre>
 halt(1).</pre>
 
-        <p>Call <c><![CDATA[escript:script_name/0]]></c> from your to
-          script to retrieve the pathname of the script (the pathname
-          is usually, but not always, absolute).</p>
+        <p>Call <seealso marker="#script_name_0">escript:script_name()</seealso>
+	from your to script to retrieve the pathname of the script
+	(the pathname is usually, but not always, absolute).</p>
 
         <p>If the file contains source code (as in the example above),
           it will be processed by the preprocessor <c>epp</c>. This
           means that you for example may use pre-defined macros (such as
-          <c><![CDATA[?MODULE]]></c>) as well as include directives like
-          the <c><![CDATA[-include_lib]]></c> directive. For instance, use</p>
+          <c>?MODULE</c>) as well as include directives like
+          the <c>-include_lib</c> directive. For instance, use</p>
         <pre>
--include_lib("kernel/include/file.hrl").        </pre>
+-include_lib("kernel/include/file.hrl").</pre>
         <p>to include the record definitions for the records used by the
-          <c><![CDATA[file:read_link_info/1]]></c> function.</p>
+          <c>file:read_link_info/1</c> function.</p>
 
         <p>The script will be checked for syntactic and semantic
           correctness before being run. If there are warnings (such as
@@ -144,7 +143,7 @@ halt(1).</pre>
           127.</p>
 
         <p>Both the module declaration and the export declaration of
-          the <c><![CDATA[main/1]]></c> function are optional.</p>
+          the <c>main/1</c> function are optional.</p>
 
         <p>By default, the script will be interpreted. You can force
           it to be compiled by including the following line somewhere
@@ -154,7 +153,10 @@ halt(1).</pre>
         <p>Execution of interpreted code is slower than compiled code.
           If much of the execution takes place in interpreted code it
           may be worthwhile to compile it, even though the compilation
-          itself will take a little while.</p>
+          itself will take a little while. It is also possible to supply
+	  <c>native</c> instead of compile, this will compile the script 
+	  using the native flag, again depending on the characteristics 
+	  of the escript this could or could not be worth while.</p>
 
         <p>As mentioned earlier, it is possible to have a script which
           contains precompiled <c>beam</c> code. In a precompiled
@@ -198,6 +200,180 @@ factorial 5 = 120
 </pre>
       </desc>
     </func>
+    <func>
+      <name>escript:create(FileOrBin, Sections) -> ok | {ok, binary()} | {error, term()}</name>
+      <fsummary>Create an escript</fsummary>
+      <type>
+        <v>FileOrBin = filename() | 'binary'</v>
+        <v>Sections = [Header] Body | Body</v>
+        <v>Header = shebang | {shebang, Shebang}
+                    | comment | {comment, Comment}
+                    | {emu_args, EmuArgs}</v>
+        <v>Shebang = string() | 'default' | 'undefined'</v>
+        <v>Comment = string() | 'default' | 'undefined'</v>
+        <v>EmuArgs = string() | 'undefined'</v>
+        <v>Body = {source, SourceCode}
+                | {beam, BeamCode}
+	        | {archive, ZipArchive}</v>
+        <v>SourceCode = BeamCode = ZipArchive = binary()</v>
+      </type>
+      <desc>
+        <p>The <marker id="create_2"></marker> <c>create/2</c>
+        function creates an escript from a list of sections. The
+        sections can be given in any order. An escript begins with an
+        optional <c>Header</c> followed by a mandatory <c>Body</c>. If
+        the header is present, it does always begin with a
+        <c>shebang</c>, possibly followed by a <c>comment</c> and
+        <c>emu_args</c>. The <c>shebang</c> defaults to
+        <c>"/usr/bin/env escript"</c>. The comment defaults to
+        <c>"This is an -*- erlang -*- file"</c>. The created escript
+        can either be returned as a binary or written to file.</p>
+
+	<p>As an example of how the function can be used, we create an
+	interpreted escript which uses emu_args to set some emulator
+	flag. In this case it happens to disable the smp_support. We
+	do also extract the different sections from the newly created
+	script:</p>
+        <pre>
+&gt; <input>Source = "%% Demo\nmain(_Args) ->\n    io:format(erlang:system_info(smp_support)).\n".</input>
+"%% Demo\nmain(_Args) ->\n    io:format(erlang:system_info(smp_support)).\n"
+&gt; <input>io:format("~s\n", [Source]).</input>
+%% Demo
+main(_Args) ->
+    io:format(erlang:system_info(smp_support)).
+
+ok
+&gt; <input>{ok, Bin} = escript:create(binary, [shebang, comment, {emu_args, "-smp disable"},
+                                      {source, list_to_binary(Source)}]).</input>
+{ok,&lt;&lt;"#!/usr/bin/env escript\n%% This is an -*- erlang -*- file\n%%!-smp disabl"...&gt;&gt;}
+&gt; <input>file:write_file("demo.escript", Bin).</input>
+ok
+&gt; <input>os:cmd("escript demo.escript").</input>
+"false"
+&gt; <input>escript:extract("demo.escript", []).</input>
+{ok,[{shebang,default}, {comment,default}, {emu_args,"-smp disable"},
+     {source,&lt;&lt;"%% Demo\nmain(_Args) ->\n    io:format(erlang:system_info(smp_su"...&gt;&gt;}]}
+	</pre>
+
+	<p>An escript without header can be created like this:</p>
+<pre>
+&gt; <input>file:write_file("demo.erl",
+                  ["%% demo.erl\n-module(demo).\n-export([main/1]).\n\n", Source]).</input>
+ok
+&gt; <input>{ok, _, BeamCode} = compile:file("demo.erl", [binary, debug_info]).</input>
+{ok,demo,
+    &lt;&lt;70,79,82,49,0,0,2,208,66,69,65,77,65,116,111,109,0,0,0,
+      79,0,0,0,9,4,100,...&gt;&gt;}
+&gt; <input>escript:create("demo.beam", [{beam, BeamCode}]).</input>
+ok
+&gt; <input>escript:extract("demo.beam", []).</input>
+{ok,[{shebang,undefined}, {comment,undefined}, {emu_args,undefined},
+     {beam,&lt;&lt;70,79,82,49,0,0,3,68,66,69,65,77,65,116,
+             111,109,0,0,0,83,0,0,0,9,...&gt;&gt;}]}
+&gt; <input>os:cmd("escript demo.beam").</input>
+"true"
+</pre>
+	<p>Here we create an archive script containing both Erlang
+	code as well as beam code. Then we iterate over all files in
+	the archive and collect their contents and some info about
+	them.
+	</p>
+<pre>
+&gt; <input>{ok, SourceCode} = file:read_file("demo.erl").</input>
+{ok,&lt;&lt;"%% demo.erl\n-module(demo).\n-export([main/1]).\n\n%% Demo\nmain(_Arg"...&gt;&gt;}
+&gt; <input>escript:create("demo.escript",
+                 [shebang,
+                  {archive, [{"demo.erl", SourceCode},
+                             {"demo.beam", BeamCode}], []}]).</input>
+ok
+&gt; <input>{ok, [{shebang,default}, {comment,undefined}, {emu_args,undefined},
+     {archive, ArchiveBin}]} = escript:extract("demo.escript", []).</input>
+{ok,[{shebang,default}, {comment,undefined}, {emu_args,undefined},
+     {{archive,&lt;&lt;80,75,3,4,20,0,0,0,8,0,118,7,98,60,105,
+                152,61,93,107,0,0,0,118,0,...&gt;&gt;}]}
+&gt; <input>file:write_file("demo.zip", ArchiveBin).</input>
+ok
+&gt; <input>zip:foldl(fun(N, I, B, A) -> [{N, I(), B()} | A] end, [], "demo.zip").</input>
+{ok,[{"demo.beam",
+      {file_info,748,regular,read_write,
+                 {{2010,3,2},{0,59,22}},
+                 {{2010,3,2},{0,59,22}},
+                 {{2010,3,2},{0,59,22}},
+                 54,1,0,0,0,0,0},
+      &lt;&lt;70,79,82,49,0,0,2,228,66,69,65,77,65,116,111,109,0,0,0,
+        83,0,0,...&gt;&gt;},
+     {"demo.erl",
+      {file_info,118,regular,read_write,
+                 {{2010,3,2},{0,59,22}},
+                 {{2010,3,2},{0,59,22}},
+                 {{2010,3,2},{0,59,22}},
+                 54,1,0,0,0,0,0},
+      &lt;&lt;"%% demo.erl\n-module(demo).\n-export([main/1]).\n\n%% Demo\nmain(_Arg"...&gt;&gt;}]}</pre>
+      </desc>
+    </func>
+    <func>
+      <name>escript:extract(File, Options) -> {ok, Sections} | {error, term()}</name>
+      <fsummary>Parses an escript and extracts its sections</fsummary>
+      <type>
+        <v>File = filename()</v>
+        <v>Options = [] | [compile_source]</v>
+        <v>Sections = Headers Body</v>
+        <v>Headers =  {shebang, Shebang}
+                      {comment, Comment}
+                      {emu_args, EmuArgs}</v>
+        <v>Shebang = string() | 'default' | 'undefined'</v>
+        <v>Comment = string() | 'default' | 'undefined'</v>
+        <v>EmuArgs = string() | 'undefined'</v>
+        <v>Body = {source, SourceCode}
+                | {source, BeamCode}
+                | {beam, BeamCode}
+	        | {archive, ZipArchive}</v>
+        <v>SourceCode = BeamCode = ZipArchive = binary()</v>
+      </type>
+      <desc>
+        <p>The <marker id="extract_2"></marker> <c>extract/2</c>
+        function parses an escript and extracts its sections. This is
+        the reverse of <c>create/2</c>.</p>
+
+	<p>All sections are returned even if they do not exist in the
+	escript. If a particular section happens to have the same
+	value as the default value, the extracted value is set to the
+	atom <c>default</c>. If a section is missing, the extracted
+	value is set to the atom <c>undefined</c>.  </p>
+
+	<p>The <c>compile_source</c> option only affects the result if
+	the escript contains <c>source</c> code. In that case the
+	Erlang code is automatically compiled and <c>{source,
+	BeamCode}</c> is returned instead of <c>{source,
+	SourceCode}</c>.</p>
+
+	<pre>
+&gt; <input>escript:create("demo.escript",
+                 [shebang, {archive, [{"demo.erl", SourceCode},
+                                      {"demo.beam", BeamCode}], []}]).</input>
+ok
+&gt; <input>{ok, [{shebang,default}, {comment,undefined}, {emu_args,undefined},
+     {archive, ArchiveBin}]} =
+              escript:extract("demo.escript", []).</input>
+{ok,[{{archive,&lt;&lt;80,75,3,4,20,0,0,0,8,0,118,7,98,60,105,
+                152,61,93,107,0,0,0,118,0,...&gt;&gt;}
+     {emu_args,undefined}]}
+	</pre>
+      </desc>
+    </func>
+    <func>
+      <name>escript:script_name() -> File</name>
+      <fsummary>Returns the name of an escript</fsummary>
+      <type>
+        <v>File = filename()</v>
+      </type>
+      <desc>
+        <p>The <marker id="script_name_0"></marker>
+        <c>script_name/0</c> function returns the name of the escript
+        being executed. If the function is invoked outside the context
+        of an escript, the behavior is undefined.</p>
+      </desc>
+    </func>
   </funcs>
 
   <section>
@@ -224,6 +400,9 @@ factorial 5 = 120
        Warnings and errors (if any) are written to the standard output, but
        the script will not be run. The exit status will be 0 if there were
        no errors, and 127 otherwise.</item>
+
+       <tag>-n</tag>
+      <item>Compile the escript using the +native flag.</item>
     </taglist>
   </section>
 </comref>
diff --git a/erts/doc/src/init.xml b/erts/doc/src/init.xml
index 33364c709a..d5c43f6e57 100644
--- a/erts/doc/src/init.xml
+++ b/erts/doc/src/init.xml
@@ -4,7 +4,7 @@
 <erlref>
   <header>
     <copyright>
-      <year>1996</year><year>2009</year>
+      <year>1996</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -47,15 +47,12 @@
   </description>
   <funcs>
     <func>
-      <name>boot(BootArgs) -> void()</name>
+      <name name="boot" arity="1"/>
       <fsummary>Start the Erlang runtime system</fsummary>
-      <type>
-        <v>BootArgs = [binary()]</v>
-      </type>
       <desc>
         <p>Starts the Erlang runtime system. This function is called
           when the emulator is started and coordinates system start-up.</p>
-        <p><c>BootArgs</c> are all command line arguments except
+        <p><c><anno>BootArgs</anno></c> are all command line arguments except
           the emulator flags, that is, flags and plain arguments. See
           <seealso marker="erts:erl">erl(1)</seealso>.</p>
         <p><c>init</c> itself interprets some of the flags, see
@@ -67,30 +64,12 @@
       </desc>
     </func>
     <func>
-      <name>get_args() -> [Arg]</name>
-      <fsummary>Get all non-flag command line arguments</fsummary>
-      <type>
-        <v>Arg = atom()</v>
-      </type>
-      <desc>
-        <p>Returns any plain command line arguments as a list of atoms
-          (possibly empty). It is recommended that
-          <c>get_plain_arguments/1</c> is used instead, because of
-          the limited length of atoms.</p>
-      </desc>
-    </func>
-    <func>
-      <name>get_argument(Flag) -> {ok, Arg} | error</name>
+      <name name="get_argument" arity="1"/>
       <fsummary>Get the values associated with a command line user flag</fsummary>
-      <type>
-        <v>Flag = atom()</v>
-        <v>Arg = [Values]</v>
-        <v>&nbsp;Values = [string()]</v>
-      </type>
       <desc>
         <p>Returns all values associated with the command line user flag
-          <c>Flag</c>. If <c>Flag</c> is provided several times, each
-          <c>Values</c> is returned in preserved order.</p>
+          <c><anno>Flag</anno></c>. If <c><anno>Flag</anno></c> is provided several times, each
+          <c><anno>Values</anno></c> is returned in preserved order.</p>
         <pre>
 % <input>erl -a b c -a d</input>
 ...
@@ -126,48 +105,37 @@
       </desc>
     </func>
     <func>
-      <name>get_arguments() -> Flags</name>
+      <name name="get_arguments" arity="0"/>
       <fsummary>Get all command line user flags</fsummary>
-      <type>
-        <v>Flags = [{Flag, Values}]</v>
-        <v>&nbsp;Flag = atom()</v>
-        <v>&nbsp;Values = [string()]</v>
-      </type>
       <desc>
         <p>Returns all command line flags, as well as the system
           defined flags, see <c>get_argument/1</c>.</p>
       </desc>
     </func>
     <func>
-      <name>get_plain_arguments() -> [Arg]</name>
+      <name name="get_plain_arguments" arity="0"/>
       <fsummary>Get all non-flag command line arguments</fsummary>
-      <type>
-        <v>Arg = string()</v>
-      </type>
       <desc>
         <p>Returns any plain command line arguments as a list of strings
           (possibly empty).</p>
       </desc>
     </func>
     <func>
-      <name>get_status() -> {InternalStatus, ProvidedStatus}</name>
+      <name name="get_status" arity="0"/>
       <fsummary>Get system status information</fsummary>
-      <type>
-        <v>InternalStatus = starting | started | stopping</v>
-        <v>ProvidedStatus = term()</v>
-      </type>
+      <type name="internal_status"/>
       <desc>
         <p>The current status of the <c>init</c> process can be
           inspected. During system startup (initialization),
-          <c>InternalStatus</c> is <c>starting</c>, and
-          <c>ProvidedStatus</c> indicates how far the boot script has
+          <c><anno>InternalStatus</anno></c> is <c>starting</c>, and
+          <c><anno>ProvidedStatus</anno></c> indicates how far the boot script has
           been interpreted. Each <c>{progress, Info}</c> term
-          interpreted in the boot script affects <c>ProvidedStatus</c>,
-          that is, <c>ProvidedStatus</c> gets the value of <c>Info</c>.</p>
+          interpreted in the boot script affects <c><anno>ProvidedStatus</anno></c>,
+          that is, <c><anno>ProvidedStatus</anno></c> gets the value of <c>Info</c>.</p>
       </desc>
     </func>
     <func>
-      <name>reboot() -> void()</name>
+      <name name="reboot" arity="0"/>
       <fsummary>Take down and restart an Erlang node smoothly</fsummary>
       <desc>
         <p>All applications are taken down smoothly, all code is
@@ -181,7 +149,7 @@
       </desc>
     </func>
     <func>
-      <name>restart() -> void()</name>
+      <name name="restart" arity="0"/>
       <fsummary>Restart the running Erlang node</fsummary>
       <desc>
         <p>The system is restarted <em>inside</em> the running Erlang
@@ -196,20 +164,17 @@
       </desc>
     </func>
     <func>
-      <name>script_id() -> Id</name>
+      <name name="script_id" arity="0"/>
       <fsummary>Get the identity of the used boot script</fsummary>
-      <type>
-        <v>Id = term()</v>
-      </type>
       <desc>
         <p>Get the identity of the boot script used to boot the system.
-          <c>Id</c> can be any Erlang term. In the delivered boot
-          scripts, <c>Id</c> is <c>{Name, Vsn}</c>. <c>Name</c> and
+          <c><anno>Id</anno></c> can be any Erlang term. In the delivered boot
+          scripts, <c><anno>Id</anno></c> is <c>{Name, Vsn}</c>. <c>Name</c> and
           <c>Vsn</c> are strings.</p>
       </desc>
     </func>
     <func>
-      <name>stop() -> void()</name>
+      <name name="stop" arity="0"/>
       <fsummary>Take down an Erlang node smoothly</fsummary>
       <desc>
         <p>All applications are taken down smoothly, all code is
@@ -223,15 +188,12 @@
       </desc>
     </func>
     <func>
-      <name>stop(Status) -> void()</name>
+      <name name="stop" arity="1"/>
       <fsummary>Take down an Erlang node smoothly</fsummary>
-      <type>
-        <v>Status = int()>=0 | string()</v>
-      </type>
       <desc>
         <p>All applications are taken down smoothly, all code is
           unloaded, and all ports are closed before the system
-          terminates by calling <c>halt(Status)</c>. If the
+          terminates by calling <c>halt(<anno>Status</anno>)</c>. If the
           <c>-heart</c> command line flag was given, the <c>heart</c>
           program is terminated before the Erlang node
           terminates. Refer to <c>heart(3)</c> for more
diff --git a/erts/doc/src/make.dep b/erts/doc/src/make.dep
deleted file mode 100644
index 98bac78235..0000000000
--- a/erts/doc/src/make.dep
+++ /dev/null
@@ -1,32 +0,0 @@
-# ----------------------------------------------------
-# >>>> Do not edit this file <<<<
-# This file was automaticly generated by
-# /home/gandalf/otp/bin/docdepend
-# ----------------------------------------------------
-
-
-# ----------------------------------------------------
-# TeX files that the DVI file depend on
-# ----------------------------------------------------
-
-book.dvi: absform.tex alt_dist.tex book.tex crash_dump.tex \
-		driver.tex driver_entry.tex epmd.tex erl.tex \
-		erl_dist_protocol.tex erl_driver.tex erl_ext_dist.tex \
-		erl_prim_loader.tex erl_set_memory_block.tex \
-		erlang.tex erlc.tex erlsrv.tex erts_alloc.tex \
-		escript.tex inet_cfg.tex init.tex match_spec.tex \
-		part.tex ref_man.tex run_erl.tex start.tex \
-		start_erl.tex tty.tex werl.tex zlib.tex
-
-# ----------------------------------------------------
-# Source inlined when transforming from source to LaTeX
-# ----------------------------------------------------
-
-book.tex: ref_man.xml
-
-# ----------------------------------------------------
-# Pictures that the DVI file depend on
-# ----------------------------------------------------
-
-book.dvi: erl_ext_fig.ps
-
diff --git a/erts/doc/src/match_spec.xml b/erts/doc/src/match_spec.xml
index b9f955e4db..bdcf9c3816 100644
--- a/erts/doc/src/match_spec.xml
+++ b/erts/doc/src/match_spec.xml
@@ -4,7 +4,7 @@
 <chapter>
   <header>
     <copyright>
-      <year>1999</year><year>2010</year>
+      <year>1999</year><year>2012</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -60,7 +60,7 @@
 
   <section>
     <title>Grammar</title>
-    <p>A match_spec can be described in this <em>informal</em> grammar:</p>
+    <p>A match_spec used in tracing can be described in this <em>informal</em> grammar:</p>
     <list type="bulleted">
       <item>MatchExpression ::= [ MatchFunction, ... ]
       </item>
@@ -75,7 +75,7 @@
       <item>MatchCondition ::= { GuardFunction } | 
        { GuardFunction, ConditionExpression, ... }
       </item>
-      <item>BoolFunction ::= <c><![CDATA[is_atom]]></c> | <c><![CDATA[is_constant]]></c> |
+      <item>BoolFunction ::= <c><![CDATA[is_atom]]></c> |
       <c><![CDATA[is_float]]></c> | <c><![CDATA[is_integer]]></c> | <c><![CDATA[is_list]]></c> |
       <c><![CDATA[is_number]]></c> | <c><![CDATA[is_pid]]></c> | <c><![CDATA[is_port]]></c> |
       <c><![CDATA[is_reference]]></c> | <c><![CDATA[is_tuple]]></c> | <c><![CDATA[is_binary]]></c> |
@@ -117,6 +117,52 @@
       <c><![CDATA[display]]></c> | <c><![CDATA[caller]]></c> | <c><![CDATA[set_tcw]]></c> |
       <c><![CDATA[silent]]></c></item>
     </list>
+
+    <p>A match_spec used in ets can be described in this <em>informal</em> grammar:</p>
+    <list type="bulleted">
+      <item>MatchExpression ::= [ MatchFunction, ... ]
+      </item>
+      <item>MatchFunction ::= { MatchHead, MatchConditions, MatchBody }
+      </item>
+      <item>MatchHead ::=  MatchVariable | <c><![CDATA['_']]></c> | { MatchHeadPart, ... }
+      </item>
+      <item>MatchHeadPart ::= term() | MatchVariable | <c><![CDATA['_']]></c></item>
+      <item>MatchVariable ::= '$&lt;number&gt;'
+      </item>
+      <item>MatchConditions ::= [ MatchCondition, ...] | <c><![CDATA[[]]]></c></item>
+      <item>MatchCondition ::= { GuardFunction } |
+       { GuardFunction, ConditionExpression, ... }
+      </item>
+      <item>BoolFunction ::= <c><![CDATA[is_atom]]></c> |
+      <c><![CDATA[is_float]]></c> | <c><![CDATA[is_integer]]></c> | <c><![CDATA[is_list]]></c> |
+      <c><![CDATA[is_number]]></c> | <c><![CDATA[is_pid]]></c> | <c><![CDATA[is_port]]></c> |
+      <c><![CDATA[is_reference]]></c> | <c><![CDATA[is_tuple]]></c> | <c><![CDATA[is_binary]]></c> |
+      <c><![CDATA[is_function]]></c> | <c><![CDATA[is_record]]></c> | <c><![CDATA[is_seq_trace]]></c> |
+      <c><![CDATA['and']]></c> | <c><![CDATA['or']]></c> | <c><![CDATA['not']]></c> | <c><![CDATA['xor']]></c> |
+      <c><![CDATA[andalso]]></c> | <c><![CDATA[orelse]]></c></item>
+      <item>ConditionExpression ::= ExprMatchVariable | { GuardFunction } |
+       { GuardFunction, ConditionExpression, ... } | TermConstruct
+      </item>
+      <item>ExprMatchVariable ::= MatchVariable (bound in the MatchHead) |
+      <c><![CDATA['$_']]></c> | <c><![CDATA['$$']]></c></item>
+      <item>TermConstruct = {{}} | {{ ConditionExpression, ... }} |
+      <c><![CDATA[[]]]></c> | [ConditionExpression, ...] | NonCompositeTerm | Constant
+      </item>
+      <item>NonCompositeTerm ::= term() (not list or tuple)
+      </item>
+      <item>Constant ::= {<c><![CDATA[const]]></c>, term()}
+      </item>
+      <item>GuardFunction ::= BoolFunction | <c><![CDATA[abs]]></c> |
+      <c><![CDATA[element]]></c> | <c><![CDATA[hd]]></c> | <c><![CDATA[length]]></c> | <c><![CDATA[node]]></c> |
+      <c><![CDATA[round]]></c> | <c><![CDATA[size]]></c> | <c><![CDATA[tl]]></c> | <c><![CDATA[trunc]]></c> |
+      <c><![CDATA['+']]></c> | <c><![CDATA['-']]></c> | <c><![CDATA['*']]></c> | <c><![CDATA['div']]></c> |
+      <c><![CDATA['rem']]></c> | <c><![CDATA['band']]></c> | <c><![CDATA['bor']]></c> | <c><![CDATA['bxor']]></c> |
+      <c><![CDATA['bnot']]></c> | <c><![CDATA['bsl']]></c> | <c><![CDATA['bsr']]></c> | <c><![CDATA['>']]></c> |
+      <c><![CDATA['>=']]></c> | <c><![CDATA['<']]></c> | <c><![CDATA['=<']]></c> | <c><![CDATA['=:=']]></c> |
+      <c><![CDATA['==']]></c> | <c><![CDATA['=/=']]></c> | <c><![CDATA['/=']]></c> | <c><![CDATA[self]]></c> |
+      <c><![CDATA[get_tcw]]></c></item>
+      <item>MatchBody ::= [ ConditionExpression, ... ]</item>
+    </list>
   </section>
 
   <section>
@@ -126,7 +172,7 @@
       <title>Functions allowed in all types of match specifications</title>
       <p>The different functions allowed in <c><![CDATA[match_spec]]></c> work like this:
         </p>
-      <p><em>is_atom, is_constant, is_float, is_integer, is_list, is_number, is_pid, is_port, is_reference, is_tuple, is_binary, is_function: </em> Like the corresponding guard tests in
+      <p><em>is_atom, is_float, is_integer, is_list, is_number, is_pid, is_port, is_reference, is_tuple, is_binary, is_function: </em> Like the corresponding guard tests in
         Erlang, return <c><![CDATA[true]]></c> or <c><![CDATA[false]]></c>.
         </p>
       <p><em>is_record: </em>Takes an additional parameter, which SHALL
@@ -453,8 +499,8 @@
   <section>
     <title>Differences between match specifications in ETS and tracing</title>
     <p>ETS match specifications are there to produce a return
-      value. Usually the expression contains one single
-      <c><![CDATA[ActionTerm]]></c> which defines the return value without having
+      value. Usually the <c><![CDATA[MatchBody]]></c> contains one single
+      <c><![CDATA[ConditionExpression]]></c> which defines the return value without having
       any side effects. Calls with side effects are not allowed in the
       ETS context.</p>
     <p>When tracing there is no return value to produce, the
@@ -530,7 +576,7 @@
       the atom 'strider' and the tuple arity is 3 and return the whole
       object.</p>
     <code type="none"><![CDATA[
-[{{strider,'_'.'_'},
+[{{strider,'_','_'},
   [],
   ['$_']}]
     ]]></code>
diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml
index 09950e865b..6b6a3374d1 100644
--- a/erts/doc/src/notes.xml
+++ b/erts/doc/src/notes.xml
@@ -4,7 +4,7 @@
 <chapter>
   <header>
     <copyright>
-      <year>2004</year><year>2010</year>
+      <year>2004</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -30,6 +30,3020 @@
   </header>
   <p>This document describes the changes made to the ERTS application.</p>
 
+<section><title>Erts 5.9.1.1</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+          <p>
+	    If threads support for the runtime system had been
+	    disabled at compile time (<c>--disable-threads</c> had
+	    been passed to <c>configure</c>), and the <c>+A</c>
+	    command line argument of <c>erl</c> was passed when
+	    starting the runtime system, <seealso
+	    marker="erl_driver#driver_system_info">driver_system_info()</seealso>
+	    erroneously claimed that the runtime system had async
+	    threads even though it had not.</p>
+          <p>
+	    Due to this bug the file driver did not split tasks into
+	    smaller chunks, but instead completed the whole task at
+	    once, i.e., the scheduler got occupied with I/O for a
+	    longer time than intended.</p>
+          <p>
+	    Own Id: OTP-10059</p>
+        </item>
+      </list>
+    </section>
+
+
+    <section><title>Improvements and New Features</title>
+      <list>
+        <item>
+          <p>
+	    A proposal for a new scheduler wakeup strategy has been
+	    implemented. For more information see the documentation
+	    of the <seealso marker="erl#+sws">+sws</seealso> command
+	    line argument of <c>erl</c>.</p>
+          <p>
+	    Own Id: OTP-10033 Aux Id: Seq12025 </p>
+        </item>
+        <item>
+          <p>
+	    A switch for configuration of busy wait length for
+	    scheduler threads has been added. For more information
+	    see the documentation of the <seealso
+	    marker="erl#+sbwt">+sbwt</seealso> command line argument
+	    of <c>erl</c>.</p>
+          <p>
+	    Own Id: OTP-10044 Aux Id: Seq11976 </p>
+        </item>
+        <item>
+          <p>
+	    The extra memory barriers introduced by bug-fix OTP-9281
+	    were unnecessarily used also on tables without the
+	    <c>write_concurrency</c> option enabled. This could
+	    unnecessarily degrade performance of ETS tables without
+	    <c>write_concurrency</c> on some hardware (e.g. PowerPC)
+	    while not effecting performance at all on other hardware
+	    (e.g. x86/x86_64).</p>
+          <p>
+	    OTP-9281 (R14B03): ETS tables using the
+	    <c>write_concurrency</c> option could potentially get
+	    into an internally inconsistent state.</p>
+          <p>
+	    Own Id: OTP-10048 Aux Id: OTP-9281 </p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.9.1</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+	    <p><c>erlang:system_profile</c> errorneous profiled the
+	    profiler process when observing runnable processes. This
+	    has been corrected. </p>
+          <p>
+	    Own Id: OTP-9849</p>
+        </item>
+        <item>
+	    <p>Calling trace_info/2 asking for information about a
+	    function that had native could could crash the run-time
+	    system.</p>
+          <p>
+	    Own Id: OTP-9886</p>
+        </item>
+        <item>
+          <p>
+	    reduce smp locking time range in erts_garbage_collect
+	    (thanks to Jovi Zhang)</p>
+          <p>
+	    Own Id: OTP-9912</p>
+        </item>
+        <item>
+          <p>
+	    Fix typo in supervisor behaviour doc (Thanks to Ricardo
+	    Catalinas Jim�nez)</p>
+          <p>
+	    Own Id: OTP-9924</p>
+        </item>
+        <item>
+          <p>
+	    Correct spelling of registered (Thanks to Richard
+	    Carlsson)</p>
+          <p>
+	    Own Id: OTP-9925</p>
+        </item>
+        <item>
+          <p>
+	    erts: Remove unused variable (Thanks to Jovi Zhang)</p>
+          <p>
+	    Own Id: OTP-9926</p>
+        </item>
+        <item>
+          <p>
+	    Fix bug in ETS with <c>compressed</c> option and
+	    insertion of term containing large integers (>2G) on
+	    64-bit machines. Seen to cause emulator crash. (Thanks to
+	    Diego Llarrull for excellent bug report)</p>
+          <p>
+	    Own Id: OTP-9932</p>
+        </item>
+        <item>
+          <p>
+	    Handle Linux OS where /sys/devices/system/node is only
+	    readable by root. Fallback to /sys/devices/system/cpu for
+	    topology info.</p>
+          <p>
+	    Own Id: OTP-9978</p>
+        </item>
+        <item>
+	    <p> When an escript ends now all printout to standard
+	    output and standard error gets out on the terminal. This
+	    bug has been corrected by changing the behaviour of
+	    erlang:halt/0,1, which should fix the same problem for
+	    other escript-like applications, i.e that data stored in
+	    the output port driver buffers got lost when printing on
+	    a TTY and exiting through erlang:halt/0,1. </p>
+	    <p> The BIF:s erlang:halt/0,1 has gotten improved
+	    semantics and there is a new BIF erlang:halt/2 to
+	    accomplish something like the old semantics. See the
+	    documentation. </p>
+	    <p> Now erlang:halt/0 and erlang:halt/1 with an integer
+	    argument will close all ports and allow all pending async
+	    threads operations to finish before exiting the emulator.
+	    Previously erlang:halt/0 and erlang:halt(0) would just
+	    wait for pending async threads operations but not close
+	    ports. And erlang:halt/1 with a non-zero integer argument
+	    would not even wait for pending async threads operations.
+	    </p>
+	    <p> To roughly the old behaviour, to not wait for ports
+	    and async threads operations when you exit the emulator,
+	    you use erlang:halt/2 with an integer first argument and
+	    an option list containing {flush,false} as the second
+	    argument. Note that now is flushing not dependant of the
+	    exit code, and you can not only flush async threads
+	    operations which we deemed as a strange behaviour anyway.
+	    </p>
+	    <p>Also, erlang:halt/1,2 has gotten a new feature: If the
+	    first argument is the atom 'abort' the emulator is
+	    aborted producing a core dump, if the operating system so
+	    allows. </p>
+          <p>
+	    Own Id: OTP-9985</p>
+        </item>
+        <item>
+          <p>
+	    Added check to inet driver to avoid building on operating
+	    systems that do not yet have IPv6 compatible socket API.
+	    (Thanks to Peer Stritzinger)</p>
+          <p>
+	    Own Id: OTP-9996</p>
+        </item>
+        <item>
+          <p>
+	    Fix bug when the number of CPUs actually found is lower
+	    than the configured value. (Thanks to Benjamin
+	    Herrenschmidt)</p>
+          <p>
+	    Own Id: OTP-10004</p>
+        </item>
+        <item>
+          <p>
+	    The runtime system without SMP support and without thread
+	    support erroneously busy waited when no work was present.
+	    This bug first appeared in <c>erts-5.9</c>.</p>
+          <p>
+	    Own Id: OTP-10019</p>
+        </item>
+        <item>
+          <p>
+	    Various typographical errors corrected in documentation
+	    for common_test, driver, erl_driver and windows
+	    installation instructions. (Thanks to Tuncer Ayaz)</p>
+          <p>
+	    Own Id: OTP-10037</p>
+        </item>
+        <item>
+          <p>
+	    Fix memory leak caused by race on exiting process</p>
+          <p>
+	    Own Id: OTP-10041</p>
+        </item>
+      </list>
+    </section>
+
+
+    <section><title>Improvements and New Features</title>
+      <list>
+        <item>
+	    <p>Add <c>erlang:statistics(scheduler_wall_time)</c> to
+	    ensure correct determination of scheduler utilization.
+	    Measuring scheduler utilization is strongly preferred
+	    over CPU utilization, since CPU utilization gives very
+	    poor indications of actual scheduler/vm usage.</p>
+          <p>
+	    Own Id: OTP-9858</p>
+        </item>
+        <item>
+          <p>
+	    ERTS internal API improvements. In some cases the amount
+	    of atomic read operations needed have been reduced due to
+	    this.</p>
+          <p>
+	    Own Id: OTP-9922</p>
+        </item>
+        <item>
+          <p>
+	    The DTrace source patch from Scott Lystig Fritchie is
+	    integrated in the source tree. Using an emulator with
+	    dtrace probe is still not supported for production use,
+	    but may be a valuable debugging tool. Configure with
+	    --with-dynamic-trace=dtrace (or
+	    --with-dynamic-trace=systemtap) to create a build with
+	    dtrace probes enabled. See runtime_tools for
+	    documentation and examples.</p>
+          <p>
+	    Own Id: OTP-10017</p>
+        </item>
+      </list>
+    </section>
+
+
+    <section><title>Known Bugs and Problems</title>
+      <list>
+        <item>
+          <p>
+	    enif_make_copy may invalidate enif_inspect_binary.</p>
+          <p>
+	    Own Id: OTP-9828</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.9.0.1</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+          <p>
+	    A feature test for the <c>lwsync</c> instruction
+	    performed on PowerPC hardware at runtime system startup
+	    got into an eternal loop if the instruction was not
+	    supported. This bug was introduced in erts-5.9/OTP-R15B.</p>
+          <p>
+	    Own Id: OTP-9843</p>
+        </item>
+        <item>
+          <p>
+	    I/O events could potentially be delayed for ever when
+	    enabling kernel-poll on a non-SMP runtime system
+	    executing on Solaris. When also combined with
+	    async-threads the runtime system hung before completing
+	    the boot phase. This bug was introduced in
+	    erts-5.9/OTP-R15B.</p>
+          <p>
+	    Own Id: OTP-9844</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.9</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+          <p>
+	    Honor option <c>packet_size</c> for http packet parsing
+	    by both TCP socket and <c>erlang:decode_packet</c>. This
+	    gives the ability to accept HTTP headers larger than the
+	    default setting, but also avoid DoS attacks by accepting
+	    lines only up to whatever length you wish to allow. For
+	    consistency, packet type <c>line</c> also honor option
+	    <c>packet_size</c>. (Thanks to Steve Vinoski)</p>
+          <p>
+	    Own Id: OTP-9389</p>
+        </item>
+        <item>
+	    <p> A few contracts in the <c>lists</c> module have been
+	    corrected. </p>
+          <p>
+	    Own Id: OTP-9616</p>
+        </item>
+        <item>
+          <p>
+	    The Unicode noncharacter code points 16#FFFE and 16#FFFE
+	    were not allowed to be encoded or decoded using the
+	    <c>unicode</c> module or bit syntax. That was
+	    inconsistent with the other noncharacters 16#FDD0 to
+	    16#FDEF that could be encoded/decoded. To resolve the
+	    inconsistency, 16#FFFE and 16#FFFE can now be encoded and
+	    decoded. (Thanks to Alisdair Sullivan.)</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-9624</p>
+        </item>
+        <item>
+          <p>
+	    Make epp search directory of current file first when
+	    including another file This completes a partial fix in
+	    R11 that only worked for include_lib(). (Thanks to
+	    Richard Carlsson)</p>
+          <p>
+	    Own Id: OTP-9645</p>
+        </item>
+        <item>
+          <p>
+	    Fixed memory leak in
+	    <c>enif_inspect_io_list_as_binary</c> when applied on a
+	    process independent environment.</p>
+          <p>
+	    Own Id: OTP-9668</p>
+        </item>
+        <item>
+	    <p>The number of beam catches allowed in code are no
+	    longer statically defined and will grow according to its
+	    need.</p>
+          <p>
+	    Own Id: OTP-9692</p>
+        </item>
+        <item>
+          <p>
+	    Add missing parenthesis in heart doc.</p>
+          <p>
+	    Add missing spaces in the Reference Manual distributed
+	    section.</p>
+          <p>
+	    In the HTML version of the doc those spaces are necessary
+	    to separate those words.</p>
+          <p>
+	    Own Id: OTP-9693</p>
+        </item>
+        <item>
+          <p>
+	    Fixes module erlang doc style: option description (Thanks
+	    to Ricardo Catalinas Jim�nez)</p>
+          <p>
+	    Own Id: OTP-9697</p>
+        </item>
+        <item>
+          <p>
+	    Specifying a scope to binary:match/3 when using multiple
+	    searchstrings resulted in faulty return values. This is
+	    now corrected.</p>
+          <p>
+	    Own Id: OTP-9701</p>
+        </item>
+        <item>
+          <p>
+	    The runtime system crashed if more than one thread tried
+	    to exit the runtime system at the same time.</p>
+          <p>
+	    Own Id: OTP-9705</p>
+        </item>
+        <item>
+          <p>
+	    Fix documentation for erlang:process_flag/2</p>
+          <p>
+	    For the subsection about process_flag(save_calls, N)
+	    there's an unrelated paragraph about process priorities
+	    which was copied from the preceeding subsection regarding
+	    process_flag(priority, Level). (Thanks to Filipe David
+	    Manana)</p>
+          <p>
+	    Own Id: OTP-9714</p>
+        </item>
+        <item>
+          <p>
+	    Calls to <c>erlang:system_flag(schedulers_online, N)</c>
+	    and/or <c>erlang:system_flag(multi_scheduling,
+	    block|unblock)</c> could cause internal data used by this
+	    functionality to get into an inconsistent state. When
+	    this happened various problems occurred. This bug was
+	    quite hard to trigger, so hopefully no-one has been
+	    effected by it.</p>
+          <p>
+	    A spinlock used by the run-queue management sometimes got
+	    heavily contended. This code has now been rewritten, and
+	    the spinlock has been removed.</p>
+          <p>
+	    Own Id: OTP-9727</p>
+        </item>
+        <item>
+          <p>
+	    Use libdlpi to get physical address (Thanks to Trond
+	    Norbye)</p>
+          <p>
+	    Own Id: OTP-9818</p>
+        </item>
+      </list>
+    </section>
+
+
+    <section><title>Improvements and New Features</title>
+      <list>
+        <item>
+	    <p> An option list argument can now be passed to
+	    <c>file:read_file_info/2, file:read_link_info/2</c> and
+	    <c>file:write_file_info/3</c> and set time type
+	    information in the call. Valid options are <c>{time,
+	    local}, {time, universal}</c> and <c>{time, posix}</c>.
+	    In the case of <c>posix</c> time no conversions are made
+	    which makes the operation a bit faster. </p>
+          <p>
+	    Own Id: OTP-7687</p>
+        </item>
+        <item>
+	    <p>A number of memory allocation optimizations have been
+	    implemented. Most optimizations reduce contention caused
+	    by synchronization between threads during allocation and
+	    deallocation of memory. Most notably:</p> <list> <item>
+	    Synchronization of memory management in scheduler
+	    specific allocator instances has been rewritten to use
+	    lock-free synchronization. </item> <item> Synchronization
+	    of memory management in scheduler specific pre-allocators
+	    has been rewritten to use lock-free synchronization.
+	    </item> <item> The 'mseg_alloc' memory segment allocator
+	    now use scheduler specific instances instead of one
+	    instance. Apart from reducing contention this also
+	    ensures that memory allocators always create memory
+	    segments on the local NUMA node on a NUMA system. </item>
+	    </list>
+          <p>
+	    Own Id: OTP-7775</p>
+        </item>
+        <item>
+          <p>
+	    The ethread atomic memory operations API used by the
+	    runtime system has been extended and improved.</p>
+          <p>
+	    The ethread library now also performs runtime tests for
+	    presence of hardware features, such as for example SSE2
+	    instructions, instead of requiring this to be determined
+	    at compile time.</p>
+          <p>
+	    All uses of the old deprecated atomic API in the runtime
+	    system have been replaced with the use of the new atomic
+	    API. In a lot of places this change imply a relaxation of
+	    memory barriers used.</p>
+          <p>
+	    Own Id: OTP-9014</p>
+        </item>
+        <item>
+	    <p>gen_sctp:open/0-2 may now return
+	    {error,eprotonosupport} if SCTP is not supported</p>
+	    <p>gen_sctp:peeloff/1 has been implemented and creates a
+	    one-to-one socket which also are supported now</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-9239</p>
+        </item>
+        <item>
+          <p>
+	    Sendfile has been added to the file module's API.
+	    sendfile/2 is used to read data from a file and send it
+	    to a tcp socket using a zero copying mechanism if
+	    available on that OS.</p>
+          <p>
+	    Thanks to Tuncer Ayaz and Steve Vinovski for original
+	    implementation</p>
+          <p>
+	    Own Id: OTP-9240</p>
+        </item>
+        <item>
+          <p>
+	    enif_get_reverse_list function added to nif API. This
+	    function should be used to reverse small lists which are
+	    deep within other structures making it impractical to do
+	    the reverse in Erlang.</p>
+          <p>
+	    Own Id: OTP-9392</p>
+        </item>
+        <item>
+          <p>
+	    The deprecated concat_binary/1 BIF has been removed. Use
+	    <c>list_to_binary</c> or <c>iolist_to_binary/1</c>
+	    instead.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-9421</p>
+        </item>
+        <item>
+	    <p>Erlang/OTP can now be built using parallel make if you
+	    limit the number of jobs, for instance using '<c>make
+	    -j6</c>' or '<c>make -j10</c>'. '<c>make -j</c>' does not
+	    work at the moment because of some missing
+	    dependencies.</p>
+          <p>
+	    Own Id: OTP-9451</p>
+        </item>
+        <item>
+	    <p>Line number and filename information are now included
+	    in exception backtraces as a fourth element in the MFA
+	    tuple. The information will be pretty-printed by the
+	    shell and used by <c>common_test</c> to provide better
+	    indication of where a test case.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-9468</p>
+        </item>
+        <item>
+	    <p>All binary constants used to be handled as heap
+	    binaries (i.e. the entire binary would be copied when
+	    sent to another process). Binary constants larger than 64
+	    bytes are now refc binaries (i.e. the actual data in the
+	    binary will not be copied when sent to another
+	    process).</p>
+          <p>
+	    Own Id: OTP-9486</p>
+        </item>
+        <item>
+          <p>
+	    If a float and an integer is compared, the integer is
+	    only converted to a float if the float datatype can
+	    contain it. Otherwise the float is converted to an
+	    integer.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-9497</p>
+        </item>
+        <item>
+          <p>
+	    Add NIF function enif_is_number</p>
+          <p>
+	    This function allows for easily determining if a term
+	    represents or not a number (integer, float, small or
+	    big).(Thanks to Filipe David Manana)</p>
+          <p>
+	    Own Id: OTP-9629</p>
+        </item>
+        <item>
+          <p>
+	    The ERTS internal system block functionality has been
+	    replaced by new functionality for blocking the system.
+	    The old system block functionality had contention issues
+	    and complexity issues. The new functionality piggy-backs
+	    on thread progress tracking functionality needed by newly
+	    introduced lock-free synchronization in the runtime
+	    system. When the functionality for blocking the system
+	    isn't used, there is more or less no overhead at all.
+	    This since the functionality for tracking thread progress
+	    is there and needed anyway.</p>
+          <p>
+	    Own Id: OTP-9631</p>
+        </item>
+        <item>
+          <p>
+	    An ERTS internal, generic, many to one, lock-free queue
+	    for communication between threads has been introduced.
+	    The many to one scenario is very common in ERTS, so it
+	    can be used in a lot of places in the future. Currently
+	    it is used by scheduling of certain jobs, and the async
+	    thread pool, but more uses are planned for the future.</p>
+          <p>
+	    Drivers using the driver_async functionality are not
+	    automatically locked to the system anymore, and can be
+	    unloaded as any dynamically linked in driver.</p>
+          <p>
+	    Scheduling of ready async jobs is now also interleaved in
+	    between other jobs. Previously all ready async jobs were
+	    performed at once.</p>
+          <p>
+	    Own Id: OTP-9632</p>
+        </item>
+        <item>
+          <p>
+	    Tuple funs (a two-element tuple with a module name and a
+	    function) are now officially deprecated and will be
+	    removed in R16. Use '<c>fun M:F/A</c>' instead. To make
+	    you aware that your system uses tuple funs, the very
+	    first time a tuple fun is applied, a warning will be sent
+	    to the error logger.</p>
+          <p>
+	    Own Id: OTP-9649</p>
+        </item>
+        <item>
+          <p>
+	    Changed the internal BIF calling convention. Will make
+	    simpler faster code and allow BIFs with an arbitrary
+	    arity.</p>
+          <p>
+	    Own Id: OTP-9662</p>
+        </item>
+        <item>
+          <p>
+	    Windows native critical sections are now used internally
+	    in the runtime system on Windows as mutex implementation.
+	    This since they perform better under extreme contention
+	    than our own implementation.</p>
+          <p>
+	    Own Id: OTP-9671</p>
+        </item>
+        <item>
+          <p>
+	    Convert some erl_nif macros into inline functions. Allow
+	    for better compile time type checking. (Thanks to Tuncer
+	    Ayaz)</p>
+          <p>
+	    Own Id: OTP-9675</p>
+        </item>
+        <item>
+          <p>
+	    The <c>+scl</c> command line flag has been added. It can
+	    be used for disabling compaction of scheduler load. For
+	    more information see the <c>erl(1)</c> documentation.</p>
+          <p>
+	    Own Id: OTP-9695</p>
+        </item>
+        <item>
+	    <p>The build system has been updated so that Erlang/OTP
+	    can be built on Mac OS X Lion systems without a GCC
+	    compiler. The INSTALL guide has been updated with
+	    instructions on how to install a GCC compiler and build
+	    Erlang/OTP with it, in order to get a run-time system
+	    with better performance.</p>
+          <p>
+	    Own Id: OTP-9712</p>
+        </item>
+        <item>
+          <p>
+	    When loading a module, the system use to run on a single
+	    scheduler during the entire loading process. This has
+	    been changed to only take down the system just before
+	    inserting the loaded code into the system tables,
+	    resulting in a much shorter disruption if a module is
+	    loaded in a busy system. (Suggested by Bob Ippolito.)</p>
+          <p>
+	    Own Id: OTP-9720</p>
+        </item>
+        <item>
+          <p>
+	    Possible to run HiPE without floating point exceptions
+	    (FPE). Useful on platforms that lack reliable FPE. Slower
+	    float operations compared to HiPE with FPE.</p>
+          <p>
+	    Own Id: OTP-9724</p>
+        </item>
+        <item>
+          <p>
+	    As of ERTS version 5.9 (OTP-R15B) the runtime system will
+	    by default <em>not</em> bind schedulers to logical
+	    processors.</p>
+          <p>
+	    If the Erlang runtime system is the only operating system
+	    process that binds threads to logical processors, this
+	    improves the performance of the runtime system. However,
+	    if other operating system processes (as for example
+	    another Erlang runtime system) also bind threads to
+	    logical processors, there might be a performance penalty
+	    instead. In some cases this performance penalty might be
+	    severe. Due to this, we change the default so that the
+	    user must make an active decision in order to bind
+	    schedulers.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-9726</p>
+        </item>
+        <item>
+          <p>
+	    The use of <c>erlang:system_flag(scheduler_bind_type,
+	    _)</c> and <c>erlang:system_flag(cpu_topology, _)</c>
+	    have been deprecated and scheduled for removal in
+	    erts-5.10/OTP-R16. For more information see the
+	    documentation of <c>erlang:system_flag/2</c>.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-9749</p>
+        </item>
+        <item>
+          <p>
+	    An ancient workaround for a Windows bug was removed from
+	    the open_port code, open_port({spawn,...}...) is now
+	    faster. Thanks to Daniel Goertzen.</p>
+          <p>
+	    Own Id: OTP-9766</p>
+        </item>
+        <item>
+          <p>
+	    The use of deprecated 32bit time_t on 32bit Windows is
+	    removed.</p>
+          <p>
+	    Own Id: OTP-9767</p>
+        </item>
+        <item>
+          <p>
+	    The NIF <c>reload</c> mechanism is deprecated. Do not use
+	    it as an upgrade method for live production systems. It
+	    might be removed in future releases. It can still serve
+	    as a development feature but a warning message will be
+	    logged each time it is used.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-9771</p>
+        </item>
+        <item>
+	    <p>The driver interface has been changed to enable 64-bit
+	    aware drivers. Most importantly the return types for
+	    ErlDrvEntry callbacks 'call' and 'control' has ben
+	    enlarged which require drivers to be changed or they will
+	    cause emulator crashes. See <seealso
+	    marker="erl_driver#rewrites_for_64_bits"> Rewrites for
+	    64-bit driver interface </seealso> in the driver manual.
+	    </p>
+	    <p>Due to this driver <seealso
+	    marker="erl_driver#version_management">version
+	    management</seealso> is now mandatory. A driver that is
+	    not written with version management or a driver that was
+	    compiled with the wrong major version will be not be
+	    loaded by the emulator.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-9795</p>
+        </item>
+        <item>
+          <p>
+	    Eliminate use of deprecated regexp module</p>
+          <p>
+	    Own Id: OTP-9810</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.8.5</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+          <p>
+	    Several bugs concerning constant binary constructions
+	    such as &lt;&lt;0:4294967295&gt;&gt; have been corrected.
+	    Depending on the actual size of the binary and the type
+	    of run-time system (32-bit, halfword, 64-bit), such
+	    expression could either crash the run-time system or make
+	    the loader refuse loading of the module.</p>
+          <p>
+	    Own Id: OTP-9284</p>
+        </item>
+        <item>
+          <p>
+	    The Erlsrv utility failed to stop the erlang machine if
+	    no StopAction was defined when the service was stopped.
+	    This is now corrected.</p>
+          <p>
+	    Own Id: OTP-9344</p>
+        </item>
+        <item>
+          <p>
+	    Due to a bug in glibc the runtime system could abort
+	    while trying to destroy a mutex. A fix for this was
+	    introduced in R14B02. This fix did, however, not solve
+	    the problem. The runtime system will now issue a warning
+	    instead of aborting.</p>
+          <p>
+	    Own Id: OTP-9373 Aux Id: OTP-9009 </p>
+        </item>
+        <item>
+          <p>
+	    Replace atom in DRV macro in prim_file with string</p>
+          <p>
+	    An experimental version of Dialyzer discovered that the
+	    atom that replaced the DRV macro in prim_file ends up in
+	    calls to erlang:open_port({spawn, Driver}, Portopts) as
+	    the Driver argument. The documentation states that this
+	    call requires a string there.</p>
+          <p>
+	    This change is also consistent with the one introduced in
+	    commit 0f03b1e9d2bef3bc830c31a369261af4c5234727 by Kostis
+	    Sagonas.</p>
+          <p>
+	    Own Id: OTP-9377</p>
+        </item>
+        <item>
+          <p>
+	    Fix typos in the epmd documentation (Thanks to Holger
+	    Wei� )</p>
+          <p>
+	    Own Id: OTP-9387</p>
+        </item>
+        <item>
+          <p>
+	    Fix faulty integer terms created by NIF API from 64-bit
+	    integers on halfword emulator. (Thanks to Paolo Negri and
+	    Paul Davis)</p>
+          <p>
+	    Own Id: OTP-9394</p>
+        </item>
+        <item>
+          <p>
+	    Fix <c>epmd</c> crash on vxworks caused by faulty
+	    argument to select() system call.</p>
+          <p>
+	    Own Id: OTP-9427 Aux Id: seq11855 </p>
+        </item>
+        <item>
+          <p>
+	    The ets:test_ms function could in rare cases truncate the
+	    error messages. This is now corrected.</p>
+          <p>
+	    Own Id: OTP-9435</p>
+        </item>
+        <item>
+          <p>
+	    Fix bug related to hibernate and HiPE (clear
+	    F_HIBERNATE_SCHED flag)</p>
+          <p>
+	    F_HIBERNATE_SCHED flag that was introduced in
+	    b7ecdcd1ae9e11b8f75e must be cleared in hipe_mode_switch
+	    as well. Otherwise, processes running HiPE code that
+	    hibernate, wake up and then trap into a BIF will not be
+	    rescheduled.(Thanks to Paul Guyot)</p>
+          <p>
+	    Own Id: OTP-9452</p>
+        </item>
+        <item>
+          <p>
+	    Fix bug in FreeBSD topology detection code (Thanks to
+	    Paul Guyot)</p>
+          <p>
+	    Own Id: OTP-9453</p>
+        </item>
+        <item>
+          <p>
+	    Fix use of logical operator &amp;&amp; with constant
+	    operand instead of bitwise &amp; (Thanks to Cristian
+	    Greco)</p>
+          <p>
+	    Own Id: OTP-9454</p>
+        </item>
+        <item>
+          <p>
+	    inet: error if fd does not match socket domain</p>
+          <p>
+	    If an IPv4 fd is opened as an IPv6 socket, unexpected
+	    behaviour can occur. For example, if an IPv4 UDP socket
+	    is opened and passed into Erlang as an IPv6 socket, the
+	    first 3 bytes (corresponding to 1 byte representing the
+	    protocol family, 2 bytes set to the port) are stripped
+	    from the payload. The cause of the UDP payload truncation
+	    happens in inet_drv.c:packet_inet_input when a call to
+	    inet_get_address fails silently because the family is set
+	    to PF_INET6 but the buffer len is the size of an IPv4
+	    struct sockaddr_in.</p>
+          <p>
+	    (Thanks to Andrew Tunnell-Jones for finding the bug and
+	    the test case!)</p>
+          <p>
+	    Own Id: OTP-9455</p>
+        </item>
+        <item>
+          <p>
+	    erts: use a union to avoid strict aliasing issues</p>
+          <p>
+	    Use a union for pointer type conversion to avoid compiler
+	    warnings about strict-aliasing violations with gcc-4.1.
+	    gcc >= 4.2 does not emit the warning. erts: adapt
+	    matrix_nif to R14 erl_nif API changes (Thanks To Tuncer
+	    Ayaz)</p>
+          <p>
+	    Own Id: OTP-9487</p>
+        </item>
+        <item>
+          <p>
+	    fix 64-bit issues in the garbage collection (Thanks to
+	    Richard Carlsson)</p>
+          <p>
+	    Own Id: OTP-9488</p>
+        </item>
+        <item>
+          <p>
+	    epmd: fix compiler warnings</p>
+          <p>
+	    Suppress compiler warnings about ignored return values.
+	    (Thanks to Michael Santos )</p>
+          <p>
+	    Own Id: OTP-9500</p>
+        </item>
+        <item>
+          <p>
+	    Fix non-existing function (erlang:disconnect/1) in
+	    distributed reference manual (Thanks to Fabian Kr�l)</p>
+          <p>
+	    Own Id: OTP-9504</p>
+        </item>
+        <item>
+          <p>
+	    Document fdatasync -lrt requirement (SunOS &lt;= 5.10)
+	    (Thanks to Tuncer Ayaz)</p>
+          <p>
+	    Own Id: OTP-9512</p>
+        </item>
+        <item>
+          <p>
+	    Let epmd ignore empty ERL_EPMD_ADDRESS</p>
+          <p>
+	    If the environment variable ERL_EPMD_ADDRESS is set to
+	    the empty string, empd now behaves like it does by
+	    default when ERL_EPMD_ADDRESS is unset. That is, in this
+	    case, epmd now listens on all available interfaces
+	    instead of using only the loopback interface, which
+	    happened because epmd added the loopback address to the
+	    (in this case empty) list of addresses specified via
+	    ERL_EPMD_ADDRESS.</p>
+          <p>
+	    Also, epmd now ignores ERL_EPMD_ADDRESS if it contains
+	    only separator characters (comma and space).</p>
+          <p>
+	    The same applies to epmd's -address option.(Thanks to
+	    Holger Wei�)</p>
+          <p>
+	    Own Id: OTP-9525</p>
+        </item>
+        <item>
+          <p>
+	    Remove dead code in erl_compile (Thanks to Tuncer Ayaz)</p>
+          <p>
+	    Own Id: OTP-9527</p>
+        </item>
+        <item>
+          <p>
+	    Add erlang:external_size/2 BIF</p>
+          <p>
+	    This BIF's second parameter is a list of options.
+	    Currently the only allowed option is {minor_version,
+	    Version} where version is either 0 (default) or 1.
+	    (Thanks to Filipe David Manana )</p>
+          <p>
+	    Own Id: OTP-9528</p>
+        </item>
+        <item>
+          <p>
+	    Fix enif_compare on 64bits machines</p>
+          <p>
+	    In 64bits machines the Sint type has a size of 8 bytes,
+	    while on 32bits machines it has a 4 bytes size.
+	    enif_compare was ignoring this and therefore returning
+	    incorrect values when the result of the CMP function
+	    (which returns a Sint value) doesn't fit in 4 bytes.
+	    (Thanks to Filipe David Manana)</p>
+          <p>
+	    Own Id: OTP-9533</p>
+        </item>
+        <item>
+          <p>
+	    Implement or fix -Werror option</p>
+          <p>
+	    If -Werror is enabled and there are warnings no output
+	    file is written. Also make sure that error/warning
+	    reporting is consistent. (Thanks to Tuncer Ayaz)</p>
+          <p>
+	    Own Id: OTP-9536</p>
+        </item>
+        <item>
+	    <p>In some rare cases we did not have a run queue when
+	    scheduling misc ops. This is now fixed.</p>
+          <p>
+	    Own Id: OTP-9537</p>
+        </item>
+        <item>
+	    <p>Remove misc. compiler warnings</p>
+          <p>
+	    Own Id: OTP-9542</p>
+        </item>
+        <item>
+          <p>
+	    Two bugs in gen_sctp has been corrected: getopts/setopts
+	    hence also send could only be called from socket owner,
+	    and options 'linger', 'rcvbuf' and 'sndbuf' was read from
+	    wrong protocol layer hence read wrong values by getopts.</p>
+          <p>
+	    Own Id: OTP-9544</p>
+        </item>
+        <item>
+          <p>
+	    Erlang/OTP can now be built on MacOS X Lion.</p>
+          <p>
+	    Own Id: OTP-9547</p>
+        </item>
+        <item>
+	    <p> XML files have been corrected. </p>
+          <p>
+	    Own Id: OTP-9550 Aux Id: OTP-9541 </p>
+        </item>
+        <item>
+          <p>
+	    Fix potential errors inspired by running cppcheck(1)
+	    (Thanks to Christian von Roques)</p>
+          <p>
+	    Own Id: OTP-9557</p>
+        </item>
+        <item>
+	    <p>When auxiliary work was enqueued on a scheduler, the
+	    wakeup of the scheduler in order to handle this work
+	    could be lost. Wakeups in order to handle ordinary work
+	    were not effected by this bug. The bug only effected
+	    runtime systems with SMP support as follows:</p> <list>
+	    <item>Deallocation of some ETS data structures could be
+	    delayed.</item> <item>On Linux systems not using the NPTL
+	    thread library (typically ancient systems with kernel
+	    versions prior to 2.6) and Windows systems, the <c>{Port,
+	    {exit_status, Status}}</c> message from a terminating
+	    port program could be delayed. That is, it only effected
+	    port programs which had been started by passing
+	    <c>exit_status</c> as an option to
+	    <c>open_port/2</c>.</item> </list>
+          <p>
+	    Own Id: OTP-9567</p>
+        </item>
+        <item>
+          <p>
+	    Handle rare race in the crypto key server functionality</p>
+          <p>
+	    Own Id: OTP-9586</p>
+        </item>
+      </list>
+    </section>
+
+
+    <section><title>Improvements and New Features</title>
+      <list>
+        <item>
+	    <p> Types and specifications have been added. </p>
+          <p>
+	    Own Id: OTP-9356</p>
+        </item>
+        <item>
+          <p>
+	    New allocator strategy "address order first fit". May
+	    ease the emptying of memory carriers and thereby real
+	    release of memory back to the OS.</p>
+          <p>
+	    Own Id: OTP-9424</p>
+        </item>
+        <item>
+          <p>
+	    The new <c>erlang:check_old_code/1</c> BIF checks whether
+	    a module has old code.</p>
+          <p>
+	    Own Id: OTP-9495</p>
+        </item>
+        <item>
+	    <p> Update documentation and specifications of some of
+	    the zlib functions. </p>
+          <p>
+	    Own Id: OTP-9506</p>
+        </item>
+        <item>
+          <p>
+	    Detect the available CPUs on IRIX</p>
+          <p>
+	    Add support for querying the number of configured and
+	    online processors on SGI systems running IRIX.(Thanks to
+	    Holger Wei�)</p>
+          <p>
+	    Own Id: OTP-9531</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.8.4</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+	    <p>Fix binary and iolist overflow problems. Typically
+	    problems arose in length calculation where the result
+	    would exceed (1 bsl 32 - 1).</p>
+          <p>
+	    Own Id: OTP-9118</p>
+        </item>
+        <item>
+          <p>
+	    Using the old erlang shell (i.e. erl instead on werl) on
+	    windows and doing several init:restart's would eventually
+	    hang the VM. That is no longer the case.</p>
+          <p>
+	    Own Id: OTP-9139</p>
+        </item>
+        <item>
+          <p>
+	    Removed recursive C code when printing Erlang terms to
+	    buffers, avoiding stack overflows that could cause VM to
+	    crash.</p>
+          <p>
+	    Own Id: OTP-9140</p>
+        </item>
+        <item>
+          <p>
+	    The send_timeout option in gen_tcp did not work properly
+	    in active mode or with {active,once} options. This is now
+	    corrected.</p>
+          <p>
+	    Own Id: OTP-9145</p>
+        </item>
+        <item>
+          <p>
+	    Fixed various typos across the documentation (Thanks to
+	    Tuncer Ayaz)</p>
+          <p>
+	    Own Id: OTP-9154</p>
+        </item>
+        <item>
+	    <p>Remove duplicate stack entries which could occur after
+	    calling certain BIFs.</p>
+          <p>
+	    Own Id: OTP-9163</p>
+        </item>
+        <item>
+          <p>
+	    A race when starting two nodes simultaneously using
+	    run_erl has been removed.</p>
+          <p>
+	    Own Id: OTP-9164</p>
+        </item>
+        <item>
+          <p>
+	    Add documentation on .erlang processing back again
+	    (Thanks to Gabor Liptak)</p>
+          <p>
+	    Own Id: OTP-9189</p>
+        </item>
+        <item>
+          <p>
+	    Remove gratuitous paren in driver_entry(Thanks to Tuncer
+	    Ayaz)</p>
+          <p>
+	    Own Id: OTP-9192</p>
+        </item>
+        <item>
+          <p>
+	    Fix some wrong pointer dereferences (Thanks to Cristian
+	    Greco)</p>
+          <p>
+	    Own Id: OTP-9194</p>
+        </item>
+        <item>
+          <p>
+	    erts: Remove unused variables (Thanks to Tuncer Ayaz)</p>
+          <p>
+	    Own Id: OTP-9205</p>
+        </item>
+        <item>
+          <p>
+	    The documentation for <c>init:get_args/0</c> has been
+	    removed. <c>init:get_args/0</c> itself was deprecated in
+	    R9C and removed in R12B. (Thanks to Eric Pailleau.)</p>
+          <p>
+	    Own Id: OTP-9209</p>
+        </item>
+        <item>
+          <p>
+	    Allow user to specify the IP address epmd binds to</p>
+          <p>
+	    The IP address(es) epmd binds to can now be specified by
+	    the user, either via epmd's new "-address" option or (if
+	    that's not used) by setting the environment variable
+	    ERL_EPMD_ADDRESS. Multiple addresses may be specified
+	    using a comma-separated list. If the loopback address is
+	    not in this list, it will be added implicitly, so that
+	    the daemon can be queried by an interactive epmd
+	    process.(Thanks to Holger Wei�)</p>
+          <p>
+	    Own Id: OTP-9213</p>
+        </item>
+        <item>
+          <p>
+	    epmd: include host address in local access check</p>
+          <p>
+	    In FreeBSD jails, the source and destination address of
+	    connections to localhost are changed to be the IP address
+	    of the jail. Consider connections from the host's IP
+	    address to itself (e.g., the source and destination
+	    address match) to be local for the access control checks.
+	    (Thanks to Michal Santos and Tom at diogunix.com)</p>
+          <p>
+	    Own Id: OTP-9214</p>
+        </item>
+        <item>
+          <p>
+	    Fix list returned by net_kernel:epmd_module</p>
+          <p>
+	    Function epmd_module of net_kernel returns a list instead
+	    of an atom, when the epmd_module-flag is used. (Thanks to
+	    Markus Knofe)</p>
+          <p>
+	    Own Id: OTP-9215</p>
+        </item>
+        <item>
+          <p>
+	    Fix epmd's dbg_perror() output</p>
+          <p>
+	    The dbg_perror() function now hands the current errno
+	    value over to dbg_gen_printf(). This fixes the problem
+	    that errno had been reset to zero by the time it was used
+	    (to print the corresponding error message) in the
+	    dbg_gen_printf() function. (Thanks to Holger Wei�)</p>
+          <p>
+	    Own Id: OTP-9223</p>
+        </item>
+        <item>
+          <p>
+	    heart: remove garbage appended to heart command</p>
+          <p>
+	    heart:get_cmd/0 is documented to return an empty string
+	    if the command is cleared. get_cmd/0 returns 2 extra
+	    bytes: 1 byte for the trailing null, 1 byte from the op
+	    (the op is an unsigned char and 2 bytes are allocated for
+	    it in the returned buffer). (Thanks to Michael Santos)</p>
+          <p>
+	    Own Id: OTP-9224</p>
+        </item>
+        <item>
+          <p>
+	    file: fix hang reading compressed files</p>
+          <p>
+	    The gzio driver goes into an infinite loop when reading
+	    past the end of a compressed file. Reported-By: Alex
+	    Morarash (Thanks to Michael Santos)</p>
+          <p>
+	    Own Id: OTP-9245</p>
+        </item>
+        <item>
+	    <p>Eliminate alias warnings from gcc 4.5.2</p>
+          <p>
+	    Own Id: OTP-9250</p>
+        </item>
+        <item>
+          <p>
+	    Unsigned integer may overflow in error message (Thanks to
+	    Michael Santos)</p>
+          <p>
+	    Own Id: OTP-9251</p>
+        </item>
+        <item>
+          <p>
+	    Driver names should be strings, not atoms</p>
+          <p>
+	    Own Id: OTP-9253</p>
+        </item>
+        <item>
+          <p>
+	    driver_entry: Remove gratuitous paren and fix typo
+	    (Thanks to Tuncer Ayaz)</p>
+          <p>
+	    Own Id: OTP-9254</p>
+        </item>
+        <item>
+          <p>
+	    Fix format specifiers in erl_exit messages</p>
+          <p>
+	    Fix an error message by using an unsigned integer
+	    specifier as seen in a tweet by @metabrew: #erlang VM
+	    crashed with "no next heap size found: -2090496108,
+	    offset 0", suddenly allocated all available RAM</p>
+          <p>
+	    Also correct mis-typed string formats in bif.c.(Thanks to
+	    Michael Santos)</p>
+          <p>
+	    Own Id: OTP-9262</p>
+        </item>
+        <item>
+          <p>
+	    net_drv: remove unused tcp request id inet_drv: remove
+	    gratuitous assignment (Thanks to Tuncer Ayaz)</p>
+          <p>
+	    Own Id: OTP-9263</p>
+        </item>
+        <item>
+          <p>
+	    Teach run_erl RUN_ERL_DISABLE_FLOWCNTRL for disabling
+	    flow control</p>
+          <p>
+	    Flow control can cause unwanted behaviour of the beam
+	    process, if accidentally hit Ctrl-S (instead of Ctrl-D to
+	    detach) the entire beam may be blocked.</p>
+          <p>
+	    Fix this problem by making it possible to turn off flow
+	    control by setting the environment variable
+	    RUN_ERL_DISABLE_FLOWCNTRL. (Thanks to Jonas Faklkevik)</p>
+          <p>
+	    Own Id: OTP-9270</p>
+        </item>
+        <item>
+	    <p>The following bugs due to missing memory barriers have
+	    been fixed:</p> <list> <item><p> ETS tables using the
+	    <c>write_concurrency</c> option could potentially get
+	    into an internally inconsistent state. </p></item>
+	    <item><p> ETS tables using the <c>ordered_set</c> option
+	    could potentially get into an internally inconsistent
+	    state. </p></item> <item><p> A number of memory barriers
+	    have been added when building with the
+	    <c>libatomic_ops</c> API (i.e. when passing
+	    <c>--with-libatomic_ops=PATH</c> to <c>configure</c>) and
+	    the tilera atomics API (i.e. when building for the tilera
+	    chip). Note that these bugs were due to erroneous usage
+	    of the APIs, and not in the implementations of the APIs.
+	    When using these APIs the following bugs where
+	    present:</p> <list> <item><p> The BIF
+	    <c>erlang:ports/0</c> could return an erroneous result.
+	    </p></item> <item><p> A thread blocking other threads
+	    during code loading, or setup of tracing could
+	    potentially read invalid data. </p></item> <item><p>
+	    Fixation of ETS tables could potentially get into an
+	    internally inconsistent state. </p></item> </list>
+	    </item> </list>
+          <p>
+	    Own Id: OTP-9281</p>
+        </item>
+        <item>
+          <p>
+	    Fix halfword bug for ETS ordered_set when doing
+	    <c>select/match</c> with partly bound key.</p>
+          <p>
+	    Own Id: OTP-9292</p>
+        </item>
+        <item>
+          <p>
+	    Fix bug in <c>code:is_module_native</c> that caused crash
+	    for deleted modules.</p>
+          <p>
+	    Own Id: OTP-9298</p>
+        </item>
+        <item>
+          <p>
+	    Calling <c>driver_async_cancel()</c> could cause a
+	    scheduler thread to enter an eternal loop doing no useful
+	    work. (Thanks to Anders Ramsell)</p>
+          <p>
+	    Own Id: OTP-9302</p>
+        </item>
+      </list>
+    </section>
+
+
+    <section><title>Improvements and New Features</title>
+      <list>
+        <item>
+          <p>
+	    New <c>enif_is_exception function</c> to allow NIFs to
+	    determine whether an <c>ERL_NIF_TERM</c> represents an
+	    exception. (Thanks to Steve Vinoski)</p>
+          <p>
+	    Own Id: OTP-9150</p>
+        </item>
+        <item>
+          <p>
+	    A process being garbage collected by another process
+	    could be scheduled on another scheduler. This prevented
+	    this scheduler from doing any useful work until the
+	    garbage collection was done. This either occurred due to
+	    a explicit call to the <c>garbage_collect/1</c> BIF, or
+	    due to a garbage collection part of code loading. A
+	    process being garbage collected like this will now not be
+	    scheduled until the garbage collection has completed.</p>
+          <p>
+	    Own Id: OTP-9211</p>
+        </item>
+        <item>
+          <p>
+	    Remove unnecessary validation copy in
+	    prim_file:drv_command/3 (Thanks to Tony Rogvall)</p>
+          <p>
+	    Own Id: OTP-9276</p>
+        </item>
+        <item>
+          <p>
+	    Symbolic link handling on windows have been slightly
+	    updated to map error conditions more consequently and
+	    correctly read directory links created outside of the
+	    Erlang environment.</p>
+          <p>
+	    Own Id: OTP-9279</p>
+        </item>
+        <item>
+          <p>
+	    Due to standard library DLL mismatches between versions
+	    of OpenSSL and Erlang/OTP, OpenSSL is now linked
+	    statically to the crypto driver on Windows. This fixes
+	    problems starting crypto when running Erlang as a service
+	    on all Windows versions.</p>
+          <p>
+	    Own Id: OTP-9280</p>
+        </item>
+        <item>
+	    <p>Halfword emulator memory handling improvements:</p>
+	    <list> <item><p>Much more of internal memory structures
+	    have been made able to use "high" memory and are no
+	    longer restricted to the 4Gb limit that still applies for
+	    all process heap data.</p> </item> <item><p>Fixed faulty
+	    values from <c>erlang:memory()</c> caused by 32-bit
+	    counter overflow.</p> </item> <item><p>New counter
+	    <c>low</c> in <c>erlang:memory()</c> that sums up all
+	    memory restricted by 4Gb limit.</p> </item> </list>
+          <p>
+	    Own Id: OTP-9291 Aux Id: seq11841 </p>
+        </item>
+        <item>
+          <p>
+	    The value set in the undocumented and unsupported
+	    ERL_version_FLAGS (e.g. ERL_R14B03_FLAGS) environment
+	    variable can now be overridden by the command line
+	    (similar to ERL_AFLAGS).</p>
+          <p>
+	    Own Id: OTP-9297</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.8.3.2</title>
+
+    <section><title>Known Bugs and Problems</title>
+      <list>
+        <item>
+          <p>
+	    Fix halfword emulator bug in <c>ets:select_delete</c> for
+	    <c>ordered_set</c> that caused emulator to crash.</p>
+          <p>
+	    Own Id: OTP-9258 Aux Id: seq11836 </p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.8.3.1</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+          <p>
+	    Ets table type ordered_set could order large integer keys
+	    wrongly on pure 64bit platforms. This is now corrected.</p>
+          <p>
+	    Own Id: OTP-9181</p>
+        </item>
+        <item>
+          <p>
+	    The status of a process was unnecessarily set to waiting
+	    before a process was enqueued in a run queue. This bug
+	    was harmless up until OTP-R14B01. In OTP-R14B02
+	    <c>erlang:hibernate/3</c> was fixed (OTP-9125). After the
+	    introduction of OTP-9125, the previously harmless process
+	    status bug sometimes caused erroneous badarg exceptions
+	    from <c>process_info()</c>.</p>
+          <p>
+	    OTP-9125 also introduced a thread unsafe access to the
+	    status field of a process which now also have been fixed.</p>
+          <p>
+	    *** INCOMPATIBILITY with noxs ***</p>
+          <p>
+	    Own Id: OTP-9197</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.8.3</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+          <p>
+	    The scroll wheel now scrolls the werl window on Windows.</p>
+          <p>
+	    Own Id: OTP-8985</p>
+        </item>
+        <item>
+          <p>
+	    Some malformed distribution messages could cause VM to
+	    crash, this is now corrected.</p>
+          <p>
+	    Own Id: OTP-8993</p>
+        </item>
+        <item>
+          <p>
+	    The OS function getifaddrs() can return NULL in some
+	    address fields for e.g PPP and tunnel devices which
+	    caused the emulator to segfault. This bug has now been
+	    corrected.</p>
+          <p>
+	    Own Id: OTP-8996</p>
+        </item>
+        <item>
+          <p>
+	    The expression &lt;&lt;A:0&gt;&gt; would always produce
+	    an empty binary, even if <c>A</c> was not an integer.
+	    Corrected to cause a <c>badarg</c> exception if the type
+	    of <c>A</c> is invalid. (Thanks to Zvi.)</p>
+          <p>
+	    Own Id: OTP-8997</p>
+        </item>
+        <item>
+          <p>
+	    A bug that potentially could cause an emulator crash when
+	    deleting an ETS-table has been fixed. A resource leak
+	    when hitting the maximum amount of ETS-tables allowed has
+	    also been fixed.</p>
+          <p>
+	    Own Id: OTP-8999</p>
+        </item>
+        <item>
+          <p>
+	    A bug in the <c>exit/2</c> BIF could potentially cause an
+	    emulator crash.</p>
+          <p>
+	    Own Id: OTP-9005</p>
+        </item>
+        <item>
+          <p>
+	    Due to a bug in glibc the runtime system could abort
+	    while trying to destroy a mutex. The runtime system will
+	    now issue a warning instead of aborting.</p>
+          <p>
+	    Own Id: OTP-9009</p>
+        </item>
+        <item>
+          <p>
+	    A bug in epmd could create strange behaviour when
+	    listen() calls failed. This is now corrected thanks to
+	    Steve Vinoski.</p>
+          <p>
+	    Own Id: OTP-9024</p>
+        </item>
+        <item>
+	    <p>When setting file_info the win32_driver will now
+	    correctly set access and modified time. Previously these
+	    entities were swapped.</p>
+          <p>
+	    Own Id: OTP-9046</p>
+        </item>
+        <item>
+          <p>
+	    Setting scheduler bind type to <c>unbound</c> failed if
+	    binding of schedulers wasn't supported, or if CPU
+	    topology wasn't present. This even though the
+	    documentation stated that it is possible to set the bind
+	    type to <c>unbound</c>.</p>
+          <p>
+	    Own Id: OTP-9056 Aux Id: Seq11779 </p>
+        </item>
+        <item>
+	    <p>Two problems were fixed in crash dump: The time left
+	    for timers are now shown as unsigned integers and the
+	    contents of ordered_set ETS tables is no longer
+	    included.</p>
+          <p>
+	    Own Id: OTP-9057</p>
+        </item>
+        <item>
+          <p>
+	    The VM could fail to set IP_TOS and SO_PRIORITY in
+	    certain situations, either because sockets were supplied
+	    as open file descriptors, or because SO_PRIORITY by
+	    default was set higher than the user can explicitly set
+	    it to. Those situations are now handled.</p>
+          <p>
+	    Own Id: OTP-9069</p>
+        </item>
+        <item>
+          <p>
+	    Wx on MacOS X generated complains on stderr about certain
+	    cocoa functions not beeing called from the "Main thread".
+	    This is now corrected.</p>
+          <p>
+	    Own Id: OTP-9081</p>
+        </item>
+        <item>
+          <p>
+	    Fix a couple typos in driver_entry(3) (thanks to Tuncer
+	    Ayaz).</p>
+          <p>
+	    Own Id: OTP-9085</p>
+        </item>
+        <item>
+          <p>
+	    Mention that "-detached" implies "-noinput"</p>
+          <p>
+	    Clarify that specifying "-noinput" is unnecessary if the
+	    "-detached" flag is given. (thanks to Holger Wei�)</p>
+          <p>
+	    Own Id: OTP-9086</p>
+        </item>
+        <item>
+          <p>
+	    A potential problem (found by code inspection) when
+	    calling a fun whose code was not loaded has been fixed.</p>
+          <p>
+	    Own Id: OTP-9095</p>
+        </item>
+        <item>
+          <p>
+	    The emulator could get into a state where it didn't check
+	    for I/O.</p>
+          <p>
+	    Own Id: OTP-9105 Aux Id: Seq11798 </p>
+        </item>
+        <item>
+          <p>
+	    Attempting to create binaries exceeding 2Gb (using for
+	    example <c>term_to_binary/1</c>) would crash the emulator
+	    with an attempt to allocate huge amounts of memory.
+	    (Thanks to Jon Meredith.)</p>
+          <p>
+	    Own Id: OTP-9117</p>
+        </item>
+        <item>
+          <p>
+	    Fix erlang:hibernate/3 on HiPE enabled emulator (Thanks
+	    to Paul Guyot)</p>
+          <p>
+	    Own Id: OTP-9125</p>
+        </item>
+      </list>
+    </section>
+
+
+    <section><title>Improvements and New Features</title>
+      <list>
+        <item>
+	    <p>From this release, the previously experimental
+	    halfword emulator is now official. It can be enabled by
+	    giving the <c>--enable-halfword-emulator</c> option to
+	    the <c>configure</c> script.</p>
+	    <p>The halfword emulator is a 64-bit application, but
+	    uses halfwords (32-bit words) for all data in Erlang
+	    processes, therefore using less memory and being faster
+	    than the standard 64-bit emulator. The total size of all
+	    BEAM code and all process data for all processes is
+	    limited to 4Gb, but ETS tables and off-heap binaries are
+	    only limited by the amount of available memory.</p>
+          <p>
+	    Own Id: OTP-8941</p>
+        </item>
+        <item>
+          <p>
+	    32-bit atomic memory operations have been introduced
+	    internally in the run time system, and are now used where
+	    appropriate. There were previously only atomic memory
+	    operations of word size available. The 32-bit atomic
+	    memory operations slightly reduce memory consumption, and
+	    slightly improve performance on 64-bit runtime systems.</p>
+          <p>
+	    Own Id: OTP-8974</p>
+        </item>
+        <item>
+          <p>
+	    Performance enhancements for looking up timer-entries and
+	    removing timers from the wheel.</p>
+          <p>
+	    Own Id: OTP-8990</p>
+        </item>
+        <item>
+          <p>
+	    Write accesses to ETS tables have been optimized by
+	    reducing the amount of atomic memory operations needed
+	    during a write access.</p>
+          <p>
+	    Own Id: OTP-9000</p>
+        </item>
+        <item>
+          <p>
+	    Strange C coding in the VM made the -D_FORTIFY_SOURCE
+	    option to gcc-4.5 react badly. The code is now cleaned up
+	    so that it's accepted by gcc-4.5.</p>
+          <p>
+	    Own Id: OTP-9025</p>
+        </item>
+        <item>
+          <p>
+	    The memory footprint for loaded code has been somewhat
+	    reduced (especially in the 64-bit BEAM machine).</p>
+          <p>
+	    Own Id: OTP-9030</p>
+        </item>
+        <item>
+          <p>
+	    The maximum number of allowed arguments for an Erlang
+	    function has been lowered from 256 to 255, so that the
+	    number of arguments can now fit in a byte.</p>
+          <p>
+	    Own Id: OTP-9049</p>
+        </item>
+        <item>
+          <p>
+	    Dependency generation for Makefiles has been added to the
+	    compiler and erlc. See the manual pages for
+	    <c>compile</c> and <c>erlc</c>. (Thanks to Jean-Sebastien
+	    Pedron.)</p>
+          <p>
+	    Own Id: OTP-9065</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.8.2</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+	    <p> Fix format_man_pages so it handles all man sections
+	    and remove warnings/errors in various man pages. </p>
+          <p>
+	    Own Id: OTP-8600</p>
+        </item>
+        <item>
+          <p>
+	    The <c>configure</c> command line argument <seealso
+	    marker="doc/installation_guide:INSTALL#How-to-Build-and-Install-ErlangOTP_A-Closer-Look-at-the-individual-Steps_Configuring">--enable-ethread-pre-pentium4-compatibility</seealso>
+	    had no effect. This option is now also automatically
+	    enabled if required on the build machine.</p>
+          <p>
+	    Own Id: OTP-8847</p>
+        </item>
+        <item>
+          <p>
+	    Windows 2003 and Windows XP pre SP3 would sometimes not
+	    start the Erlang R14B VM at all due to a bug in the cpu
+	    topology detection. The bug affects Windows only, no
+	    other platform is even remotely affected. The bug is now
+	    corrected.</p>
+          <p>
+	    Own Id: OTP-8876</p>
+        </item>
+        <item>
+          <p>
+	    The HiPE run-time in the 64-bit emulator could do a
+	    64-bit write to a 32-bit struct field. It happened to be
+	    harmless on Intel/AMD processors. Corrected. (Thanks to
+	    Mikael Pettersson.)</p>
+          <p>
+	    Own Id: OTP-8877</p>
+        </item>
+        <item>
+          <p>
+	    A bug in <seealso
+	    marker="erl_driver#erl_drv_tsd_get">erl_drv_tsd_get()</seealso>
+	    and <seealso
+	    marker="erl_nif#enif_tsd_get">enif_tsd_get()</seealso>
+	    could cause an emulator crash. These functions are
+	    currently not used in OTP. That is, the crash only occur
+	    on systems with user implemented NIF libraries, or
+	    drivers that use one of these functions.</p>
+          <p>
+	    Own Id: OTP-8889</p>
+        </item>
+        <item>
+          <p>
+	    Calling <c>erlang:system_info({cpu_topology,
+	    CpuTopologyType})</c> with another <c>CpuTopologyType</c>
+	    element than one of the documented atoms <c>defined</c>,
+	    <c>detected</c>, or <c>used</c> caused an emulator crash.
+	    (Thanks to Paul Guyot)</p>
+          <p>
+	    Own Id: OTP-8914</p>
+        </item>
+        <item>
+          <p>
+	    The ERTS internal rwlock implementation could get into an
+	    inconsistent state. This bug was very seldom triggered,
+	    but could be during heavy contention. The bug was
+	    introduced in R14B (erts-5.8.1).</p>
+          <p>
+	    The bug was most likely to be triggered when using the
+	    <c>read_concurrency</c> option on an ETS table that was
+	    frequently accessed from multiple processes doing lots of
+	    writes and reads. That is, in a situation where you
+	    typically don't want to use the <c>read_concurrency</c>
+	    option in the first place.</p>
+          <p>
+	    Own Id: OTP-8925 Aux Id: OTP-8544 </p>
+        </item>
+        <item>
+          <p>
+	    Tracing to port could cause an emulator crash when
+	    unloading the trace driver.</p>
+          <p>
+	    Own Id: OTP-8932</p>
+        </item>
+        <item>
+          <p>
+	    Removed use of CancelIoEx on Windows that had been shown
+	    to cause problems with some drivers.</p>
+          <p>
+	    Own Id: OTP-8937</p>
+        </item>
+        <item>
+          <p>
+	    The fallback implementation used when no native atomic
+	    implementation was found did not compile. (Thanks to
+	    Patrick Baggett, and Tuncer Ayaz)</p>
+          <p>
+	    Own Id: OTP-8944</p>
+        </item>
+        <item>
+          <p>
+	    Some integer values used during load balancing could
+	    under rare circumstances wrap causing a load unbalance
+	    between schedulers.</p>
+          <p>
+	    Own Id: OTP-8950</p>
+        </item>
+        <item>
+          <p>
+	    The windows VM now correctly handles appending to large
+	    files (> 4GB).</p>
+          <p>
+	    Own Id: OTP-8958</p>
+        </item>
+        <item>
+          <p>
+	    Name resolving of IPv6 addresses has been implemented for
+	    Windows versions that support it. The use of ancient
+	    resolver flags (AI_V4MAPPED | AI_ADDRCONFIG) to the
+	    getaddrinfo() function has been removed since e.g FreeBSD
+	    regard mapped IPv4 addresses to be a security problem and
+	    the semantics of the address configured flag is
+	    uncertain.</p>
+          <p>
+	    Own Id: OTP-8969</p>
+        </item>
+      </list>
+    </section>
+
+
+    <section><title>Improvements and New Features</title>
+      <list>
+        <item>
+          <p>
+	    The help texts produced by the <c>configure</c> scripts
+	    in the top directory and in the erts directory have been
+	    aligned and cleaned up.</p>
+          <p>
+	    Own Id: OTP-8859</p>
+        </item>
+        <item>
+          <p>
+	    When the runtime system had fewer schedulers than logical
+	    processors, the system could get an unnecessarily large
+	    amount reader groups.</p>
+          <p>
+	    Own Id: OTP-8861</p>
+        </item>
+        <item>
+          <p>
+	    <c>run_rel</c> has been updated to support Solaris's
+	    /dev/ptmx device and to load the necessary STREAMS
+	    modules so that <c>to_erl</c> can provide terminal echo
+	    of keyboard input. (Thanks to Ryan Tilder.)</p>
+          <p>
+	    Own Id: OTP-8878</p>
+        </item>
+        <item>
+          <p>
+	    The Erlang VM now supports Unicode filenames. The feature
+	    is turned on by default on systems where Unicode
+	    filenames are mandatory (Windows and MacOSX), but can be
+	    enabled on other systems with the '+fnu' emulator option.
+	    Enabling the Unicode filename feature on systems where it
+	    is not default is however considered experimental and not
+	    to be used for production. Together with the Unicode file
+	    name support, the concept of "raw filenames" is
+	    introduced, which means filenames provided without
+	    implicit unicode encoding translation. Raw filenames are
+	    provided as binaries, not lists. For further information,
+	    see stdlib users guide and the chapter about using
+	    Unicode in Erlang. Also see the file module manual page.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8887</p>
+        </item>
+        <item>
+	    <p>Buffer overflows have been prevented in <c>erlc</c>,
+	    <c>dialyzer</c>, <c>typer</c>, <c>run_test</c>,
+	    <c>heart</c>, <c>escript</c>, and <c>erlexec</c>.</p>
+	    (Thanks to Michael Santos.)
+          <p>
+	    Own Id: OTP-8892</p>
+        </item>
+        <item>
+          <p>
+	    The runtime system is now less eager to suspend processes
+	    sending messages over the distribution. The default value
+	    of the distribution buffer busy limit has also been
+	    increased from 128 KB to 1 MB. This in order to improve
+	    throughput.</p>
+          <p>
+	    Own Id: OTP-8901</p>
+        </item>
+        <item>
+          <p>
+	    The distribution buffer busy limit can now be configured
+	    at system startup. For more information see the
+	    documentation of the <c>erl</c> <seealso
+	    marker="erl#+zdbbl">+zdbbl</seealso> command line flag.
+	    (Thanks to Scott Lystig Fritchie)</p>
+          <p>
+	    Own Id: OTP-8912</p>
+        </item>
+        <item>
+          <p>
+	    The inet driver internal buffer stack implementation has
+	    been rewritten in order to reduce lock contention.</p>
+          <p>
+	    Own Id: OTP-8916</p>
+        </item>
+        <item>
+          <p>
+	    New ETS option <c>compressed</c>, to enable a more
+	    compact storage format at the expence of heavier table
+	    operations. For test and evaluation, <c>erl +ec</c> can
+	    be used to force compression on all ETS tables.</p>
+          <p>
+	    Own Id: OTP-8922 Aux Id: seq11658 </p>
+        </item>
+        <item>
+          <p>
+	    There is now a new function inet:getifaddrs/0 modeled
+	    after C library function getifaddrs() on BSD and LInux
+	    that reports existing interfaces and their addresses on
+	    the host. This replaces the undocumented and unsupported
+	    inet:getiflist/0 and inet:ifget/2.</p>
+          <p>
+	    Own Id: OTP-8926</p>
+        </item>
+        <item>
+          <p>
+	    Support for detection of CPU topology and binding of
+	    schedulers on FreeBSD 8 have been added. (Thanks to Paul
+	    Guyot)</p>
+          <p>
+	    Own Id: OTP-8939</p>
+        </item>
+        <item>
+          <p>
+	    Several bugs related to hibernate/3 and HiPE have been
+	    corrected. (Thanks to Paul Guyot.)</p>
+          <p>
+	    Own Id: OTP-8952</p>
+        </item>
+        <item>
+          <p>
+	    Support for soft and hard links on Windows versions and
+	    filesystems that support them is added.</p>
+          <p>
+	    Own Id: OTP-8955</p>
+        </item>
+        <item>
+          <p>
+	    The win32 virtual machine is now linked large address
+	    aware. his allows the Erlang VM to use up to 3 gigs of
+	    address space on Windows instead of the default of 2
+	    gigs.</p>
+          <p>
+	    Own Id: OTP-8956</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.8.1.2</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+	    <p> Fix that the documentation top index generator can
+	    handle an Ericsson internal application group. </p>
+          <p>
+	    Own Id: OTP-8875</p>
+        </item>
+        <item>
+	    <p>In embedded mode, on_load handlers that called
+	    <c>code:priv_dir/1</c> or other functions in <c>code</c>
+	    would hang the system. Since the <c>crypto</c>
+	    application now contains an on_loader handler that calls
+	    <c>code:priv_dir/1</c>, including the <c>crypto</c>
+	    application in the boot file would prevent the system
+	    from starting.</p>
+	    <p>Also extended the <c>-init_debug</c> option to print
+	    information about on_load handlers being run to
+	    facilitate debugging.</p>
+          <p>
+	    Own Id: OTP-8902 Aux Id: seq11703 </p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.8.1.1</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+          <p>
+	    Windows 2003 and Windows XP pre SP3 would sometimes not
+	    start the Erlang R14B VM at all due to a bug in the cpu
+	    topology detection. The bug affects Windows only, no
+	    other platform is even remotely affected. The bug is now
+	    corrected.</p>
+          <p>
+	    Own Id: OTP-8876</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.8.1</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+          <p>
+	    Very small floating point numbers generated errors when
+	    converting from list to float in some versions of the VM,
+	    this is now corrected so that i.e.
+	    list_to_float("1.0e-324"). returns 0.0 in all versions of
+	    Erlang.</p>
+          <p>
+	    Own Id: OTP-7178</p>
+        </item>
+        <item>
+          <p>
+	    Windows Vista and Windows 7 file system virtualization,
+	    which makes "old style" windows programs execute in a
+	    file system sandbox, was previously unintentionally
+	    turned on for the Erlang VM. This is now corrected so
+	    that i.e. writes to C:\Program Files\. without
+	    administrator privileges will fail.</p>
+          <p>
+	    Own Id: OTP-7405</p>
+        </item>
+        <item>
+          <p>
+	    Fix faulty 64-bit integer term output from drivers. Large
+	    64-bits integers did not generate correct bignums and
+	    could even cause emulator crash. Only affects drivers
+	    using ERL_DRV_INT64 or ERL_DRV_UINT64, introduced in
+	    R13B03.</p>
+          <p>
+	    Own Id: OTP-8716</p>
+        </item>
+        <item>
+          <p>
+	    Fixed: inet:setopts(S, [{linger,{true,2}}]) returned
+	    {error,einval} for SCTP sockets. The inet_drv had a bug
+	    when checking the option size.</p>
+          <p>
+	    Own Id: OTP-8726 Aux Id: seq11617 </p>
+        </item>
+        <item>
+	    <p>Fix libm linking with --as-needed flag
+          <p>
+	    When building with "--as-needed" linker flags on Linux
+	    the build will fail. This has now been fixed.</p>
+          <p>
+	    (Thanks to Christian Faulhammer)</p></p>
+          <p>
+	    Own Id: OTP-8728</p>
+        </item>
+        <item>
+          <p>
+	    gen_udp:connect/3 was broken for SCTP enabled builds. It
+	    did not detect remote end errors as it should.</p>
+          <p>
+	    Own Id: OTP-8729</p>
+        </item>
+        <item>
+	    <p>Reduce the risk of integer wrapping in bin vheap size
+	    counting.</p> <p>The vheap size series will now use the
+	    golden ratio instead of doubling and fibonacci
+	    sequences.</p>
+          <p>
+	    Own Id: OTP-8730</p>
+        </item>
+        <item>
+          <p>
+	    ETS ordered_set containing <c>[]</c> as key could cause
+	    strange thing to happen, like an infinite hanging
+	    <c>ets:select</c>.</p>
+          <p>
+	    Own Id: OTP-8732</p>
+        </item>
+        <item>
+	    <p>reference() has been substituted for ref() in the
+	    documentation.</p>
+          <p>
+	    Own Id: OTP-8733</p>
+        </item>
+        <item>
+          <p>
+	    When a native compiled module called a not loaded
+	    non-native compiled module that had an on_load function,
+	    the export entries were trashed after code loading so on
+	    the next call from the native compiled module to the
+	    non-native compiled the emulator crashed. This bug has
+	    now been fixed.</p>
+          <p>
+	    Own Id: OTP-8736</p>
+        </item>
+        <item>
+          <p>
+	    HiPE-enabled Erlang VMs running on BSD systems sometimes
+	    generated messages like "Yikes! erts_alloc() returned
+	    misaligned address 0x8016a512c". Fixed. (Thanks to Mikael
+	    Pettersson.)</p>
+          <p>
+	    Own Id: OTP-8769</p>
+        </item>
+        <item>
+          <p>
+	    A race condition in <c>erts_poll()</c> could cause delay
+	    of poll for I/O.</p>
+          <p>
+	    Own Id: OTP-8773</p>
+        </item>
+        <item>
+          <p>
+	    Removed some potential vulnerabilities from the Erlang
+	    Port Mapper Daemon (epmd) and straightened up access
+	    control. Also removed hazardous interfaces allowing
+	    anyone on a machine to forcefully unregister other nodes.
+	    This means that the ei_unregister/erl_unregister
+	    interfaces in erl_interface is rendered not only error
+	    prone and mystifying as before, but totally ineffective.
+	    The old behaviour of unchecked node unregistering can be
+	    restored if needed, see epmd documentation for details.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8780</p>
+        </item>
+        <item>
+	    <p> Building in a source tree without prebuilt platform
+	    independent build results failed on the SSL examples
+	    when: </p> <list><item> cross building. This has been
+	    solved by not building the SSL examples during a cross
+	    build. </item><item> building on Windows. </item></list>
+          <p>
+	    Own Id: OTP-8791</p>
+        </item>
+        <item>
+          <p>
+	    inet:getsockopt for SCTP sctp_default_send_param had a
+	    bug to not initialize required feilds causing random
+	    answers. It is now corrected.</p>
+          <p>
+	    Own Id: OTP-8795 Aux Id: seq11655 </p>
+        </item>
+        <item>
+          <p>
+	    The hipe_bifs:get_hrvtime/0 BIF now always returns a real
+	    value even if the "perfctr" Linux kernel extension is not
+	    available. It used to return a dummy value. (Thanks to
+	    Mikael Pettersson.)</p>
+          <p>
+	    Own Id: OTP-8798</p>
+        </item>
+        <item>
+          <p>
+	    Calling a native-code compiled module with an
+	    <c>on_load</c> function could cause a crash. (Thanks to
+	    Mikael Pettersson.)</p>
+          <p>
+	    Own Id: OTP-8799</p>
+        </item>
+        <item>
+	    <p>The emulator could crash while writing a crash dump if
+	    native-compiled modules had been loaded. (Thanks to Paul
+	    Guyot.)</p>
+          <p>
+	    Own Id: OTP-8801</p>
+        </item>
+        <item>
+          <p>
+	    The garbage collector could crash if invoked from
+	    native-compiled code after a call to a BIF. (Thanks to
+	    Paul Guyot.)</p>
+          <p>
+	    Own Id: OTP-8821</p>
+        </item>
+        <item>
+          <p>
+	    A rare memory leak in binary:matches is removed</p>
+          <p>
+	    Own Id: OTP-8823</p>
+        </item>
+        <item>
+	    <p>For a socket in the HTTP packet mode, the return value
+	    from <c>gen_tcp:recv/2,3</c> if there is an error in the
+	    header will be <c>{ok,{http_error,String}}</c> instead of
+	    <c>{error,{http_error,String}}</c> to be consistent with
+	    <c>ssl:recv/2,3</c>.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8831</p>
+        </item>
+      </list>
+    </section>
+
+
+    <section><title>Improvements and New Features</title>
+      <list>
+        <item>
+          <p>
+	    ets:select_reverse/{1,2,3} are now documented.</p>
+          <p>
+	    Own Id: OTP-7863</p>
+        </item>
+        <item>
+          <p>
+	    External format of integers changed to make full use of
+	    all 32 bits of INTEGER_EXT. This is a compatible change
+	    as old code can read full 32-bit integers but only
+	    produce 28-bit integers as INTEGER_EXT.</p>
+          <p>
+	    Own Id: OTP-8540 Aux Id: seq11534 </p>
+        </item>
+        <item>
+          <p>
+	    Large parts of the <c>ethread</c> library have been
+	    rewritten. The <c>ethread</c> library is an Erlang
+	    runtime system internal, portable thread library used by
+	    the runtime system itself.</p>
+          <p>
+	    Most notable improvement is a reader optimized rwlock
+	    implementation which dramatically improve the performance
+	    of read-lock/read-unlock operations on multi processor
+	    systems by avoiding ping-ponging of the rwlock cache
+	    lines. The reader optimized rwlock implementation is used
+	    by miscellaneous rwlocks in the runtime system that are
+	    known to be read-locked frequently, and can be enabled on
+	    ETS tables by passing the <seealso
+	    marker="stdlib:ets#new_2_read_concurrency">{read_concurrency,
+	    true}</seealso> option upon table creation. See the
+	    documentation of <seealso
+	    marker="stdlib:ets#new/2">ets:new/2</seealso> for more
+	    information. The reader optimized rwlock implementation
+	    can be fine tuned when starting the runtime system. For
+	    more information, see the documentation of the <seealso
+	    marker="erts:erl#+rg">+rg</seealso> command line argument
+	    of <c>erl</c>.</p>
+          <p>
+	    There is also a new implementation of rwlocks that is not
+	    optimized for readers. Both implementations interleaves
+	    readers and writers during contention as opposed to,
+	    e.g., the NPTL (Linux) pthread rwlock implementation
+	    which use either a reader or writer preferred strategy.
+	    The reader/writer preferred strategy is problematic since
+	    it starves threads doing the non-preferred operation.</p>
+          <p>
+	    The new rwlock implementations in general performs better
+	    in ERTS than common pthread implementations. However, in
+	    some extremely heavily contended cases this is not the
+	    case. Such heavy contention can more or less only appear
+	    on ETS tables. This when multiple processes do very large
+	    amounts of write locked operations simultaneously on the
+	    same table. Such use of ETS is bad regardless of rwlock
+	    implementation, will never scale, and is something we
+	    strongly advise against.</p>
+          <p>
+	    The new rwlock implementations depend on atomic
+	    operations. If no native atomic implementation is found,
+	    a fallback solution will be used. Using the fallback
+	    implies a performance degradation. That is, it is more
+	    important now than before to build OTP with a native
+	    atomic implementation.</p>
+          <p>
+	    The <c>ethread</c> library contains native atomic
+	    implementations for, x86 (32 and 64 bit), powerpc (32
+	    bit), sparc V9 (32 and 64 bit), and tilera (32 bit). On
+	    other hardware gcc's builtin support for atomic memory
+	    access will be used if such exists. If no such support is
+	    found, <c>configure</c> will warn about no atomic
+	    implementation available.</p>
+          <p>
+	    The <c>ethread</c> library can now also use the
+	    <c>libatomic_ops</c> library for atomic memory accesses.
+	    This makes it possible for the Erlang runtime system to
+	    utilize optimized native atomic operations on more
+	    platforms than before. If <c>configure</c> warns about no
+	    atomic implementation available, try using the
+	    <c>libatomic_ops</c> library. Use the <seealso
+	    marker="doc/installation_guide:INSTALL#How-to-Build-and-Install-ErlangOTP_A-Closer-Look-at-the-individual-Steps_Configuring">--with-libatomic_ops=PATH</seealso>
+	    <c>configure</c> command line argument when specifying
+	    where the <c>libatomic_ops</c> installation is located.
+	    The <c>libatomic_ops</c> library can be downloaded from:
+	    <url
+	    href="http://www.hpl.hp.com/research/linux/atomic_ops/">http://www.hpl.hp.com/research/linux/atomic_ops/</url></p>
+          <p>
+	    The changed API of the <c>ethread</c> library has also
+	    caused modifications in the Erlang runtime system.
+	    Preparations for the to come "delayed deallocation"
+	    feature has also been done since it depends on the
+	    <c>ethread</c> library.</p>
+          <p>
+	    <em>Note</em>: When building for x86, the <c>ethread</c>
+	    library will now use instructions that first appeared on
+	    the pentium 4 processor. If you want the runtime system
+	    to be compatible with older processors (back to 486) you
+	    need to pass the <seealso
+	    marker="doc/installation_guide:INSTALL#How-to-Build-and-Install-ErlangOTP_A-Closer-Look-at-the-individual-Steps_Configuring">--enable-ethread-pre-pentium4-compatibility</seealso>
+	    <c>configure</c> command line argument when configuring
+	    the system.</p>
+          <p>
+	    Own Id: OTP-8544</p>
+        </item>
+        <item>
+          <p>
+	    erlang:localtime_to_universaltime({{2008, 8, 1}, {0, 0,
+	    0}},true) when TZ=UTC now behaves consistently on all
+	    Unix platforms.</p>
+          <p>
+	    The problem fixed was originally reported by Paul Guyot
+	    on erlang-bugs mailing list:</p>
+          <p>
+	    http://www.erlang.org/pipermail/erlang-bugs/2008-November/001077.html</p>
+          <p>
+	    Own Id: OTP-8580</p>
+        </item>
+        <item>
+          <p>
+	    Optimization reducing memory consumption by two words per
+	    ETS object.</p>
+          <p>
+	    Own Id: OTP-8737</p>
+        </item>
+        <item>
+          <p>
+	    Fixes for unsupported halfword-emulator</p>
+          <p>
+	    Own Id: OTP-8745</p>
+        </item>
+        <item>
+          <p>
+	    NIF 64-bit integer support; <c>enif_get_int64</c>,
+	    <c>enif_get_uint64</c>, <c>enif_make_int64</c>,
+	    <c>enif_make_uint64</c>.</p>
+          <p>
+	    Own Id: OTP-8746</p>
+        </item>
+        <item>
+          <p>
+	    Alignment of trailing data in messages has been adjusted.
+	    This in order to be able to pass data of any type as
+	    trailing data in the future.</p>
+          <p>
+	    Own Id: OTP-8754</p>
+        </item>
+        <item>
+          <p>
+	    The obsolete/driver.h header file has been removed. It
+	    has been obsolete and deprecated since R8B. Drivers that
+	    still include obsolete/driver.h must be updated to
+	    include erl_driver.h.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8758</p>
+        </item>
+        <item>
+          <p>
+	    Added erlang:system_info(build_type) which makes it
+	    easier to chose drivers, NIF libraries, etc based on
+	    build type of the runtime system.</p>
+          <p>
+	    The NIF library for crypto can now be built for valgrind
+	    and/or debug as separate NIF libraries that will be
+	    automatically loaded if the runtime system has been built
+	    with a matching build type.</p>
+          <p>
+	    Own Id: OTP-8760</p>
+        </item>
+        <item>
+          <p>
+	    Further lessened the memory requirements of ETS objects.</p>
+          <p>
+	    Own Id: OTP-8762</p>
+        </item>
+        <item>
+	    <p>The broken elib_malloc alternate memory allocator has
+	    been removed. <c>erlang:system_info(elib_malloc)</c> will
+	    always return <c>false</c>, and in R15,
+	    <c>erlang:system_info(elib_malloc)</c> will fail with a
+	    <c>badarg</c> exception.</p>
+          <p>
+	    Own Id: OTP-8764</p>
+        </item>
+        <item>
+          <p>
+	    Calling <c>erlang:system_info/1</c> with the new argument
+	    <c>update_cpu_info</c> will make the runtime system
+	    reread and update the internally stored CPU information.
+	    For more information see the documentation of <seealso
+	    marker="erlang#update_cpu_info">erlang:system_info(update_cpu_info)</seealso>.</p>
+          <p>
+	    The CPU topology is now automatically detected on Windows
+	    systems with less than 33 logical processors. The runtime
+	    system will now, also on Windows, by default bind
+	    schedulers to logical processors using the
+	    <c>default_bind</c> bind type if the amount of schedulers
+	    is at least equal to the amount of logical processors
+	    configured, binding of schedulers is supported, and a CPU
+	    topology is available at startup.</p>
+          <p>
+	    Own Id: OTP-8765</p>
+        </item>
+        <item>
+          <p>
+	    The SMP ERTS internal child waiter thread used on Linux
+	    system with NPTL was unintentionally disabled during
+	    cross compilation rewrites (OTP-8323 in R13B03). It has
+	    now been re-enabled. Enabling it again gives a slight
+	    performance improvement.</p>
+          <p>
+	    Own Id: OTP-8774</p>
+        </item>
+        <item>
+          <p>
+	    <c>epmd</c> used to generate a message to the syslog when
+	    it started up, which could be annoying. This has been
+	    changed to only generate the message if the debug swith
+	    is given. (Thanks to Michael Santos.)</p>
+          <p>
+	    Own Id: OTP-8775</p>
+        </item>
+        <item>
+          <p>
+	    The scheduler wakeup threshold is now possible to adjust
+	    at system boot. For more information see the <seealso
+	    marker="erl#+swt">+swt</seealso> command line argument of
+	    <c>erl</c>.</p>
+          <p>
+	    Own Id: OTP-8811</p>
+        </item>
+        <item>
+          <p>
+	    The undocumented function inet:ifget/2 has been improved
+	    to return interface hardware address (MAC) on platforms
+	    supporting getaddrinfo() (such as BSD unixes). Note it
+	    still does not work on all platforms for example not
+	    Windows nor Solaris, so the function is still
+	    undocumented.</p>
+          <p>
+	    Buffer overflow and field init bugs for inet:ifget/2 and
+	    inet:getservbyname/2 has also been fixed.</p>
+          <p>
+	    Thanks to Michael Santos.</p>
+          <p>
+	    Own Id: OTP-8816</p>
+        </item>
+        <item>
+          <p>
+	    Optimizations for MIPS when using gcc atomics. (Thanks to
+	    Steve Vinoski)</p>
+          <p>
+	    Own Id: OTP-8834</p>
+        </item>
+        <item>
+          <p>
+	    Lock optimization in timer functionality.</p>
+          <p>
+	    Own Id: OTP-8835</p>
+        </item>
+      </list>
+    </section>
+
+
+    <section><title>Known Bugs and Problems</title>
+      <list>
+        <item>
+	    <p>Fix epmd and build environment to build on VxWorks</p>
+          <p>
+	    Own Id: OTP-8838</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
+<section><title>Erts 5.8</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+          <p>
+	    Driver threads, such as async threads, using <seealso
+	    marker="erl_driver#ErlDrvPDL">port data locks</seealso>
+	    peeked at the port status field without proper locking
+	    when looking up the driver queue.</p>
+          <p>
+	    Own Id: OTP-8475</p>
+        </item>
+        <item>
+          <p>
+	    The use of <c>mmap()</c> was unnecessarily disabled when
+	    cross compiling.</p>
+          <p>
+	    The <c>configure</c> arguments <c>--with-ssl</c>, and
+	    <c>--with-odbc</c> refused to accept libraries outside of
+	    <c>$erl_xcomp_sysroot</c> when cross compiling for no
+	    good reason.</p>
+          <p>
+	    The <c>configure</c> argument <c>--with-odbc</c> didn't
+	    handle the value <c>yes</c> correct.</p>
+          <p>
+	    The <c>configure</c> arguments <c>--with-odbc</c>, and
+	    <c>--without-odbc</c> have also been added to the
+	    configure help.</p>
+          <p>
+	    (Thanks to Steve Vinoski for reporting these issues)</p>
+          <p>
+	    Own Id: OTP-8484</p>
+        </item>
+        <item>
+          <p>
+	    A call to the BIF <c>unregister(RegName)</c> when a port
+	    had the name <c>RegName</c> registered in the runtime
+	    system without SMP support caused a runtime system crash.
+	    (Thanks to Per Hedeland for the bugfix and test case.)</p>
+          <p>
+	    Own Id: OTP-8487</p>
+        </item>
+        <item>
+          <p>
+	    The runtime system crashed if fewer logical processors
+	    were found than reported by <c>sysconf(
+	    SC_NPROCESSORS_CONF)</c>.</p>
+          <p>
+	    Own Id: OTP-8549</p>
+        </item>
+        <item>
+          <p>
+	    Fix memory management bug causing crash of non-SMP
+	    emulator with async threads enabled. The bug did first
+	    appear in R13B03.</p>
+          <p>
+	    Own Id: OTP-8591 Aux Id: seq11554 </p>
+        </item>
+        <item>
+          <p>
+	    Port locks could be prematurely destroyed.</p>
+          <p>
+	    Own Id: OTP-8612</p>
+        </item>
+        <item>
+	    <p>The <c>empd</c> program could loop and consume 100%
+	    CPU time if an unexpected error ocurred in
+	    <c>listen()</c> or <c>accept()</c>. Now <c>epmd</c> will
+	    terminate if a non-recoverable error occurs. (Thanks to
+	    Michael Santos.)</p>
+          <p>
+	    Own Id: OTP-8618</p>
+        </item>
+        <item>
+          <p>
+	    When kernel poll has been enabled, a livelock could in
+	    rare circumstances occur. Problem reported by Chetan
+	    Ahuja, fix by Mikael Pettersson.</p>
+          <p>
+	    Own Id: OTP-8632</p>
+        </item>
+        <item>
+          <p>
+	    Windows: Closing port of program that stalled without
+	    reading all data could deadlock scheduler thread.</p>
+          <p>
+	    Own Id: OTP-8641</p>
+        </item>
+        <item>
+          <p>
+	    On some combination of Montavista Linux on Cavium Octeon
+	    processors, some socket-related system calls returned
+	    other numbers than -1 for errors. This caused a core dump
+	    in inet_drv.c. Now the code works around this problem.</p>
+          <p>
+	    Own Id: OTP-8654</p>
+        </item>
+        <item>
+          <p>
+	    Missing memory barriers in <c>erts_poll()</c> could cause
+	    the runtime system to hang indefinitely.</p>
+          <p>
+	    Own Id: OTP-8658</p>
+        </item>
+        <item>
+          <p>
+	    <c>ethr_rwmutex_tryrlock()</c> acquired and refused to
+	    acquire a lock with inverted logic. The lock was however
+	    never acquired in a thread unsafe manner. (Thanks to JR
+	    Zhang for noting this issue)</p>
+          <p>
+	    Own Id: OTP-8662</p>
+        </item>
+        <item>
+          <p>
+	    Extreme combinations of register/unregister in a highly
+	    parallell SMP application could crash the VM. The error
+	    is corrected.</p>
+          <p>
+	    Own Id: OTP-8663</p>
+        </item>
+        <item>
+          <p>
+	    On Windows, files are now opened with FILE_SHARE_DELETE
+	    to get closer to Unix semantics.</p>
+          <p>
+	    Own Id: OTP-8667</p>
+        </item>
+        <item>
+          <p>
+	    <c>erlang:system_info(multi_scheduling)</c> sometimes
+	    erroneously returned <c>enabled</c> when it should have
+	    returned <c>blocked</c>.</p>
+          <p>
+	    Own Id: OTP-8675</p>
+        </item>
+        <item>
+          <p>
+	    Fix bug causing <c>erlang:decode_packet</c> and
+	    <c>enif_make_string</c> to generate faulty strings with
+	    negative character values for ascii values larger than
+	    127. (Thanks to Paul Guyot)</p>
+          <p>
+	    Own Id: OTP-8685</p>
+        </item>
+        <item>
+          <p>
+	    <c>open_port/2</c> with the <c>spawn</c> and
+	    <c>spawn_executable</c> options can include an
+	    <c>{env,Env}</c> option. In some cases unsetting
+	    variables would not work on Unix (typically if more
+	    variables were unset than were actually present in the
+	    environment).</p>
+          <p>
+	    Own Id: OTP-8701</p>
+        </item>
+        <item>
+          <p>
+	    A user defined CPU topology set via a call to <seealso
+	    marker="erlang#system_flag_cpu_topology">erlang:system_flag(cpu_topology,
+	    CPUTopology)</seealso> was not properly verified, and
+	    could in worst case cause an emulator crash. The emulator
+	    crash could only occur when a user defined CPU topology
+	    already existed and was redefined.</p>
+          <p>
+	    Own Id: OTP-8710</p>
+        </item>
+      </list>
+    </section>
+
+
+    <section><title>Improvements and New Features</title>
+      <list>
+        <item>
+          <p>
+	    The grammar for match specifications in ERTS users guide
+	    only described the tracing dialect of match
+	    specifications. An informal grammar for the ETS dialect
+	    is added.</p>
+          <p>
+	    Own Id: OTP-8086 Aux Id: seq11333 </p>
+        </item>
+        <item>
+          <p>
+	    The module binary from EEP31 (and EEP9) is implemented.</p>
+          <p>
+	    Own Id: OTP-8217</p>
+        </item>
+        <item>
+          <p>
+	    New NIF API function <c>enif_make_new_binary</c></p>
+          <p>
+	    Own Id: OTP-8474</p>
+        </item>
+        <item>
+          <p>
+	    The guard BIF <c>is_boolean/1</c> (introduced in R10B)
+	    has now been included in the lists of BIFs allowed in
+	    guards in the Reference Manual.</p>
+          <p>
+	    Own Id: OTP-8477</p>
+        </item>
+        <item>
+          <p>
+	    Added function <c>zip:foldl/3</c> to iterate over zip
+	    archives.</p>
+          <p>
+	    Added functions to create and extract escripts. See
+	    <c>escript:create/2</c> and <c>escript:extract/2</c>.</p>
+          <p>
+	    The undocumented function <c>escript:foldl/3</c> has been
+	    removed. The same functionality can be achieved with the
+	    more flexible functions <c>escript:extract/2</c> and
+	    <c>zip:foldl/3</c>.</p>
+          <p>
+	    Record fields has been annotated with type info. Source
+	    files as been adapted to fit within 80 chars and trailing
+	    whitespace has been removed.</p>
+          <p>
+	    Own Id: OTP-8521</p>
+        </item>
+        <item>
+	    <p>A regular expression with many levels of parenthesis
+	    could cause a buffer overflow. That has been corrected.
+	    (Thanks to Michael Santos.)</p>
+          <p>
+	    Own Id: OTP-8539</p>
+        </item>
+        <item>
+          <p>
+	    <c>erlang:decode_packet(httph_bin,..)</c> could return
+	    corrupt header strings or even crash the VM. This has
+	    been fixed. It only happened on 32-bit VM if the header
+	    name was unknown and between 16 and 20 characters long.
+	    Sockets with simular <c>packet</c> option did not suffer
+	    from this bug.</p>
+          <p>
+	    Own Id: OTP-8548</p>
+        </item>
+        <item>
+	    <p>New NIF features:</p> <list><item> Send messages from
+	    a NIF, or from thread created by NIF, to any local
+	    process (<c>enif_send</c>) </item><item> Store terms
+	    between NIF calls (<c>enif_alloc_env</c>,
+	    <c>enif_make_copy</c>) </item><item> Create binary terms
+	    with user defined memory management
+	    (<c>enif_make_resource_binary</c>) </item></list> <p>And
+	    some incompatible changes made to the API. For more
+	    information see the warning text in <seealso
+	    marker="erl_nif">erl_nif(3)</seealso>.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8555</p>
+        </item>
+        <item>
+	    <p>If the '<c>fop</c>' program (needed for building PDF
+	    files) cannot not be found, it is now possible to build
+	    the HTML and man pages anyway (there will also be dummy
+	    PDF files with no real content created). (Thanks to
+	    Tuncer Ayaz.)</p>
+          <p>
+	    Own Id: OTP-8559</p>
+        </item>
+        <item>
+	    <p>When defining macros the closing right parenthesis
+	    before the dot is now mandatory.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8562</p>
+        </item>
+        <item>
+	    <p>Local and imported functions now override the
+	    auto-imported BIFs when the names clash. The pre R14
+	    behaviour was that auto-imported BIFs would override
+	    local functions. To avoid that old programs change
+	    behaviour, the following will generate an error:</p>
+	    <list><item><p>Doing a call without explicit module name
+	    to a local function having a name clashing with the name
+	    of an auto-imported BIF that was present (and
+	    auto-imported) before OTP R14A</p></item>
+	    <item><p>Explicitly importing a function having a name
+	    clashing with the name of an autoimported BIF that was
+	    present (and autoimported) before OTP R14A</p></item>
+	    <item><p>Using any form of the old compiler directive
+	    <c>nowarn_bif_clash</c></p></item> </list> <p>If the BIF
+	    was added or auto-imported in OTP R14A or later,
+	    overriding it with an import or a local function will
+	    only result in a warning,</p> <p>To resolve clashes, you
+	    can either use the explicit module name <c>erlang</c> to
+	    call the BIF, or you can remove the auto-import of that
+	    specific BIF by using the new compiler directive
+	    <c>-compile({no_auto_import,[F/A]}).</c>, which makes all
+	    calls to the local or imported function without explicit
+	    module name pass without warnings or errors.</p> <p>The
+	    change makes it possible to add auto-imported BIFs
+	    without breaking or silently changing old code in the
+	    future. However some current code ingeniously utilizing
+	    the old behaviour or the <c>nowarn_bif_clash</c> compiler
+	    directive, might need changing to be accepted by the
+	    compiler.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8579</p>
+        </item>
+        <item>
+          <p>
+	    A bug in re that could cause certain regular expression
+	    matches never to terminate is corrected. (Thanks to
+	    Michael Santos and Gordon Guthrie.)</p>
+          <p>
+	    Own Id: OTP-8589</p>
+        </item>
+        <item>
+	    <p>The erlang:open_port spawn and spawn_executable
+	    directives can include an <c>{env,Env}</c> directive to
+	    set up environment variables for the spawned process. A
+	    bug prevented applications from using <c>{env,Env}</c> to
+	    set an environment variable whose value ended with a
+	    '<c>=</c>' (equal sign) character; the trailing equal
+	    sign was mistaken as an indication that an environment
+	    variable was to be cleared from the environment of the
+	    spawned process. (Thanks to Steve Vinoski.)</p>
+          <p>
+	    Own Id: OTP-8614</p>
+        </item>
+        <item>
+	    <p><c>receive</c> statements that can only read out a
+	    newly created reference are now specially optimized so
+	    that it will execute in constant time regardless of the
+	    number of messages in the receive queue for the process.
+	    That optimization will benefit calls to
+	    <c>gen_server:call()</c>. (See <c>gen:do_call/4</c> for
+	    an example of a receive statement that will be
+	    optimized.)</p>
+          <p>
+	    Own Id: OTP-8623</p>
+        </item>
+        <item>
+          <p>
+	    The functions file:advise/4 and file:datasync/1 have been
+	    added. (Thanks to Filipe David Manana.)</p>
+          <p>
+	    Own Id: OTP-8637</p>
+        </item>
+        <item>
+          <p>
+	    New NIF API functions: <c>enif_make_atom_len</c>,
+	    <c>enif_make_existing_atom_len</c>,
+	    <c>enif_make_string_len</c>, <c>enif_get_atom_length</c>,
+	    <c>enif_get_list_length</c>, <c>enif_is_list</c>,
+	    <c>enif_is_tuple</c> (by Tuncer Ayaz)</p>
+          <p>
+	    Own Id: OTP-8640</p>
+        </item>
+        <item>
+          <p>
+	    Support for using gcc's built-in functions for atomic
+	    memory access has been added. This functionallity will be
+	    used if available and no other native atomic
+	    implementation in ERTS is available.</p>
+          <p>
+	    Own Id: OTP-8659</p>
+        </item>
+        <item>
+          <p>
+	    The number of spinlocks used when implementing atomic
+	    fall-backs when no native atomic implementation is
+	    available has been increased from 16 to 1024.</p>
+          <p>
+	    Own Id: OTP-8660</p>
+        </item>
+        <item>
+          <p>
+	    Writer preferred pthread read/write locks has been
+	    enabled on Linux.</p>
+          <p>
+	    Own Id: OTP-8661</p>
+        </item>
+        <item>
+          <p>
+	    The runtime system will by default bind schedulers to
+	    logical processors using the <c>default_bind</c> bind
+	    type if the amount of schedulers are at least equal to
+	    the amount of logical processors configured, binding of
+	    schedulers is supported, and a CPU topology is available
+	    at startup. </p>
+          <p>
+	    <em>NOTE:</em> If the Erlang runtime system is the only
+	    operating system process that binds threads to logical
+	    processors, this improves the performance of the runtime
+	    system. However, if other operating system processes (as
+	    for example another Erlang runtime system) also bind
+	    threads to logical processors, there might be a
+	    performance penalty instead. If this is the case you, are
+	    are advised to unbind the schedulers using the <seealso
+	    marker="erl#+sbt">+sbtu</seealso> command line argument,
+	    or by invoking <seealso
+	    marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type,
+	    unbound)</seealso>.</p>
+          <p>
+	    Own Id: OTP-8666</p>
+        </item>
+        <item>
+          <p>
+	    The recently added BIFs erlang:min/2, erlang:max/2 and
+	    erlang:port_command/3 are now auto-imported (as they were
+	    originally intended to be). Due to the recent compiler
+	    change (OTP-8579), the only impact on old code defining
+	    it's own min/2, max/2 or port_command/3 functions will be
+	    a warning, the local functions will still be used. The
+	    warning can be removed by using
+	    -compile({no_auto_import,[min/2,max/2,port_command/3]}).
+	    in the source file.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8669 Aux Id: OTP-8579 </p>
+        </item>
+        <item>
+          <p>
+	    There is a new option 'exclusive' to file:open/2 that
+	    uses the OS O_EXCL flag where supported to open the file
+	    in exclusive mode.</p>
+          <p>
+	    Own Id: OTP-8670</p>
+        </item>
+        <item>
+          <p>
+	    Now, binary_to_term/2 is auto-imported. This will cause a
+	    compile warning if and only if a module has got a local
+	    function with that name.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8671</p>
+        </item>
+        <item>
+          <p>
+	    Alignment of scheduler data and run queues were adjusted.</p>
+          <p>
+	    Own Id: OTP-8673</p>
+        </item>
+        <item>
+	    <p>Call time breakpoint tracing</p> <list><item>Introduce
+	    a <c>call_time</c> option to
+	    <c>erlang:trace_pattern/3</c>.This option enables call
+	    time breakpoint tracing on code that is executed by
+	    processes with call tracing enabled. Call time tracing
+	    stores the number of calls and the time spent of each
+	    function with this trace pattern enabled. The information
+	    can be retrieved with <c>erlang:trace_info/2</c></item>
+	    <item>Add a scheduler array for BpData. To solve the
+	    issue of multiple schedulers constantly updating the head
+	    pointer to the bp data wheel, each scheduler now has its
+	    own entrypoint to the wheel. This head pointer can be
+	    updated without a locking being taken.</item></list>
+	    <p>Teach call count tracing to use atomics</p>
+	    <list><item>Call count previously used a global lock for
+	    accessing and writing its counter in the breakpoint. This
+	    is now changed to atomics instead.</item> <item>The
+	    change will let call count tracing and cprof to scale
+	    better when increasing the number of
+	    schedulers.</item></list>
+          <p>
+	    Own Id: OTP-8677</p>
+        </item>
+        <item>
+	    <p><c>eprof</c> has been reimplemented with support in
+	    the Erlang virtual machine and is now both faster (i.e.
+	    slows down the code being measured less) and scales much
+	    better. In measurements we saw speed-ups compared to the
+	    old eprof ranging from 6 times (for sequential code that
+	    only uses one scheduler/core) up to 84 times (for
+	    parallel code that uses 8 cores).</p>
+	    <p>Note: The API for the <c>eprof</c> has been cleaned up
+	    and extended. See the documentation.</p>
+          <p>
+	    *** POTENTIAL INCOMPATIBILITY ***</p>
+          <p>
+	    Own Id: OTP-8706</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
 <section><title>Erts 5.7.5.2</title>
 
     <section><title>Known Bugs and Problems</title>
@@ -162,7 +3176,7 @@
         <item>
 	    <p>A number of bugs concerning re and unicode are
 	    corrected:</p>
-	    <p>re:compile no longer looses unicode option, which also
+	    <p>re:compile no longer loses unicode option, which also
 	    fixes bug in re:split.</p>
 	    <p>re:replace now handles unicode charlist replacement
 	    argument</p>
@@ -2633,7 +5647,7 @@
 	    The race occurred when a process removed a table during
 	    termination simultaneously as another process removed the
 	    same table via <c>ets:delete/1</c> and a third process
-	    created a table that accidentaly got the same internal
+	    created a table that accidentally got the same internal
 	    table index as the table being removed.</p>
           <p>
 	    Own Id: OTP-7349</p>
@@ -4734,7 +7748,7 @@
         </item>
         <item>
           <p>The runtime system with SMP support did not slowly adjust
-            it's view of time when the system time suddenly changed.</p>
+            its view of time when the system time suddenly changed.</p>
           <p>Timeouts could sometimes timeout too early on the runtime
             system with SMP support.</p>
           <p>Own Id: OTP-6202</p>
diff --git a/erts/doc/src/run_erl.xml b/erts/doc/src/run_erl.xml
index 7bf7f559c5..c9784299b3 100644
--- a/erts/doc/src/run_erl.xml
+++ b/erts/doc/src/run_erl.xml
@@ -4,7 +4,7 @@
 <comref>
   <header>
     <copyright>
-      <year>1999</year><year>2009</year>
+      <year>1999</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -144,6 +144,14 @@
       <item>The size (in bytes) of a log file before switching to a
        new log file. Default is 100000, minimum is 1000 and maximum is
        approximately 2^30.</item>
+      <tag>RUN_ERL_DISABLE_FLOWCNTRL</tag>
+      <item>If defined, disables input and output flow control for the pty opend by run_erl.
+       Useful if you want to remove any risk of accidentally blocking the flow control
+       by hit Ctrl-S (instead of Ctrl-D to detach).
+       Which may result in blocking of the entire beam process
+       and in the case of running heart as supervisor
+       even the heart process will be blocked when writing log message to terminal.
+       Leaving the heart process unable to do its work.</item>
     </taglist>
   </section>
 
diff --git a/erts/doc/src/specs.xml b/erts/doc/src/specs.xml
new file mode 100644
index 0000000000..e5c2f4783f
--- /dev/null
+++ b/erts/doc/src/specs.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="latin1" ?>
+<specs xmlns:xi="http://www.w3.org/2001/XInclude">
+  <xi:include href="../specs/specs_erl_prim_loader.xml"/>
+  <xi:include href="../specs/specs_erlang.xml"/>
+  <xi:include href="../specs/specs_init.xml"/>
+  <xi:include href="../specs/specs_zlib.xml"/>
+</specs>
diff --git a/erts/doc/src/start_erl.xml b/erts/doc/src/start_erl.xml
index 21cc901f52..92d87b095a 100644
--- a/erts/doc/src/start_erl.xml
+++ b/erts/doc/src/start_erl.xml
@@ -4,7 +4,7 @@
 <comref>
   <header>
     <copyright>
-      <year>1998</year><year>2009</year>
+      <year>1998</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -69,12 +69,29 @@
           <c><![CDATA[erl]]></c> program. Everything <em>after</em><c><![CDATA[++]]></c> is
            interpreted as options to <c><![CDATA[start_erl]]></c> itself.</item>
           <tag>-reldir &lt;release root&gt;</tag>
-          <item>Mandatory if the environment variable <c><![CDATA[RELDIR]]></c> is not
-           specified. Tells start_erl where the root of the
-           release tree is placed in the file-system 
-           (like &lt;Erlang root&gt;\\releases). The
-          <c><![CDATA[start_erl.data]]></c> file is expected to be placed in
-           this directory (if not otherwise specified).</item>
+
+          <item>Mandatory if the environment variable
+          <c><![CDATA[RELDIR]]></c> is not specified and no
+          <c>-rootdir</c> option is given. Tells start_erl where the
+          root of the release tree is placed in the file-system (typically
+          &lt;Erlang root&gt;\\releases). The
+          <c><![CDATA[start_erl.data]]></c> file is expected to be
+          placed in this directory (if not otherwise specified). If
+          only the <c>-rootdir</c> option is given, the directory is
+          assumed to be &lt;Erlang root&gt;\\releases.</item>
+
+	  <tag>-rootdir &lt;Erlang root directory&gt;</tag> 
+
+	  <item>Mandatory if <c>-reldir</c> is not given and there is
+	  no <c><![CDATA[RELDIR]]></c> in the environment. This
+	  specifies the Erlang installation root directory (under
+	  which the <c>lib</c>, <c>releases</c> and
+	  <c>erts-&lt;Version&gt;</c> directories are placed). If only
+	  <c>-reldir</c> (or the environment variable
+	  <c><![CDATA[RELDIR]]></c>) is given, the Erlang root is assumed to
+	  be the directory exactly one level above the release
+	  directory.</item>
+
           <tag>-data &lt;data file name&gt;</tag>
           <item>Optional, specifies another data file than start_erl.data
            in the &lt;release root&gt;. It is specified relative to the
diff --git a/erts/doc/src/zlib.xml b/erts/doc/src/zlib.xml
index b1e768bce9..8917ab5c3a 100644
--- a/erts/doc/src/zlib.xml
+++ b/erts/doc/src/zlib.xml
@@ -4,7 +4,7 @@
 <erlref>
   <header>
     <copyright>
-      <year>2005</year><year>2010</year>
+      <year>2005</year><year>2011</year>
       <holder>Ericsson AB. All Rights Reserved.</holder>
     </copyright>
     <legalnotice>
@@ -76,96 +76,92 @@ list_to_binary([Compressed|Last])</pre>
     </taglist>
   </description>
 
-  <section>
-    <title>DATA TYPES</title>
-    <code type="none">
-iodata = iolist() | binary()
-
-iolist = [char() | binary() | iolist()]
-  a binary is allowed as the tail of the list
-
-zstream = a zlib stream, see open/0</code>
-  </section>
+  <datatypes>
+    <datatype>
+      <name name="zstream"/>
+      <desc>
+        <p>A zlib stream, see <seealso marker="#open/0">open/0</seealso>.
+        </p>
+       </desc>
+    </datatype>
+    <datatype>
+      <name name="zlevel"/>
+    </datatype>
+    <datatype>
+      <name name="zmemlevel"/>
+    </datatype>
+    <datatype>
+      <name name="zmethod"/>
+    </datatype>
+    <datatype>
+      <name name="zstrategy"/>
+    </datatype>
+    <datatype>
+      <name name="zwindowbits"/>
+      <desc>
+        <p>Normally in the range <c>-15..-9 | 9..15</c>.</p>
+      </desc>
+    </datatype>
+  </datatypes>
   <funcs>
     <func>
-      <name>open() -> Z </name>
+      <name name="open" arity="0"/>
       <fsummary>Open a stream and return a stream reference</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-      </type>
       <desc>
         <p>Open a zlib stream.</p>
       </desc>
     </func>
     <func>
-      <name>close(Z)  -> ok</name>
+      <name name="close" arity="1"/>
       <fsummary>Close a stream</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-      </type>
       <desc>
-        <p>Closes the stream referenced by <c>Z</c>.</p>
+        <p>Closes the stream referenced by <c><anno>Z</anno></c>.</p>
       </desc>
     </func>
     <func>
-      <name>deflateInit(Z) -> ok</name>
+      <name name="deflateInit" arity="1"/>
       <fsummary>Initialize a session for compression</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-      </type>
       <desc>
-        <p>Same as <c>zlib:deflateInit(Z, default)</c>.</p>
+        <p>Same as <c>zlib:deflateInit(<anno>Z</anno>, default)</c>.</p>
       </desc>
     </func>
     <func>
-      <name>deflateInit(Z, Level) -> ok</name>
+      <name name="deflateInit" arity="2"/>
       <fsummary>Initialize a session for compression</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-        <v>Level = none | default | best_speed | best_compression | 0..9</v>
-      </type>
       <desc>
         <p>Initialize a zlib stream for compression.</p>
-        <p><c>Level</c> decides the compression level to be used, 0
+        <p><c><anno>Level</anno></c> decides the compression level to be used, 0
           (<c>none</c>), gives no compression at all, 1
           (<c>best_speed</c>) gives best speed and 9
           (<c>best_compression</c>) gives best compression.</p>
       </desc>
     </func>
     <func>
-      <name>deflateInit(Z, Level, Method, WindowBits, MemLevel, Strategy) -> ok</name>
+      <name name="deflateInit" arity="6"/>
       <fsummary>Initialize a session for compression</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-        <v>Level = none | default | best_speed | best_compression | 0..9</v>
-        <v>Method = deflated</v>
-        <v>WindowBits = 9..15|-9..-15</v>
-        <v>MemLevel = 1..9</v>
-        <v>Strategy = default|filtered|huffman_only</v>
-      </type>
       <desc>
         <p>Initiates a zlib stream for compression.</p>
-        <p>The <c>Level</c> parameter decides the compression level to be
+        <p>The <c><anno>Level</anno></c> parameter decides the compression level to be
           used, 0 (<c>none</c>), gives no compression at all, 1
           (<c>best_speed</c>) gives best speed and 9
           (<c>best_compression</c>) gives best compression.</p>
-        <p>The <c>Method</c> parameter decides which compression method to use,
+        <p>The <c><anno>Method</anno></c> parameter decides which compression method to use,
           currently the only supported method is <c>deflated</c>.</p>
-        <p>The <c>WindowBits</c> parameter is the base two logarithm
+        <p>The <c><anno>WindowBits</anno></c> parameter is the base two logarithm
           of the window size (the size of the history buffer).  It
           should be in the range 9 through 15. Larger values
           of this parameter result in better compression at the
           expense of memory usage. The default value is 15 if
-          <c>deflateInit/2</c>. A negative <c>WindowBits</c>
+          <c>deflateInit/2</c>. A negative <c><anno>WindowBits</anno></c>
           value suppresses the zlib header (and checksum) from the
           stream. Note that the zlib source mentions this only as a
           undocumented feature.</p>
-        <p>The <c>MemLevel</c> parameter specifies how much memory
+        <p>The <c><anno>MemLevel</anno></c> parameter specifies how much memory
           should be allocated for the internal compression
-          state. <c>MemLevel</c>=1 uses minimum memory but is slow and
-          reduces compression ratio; <c>MemLevel</c>=9 uses maximum
+          state. <c><anno>MemLevel</anno></c>=1 uses minimum memory but is slow and
+          reduces compression ratio; <c><anno>MemLevel</anno></c>=9 uses maximum
           memory for optimal speed. The default value is 8.</p>
-        <p>The <c>Strategy</c> parameter is used to tune the
+        <p>The <c><anno>Strategy</anno></c> parameter is used to tune the
           compression algorithm. Use the value <c>default</c> for
           normal data, <c>filtered</c> for data produced by a filter
           (or predictor), or <c>huffman_only</c> to force Huffman
@@ -175,54 +171,43 @@ zstream = a zlib stream, see open/0</code>
           tuned to compress them better. The effect of
           <c>filtered</c>is to force more Huffman coding and less
           string matching; it is somewhat intermediate between
-          <c>default</c> and <c>huffman_only</c>. The <c>Strategy</c>
+          <c>default</c> and <c>huffman_only</c>. The <c><anno>Strategy</anno></c>
           parameter only affects the compression ratio but not the
           correctness of the compressed output even if it is not set
           appropriately.</p>
       </desc>
     </func>
     <func>
-      <name>deflate(Z, Data)  -> Compressed</name>
+      <name name="deflate" arity="2"/>
       <fsummary>Compress data</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-        <v>Data = iodata()</v>
-        <v>Compressed = iolist()</v>
-      </type>
       <desc>
-        <p>Same as <c>deflate(Z, Data, none)</c>.</p>
+        <p>Same as <c>deflate(<anno>Z</anno>, <anno>Data</anno>, none)</c>.</p>
       </desc>
     </func>
     <func>
-      <name>deflate(Z, Data, Flush)  -> </name>
+      <name name="deflate" arity="3"/>
       <fsummary>Compress data</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-        <v>Data = iodata()</v>
-        <v>Flush = none | sync | full | finish</v>
-        <v>Compressed = iolist()</v>
-      </type>
       <desc>
         <p><c>deflate/3</c> compresses as much data as possible, and
           stops when the input buffer becomes empty. It may introduce
           some output latency (reading input without producing any
           output) except when forced to flush.</p>
-        <p>If the parameter <c>Flush</c> is set to <c>sync</c>, all
+        <p>If the parameter <c><anno>Flush</anno></c> is set to <c>sync</c>, all
           pending output is flushed to the output buffer and the
           output is aligned on a byte boundary, so that the
           decompressor can get all input data available so far. 
           Flushing may degrade compression for some compression algorithms and so
           it should be used only when necessary.</p>
-        <p>If <c>Flush</c> is set to <c>full</c>, all output is flushed as with
+        <p>If <c><anno>Flush</anno></c> is set to <c>full</c>, all output is flushed as with
           <c>sync</c>, and the compression state is reset so that decompression can
           restart from this point if previous compressed data has been damaged or if
           random access is desired. Using <c>full</c> too often can seriously degrade
           the compression.</p>
-        <p>If the parameter <c>Flush</c> is set to <c>finish</c>,
+        <p>If the parameter <c><anno>Flush</anno></c> is set to <c>finish</c>,
           pending input is processed, pending output is flushed and
           <c>deflate/3</c> returns. Afterwards the only possible
           operations on the stream are <c>deflateReset/1</c> or <c>deflateEnd/1</c>.</p>
-        <p><c>Flush</c> can be set to <c>finish</c> immediately after 
+        <p><c><anno>Flush</anno></c> can be set to <c>finish</c> immediately after
           <c>deflateInit</c> if all compression is to be done in one step.</p>
         <pre>
  
@@ -234,13 +219,8 @@ list_to_binary([B1,B2])</pre>
       </desc>
     </func>
     <func>
-      <name>deflateSetDictionary(Z, Dictionary) -> Adler32</name>
+      <name name="deflateSetDictionary" arity="2"/>
       <fsummary>Initialize the compression dictionary</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-        <v>Dictionary = binary()</v>
-        <v>Adler32 = integer()</v>
-      </type>
       <desc>
         <p>Initializes the compression dictionary from the given byte
           sequence without producing any compressed output. This
@@ -253,11 +233,8 @@ list_to_binary([B1,B2])</pre>
       </desc>
     </func>
     <func>
-      <name>deflateReset(Z) -> ok</name>
+      <name name="deflateReset" arity="1"/>
       <fsummary>Reset the deflate session</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-      </type>
       <desc>
         <p>This function is equivalent to <c>deflateEnd/1</c>
           followed by <c>deflateInit/[1|2|6]</c>, but does not free
@@ -267,34 +244,26 @@ list_to_binary([B1,B2])</pre>
       </desc>
     </func>
     <func>
-      <name>deflateParams(Z, Level, Strategy) -> ok </name>
+      <name name="deflateParams" arity="3"/>
       <fsummary>Dynamicly update deflate parameters</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-        <v>Level = none | default | best_speed | best_compression | 0..9</v>
-        <v>Strategy = default|filtered|huffman_only</v>
-      </type>
       <desc>
         <p>Dynamically update the compression level and compression
-          strategy.  The interpretation of <c>Level</c> and
-          <c>Strategy</c> is as in <c>deflateInit/6</c>.  This can be
+          strategy.  The interpretation of <c><anno>Level</anno></c> and
+          <c><anno>Strategy</anno></c> is as in <c>deflateInit/6</c>.  This can be
           used to switch between compression and straight copy of the
           input data, or to switch to a different kind of input data
           requiring a different strategy. If the compression level is
           changed, the input available so far is compressed with the
           old level (and may be flushed); the new level will take
           effect only at the next call of <c>deflate/3</c>.</p>
-        <p>Before the call of deflateParams, the stream state must be set as for
+        <p>Before the call of <c>deflateParams</c>, the stream state must be set as for
           a call of <c>deflate/3</c>, since the currently available input may have to
           be compressed and flushed.</p>
       </desc>
     </func>
     <func>
-      <name>deflateEnd(Z)  -> ok</name>
+      <name name="deflateEnd" arity="1"/>
       <fsummary>End deflate session</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-      </type>
       <desc>
         <p>End the deflate session and cleans all data used.
           Note that this function will throw an <c>data_error</c>
@@ -304,43 +273,31 @@ list_to_binary([B1,B2])</pre>
       </desc>
     </func>
     <func>
-      <name>inflateInit(Z)  -> ok </name>
+      <name name="inflateInit" arity="1"/>
       <fsummary>Initialize a session for decompression</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-      </type>
       <desc>
         <p>Initialize a zlib stream for decompression.</p>
       </desc>
     </func>
     <func>
-      <name>inflateInit(Z, WindowBits)  -> ok </name>
+      <name name="inflateInit" arity="2"/>
       <fsummary>Initialize a session for decompression</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-        <v>WindowBits = 9..15|-9..-15</v>
-      </type>
       <desc>
         <p>Initialize decompression session on zlib stream.</p>
-        <p>The <c>WindowBits</c> parameter is the base two logarithm
+        <p>The <c><anno>WindowBits</anno></c> parameter is the base two logarithm
           of the maximum window size (the size of the history buffer).
           It should be in the range 9 through 15.
           The default value is 15 if <c>inflateInit/1</c> is used.
           If a compressed stream with a larger window size is
           given as input, inflate() will throw the <c>data_error</c>
-          exception. A negative <c>WindowBits</c> value makes zlib ignore the
+          exception. A negative <c><anno>WindowBits</anno></c> value makes zlib ignore the
           zlib header (and checksum) from the stream.  Note that the zlib
           source mentions this only as a undocumented feature.</p>
       </desc>
     </func>
     <func>
-      <name>inflate(Z, Data)  -> DeCompressed </name>
+      <name name="inflate" arity="2"/>
       <fsummary>Decompress data</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-        <v>Data = iodata()</v>
-        <v>DeCompressed = iolist()</v>
-      </type>
       <desc>
         <p><c>inflate/2</c> decompresses as much data as possible.
           It may some introduce some output latency (reading
@@ -353,12 +310,8 @@ list_to_binary([B1,B2])</pre>
       </desc>
     </func>
     <func>
-      <name>inflateSetDictionary(Z, Dictionary)  -> ok</name>
+      <name name="inflateSetDictionary" arity="2"/>
       <fsummary>Initialize the decompression dictionary</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-        <v>Dictionary = binary()</v>
-      </type>
       <desc>
         <p>Initializes the decompression dictionary from the given
           uncompressed byte sequence. This function must be called
@@ -381,11 +334,8 @@ unpack(Z, Compressed, Dict) ->
       </desc>
     </func>
     <func>
-      <name>inflateReset(Z)  -> ok</name>
+      <name name="inflateReset" arity="1"/>
       <fsummary>>Reset the inflate session</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-      </type>
       <desc>
         <p>This function is equivalent to <c>inflateEnd/1</c> followed
           by <c>inflateInit/1</c>, but does not free and reallocate all
@@ -394,11 +344,8 @@ unpack(Z, Compressed, Dict) ->
       </desc>
     </func>
     <func>
-      <name>inflateEnd(Z)  -> ok</name>
+      <name name="inflateEnd" arity="1"/>
       <fsummary>End inflate session</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-      </type>
       <desc>
         <p>End the inflate session and cleans all data used. Note
           that this function will throw a <c>data_error</c> exception
@@ -407,198 +354,132 @@ unpack(Z, Compressed, Dict) ->
       </desc>
     </func>
     <func>
-      <name>setBufSize(Z, Size)  -> ok</name>
+      <name name="setBufSize" arity="2"/>
       <fsummary>Set buffer size</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-        <v>Size = integer()</v>
-      </type>
       <desc>
         <p>Sets the intermediate buffer size.</p>
       </desc>
     </func>
     <func>
-      <name>getBufSize(Z)  -> Size</name>
+      <name name="getBufSize" arity="1"/>
       <fsummary>Get buffer size</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-        <v>Size = integer()</v>
-      </type>
       <desc>
         <p>Get the size of intermediate buffer.</p>
       </desc>
     </func>
     <func>
-      <name>crc32(Z)  -> CRC</name>
+      <name name="crc32" arity="1"/>
       <fsummary>Get current CRC</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-        <v>CRC = integer()</v>
-      </type>
       <desc>
         <p>Get the current calculated CRC checksum.</p>
       </desc>
     </func>
     <func>
-      <name>crc32(Z, Binary)  -> CRC</name>
+      <name name="crc32" arity="2"/>
       <fsummary>Calculate CRC</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-        <v>Binary = binary()</v>
-        <v>CRC = integer()</v>
-      </type>
       <desc>
-        <p>Calculate the CRC checksum for <c>Binary</c>.</p>
+        <p>Calculate the CRC checksum for <c><anno>Data</anno></c>.</p>
       </desc>
     </func>
     <func>
-      <name>crc32(Z, PrevCRC, Binary)  -> CRC </name>
+      <name name="crc32" arity="3"/>
       <fsummary>Calculate CRC</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-        <v>PrevCRC = integer()</v>
-        <v>Binary = binary()</v>
-        <v>CRC = integer()</v>
-      </type>
-      <desc>
-        <p>Update a running CRC checksum for <c>Binary</c>.
-          If <c>Binary</c> is the empty binary, this function returns
+      <desc>
+        <p>Update a running CRC checksum for <c><anno>Data</anno></c>.
+          If <c><anno>Data</anno></c> is the empty binary or the empty iolist, this function returns
           the required initial value for the crc.</p>
         <pre>
-Crc = lists:foldl(fun(Bin,Crc0) ->  
-                      zlib:crc32(Z, Crc0, Bin),
-                  end, zlib:crc32(Z,&lt;&lt; &gt;&gt;), Bins)</pre>
+Crc = lists:foldl(fun(Data,Crc0) ->
+                      zlib:crc32(Z, Crc0, Data),
+                  end, zlib:crc32(Z,&lt;&lt; &gt;&gt;), Datas)</pre>
       </desc>
     </func>
     <func>
-      <name>crc32_combine(Z, CRC1, CRC2, Size2)  -> CRC </name>
+      <name name="crc32_combine" arity="4"/>
       <fsummary>Combine two CRC's</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-	<v>CRC = integer()</v>
-        <v>CRC1 = integer()</v>
-        <v>CRC2 = integer()</v>
-        <v>Size2 = integer()</v>
-      </type>
-      <desc>
-	<p>Combine two CRC checksums into one.  For two binaries,
-	  <c>Bin1</c> and <c>Bin2</c> with sizes of <c>Size1</c> and
-	  <c>Size2</c>, with CRC checksums <c>CRC1</c> and
-	  <c>CRC2</c>.  <c>crc32_combine/4</c> returns the <c>CRC</c>
-	  checksum of <c>&lt;&lt;Bin1/binary,Bin2/binary&gt;&gt;</c>, requiring
-	  only <c>CRC1</c>, <c>CRC2</c>, and <c>Size2</c>.
+      <desc>
+	<p>Combine two CRC checksums into one.  For two binaries or iolists,
+	  <c>Data1</c> and <c>Data2</c> with sizes of <c>Size1</c> and
+	  <c><anno>Size2</anno></c>, with CRC checksums <c><anno>CRC1</anno></c> and
+	  <c><anno>CRC2</anno></c>.  <c>crc32_combine/4</c> returns the <c><anno>CRC</anno></c>
+	  checksum of <c>[Data1,Data2]</c>, requiring
+	  only <c><anno>CRC1</anno></c>, <c><anno>CRC2</anno></c>, and <c><anno>Size2</anno></c>.
 	</p>
       </desc>
     </func>
     <func>
-      <name>adler32(Z, Binary)  -> Checksum</name>
+      <name name="adler32" arity="2"/>
       <fsummary>Calculate the adler checksum</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-        <v>Binary = binary()</v>
-        <v>Checksum = integer()</v>
-      </type>
       <desc>
-        <p>Calculate the Adler-32 checksum for <c>Binary</c>.</p>
+        <p>Calculate the Adler-32 checksum for <c><anno>Data</anno></c>.</p>
       </desc>
     </func>
     <func>
-      <name>adler32(Z, PrevAdler, Binary)  -> Checksum</name>
+      <name name="adler32" arity="3"/>
       <fsummary>Calculate the adler checksum</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-        <v>PrevAdler = integer()</v>
-        <v>Binary = binary()</v>
-        <v>Checksum = integer()</v>
-      </type>
-      <desc>
-        <p>Update a running Adler-32 checksum for <c>Binary</c>.
-          If <c>Binary</c> is the empty binary, this function returns
+      <desc>
+        <p>Update a running Adler-32 checksum for <c><anno>Data</anno></c>.
+          If <c><anno>Data</anno></c> is the empty binary or the empty iolist, this function returns
           the required initial value for the checksum.</p>
         <pre>
-Crc = lists:foldl(fun(Bin,Crc0) ->  
-                      zlib:adler32(Z, Crc0, Bin),
-                  end, zlib:adler32(Z,&lt;&lt; &gt;&gt;), Bins)</pre>
+Crc = lists:foldl(fun(Data,Crc0) ->
+                      zlib:adler32(Z, Crc0, Data),
+                  end, zlib:adler32(Z,&lt;&lt; &gt;&gt;), Datas)</pre>
       </desc>
     </func>
     <func>
-      <name>adler32_combine(Z, Adler1, Adler2, Size2)  -> Adler </name>
+      <name name="adler32_combine" arity="4"/>
       <fsummary>Combine two Adler-32 checksums</fsummary>
-      <type>
-        <v>Z = zstream()</v>
-        <v>Adler = integer()</v>
-        <v>Adler1 = integer()</v>
-        <v>Adler2 = integer()</v>
-        <v>Size2 = integer()</v>
-      </type>
-      <desc>
-	<p>Combine two Adler-32 checksums into one.  For two binaries,
-	  <c>Bin1</c> and <c>Bin2</c> with sizes of <c>Size1</c> and
-	  <c>Size2</c>, with Adler-32 checksums <c>Adler1</c> and
-	  <c>Adler2</c>.  <c>adler32_combine/4</c> returns the <c>Adler</c>
-	  checksum of <c>&lt;&lt;Bin1/binary,Bin2/binary&gt;&gt;</c>, requiring
-	  only <c>Adler1</c>, <c>Adler2</c>, and <c>Size2</c>.
+      <desc>
+	<p>Combine two Adler-32 checksums into one.  For two binaries or iolists,
+	  <c>Data1</c> and <c>Data2</c> with sizes of <c>Size1</c> and
+	  <c><anno>Size2</anno></c>, with Adler-32 checksums <c><anno>Adler1</anno></c> and
+	  <c><anno>Adler2</anno></c>.  <c>adler32_combine/4</c> returns the <c><anno>Adler</anno></c>
+	  checksum of <c>[Data1,Data2]</c>, requiring
+	  only <c><anno>Adler1</anno></c>, <c><anno>Adler2</anno></c>, and <c><anno>Size2</anno></c>.
 	</p>
       </desc>
     </func>
     <func>
-      <name>compress(Binary)  -> Compressed </name>
-      <fsummary>Compress a binary with standard zlib functionality</fsummary>
-      <type>
-        <v>Binary = Compressed = binary()</v>
-      </type>
+      <name name="compress" arity="1"/>
+      <fsummary>Compress data with standard zlib functionality</fsummary>
       <desc>
-        <p>Compress a binary (with zlib headers and checksum).</p>
+        <p>Compress data (with zlib headers and checksum).</p>
       </desc>
     </func>
     <func>
-      <name>uncompress(Binary)  -> Decompressed</name>
-      <fsummary>Uncompress a binary with standard zlib functionality</fsummary>
-      <type>
-        <v>Binary = Decompressed = binary()</v>
-      </type>
+      <name name="uncompress" arity="1"/>
+      <fsummary>Uncompress data with standard zlib functionality</fsummary>
       <desc>
-        <p>Uncompress a binary (with zlib headers and checksum).</p>
+        <p>Uncompress data (with zlib headers and checksum).</p>
       </desc>
     </func>
     <func>
-      <name>zip(Binary)  -> Compressed</name>
-      <fsummary>Compress a binary without the zlib headers</fsummary>
-      <type>
-        <v>Binary = Compressed = binary()</v>
-      </type>
+      <name name="zip" arity="1"/>
+      <fsummary>Compress data without the zlib headers</fsummary>
       <desc>
-        <p>Compress a binary (without zlib headers and checksum).</p>
+        <p>Compress data (without zlib headers and checksum).</p>
       </desc>
     </func>
     <func>
-      <name>unzip(Binary)  -> Decompressed</name>
-      <fsummary>Uncompress a binary without the zlib headers</fsummary>
-      <type>
-        <v>Binary = Decompressed = binary()</v>
-      </type>
+      <name name="unzip" arity="1"/>
+      <fsummary>Uncompress data without the zlib headers</fsummary>
       <desc>
-        <p>Uncompress a binary (without zlib headers and checksum).</p>
+        <p>Uncompress data (without zlib headers and checksum).</p>
       </desc>
     </func>
     <func>
-      <name>gzip(Data)  -> Compressed</name>
-      <fsummary>Compress a binary with gz header</fsummary>
-      <type>
-        <v>Binary = Compressed = binary()</v>
-      </type>
+      <name name="gzip" arity="1"/>
+      <fsummary>Compress data with gz header</fsummary>
       <desc>
-        <p>Compress a binary (with gz headers and checksum).</p>
+        <p>Compress data (with gz headers and checksum).</p>
       </desc>
     </func>
     <func>
-      <name>gunzip(Bin) -> Decompressed</name>
-      <fsummary>Uncompress a binary with gz header</fsummary>
-      <type>
-        <v>Binary = Decompressed = binary()</v>
-      </type>
+      <name name="gunzip" arity="1"/>
+      <fsummary>Uncompress data with gz header</fsummary>
       <desc>
-        <p>Uncompress a binary (with gz headers and checksum).</p>
+        <p>Uncompress data (with gz headers and checksum).</p>
       </desc>
     </func>
   </funcs>
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index a81ab28847..985ef72517 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1996-2010. All Rights Reserved.
+# Copyright Ericsson AB 1996-2012. All Rights Reserved.
 #
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -23,26 +23,34 @@ include $(ERL_TOP)/make/$(TARGET)/otp.mk
 
 ENABLE_ALLOC_TYPE_VARS = @ENABLE_ALLOC_TYPE_VARS@
 HIPE_ENABLED=@HIPE_ENABLED@
+DTRACE_ENABLED=@DTRACE_ENABLED@
+DTRACE_ENABLED_2STEP=@DTRACE_ENABLED_2STEP@
+USE_VM_PROBES=@USE_VM_PROBES@
 LIBS = @LIBS@
 Z_LIB=@Z_LIB@
 NO_INLINE_FUNCTIONS=false
 OPCODE_TABLES = $(ERL_TOP)/lib/compiler/src/genop.tab beam/ops.tab
 
+DEBUG_CFLAGS = @DEBUG_CFLAGS@
+CONFIGURE_CFLAGS = @CFLAGS@
+
 #
 # Run this make file with TYPE set to the type of emulator you want.
 # Different versions of the emulator for different uses. The default
 # is "debug". For a normal version use "opt".
 #
+DEFS=@DEFS@
 THR_DEFS=@EMU_THR_DEFS@
 M4FLAGS=
 CREATE_DIRS=
 
 LDFLAGS=@LDFLAGS@
+ARFLAGS=rc
 
 ifeq ($(TYPE),debug)
 PURIFY =
 TYPEMARKER = .debug
-TYPE_FLAGS = @DEBUG_CFLAGS@ -DDEBUG
+TYPE_FLAGS = $(DEBUG_CFLAGS) -DDEBUG
 ENABLE_ALLOC_TYPE_VARS += debug
 ifeq ($(TARGET),win32)
 TYPE_FLAGS += -DNO_JUMP_TABLE
@@ -53,7 +61,7 @@ else
 ifeq ($(TYPE),purify)
 PURIFY = purify $(PURIFY_BUILD_OPTIONS)
 TYPEMARKER = .purify
-TYPE_FLAGS = @DEBUG_CFLAGS@ -DPURIFY -DNO_JUMP_TABLE -DERTS_MSEG_FAKE_SEGMENTS
+TYPE_FLAGS = $(DEBUG_CFLAGS) -DPURIFY -DNO_JUMP_TABLE -DERTS_MSEG_FAKE_SEGMENTS
 ENABLE_ALLOC_TYPE_VARS += purify
 else
 
@@ -61,24 +69,20 @@ ifeq ($(TYPE),quantify)
 PURIFY = quantify $(QUANTIFY_BUILD_OPTIONS)
 TYPEMARKER = .quantify
 ENABLE_ALLOC_TYPE_VARS += quantify
-ifeq ($(findstring ose,$(TARGET)),ose)
-  TYPE_FLAGS = @CFLAGS@ -DQUANTIFY
-else
   TYPE_FLAGS = @CFLAGS@ -g -O2 -DQUANTIFY -DNO_JUMP_TABLE
-endif
 else
 
 ifeq ($(TYPE),purecov)
 PURIFY = purecov --follow-child-processes=yes $(PURECOV_BUILD_OPTIONS)
 TYPEMARKER = .purecov
-TYPE_FLAGS = @DEBUG_CFLAGS@ -DPURECOV -DNO_JUMP_TABLE
+TYPE_FLAGS = $(DEBUG_CFLAGS) -DPURECOV -DNO_JUMP_TABLE
 ENABLE_ALLOC_TYPE_VARS += purecov
 else
 
 ifeq ($(TYPE),gcov)
 PURIFY = 
 TYPEMARKER = .gcov
-TYPE_FLAGS = @DEBUG_CFLAGS@ -DNO_JUMP_TABLE -fprofile-arcs -ftest-coverage -O0 -DERTS_CAN_INLINE=0 -DERTS_INLINE=
+TYPE_FLAGS = $(DEBUG_CFLAGS) -DERTS_GCOV -DNO_JUMP_TABLE -fprofile-arcs -ftest-coverage -O0 -DERTS_CAN_INLINE=0 -DERTS_INLINE=
 ifneq ($(findstring solaris,$(TARGET)),solaris)
 LIBS += -lgcov
 endif
@@ -88,7 +92,7 @@ else
 ifeq ($(TYPE),valgrind)
 PURIFY = 
 TYPEMARKER = .valgrind
-TYPE_FLAGS = @DEBUG_CFLAGS@ -DVALGRIND -DNO_JUMP_TABLE -DERTS_MSEG_FAKE_SEGMENTS
+TYPE_FLAGS = $(DEBUG_CFLAGS) -DVALGRIND -DNO_JUMP_TABLE -DERTS_MSEG_FAKE_SEGMENTS
 ENABLE_ALLOC_TYPE_VARS += valgrind
 else
 
@@ -132,13 +136,6 @@ ENABLE_ALLOC_TYPE_VARS += smp nofrag
 M4FLAGS += -DERTS_SMP=1
 else
 
-ifeq ($(FLAVOR),hybrid)
-FLAVOR_MARKER=.hybrid
-FLAVOR_FLAGS=-DHYBRID
-ENABLE_ALLOC_TYPE_VARS += hybrid
-else
-
-
 # If flavor isn't one of the above, it *is* plain flavor...
 override FLAVOR=plain
 FLAVOR_MARKER=
@@ -147,10 +144,18 @@ ENABLE_ALLOC_TYPE_VARS += nofrag
 M4FLAGS +=
 
 endif
-endif
 
 TF_MARKER=$(TYPEMARKER)$(FLAVOR_MARKER)
 
+ifeq ($(FLAVOR)-@ERTS_BUILD_SMP_EMU@,smp-no)
+VOID_EMULATOR = '*** SMP emulator disabled by configure'
+else
+ifeq ($(TYPE)-@HAVE_VALGRIND@,valgrind-no)
+VOID_EMULATOR = '*** valgrind emulator disabled by configure'
+else
+VOID_EMULATOR =
+endif
+endif
 
 OPSYS=@OPSYS@
 sol2CFLAGS=
@@ -191,7 +196,7 @@ else
 EMU_CC  = @EMU_CC@
 endif
 WFLAGS  = @WFLAGS@
-CFLAGS  = @STATIC_CFLAGS@ $(TYPE_FLAGS) $(FLAVOR_FLAGS) @DEFS@ $(WFLAGS) $(THR_DEFS) $(ARCHCFLAGS)
+CFLAGS  = @STATIC_CFLAGS@ $(TYPE_FLAGS) $(FLAVOR_FLAGS) $(DEFS) $(WFLAGS) $(THR_DEFS) $(ARCHCFLAGS)
 HCC     = @HCC@
 LD      = @LD@
 DEXPORT = @DEXPORT@
@@ -202,6 +207,15 @@ RM	= @RM@
 MKDIR	= @MKDIR@
 
 USING_MINGW=@MIXED_CYGWIN_MINGW@
+MIXED_MSYS=@MIXED_MSYS@
+
+ifeq ($(TARGET),win32)
+LIB_PREFIX=
+LIB_SUFFIX=.lib
+else
+LIB_PREFIX=lib
+LIB_SUFFIX=.a
+endif
 
 OMIT_OMIT_FP=no
 
@@ -233,9 +247,12 @@ ifeq ($(TARGET), win32)
 GEN_OPT_FLGS = $(OPT_LEVEL)
 UNROLL_FLG = 
 RC=rc.sh
+ifeq ($(MIXED_MSYS), yes)
+MAKE_PRELOAD_EXTRA = -msys
+endif
 ifeq ($(USING_MINGW), yes)
-RES_EXT=@OBJEXT@
-MAKE_PRELOAD_EXTRA=-windres
+RES_EXT = @OBJEXT@
+MAKE_PRELOAD_EXTRA += " -windres"
 else
 RES_EXT=res
 endif
@@ -255,44 +272,39 @@ CS_PURIFY =
 CS_TYPE_FLAGS = $(subst QUANTIFY,FAKE_QUANTIFY, \
 		$(subst PURIFY,FAKE_PURIFY, $(TYPE_FLAGS)))
 endif
-CS_CFLAGS_ = $(CS_TYPE_FLAGS) @DEFS@ $(WFLAGS) 
+CS_CFLAGS_ = $(CS_TYPE_FLAGS) $(DEFS) $(WFLAGS)
 ifeq ($(GCC),yes)
 CS_CFLAGS = $(subst -O2, $(GEN_OPT_FLGS) $(UNROLL_FLG), $(CS_CFLAGS_))
 else
 CS_CFLAGS = $(CS_CFLAGS_)
 endif
 CS_LDFLAGS = $(LDFLAGS)
-CS_LIBS = -L../lib/internal/$(TARGET) -lerts_internal @ERTS_INTERNAL_X_LIBS@
+CS_LIBS = -L../lib/internal/$(TARGET) -lerts_internal$(TYPEMARKER) @ERTS_INTERNAL_X_LIBS@
 
 LIBS	+= @TERMCAP_LIB@ -L../lib/internal/$(TARGET) @ERTS_INTERNAL_X_LIBS@
 
 ifdef Z_LIB
 # Use shared zlib library
 LIBS += $(Z_LIB)
+DEPLIBS =
 else
+DEPLIBS=$(ZLIB_LIBRARY)
 ifeq ($(TARGET),win32)
-LIBS    += -L$(ERL_TOP)/erts/emulator/zlib/obj/$(TARGET)/$(TYPE) -lz
-DEPLIBS = $(ERL_TOP)/erts/emulator/zlib/obj/$(TARGET)/$(TYPE)/z.lib
+LIBS    += -L$(ZLIB_OBJDIR) -lz
 else
 # Build on darwin fails if -lz is used
-LIBS += $(ERL_TOP)/erts/emulator/zlib/obj/$(TARGET)/$(TYPE)/libz.a
-DEPLIBS = $(ERL_TOP)/erts/emulator/zlib/obj/$(TARGET)/$(TYPE)/libz.a
+LIBS += $(ZLIB_LIBRARY)
 endif
-
 endif
 
 ifeq ($(TARGET),win32)
 LIBS    += -L$(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE) -lepcre
-DEPLIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/epcre.lib
 else
-LIBS    += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/libepcre.a
-DEPLIBS += \
-	$(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/libepcre.a \
-	$(ERL_TOP)/erts/lib/internal/$(TARGET)/liberts_internal.a 
-# rem liberts_internal.a
+LIBS    += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/$(LIB_PREFIX)epcre$(LIB_SUFFIX)
 endif
 
-ELIB_FLAGS = -DENABLE_ELIB_MALLOC -DELIB_ALLOC_IS_CLIB -DELIB_HEAP_SBRK
+EPCRE_LIB = $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/$(LIB_PREFIX)epcre$(LIB_SUFFIX)
+DEPLIBS += $(EPCRE_LIB)
 
 PERFCTR_PATH=@PERFCTR_PATH@
 USE_PERFCTR=@USE_PERFCTR@
@@ -309,7 +321,16 @@ LIBSCTP = @LIBSCTP@
 ORG_THR_LIBS=@EMU_THR_LIBS@
 THR_LIB_NAME=@EMU_THR_LIB_NAME@
 
-THR_LIBS=$(subst -l$(THR_LIB_NAME),-l$(THR_LIB_NAME)$(TYPEMARKER),$(ORG_THR_LIBS))
+ERTS_LIB_DIR = $(ERL_TOP)/erts/lib_src
+ERTS_LIB = $(ERTS_LIB_DIR)/obj/$(TARGET)/$(TYPE)/MADE
+DEPLIBS += $(ERTS_LIB)
+
+THR_LIBS=$(subst -l$(THR_LIB_NAME),-l$(THR_LIB_NAME)$(TYPEMARKER), \
+	   $(subst -lerts_internal_r,-lerts_internal_r$(TYPEMARKER),$(ORG_THR_LIBS)))
+
+LIBS += $(THR_LIBS)
+
+ifneq ($(findstring erts_internal_r, $(THR_LIBS)),erts_internal_r)
 
 ifeq ($(findstring vxworks,$(TARGET)),vxworks)
 ERTS_INTERNAL_LIB=erts_internal
@@ -321,7 +342,9 @@ ERTS_INTERNAL_LIB=erts_internal
 endif
 endif
 
-LIBS += $(THR_LIBS) -l$(ERTS_INTERNAL_LIB)$(TYPEMARKER)
+LIBS += -l$(ERTS_INTERNAL_LIB)$(TYPEMARKER)
+
+endif # erts_internal_r
 
 LIBS += @LIBRT@
 
@@ -336,8 +359,7 @@ OBJDIR = obj/$(TTF_DIR)
 
 CREATE_DIRS += $(OBJDIR) \
 	pcre/obj/$(TARGET)/$(TYPE) \
-	zlib/obj/$(TARGET)/$(TYPE)
-
+	$(ZLIB_OBJDIR)
 
 BINDIR = $(ERL_TOP)/bin/$(TARGET)
 
@@ -345,8 +367,6 @@ ERLANG_OSTYPE = @ERLANG_OSTYPE@
 
 ENABLE_ALLOC_TYPE_VARS += @ERLANG_OSTYPE@
 
-EMULATOR_EXECUTABLE_SAE = beam_evm$(TF_MARKER)
-EMULATOR_EXECUTABLE_ELIB = beam.elib$(TF_MARKER)
 ifeq ($(TARGET), win32)
 EMULATOR_EXECUTABLE = beam$(TF_MARKER).dll
 else
@@ -362,15 +382,12 @@ else
 UNIX_ONLY_BUILDS =
 endif
 
-ifeq ($(TYPE)-@HAVE_VALGRIND@,valgrind-no)
-all:
-	@echo '*** valgrind not found by configure'
-else
-ifeq ($(FLAVOR)-@ERTS_BUILD_SMP_EMU@,smp-no)
+.PHONY: all
+ifdef VOID_EMULATOR
 all:
-	@echo '*** Omitted build of emulator with smp support'
+	@echo $(VOID_EMULATOR)' - omitted target all'
 else
-all: generate erts_lib zlib pcre $(BINDIR)/$(EMULATOR_EXECUTABLE) $(UNIX_ONLY_BUILDS)
+all: $(BINDIR)/$(EMULATOR_EXECUTABLE) $(UNIX_ONLY_BUILDS)
 ifeq ($(OMIT_OMIT_FP),yes)
 	@echo '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *'
 	@echo '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *'
@@ -381,37 +398,25 @@ ifeq ($(OMIT_OMIT_FP),yes)
 	@echo '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *'
 endif
 endif
-endif
 
-ifdef Z_LIB
-zlib:
-	@echo 'Skip zlib directory, use shared library'
-else
-zlib:
-	@set -e ; cd zlib && $(MAKE) TYPE=$(TYPE) $(TYPE)
-endif
+include zlib/zlib.mk
+include pcre/pcre.mk
 
-pcre:
-	@set -e ; cd pcre && $(MAKE) TYPE=$(TYPE) $(TYPE)
-
-erts_lib:
-	cd $(ERL_TOP)/erts/lib_src && $(MAKE) $(TYPE)
+$(ERTS_LIB):
+	cd $(ERTS_LIB_DIR) && $(MAKE) $(TYPE)
 
+.PHONY: clean
 clean:
-ifeq ($(TARGET),win32)
-	$(RM) -f $(TARGET)/beams.rc
-endif
-	$(RM) -f $(TARGET)/*.c $(TARGET)/*.h $(TARGET)/depend.mk
-	$(RM) -f $(TARGET)/*/*/*.c $(TARGET)/*/*/*.h $(TARGET)/*/*/*.S
-	$(RM) -f $(ERL_TOP)/erts/emulator/obj/$(TARGET)/*/*/*.o
-	$(RM) -f $(BINDIR)/beam $(BINDIR)/beam.*
-	$(RM) -rf $(BINDIR)/child_setup $(BINDIR)/child_setup.*
-	$(RM) -f $(BINDIR)/hipe_mkliterals $(BINDIR)/hipe_mkliterals.*
-	@set -e ; cd zlib && $(MAKE) clean
-	@set -e ; cd pcre && $(MAKE) clean
-
-.PHONY: all sae zlib pcre clean
-
+	$(RM) -f $(GENERATE)
+	$(RM) -rf $(TARGET)/*.c $(TARGET)/*.h $(TARGET)/*-GENERATED
+	$(RM) -rf $(TARGET)/*/*
+	$(RM) -rf obj/$(TARGET)
+	$(RM) -rf pcre/obj/$(TARGET) $(PCRE_GENINC)
+	$(RM) -rf zlib/obj/$(TARGET)
+	$(RM) -rf bin/$(TARGET)
+	cd $(ERTS_LIB_DIR) && $(MAKE) clean
+
+.PHONY: docs
 docs:
 
 # ----------------------------------------------------------------------
@@ -423,100 +428,93 @@ RELEASE_INCLUDES = beam/erl_driver.h sys/$(ERLANG_OSTYPE)/driver_int.h beam/erl_
 ifeq ($(TARGET),win32)
 RELEASE_INCLUDES += sys/$(ERLANG_OSTYPE)/erl_win_dyn_driver.h
 endif
-ifeq ($(findstring ose,$(TARGET)),ose)
-RELEASE_INCLUDES += sys/$(ERLANG_OSTYPE)/erl_port_signals.sig \
-		    sys/$(ERLANG_OSTYPE)/ose_erl_port_prog.h  \
-		    drivers/$(ERLANG_OSTYPE)/ose_erl_driver.h
 
-endif
 
-ifeq ($(TYPE)-@HAVE_VALGRIND@,valgrind-no)
+.PHONY: release_spec
+ifdef VOID_EMULATOR
 release_spec:
-	@echo '*** valgrind not found by configure'
-else
-ifeq ($(FLAVOR)-@ERTS_BUILD_SMP_EMU@,smp-no)
-release_spec:
-	@echo '*** No emulator with smp support to install'
+	@echo $(VOID_EMULATOR)' - omitted target release_spec (install)'
 else
 release_spec: all
-	$(INSTALL_DIR) $(RELSYSDIR)
-	$(INSTALL_DIR) $(RELSYSDIR)/src
-	$(INSTALL_DIR) $(RELSYSDIR)/include
-	$(INSTALL_DIR) $(RELSYSDIR)/man
-	$(INSTALL_DIR) $(RELSYSDIR)/doc
-	$(INSTALL_DIR) $(RELSYSDIR)/bin
-	$(INSTALL_DIR) $(RELEASE_PATH)/usr/include
-	$(INSTALL_DATA) $(RELEASE_INCLUDES) $(RELEASE_PATH)/usr/include
-	$(INSTALL_DATA) $(RELEASE_INCLUDES) $(RELSYSDIR)/include
-	$(INSTALL_PROGRAM) $(BINDIR)/$(EMULATOR_EXECUTABLE) $(RELSYSDIR)/bin
+	$(INSTALL_DIR) "$(RELSYSDIR)"
+	$(INSTALL_DIR) "$(RELSYSDIR)/src"
+	$(INSTALL_DIR) "$(RELSYSDIR)/include"
+	$(INSTALL_DIR) "$(RELSYSDIR)/man"
+	$(INSTALL_DIR) "$(RELSYSDIR)/doc"
+	$(INSTALL_DIR) "$(RELSYSDIR)/bin"
+	$(INSTALL_DIR) "$(RELEASE_PATH)/usr/include"
+	$(INSTALL_DATA) $(RELEASE_INCLUDES) "$(RELEASE_PATH)/usr/include"
+	$(INSTALL_DATA) $(RELEASE_INCLUDES) "$(RELSYSDIR)/include"
+	$(INSTALL_PROGRAM) $(BINDIR)/$(EMULATOR_EXECUTABLE) "$(RELSYSDIR)/bin"
 ifeq ($(ERLANG_OSTYPE), unix)
-	$(INSTALL_PROGRAM) $(BINDIR)/$(CS_EXECUTABLE) $(RELSYSDIR)/bin
-endif
-	$(INSTALL_DIR) $(RELEASE_PATH)/usr/include/obsolete
-	$(INSTALL_DATA) obsolete/driver.h $(RELEASE_PATH)/usr/include/obsolete
+	$(INSTALL_PROGRAM) $(BINDIR)/$(CS_EXECUTABLE) "$(RELSYSDIR)/bin"
 endif
 endif
 
+.PHONY: release_docs_spec
 release_docs_spec:
 
 # ----------------------------------------------------------------------
 # Generated source code. Put in $(TARGET) directory
 #
 
-.PHONY : generate
-
-GENERATE=	$(CREATE_DIRS) \
-		$(TTF_DIR)/beam_opcodes.h \
-		$(TARGET)/erl_bif_table.c \
-		$(TARGET)/erl_version.h \
-		$(TTF_DIR)/driver_tab.c \
-		$(TTF_DIR)/erl_alloc_types.h
+_create_dirs := $(shell mkdir -p $(CREATE_DIRS))
 
-ifeq ($(TARGET),win32)
-GENERATE +=	$(TARGET)/beams.rc
-else
-GENERATE +=	$(TARGET)/preload.c
-endif
+GENERATE =
+HIPE_ASM =
 
 ifeq ($(findstring vxworks,$(TARGET)),vxworks)
 else
-ifeq ($(findstring ose,$(TARGET)),ose)
-else
 ifdef HIPE_ENABLED
-GENERATE +=	$(TTF_DIR)/hipe_x86_asm.h \
+HIPE_ASM +=	$(TTF_DIR)/hipe_x86_asm.h \
 		$(TTF_DIR)/hipe_amd64_asm.h \
 		$(TTF_DIR)/hipe_sparc_asm.h \
 		$(TTF_DIR)/hipe_ppc_asm.h \
-		$(TTF_DIR)/hipe_arm_asm.h \
+		$(TTF_DIR)/hipe_arm_asm.h
+
+GENERATE +=	$(HIPE_ASM) \
 		$(TTF_DIR)/hipe_literals.h \
 		$(BINDIR)/hipe_mkliterals$(TF_MARKER)
 endif
 endif
-endif
-
-ifeq ($(FLAVOR)-@ERTS_BUILD_SMP_EMU@,smp-no)
-GENERATE=
-endif
 
+ifdef DTRACE_ENABLED
+# global.h causes problems by including dtrace-wrapper.h which includes
+# the autogenerated erlang_dtrace.h ... so make erlang_dtrace.h very early.
+generate: $(TARGET)/erlang_dtrace.h $(GENERATE)
+else
 generate: $(GENERATE)
+endif
 
 ifdef HIPE_ENABLED
 OPCODE_TABLES += hipe/hipe_ops.tab
 endif
 
-$(TTF_DIR)/beam_opcodes.h $(TTF_DIR)/beam_opcodes.c: $(OPCODE_TABLES)
-	LANG=C $(PERL) utils/beam_makeops -outdir $(TTF_DIR) \
-		-emulator $(OPCODE_TABLES)
+$(TTF_DIR)/beam_cold.h \
+$(TTF_DIR)/beam_hot.h \
+$(TTF_DIR)/beam_opcodes.c \
+$(TTF_DIR)/beam_opcodes.h \
+$(TTF_DIR)/beam_pred_funcs.h \
+$(TTF_DIR)/beam_tr_funcs.h \
+	: $(TTF_DIR)/OPCODES-GENERATED
+$(TTF_DIR)/OPCODES-GENERATED: $(OPCODE_TABLES) utils/beam_makeops
+	LANG=C $(PERL) utils/beam_makeops \
+		-wordsize @EXTERNAL_WORD_SIZE@ \
+		-outdir $(TTF_DIR) \
+		-DUSE_VM_PROBES=$(if $(USE_VM_PROBES),1,0) \
+		-emulator $(OPCODE_TABLES) && echo $? >$(TTF_DIR)/OPCODES-GENERATED
+GENERATE += $(TTF_DIR)/OPCODES-GENERATED
 
 # bif and atom table
 ATOMS=	beam/atom.names
 BIFS =	beam/bif.tab
 ifdef HIPE_ENABLED
+HIPE_ARCH64_TAB=hipe/hipe_bif64.tab
 HIPE_x86_TAB=hipe/hipe_x86.tab
-HIPE_amd64_TAB=hipe/hipe_amd64.tab
+HIPE_amd64_TAB=hipe/hipe_amd64.tab $(HIPE_ARCH64_TAB)
 HIPE_ultrasparc_TAB=hipe/hipe_sparc.tab
 HIPE_ppc_TAB=hipe/hipe_ppc.tab
-HIPE_ppc64_TAB=hipe/hipe_ppc64.tab
+HIPE_ppc64_TAB=hipe/hipe_ppc64.tab $(HIPE_ARCH64_TAB)
 HIPE_arm_TAB=hipe/hipe_arm.tab
 HIPE_ARCH_TAB=$(HIPE_$(ARCH)_TAB)
 BIFS += hipe/hipe_bif0.tab hipe/hipe_bif1.tab hipe/hipe_bif2.tab $(HIPE_ARCH_TAB)
@@ -525,31 +523,44 @@ BIFS += hipe/hipe_perfctr.tab
 endif
 endif
 
-TABLES=	$(TARGET)/erl_bif_table.c $(TARGET)/erl_bif_table.h	\
-	$(TARGET)/erl_bif_wrap.c $(TARGET)/erl_bif_list.h	\
-	$(TARGET)/erl_atom_table.c $(TARGET)/erl_atom_table.h   \
-	$(TARGET)/erl_pbifs.c
-
-$(TABLES):	$(ATOMS) $(BIFS)
-	LANG=C $(PERL) utils/make_tables -src $(TARGET) -include $(TARGET) $^
+$(TARGET)/erl_bif_table.c \
+$(TARGET)/erl_bif_table.h \
+$(TARGET)/erl_bif_wrap.c \
+$(TARGET)/erl_bif_list.h \
+$(TARGET)/erl_atom_table.c \
+$(TARGET)/erl_atom_table.h \
+$(TARGET)/erl_pbifs.c \
+	: $(TARGET)/TABLES-GENERATED
+$(TARGET)/TABLES-GENERATED: $(ATOMS) $(BIFS) utils/make_tables
+	LANG=C $(PERL) utils/make_tables -src $(TARGET) -include $(TARGET)\
+		$(ATOMS) $(BIFS) && echo $? >$(TARGET)/TABLES-GENERATED
+GENERATE += $(TARGET)/TABLES-GENERATED
 
 $(TTF_DIR)/erl_alloc_types.h: beam/erl_alloc.types utils/make_alloc_types
 	LANG=C $(PERL) utils/make_alloc_types -src $< -dst $@ $(ENABLE_ALLOC_TYPE_VARS)
+GENERATE += $(TTF_DIR)/erl_alloc_types.h
 
 # version include file
 $(TARGET)/erl_version.h: ../vsn.mk
 	LANG=C $(PERL) utils/make_version -o $@ $(SYSTEM_VSN) $(VSN)$(SERIALNO) $(TARGET)
+GENERATE += $(TARGET)/erl_version.h
 
 # driver table
 $(TTF_DIR)/driver_tab.c: Makefile.in
 	LANG=C $(PERL) utils/make_driver_tab -o $@ $(DRV_OBJS)
+GENERATE += $(TTF_DIR)/driver_tab.c
+
+
 
 # Preloaded code.
 # 
 # This list must be consistent with PRE_LOADED_MODULES in
 # lib/kernel/src/Makefile.
 ifeq ($(TARGET),win32)
-$(TARGET)/beams.rc: $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \
+# On windows the preloaded objects are in a resource object.
+PRELOAD_OBJ = $(OBJDIR)/beams.$(RES_EXT)
+PRELOAD_SRC = $(TARGET)/beams.rc
+$(PRELOAD_SRC): $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/init.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \
@@ -559,7 +570,9 @@ $(TARGET)/beams.rc: $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/erlang.beam
 	LANG=C $(PERL) utils/make_preload $(MAKE_PRELOAD_EXTRA) -rc $^ > $@
 else
-$(TARGET)/preload.c: $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \
+PRELOAD_OBJ = $(OBJDIR)/preload.o
+PRELOAD_SRC = $(TARGET)/preload.c
+$(PRELOAD_SRC): $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/init.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \
 			$(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \
@@ -570,6 +583,22 @@ $(TARGET)/preload.c: $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \
 	LANG=C $(PERL) utils/make_preload -old $^ > $@
 endif
 
+.PHONY : generate
+ifdef VOID_EMULATOR
+generate:
+	@echo $(VOID_EMULATOR)' - omitted target generate'
+else
+generate: $(TTF_DIR)/GENERATED $(PRELOAD_SRC)
+
+$(TTF_DIR)/GENERATED: $(GENERATE)
+	echo $? >$(TTF_DIR)/GENERATED
+endif
+
+$(TARGET)/erlang_dtrace.h: beam/erlang_dtrace.d
+	dtrace -h -C -Ibeam -s $< -o ./erlang_dtrace.tmp
+	sed -e '/^#define[ 	]*ERLANG_[A-Z0-9_]*(.*)/y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/' ./erlang_dtrace.tmp > $@
+	rm ./erlang_dtrace.tmp
+
 # ----------------------------------------------------------------------
 # Pattern rules
 #
@@ -592,13 +621,7 @@ ifeq ($(findstring vxworks,$(TARGET)),vxworks)
 INCLUDES += -I$(ERL_TOP)/erts/etc/vxworks
 endif
 
-ifneq ($(findstring ose,$(TARGET)),ose)
 ifeq ($(TARGET),win32)
-# Usually the same as the default rule, but certain platforms (i.e. win32) mix
-# different compilers
-$(OBJDIR)/beam_emu.o: beam/beam_emu.c
-	$(EMU_CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
-
 $(OBJDIR)/dll_sys.o: sys/$(ERLANG_OSTYPE)/sys.c
 	$(CC) $(CFLAGS) -DERL_RUN_SHARED_LIB=1 $(INCLUDES) -c $< -o $@
 
@@ -612,38 +635,15 @@ $(OBJDIR)/beam_emu.o: beam/beam_emu.c
 	$(CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) \
          -OPT:Olimit=0 -WOPT:lpre=off:spre=off:epre=off \
          $(INCLUDES) -c $< -o $@
-endif
-
-
-$(OBJDIR)/%.o: beam/%.c
-	$(CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
-
-else
-
-INCLUDES += -Idrivers/ose
-
-ifeq ($(TYPE),debug)
-$(OBJDIR)/%.o: beam/%.c
-	  $(CC) $(CFLAGS) -DNO_JUMP_TABLE $(INCLUDES) -c $< -o $@
 else
-
-VXCC=@VXCC@
-VXCFLAGS=@VXCFLAGS@
-CFLAGS_NOOPT=@CFLAGS_NOOPT@ @DEFS@ $(WFLAGS) $(THR_DEFS) $(ARCHCFLAGS)
-
-# we want to use jump table
+# Usually the same as the default rule, but certain platforms (e.g. win32) mix
+# different compilers
 $(OBJDIR)/beam_emu.o: beam/beam_emu.c
-	$(VXCC) $(VXCFLAGS) $(INCLUDES) -c $< -o $@
-
-# erl_process does not work properly with DIAB's -XO option,
-# we'll compile it with gcc instead
-$(OBJDIR)/erl_process.o: beam/erl_process.c
-	$(VXCC) $(VXCFLAGS) $(INCLUDES) -c $< -o $@
+	$(EMU_CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
+endif
 
 $(OBJDIR)/%.o: beam/%.c
-	$(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@
-endif
-endif
+	$(CC) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
 
 $(OBJDIR)/%.o: $(TARGET)/%.c
 	$(CC) $(CFLAGS) $(INCLUDES) -Idrivers/common -c $< -o $@
@@ -663,32 +663,21 @@ $(OBJDIR)/%.o: drivers/common/%.c
 $(OBJDIR)/%.o: drivers/$(ERLANG_OSTYPE)/%.c
 	$(CC) $(CFLAGS) $(INCLUDES) -Idrivers/common -Idrivers/$(ERLANG_OSTYPE) -I../etc/$(ERLANG_OSTYPE) -c $< -o $@
 
-# VxWorks and OSE uses unix drivers too...
+# VxWorks uses unix drivers too...
 ifeq ($(findstring vxworks,$(TARGET)),vxworks)
 $(OBJDIR)/%.o: drivers/unix/%.c
 	$(CC) $(CFLAGS) $(INCLUDES) -Idrivers/common -c $< -o $@
 endif
-ifeq ($(findstring ose,$(TARGET)),ose)
-$(OBJDIR)/%.o: drivers/unix/%.c
-	$(CC) $(CFLAGS) $(INCLUDES) -Idrivers/common -c $< -o $@
-endif
 
 # ----------------------------------------------------------------------
 # Specials
 #
 CS_SRC = sys/$(ERLANG_OSTYPE)/erl_child_setup.c
 
-$(BINDIR)/$(CS_EXECUTABLE): $(CS_SRC)
+$(BINDIR)/$(CS_EXECUTABLE): $(TTF_DIR)/GENERATED $(PRELOAD_SRC) $(CS_SRC) $(ERTS_LIB)
 	$(CS_PURIFY) $(CC) $(CS_LDFLAGS) -o $(BINDIR)/$(CS_EXECUTABLE) \
                $(CS_CFLAGS) $(COMMON_INCLUDES) $(CS_SRC) $(CS_LIBS)
 
-$(OBJDIR)/%.elib.o: beam/%.c
-	$(CC) $(ELIB_FLAGS) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
-
-# Disable vfork() for sae (then we don't need the child_setup program)
-$(OBJDIR)/sys_sae.o: sys/$(ERLANG_OSTYPE)/sys.c
-	$(CC) -DDISABLE_VFORK=1 $(CFLAGS) $(INCLUDES) -c $< -o $@
-
 $(OBJDIR)/%.kp.o: sys/common/%.c
 	$(CC) -DERTS_KERNEL_POLL_VERSION $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@
 
@@ -697,9 +686,6 @@ $(OBJDIR)/%.nkp.o: sys/common/%.c
 
 ifeq ($(GCC),yes)
 
-$(OBJDIR)/erl_obsolete.o: beam/erl_obsolete.c
-	$(CC) $(subst -Wstrict-prototypes, , $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS))) $(INCLUDES) -c $< -o $@
-
 $(OBJDIR)/erl_goodfit_alloc.o: beam/erl_goodfit_alloc.c
 	$(CC) $(subst -O2, $(GEN_OPT_FLGS) $(UNROLL_FLG), $(CFLAGS)) $(INCLUDES) -c $< -o $@
 endif
@@ -709,32 +695,15 @@ endif
 # rebuilding (is this a good idea?) add a dummy dependency to this target.
 #
 
-ifeq ($(findstring clearmake,$(MAKE)),clearmake)
-BEAMFILE_MAKEFLAG=-T
-else
-BEAMFILE_MAKEFLAG=
-endif
-
 $(ERL_TOP)/lib/%.beam:
-	cd $(@D)/../src && $(MAKE) $(BEAMFILE_MAKEFLAG) ../ebin/$(@F)
+	cd $(@D)/../src && $(MAKE) ../ebin/$(@F)
 
 
 # ----------------------------------------------------------------------
 # Object files
 #
 
-# On windows the preloaded objects are in a resource object.
-
-ifeq ($(TARGET),win32)
-PRELOAD =	$(OBJDIR)/beams.$(RES_EXT)
-else
-PRELOAD = 	$(OBJDIR)/preload.o
-endif
-
-
-INIT_OBJS = $(OBJDIR)/erl_main.o		$(PRELOAD)
-
-INIT_OBJS_SAE = $(OBJDIR)/erl9_start.o
+INIT_OBJS = $(OBJDIR)/erl_main.o	$(PRELOAD_OBJ)
 
 EMU_OBJS = \
 	$(OBJDIR)/beam_emu.o		$(OBJDIR)/beam_opcodes.o \
@@ -764,23 +733,25 @@ RUN_OBJS = \
 	$(OBJDIR)/external.o		$(OBJDIR)/dist.o \
 	$(OBJDIR)/binary.o		$(OBJDIR)/erl_db.o \
 	$(OBJDIR)/erl_db_util.o		$(OBJDIR)/erl_db_hash.o \
-	$(OBJDIR)/erl_db_tree.o		$(OBJDIR)/fix_alloc.o \
+	$(OBJDIR)/erl_db_tree.o		$(OBJDIR)/erl_thr_progress.o \
 	$(OBJDIR)/big.o			$(OBJDIR)/hash.o \
 	$(OBJDIR)/index.o		$(OBJDIR)/atom.o \
 	$(OBJDIR)/module.o		$(OBJDIR)/export.o \
 	$(OBJDIR)/register.o		$(OBJDIR)/break.o \
 	$(OBJDIR)/erl_async.o		$(OBJDIR)/erl_lock_check.o \
 	$(OBJDIR)/erl_gc.o 		$(OBJDIR)/erl_lock_count.o \
-	$(OBJDIR)/erl_nmgc.o		$(OBJDIR)/erl_posix_str.o \
+	$(OBJDIR)/erl_posix_str.o \
 	$(OBJDIR)/erl_bits.o 		$(OBJDIR)/erl_math.o \
 	$(OBJDIR)/erl_fun.o             $(OBJDIR)/erl_bif_port.o \
 	$(OBJDIR)/erl_term.o 		$(OBJDIR)/erl_node_tables.o \
 	$(OBJDIR)/erl_monitors.o	$(OBJDIR)/erl_process_dump.o \
-	$(OBJDIR)/erl_obsolete.o	$(OBJDIR)/erl_bif_timer.o \
+	$(OBJDIR)/erl_bif_timer.o	$(OBJDIR)/erl_cpu_topology.o \
 	$(OBJDIR)/erl_drv_thread.o      $(OBJDIR)/erl_bif_chksum.o \
 	$(OBJDIR)/erl_bif_re.o		$(OBJDIR)/erl_unicode.o \
 	$(OBJDIR)/packet_parser.o	$(OBJDIR)/safe_hash.o \
-	$(OBJDIR)/erl_zlib.o		$(OBJDIR)/erl_nif.o
+	$(OBJDIR)/erl_zlib.o		$(OBJDIR)/erl_nif.o \
+	$(OBJDIR)/erl_bif_binary.o      $(OBJDIR)/erl_ao_firstfit_alloc.o \
+	$(OBJDIR)/erl_thr_queue.o	$(OBJDIR)/erl_sched_spec_pre_alloc.o
 
 ifeq ($(TARGET),win32)
 DRV_OBJS = \
@@ -799,29 +770,21 @@ OS_OBJS = \
 	$(OBJDIR)/sys_time.o \
 	$(OBJDIR)/sys_interrupt.o \
 	$(OBJDIR)/sys_env.o \
-	$(OBJDIR)/dosmap.o \
-	$(OBJDIR)/elib_malloc.o
+	$(OBJDIR)/dosmap.o
 else
 OS_OBJS = \
 	$(OBJDIR)/sys.o \
 	$(OBJDIR)/driver_tab.o \
 	$(OBJDIR)/unix_efile.o \
 	$(OBJDIR)/gzio.o \
-	$(OBJDIR)/elib_malloc.o \
 	$(OBJDIR)/elib_memmove.o
 
-ifeq ($(findstring ose,$(TARGET)),ose)
-	OS_OBJS += $(OBJDIR)/erl_port_init.o \
-		   $(OBJDIR)/ose_inet_sock_select.o \
-		   $(OBJDIR)/ose_sfp.o
-else
 ifeq ($(findstring vxworks,$(TARGET)),vxworks)
 	OS_OBJS += $(OBJDIR)/int64.o 
 else
 	OS_OBJS += $(OBJDIR)/sys_float.o \
                    $(OBJDIR)/sys_time.o
 endif
-endif
 DRV_OBJS = \
 	$(OBJDIR)/efile_drv.o \
 	$(OBJDIR)/inet_drv.o \
@@ -830,11 +793,7 @@ DRV_OBJS = \
 endif
 
 ifneq ($(findstring vxworks,$(TARGET)),vxworks)
-  ifeq ($(findstring ose,$(TARGET)),ose)
-	DRV_OBJS += $(OBJDIR)/ose_inet_drv.o
-  else
 	DRV_OBJS += $(OBJDIR)/ttsl_drv.o
-  endif
 endif
 
 ifeq ($(ERTS_ENABLE_KERNEL_POLL),yes)
@@ -849,14 +808,17 @@ endif
 
 OS_OBJS +=	$(OBJDIR)/erl_mseg.o \
 		$(OBJDIR)/erl_$(ERLANG_OSTYPE)_sys_ddll.o \
-		$(OBJDIR)/erl_mtrace_sys_wrap.o
+		$(OBJDIR)/erl_mtrace_sys_wrap.o \
+		$(OBJDIR)/erl_sys_common_misc.o
+
+HIPE_ARCH64_OBJS=$(OBJDIR)/hipe_bif64.o
 
 HIPE_x86_OS_OBJS=$(HIPE_x86_$(OPSYS)_OBJS)
 HIPE_x86_OBJS=$(OBJDIR)/hipe_x86.o $(OBJDIR)/hipe_x86_glue.o $(OBJDIR)/hipe_x86_bifs.o $(OBJDIR)/hipe_x86_signal.o $(OBJDIR)/hipe_x86_stack.o $(HIPE_x86_OS_OBJS)
-HIPE_amd64_OBJS=$(OBJDIR)/hipe_amd64.o $(OBJDIR)/hipe_amd64_glue.o $(OBJDIR)/hipe_amd64_bifs.o $(OBJDIR)/hipe_x86_signal.o $(OBJDIR)/hipe_x86_stack.o
+HIPE_amd64_OBJS=$(OBJDIR)/hipe_amd64.o $(OBJDIR)/hipe_amd64_glue.o $(OBJDIR)/hipe_amd64_bifs.o $(OBJDIR)/hipe_x86_signal.o $(OBJDIR)/hipe_x86_stack.o $(HIPE_ARCH64_OBJS)
 HIPE_ultrasparc_OBJS=$(OBJDIR)/hipe_sparc.o $(OBJDIR)/hipe_sparc_glue.o $(OBJDIR)/hipe_sparc_bifs.o $(OBJDIR)/hipe_risc_stack.o
 HIPE_ppc_OBJS=$(OBJDIR)/hipe_ppc.o $(OBJDIR)/hipe_ppc_glue.o $(OBJDIR)/hipe_ppc_bifs.o $(OBJDIR)/hipe_risc_stack.o
-HIPE_ppc64_OBJS=$(HIPE_ppc_OBJS)
+HIPE_ppc64_OBJS=$(HIPE_ppc_OBJS) $(HIPE_ARCH64_OBJS)
 HIPE_arm_OBJS=$(OBJDIR)/hipe_arm.o $(OBJDIR)/hipe_arm_glue.o $(OBJDIR)/hipe_arm_bifs.o $(OBJDIR)/hipe_risc_stack.o
 HIPE_noarch_OBJS=
 HIPE_ARCH_OBJS=$(HIPE_$(ARCH)_OBJS)
@@ -879,18 +841,21 @@ endif
 
 BASE_OBJS = $(RUN_OBJS) $(EMU_OBJS) $(OS_OBJS) $(EXTRA_BASE_OBJS)
 
-OBJS =	$(BASE_OBJS) $(DRV_OBJS)
-OBJS_SAE =  $(subst sys.o,sys_sae.o,$(OBJS))
+before_DTrace_OBJS =	$(BASE_OBJS) $(DRV_OBJS)
 
-ELIB_C_FILES =	beam/elib_malloc.c \
-		beam/elib_memmove.c \
-		beam/erl_bif_info.c \
-		beam/utils.c \
-		beam/erl_alloc.c
+DTRACE_OBJS =
+ifdef DTRACE_ENABLED_2STEP
+DTRACE_OBJS = $(OBJDIR)/erlang_dtrace.o
+$(OBJDIR)/erlang_dtrace.o: $(before_DTrace_OBJS) $(TARGET)/erlang_dtrace.h
+	dtrace -G -C -Ibeam \
+	  -s beam/erlang_dtrace.d \
+	  -o $@ $(before_DTrace_OBJS)
+endif
+
+OBJS = $(before_DTrace_OBJS) $(DTRACE_OBJS)
 
-MOD_OBJS_ELIB =	$(patsubst %.c,$(OBJDIR)/%.o,$(notdir $(ELIB_C_FILES)))
-OBJS_ELIB = 	$(patsubst %.o,%.elib.o,$(MOD_OBJS_ELIB)) \
-		$(filter-out $(MOD_OBJS_ELIB),$(OBJS))
+$(INIT_OBJS): $(TTF_DIR)/GENERATED
+$(OBJS): $(TTF_DIR)/GENERATED
 
 ########################################
 # HiPE section
@@ -915,30 +880,54 @@ $(OBJDIR)/%.o: hipe/%.c
 $(BINDIR)/hipe_mkliterals$(TF_MARKER):	$(OBJDIR)/hipe_mkliterals.o
 	$(CC) $(CFLAGS) $(INCLUDES) -o $@ $<
 
-$(OBJDIR)/hipe_mkliterals.o:	$(TTF_DIR)/hipe_x86_asm.h $(TTF_DIR)/hipe_ppc_asm.h
+$(OBJDIR)/hipe_mkliterals.o:	$(HIPE_ASM) $(TTF_DIR)/erl_alloc_types.h \
+	$(TTF_DIR)/OPCODES-GENERATED $(TARGET)/TABLES-GENERATED
 
 $(TTF_DIR)/hipe_literals.h:	$(BINDIR)/hipe_mkliterals$(TF_MARKER)
 	$(BINDIR)/hipe_mkliterals$(TF_MARKER) -c > $@
 
-$(OBJDIR)/hipe_x86_glue.o:	hipe/hipe_x86_glue.S $(TTF_DIR)/hipe_x86_asm.h $(TTF_DIR)/hipe_literals.h hipe/hipe_mode_switch.h
-$(TTF_DIR)/hipe_x86_bifs.S:	hipe/hipe_x86_bifs.m4 hipe/hipe_x86_asm.m4 hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h
-$(OBJDIR)/hipe_x86_bifs.o:	$(TTF_DIR)/hipe_x86_bifs.S $(TTF_DIR)/hipe_literals.h
-
-$(OBJDIR)/hipe_amd64_glue.o:	hipe/hipe_amd64_glue.S $(TTF_DIR)/hipe_amd64_asm.h $(TTF_DIR)/hipe_literals.h hipe/hipe_mode_switch.h
-$(TTF_DIR)/hipe_amd64_bifs.S:	hipe/hipe_amd64_bifs.m4 hipe/hipe_amd64_asm.m4 hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h
-$(OBJDIR)/hipe_amd64_bifs.o:	$(TTF_DIR)/hipe_amd64_bifs.S $(TTF_DIR)/hipe_literals.h
-
-$(OBJDIR)/hipe_sparc_glue.o:	hipe/hipe_sparc_glue.S $(TTF_DIR)/hipe_sparc_asm.h hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_literals.h
-$(TTF_DIR)/hipe_sparc_bifs.S:	hipe/hipe_sparc_bifs.m4 hipe/hipe_sparc_asm.m4 hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h
-$(OBJDIR)/hipe_sparc_bifs.o:	$(TTF_DIR)/hipe_sparc_bifs.S $(TTF_DIR)/hipe_literals.h
-
-$(OBJDIR)/hipe_ppc_glue.o:	hipe/hipe_ppc_glue.S $(TTF_DIR)/hipe_ppc_asm.h hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_literals.h
-$(TTF_DIR)/hipe_ppc_bifs.S:	hipe/hipe_ppc_bifs.m4 hipe/hipe_ppc_asm.m4 hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h
-$(OBJDIR)/hipe_ppc_bifs.o:	$(TTF_DIR)/hipe_ppc_bifs.S $(TTF_DIR)/hipe_literals.h
-
-$(OBJDIR)/hipe_arm_glue.o:	hipe/hipe_arm_glue.S $(TTF_DIR)/hipe_arm_asm.h hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_literals.h
-$(TTF_DIR)/hipe_arm_bifs.S:	hipe/hipe_arm_bifs.m4 hipe/hipe_arm_asm.m4 hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h
-$(OBJDIR)/hipe_arm_bifs.o:	$(TTF_DIR)/hipe_arm_bifs.S $(TTF_DIR)/hipe_literals.h
+$(OBJDIR)/hipe_x86_glue.o:	hipe/hipe_x86_glue.S \
+	$(TTF_DIR)/hipe_x86_asm.h $(TTF_DIR)/hipe_literals.h \
+	hipe/hipe_mode_switch.h
+$(TTF_DIR)/hipe_x86_bifs.S:	hipe/hipe_x86_bifs.m4 hipe/hipe_x86_asm.m4 \
+	hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h
+$(OBJDIR)/hipe_x86_bifs.o:	$(TTF_DIR)/hipe_x86_bifs.S \
+	$(TTF_DIR)/hipe_literals.h
+
+$(OBJDIR)/hipe_amd64_glue.o:	hipe/hipe_amd64_glue.S \
+	$(TTF_DIR)/hipe_amd64_asm.h $(TTF_DIR)/hipe_literals.h \
+	hipe/hipe_mode_switch.h
+$(TTF_DIR)/hipe_amd64_bifs.S:	hipe/hipe_amd64_bifs.m4 hipe/hipe_amd64_asm.m4 \
+	hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h
+$(OBJDIR)/hipe_amd64_bifs.o:	$(TTF_DIR)/hipe_amd64_bifs.S \
+	$(TTF_DIR)/hipe_literals.h
+
+$(OBJDIR)/hipe_sparc_glue.o:	hipe/hipe_sparc_glue.S \
+	$(TTF_DIR)/hipe_sparc_asm.h hipe/hipe_mode_switch.h \
+	$(TTF_DIR)/hipe_literals.h
+$(TTF_DIR)/hipe_sparc_bifs.S:	hipe/hipe_sparc_bifs.m4 hipe/hipe_sparc_asm.m4 \
+	hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h
+$(OBJDIR)/hipe_sparc_bifs.o:	$(TTF_DIR)/hipe_sparc_bifs.S \
+	$(TTF_DIR)/hipe_literals.h
+
+$(OBJDIR)/hipe_ppc_glue.o:	hipe/hipe_ppc_glue.S $(TTF_DIR)/hipe_ppc_asm.h \
+	hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_literals.h
+$(TTF_DIR)/hipe_ppc_bifs.S:	hipe/hipe_ppc_bifs.m4 hipe/hipe_ppc_asm.m4 \
+	hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h
+$(OBJDIR)/hipe_ppc_bifs.o:	$(TTF_DIR)/hipe_ppc_bifs.S \
+	$(TTF_DIR)/hipe_literals.h
+
+$(OBJDIR)/hipe_arm_glue.o:	hipe/hipe_arm_glue.S $(TTF_DIR)/hipe_arm_asm.h \
+	hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_literals.h
+$(TTF_DIR)/hipe_arm_bifs.S:	hipe/hipe_arm_bifs.m4 hipe/hipe_arm_asm.m4 \
+	hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h
+$(OBJDIR)/hipe_arm_bifs.o:	$(TTF_DIR)/hipe_arm_bifs.S \
+	$(TTF_DIR)/hipe_literals.h
+
+# Use -fomit-frame-pointer to work around gcc (v4.5.2) bug causing
+# "error: r7 cannot be used in asm here" for DEBUG build.
+$(OBJDIR)/hipe_arm.o:           hipe/hipe_arm.c
+	$(CC) $(subst O2,O3, $(CFLAGS)) -fomit-frame-pointer $(INCLUDES) -c $< -o $@
 
 # end of HiPE section
 ########################################
@@ -968,33 +957,6 @@ $(TARGET)/int64.c:
 
 endif
 
-ifeq ($(findstring ose,$(TARGET)),ose)
-# Extract soft float functions from libgcc.a (for beam_emu)
-VXCC=@VXCC@
-VXCFLAGS=@VXCFLAGS@
-VXLD=@VXLD@
-VXLDFLAGS=@VXLDFLAGS@
-VXCCLIBFLAGS=@VXCCLIBFLAGS@
-STRIP=@STRIP@
-SYMPREFIX=@SYMPREFIX@
-
-NEEDFUNCTIONS=__floatsidf __adddf3 __negdf2 __muldf3 __divdf3 __subdf3
-KEEPSYMS=$(NEEDFUNCTIONS:%=-K $(SYMPREFIX)%)
-
-$(OBJDIR)/ose_sfp.o: $(TARGET)/ose_sfp.c
-	$(VXCC) $(VXCFLAGS) -o $(OBJDIR)/ose_sfp_tmp.o -c $(TARGET)/ose_sfp.c
-	$(VXLD) -o $(OBJDIR)/ose_sfp.o $(OBJDIR)/ose_sfp_tmp.o $(VXLDFLAGS) $(VXCCLIBFLAGS)
-	$(STRIP) $(KEEPSYMS) $(OBJDIR)/ose_sfp.o
-
-$(TARGET)/ose_sfp.c:
-	echo 'void dummy(void); void dummy(void) {' > $(TARGET)/ose_sfp.c
-	for x in $(NEEDFUNCTIONS); do echo 'extern void '$$x'();' \
-		>> $(TARGET)/ose_sfp.c; done 
-	for x in $(NEEDFUNCTIONS); do echo $$x'();' >> $(TARGET)/ose_sfp.c; done
-	echo '}'  >> $(TARGET)/ose_sfp.c 
-
-endif
-
 # ----------------------------------------------------------------------
 # The emulator itself
 
@@ -1010,23 +972,8 @@ $(BINDIR)/$(EMULATOR_EXECUTABLE): $(INIT_OBJS) $(OBJS) $(DEPLIBS)
 	$(PURIFY) $(LD) -o $(BINDIR)/$(EMULATOR_EXECUTABLE) \
 	$(HIPEBEAMLDFLAGS) $(LDFLAGS) $(DEXPORT) $(INIT_OBJS) $(OBJS) $(LIBS)
 
-$(BINDIR)/$(EMULATOR_EXECUTABLE_ELIB): $(INIT_OBJS) $(OBJS_ELIB) $(DEPLIBS)
-	$(PURIFY) $(LD) -o $(BINDIR)/$(EMULATOR_EXECUTABLE_ELIB) \
-	$(LDFLAGS) $(DEXPORT) $(INIT_OBJS) $(OBJS_ELIB) $(LIBS)
-
-$(BINDIR)/$(EMULATOR_EXECUTABLE_SAE): $(INIT_OBJS_SAE) $(OBJS_SAE) $(DEPLIBS)
-	$(PURIFY) $(LD) -o $(BINDIR)/$(EMULATOR_EXECUTABLE_SAE) \
-	$(LDFLAGS) $(DEXPORT) $(INIT_OBJS_SAE) $(OBJS_SAE) $(LIBS)
-
 endif
 
-#
-# Create directories
-#
-
-$(CREATE_DIRS):
-	$(MKDIR) -p $@
-
 # ----------------------------------------------------------------------
 # Dependencies
 #
@@ -1038,7 +985,7 @@ $(TARGET)/Makefile: Makefile.in
 
 #SED_REPL_WIN_DRIVE=s|\([ 	]\)\([A-Za-z]\):|\1/cygdrive/\2|g;s|^\([A-Za-z]\):|/cygdrive/\1|g
 SED_REPL_O=s|^\([^:]*:\)|$$(OBJDIR)/\1|g
-SED_REPL_ELIB_O=s|^\([^:]*\).o[ 	]*:|$$(OBJDIR)/\1.elib.o:|g
+SED_REPL_O_ZLIB=s|^\([^:]*:\)|$$(ZLIB_OBJDIR)/\1|g
 SED_REPL_TTF_DIR=s|$(TTF_DIR)/|$$(TTF_DIR)/|g
 SED_REPL_ERL_TOP=s|\([ 	]\)$(ERL_TOP)/|\1$$(ERL_TOP)/|g;s|^$(ERL_TOP)/|$$(ERL_TOP)/|g
 SED_REPL_POLL=s|$$(OBJDIR)/erl_poll.o|$$(OBJDIR)/erl_poll.kp.o $$(OBJDIR)/erl_poll.nkp.o|g
@@ -1058,7 +1005,7 @@ SED_SUFFIX=
 endif
 
 SED_DEPEND=sed '$(SED_PREFIX)$(SED_REPL_O);$(SED_REPL_TTF_DIR);$(SED_REPL_ERL_TOP)$(SED_SUFFIX)'
-SED_ELIB_DEPEND=sed '$(SED_PREFIX)$(SED_REPL_ELIB_O);$(SED_REPL_TTF_DIR);$(SED_REPL_ERL_TOP)$(SED_SUFFIX)'
+SED_DEPEND_ZLIB=sed '$(SED_PREFIX)$(SED_REPL_O_ZLIB)'
 
 ifdef HIPE_ENABLED
 HIPE_SRC=$(wildcard hipe/*.c)
@@ -1067,7 +1014,8 @@ HIPE_SRC=
 endif
 
 BEAM_SRC=$(wildcard beam/*.c)
-DRV_SRC=$(wildcard drivers/common/*.c) $(wildcard drivers/$(ERLANG_OSTYPE)/*.c)
+DRV_COMMON_SRC=$(wildcard drivers/common/*.c)
+DRV_OSTYPE_SRC=$(wildcard drivers/$(ERLANG_OSTYPE)/*.c)
 ALL_SYS_SRC=$(wildcard sys/$(ERLANG_OSTYPE)/*.c) $(wildcard sys/common/*.c)
 TARGET_SRC=$(wildcard $(TARGET)/*.c) $(wildcard $(TTF_DIR)/*.c)
 
@@ -1078,7 +1026,7 @@ ifeq ($(TARGET),win32)
 
 #DEP_CC=$(EMU_CC)
 DEP_CC=$(CC)
-DEP_FLAGS=-MM  $(subst -O2,,$(CFLAGS)) $(INCLUDES) -I../etc/win32 -Idrivers/common
+DEP_FLAGS=-MM  $(subst -O2,,$(CFLAGS)) $(INCLUDES) -I../etc/win32 -Idrivers/common -Idrivers/$(ERLANG_OSTYPE)
 # ifeq (@MIXED_CYGWIN_VC@,yes)
 # VC++ used for compiling. If __GNUC__ is defined we will include
 # other headers then when compiling which will result in faulty
@@ -1098,28 +1046,40 @@ MG_FLAG=-MG
 endif
 
 DEP_CC=$(CC)
-DEP_FLAGS=-MM $(MG_FLAG) $(CFLAGS) $(INCLUDES) -Idrivers/common
+DEP_FLAGS=-MM $(MG_FLAG) $(CFLAGS) $(INCLUDES) -Idrivers/common -Idrivers/$(ERLANG_OSTYPE)
 SYS_SRC=$(ALL_SYS_SRC)
 endif
 
+.PHONY: depend
+ifdef VOID_EMULATOR
 depend:
+	@echo $(VOID_EMULATOR)' - omitted target depend'
+else
+depend: $(TTF_DIR)/depend.mk
+$(TTF_DIR)/depend.mk: $(TTF_DIR)/GENERATED $(PRELOAD_SRC)
 	$(DEP_CC) $(DEP_FLAGS) $(BEAM_SRC) \
-		| $(SED_DEPEND) > $(TARGET)/depend.mk
-	$(DEP_CC) $(DEP_FLAGS) $(DRV_SRC) \
-		| $(SED_DEPEND) >> $(TARGET)/depend.mk
+		| $(SED_DEPEND) > $(TTF_DIR)/depend.mk
+	$(DEP_CC) $(DEP_FLAGS) -DLIBSCTP=$(LIBSCTP) $(DRV_COMMON_SRC) \
+		| $(SED_DEPEND) >> $(TTF_DIR)/depend.mk
+	$(DEP_CC) $(DEP_FLAGS) -I../etc/$(ERLANG_OSTYPE) $(DRV_OSTYPE_SRC) \
+		| $(SED_DEPEND) >> $(TTF_DIR)/depend.mk
 	$(DEP_CC) $(DEP_FLAGS) $(SYS_SRC) \
-		| $(SED_DEPEND) >> $(TARGET)/depend.mk
+		| $(SED_DEPEND) >> $(TTF_DIR)/depend.mk
 	$(DEP_CC) $(DEP_FLAGS) $(TARGET_SRC) \
-		| $(SED_DEPEND) >> $(TARGET)/depend.mk
-ifneq ($(TARGET),win32)
-	$(DEP_CC) $(DEP_FLAGS) $(ELIB_FLAGS) $(ELIB_C_FILES) \
-		| $(SED_ELIB_DEPEND) >> $(TARGET)/depend.mk
-endif
+		| $(SED_DEPEND) >> $(TTF_DIR)/depend.mk
+	$(DEP_CC) $(DEP_FLAGS) $(ZLIB_SRC) \
+		| $(SED_DEPEND_ZLIB) >> $(TTF_DIR)/depend.mk
 ifdef HIPE_ENABLED
 	$(DEP_CC) $(DEP_FLAGS) $(HIPE_SRC) \
-		| $(SED_DEPEND) >> $(TARGET)/depend.mk
+		| $(SED_DEPEND) >> $(TTF_DIR)/depend.mk
+endif
+	cd $(ERTS_LIB_DIR) && $(MAKE) depend
 endif
--include $(TARGET)/depend.mk
-
-
 
+ifneq ($(MAKECMDGOALS),clean)
+ifneq ($(MAKECMDGOALS),generate)
+ifndef VOID_EMULATOR
+-include $(TTF_DIR)/depend.mk
+endif
+endif
+endif
diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index e2a79d6e4f..d7c7f117cf 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -41,8 +41,7 @@ static erts_smp_rwmtx_t atom_table_lock;
 #define atom_read_unlock()	erts_smp_rwmtx_runlock(&atom_table_lock)
 #define atom_write_lock()	erts_smp_rwmtx_rwlock(&atom_table_lock)
 #define atom_write_unlock()	erts_smp_rwmtx_rwunlock(&atom_table_lock)
-#define atom_init_lock()	erts_smp_rwmtx_init(&atom_table_lock, \
-						    "atom_tab")
+
 #if 0
 #define ERTS_ATOM_PUT_OPS_STAT
 #endif
@@ -76,7 +75,7 @@ void atom_info(int to, void *to_arg)
     index_info(to, to_arg, &erts_atom_table);
 #ifdef ERTS_ATOM_PUT_OPS_STAT
     erts_print(to, to_arg, "atom_put_ops: %ld\n",
-	       erts_smp_atomic_read(&atom_put_ops));
+	       erts_smp_atomic_read_nob(&atom_put_ops));
 #endif
 
     if (lock)
@@ -214,7 +213,7 @@ am_atom_put(const char* name, int len)
 	len = MAX_ATOM_LENGTH;
     }
 #ifdef ERTS_ATOM_PUT_OPS_STAT
-    erts_smp_atomic_inc(&atom_put_ops);
+    erts_smp_atomic_inc_nob(&atom_put_ops);
 #endif
     a.len = len;
     a.name = (byte*)name;
@@ -304,12 +303,17 @@ init_atom_table(void)
     HashFunctions f;
     int i;
     Atom a;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+
+    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
 #ifdef ERTS_ATOM_PUT_OPS_STAT
-    erts_smp_atomic_init(&atom_put_ops, 0);
+    erts_smp_atomic_init_nob(&atom_put_ops, 0);
 #endif
 
-    atom_init_lock();
+    erts_smp_rwmtx_init_opt(&atom_table_lock, &rwmtx_opt, "atom_tab");
+
     f.hash = (H_FUN) atom_hash;
     f.cmp  = (HCMP_FUN) atom_cmp;
     f.alloc = (HALLOC_FUN) atom_alloc;
diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h
index cb245a87b1..fd9c04d3d0 100644
--- a/erts/emulator/beam/atom.h
+++ b/erts/emulator/beam/atom.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -34,7 +34,7 @@
 /* Internal atom cache needs MAX_ATOM_TABLE_SIZE to be less than an
    unsigned 32 bit integer. See external.c(erts_encode_ext_dist_header_setup)
    for more details. */
-#define MAX_ATOM_TABLE_SIZE ((MAX_ATOM_INDEX + 1 < (1UL << 32)) ? MAX_ATOM_INDEX + 1 : (1UL << 32))
+#define MAX_ATOM_TABLE_SIZE ((MAX_ATOM_INDEX + 1 < (UWORD_CONSTANT(1) << 32)) ? MAX_ATOM_INDEX + 1 : (UWORD_CONSTANT(1) << 32))
 #else
 #define MAX_ATOM_TABLE_SIZE (MAX_ATOM_INDEX + 1)
 #endif
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 57c8b08223..106fad030b 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1996-2010. All Rights Reserved.
+# Copyright Ericsson AB 1996-2012. All Rights Reserved.
 #
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -65,9 +65,12 @@ atom EXIT='EXIT'
 atom aborted
 atom abs_path
 atom absoluteURI
+atom ac
 atom active
 atom all
 atom all_but_first
+atom alloc_info
+atom alloc_sizes
 atom allocated
 atom allocated_areas
 atom allocator
@@ -75,6 +78,7 @@ atom allocator_sizes
 atom alloc_util_allocators
 atom allow_passive_connect
 atom already_loaded
+atom amd64
 atom anchored
 atom and
 atom andalso
@@ -91,6 +95,7 @@ atom atom
 atom atom_used
 atom attributes
 atom await_proc_exit
+atom await_sched_wall_time_modifications
 atom awaiting_load
 atom awaiting_unload
 atom backtrace backtrace_depth
@@ -100,8 +105,15 @@ atom band
 atom big
 atom bif_return_trap
 atom binary
+atom binary_bin_to_list_trap
+atom binary_copy_trap
+atom binary_longest_prefix_trap
+atom binary_longest_suffix_trap
+atom binary_match_trap
+atom binary_matches_trap
 atom block
 atom blocked
+atom bm
 atom bnot
 atom bor
 atom bxor
@@ -111,10 +123,12 @@ atom bsl
 atom bsr
 atom bsr_anycrlf
 atom bsr_unicode
+atom build_type
 atom busy_dist_port
 atom busy_port
 atom call
 atom call_count
+atom call_time
 atom caller
 atom capture
 atom case_clause
@@ -145,6 +159,8 @@ atom cr
 atom crlf
 atom creation
 atom current_function
+atom current_location
+atom current_stacktrace
 atom data
 atom debug_flags
 atom delay_trap
@@ -224,13 +240,14 @@ atom generational
 atom get_seq_token
 atom get_tcw
 atom getenv
+atom gather_sched_wall_time_result
 atom getting_linked
 atom getting_unlinked
 atom global
-atom global_heaps_size
 atom Gt='>'
 atom grun
 atom group_leader
+atom have_dt_utag
 atom heap_block_size
 atom heap_size
 atom heap_sizes
@@ -241,7 +258,6 @@ atom hide
 atom high
 atom hipe_architecture
 atom http httph https http_response http_request http_header http_eoh http_error http_bin httph_bin
-atom hybrid
 atom id
 atom if_clause
 atom imports
@@ -256,6 +272,7 @@ atom info
 atom info_msg
 atom initial_call
 atom input
+atom internal
 atom internal_error
 atom internal_status
 atom instruction_counts
@@ -368,9 +385,11 @@ atom old_heap_size
 atom on_load
 atom open
 atom open_error
+atom opt
 atom or
 atom ordered_set
 atom orelse
+atom os_pid
 atom os_type
 atom os_version
 atom ose_bg_proc
@@ -422,6 +441,7 @@ atom raw
 atom re
 atom re_pattern
 atom re_run_trap
+atom read_concurrency
 atom ready_input
 atom ready_output
 atom ready_async
@@ -453,6 +473,7 @@ atom scheduler
 atom scheduler_id
 atom schedulers_online
 atom scheme
+atom scope
 atom sensitive
 atom sequential_tracer
 atom sequential_trace_token
@@ -535,8 +556,10 @@ atom waiting
 atom wall_clock
 atom warning
 atom warning_msg
+atom scheduler_wall_time
 atom wordsize
 atom write_concurrency
 atom xor
+atom x86
 atom yes
 atom yield
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index b1feec7074..ada2d152b7 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -33,27 +33,29 @@
 #include "beam_catches.h"
 #include "erl_binary.h"
 #include "erl_nif.h"
+#include "erl_thr_progress.h"
 
 static void set_default_trace_pattern(Eterm module);
 static Eterm check_process_code(Process* rp, Module* modp);
 static void delete_code(Process *c_p, ErtsProcLocks c_p_locks, Module* modp);
 static void delete_export_references(Eterm module);
 static int purge_module(int module);
-static int is_native(Eterm* code);
+static void decrement_refc(BeamInstr* code);
+static int is_native(BeamInstr* code);
 static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
 static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
-static void remove_from_address_table(Eterm* code);
+static void remove_from_address_table(BeamInstr* code);
 
 Eterm
 load_module_2(BIF_ALIST_2)
 {
     Eterm   reason;
     Eterm*  hp;
-    int      i;
     int      sz;
     byte*    code;
     Eterm res;
     byte* temp_alloc = NULL;
+    struct LoaderState* stp;
 
     if (is_not_atom(BIF_ARG_1)) {
     error:
@@ -63,49 +65,37 @@ load_module_2(BIF_ALIST_2)
     if ((code = erts_get_aligned_binary_bytes(BIF_ARG_2, &temp_alloc)) == NULL) {
 	goto error;
     }
-    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-    erts_smp_block_system(0);
-
-    erts_export_consolidate();
-
     hp = HAlloc(BIF_P, 3);
+
+    /*
+     * Read the BEAM file and prepare the module for loading.
+     */
+    stp = erts_alloc_loader_state();
     sz = binary_size(BIF_ARG_2);
-    if ((i = erts_load_module(BIF_P, 0,
-			      BIF_P->group_leader, &BIF_ARG_1, code, sz)) < 0) { 
-	switch (i) {
-	case -1: reason = am_badfile; break; 
-	case -2: reason = am_nofile; break;
-	case -3: reason = am_not_purged; break;
-	case -4:
-	    reason = am_atom_put("native_code", sizeof("native_code")-1);
-	    break;
-	case -5:
-	    {
-		/*
-		 * The module contains an on_load function. The loader
-		 * has loaded the module as usual, except that the
-		 * export entries does not point into the module, so it
-		 * is not possible to call any code in the module.
-		 */
-
-		ERTS_DECL_AM(on_load);
-		reason = AM_on_load;
-		break;
-	    }
-	default: reason = am_badfile; break;
-	}
+    reason = erts_prepare_loading(stp, BIF_P, BIF_P->group_leader,
+				  &BIF_ARG_1, code, sz);
+    erts_free_aligned_binary_bytes(temp_alloc);
+    if (reason != NIL) {
 	res = TUPLE2(hp, am_error, reason);
-	goto done;
+	BIF_RET(res);
     }
 
-    set_default_trace_pattern(BIF_ARG_1);
-    res = TUPLE2(hp, am_module, BIF_ARG_1);
+    /*
+     * Stop all other processes and finish the loading of the module.
+     */
+    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_smp_thr_progress_block();
 
- done:
-    erts_free_aligned_binary_bytes(temp_alloc);
-    erts_smp_release_system();
-    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    reason = erts_finish_loading(stp, BIF_P, 0, &BIF_ARG_1);
+    if (reason != NIL) {
+	res = TUPLE2(hp, am_error, reason);
+    } else {
+	set_default_trace_pattern(BIF_ARG_1);
+	res = TUPLE2(hp, am_module, BIF_ARG_1);
+    }
 
+    erts_smp_thr_progress_unblock();
+    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
     BIF_RET(res);
 }
 
@@ -118,12 +108,12 @@ BIF_RETTYPE purge_module_1(BIF_ALIST_1)
     }
 
     erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-    erts_smp_block_system(0);
+    erts_smp_thr_progress_block();
 
     erts_export_consolidate();
     purge_res = purge_module(atom_val(BIF_ARG_1));
 
-    erts_smp_release_system();
+    erts_smp_thr_progress_unblock();
     erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
     if (purge_res < 0) {
@@ -142,7 +132,7 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1)
     if ((modp = erts_get_module(BIF_ARG_1)) == NULL) {
 	return am_undefined;
     }
-    return (is_native(modp->code) ||
+    return ((modp->code && is_native(modp->code)) ||
 	    (modp->old_code != 0 && is_native(modp->old_code))) ?
 		am_true : am_false;
 }
@@ -152,16 +142,33 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
     Eterm res;
 
     erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-    erts_smp_block_system(0);
+    erts_smp_thr_progress_block();
 
     erts_export_consolidate();
     res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
 
-    erts_smp_release_system();
+    erts_smp_thr_progress_unblock();
     erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
     return res;
 }
 
+BIF_RETTYPE
+check_old_code_1(BIF_ALIST_1)
+{
+    Module* modp;
+
+    if (is_not_atom(BIF_ARG_1)) {
+	BIF_ERROR(BIF_P, BADARG);
+    }
+    modp = erts_get_module(BIF_ARG_1);
+    if (modp == NULL) {		/* Doesn't exist. */
+	BIF_RET(am_false);
+    } else if (modp->old_code == NULL) { /* No old code. */
+	BIF_RET(am_false);
+    }
+    BIF_RET(am_true);
+}
+
 Eterm
 check_process_code_2(BIF_ALIST_2)
 {
@@ -175,8 +182,19 @@ check_process_code_2(BIF_ALIST_2)
 	Eterm res;
 	if (internal_pid_index(BIF_ARG_1) >= erts_max_processes)
 	    goto error;
-	rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN,
-				       BIF_ARG_1, ERTS_PROC_LOCK_MAIN);
+	modp = erts_get_module(BIF_ARG_2);
+	if (modp == NULL) {		/* Doesn't exist. */
+	    return am_false;
+	} else if (modp->old_code == NULL) { /* No old code. */
+	    return am_false;
+	}
+	
+#ifdef ERTS_SMP
+	rp = erts_pid2proc_suspend(BIF_P, ERTS_PROC_LOCK_MAIN,
+				   BIF_ARG_1, ERTS_PROC_LOCK_MAIN);
+#else
+	rp = erts_pid2proc(BIF_P, 0, BIF_ARG_1, 0);
+#endif
 	if (!rp) {
 	    BIF_RET(am_false);
 	}
@@ -184,11 +202,12 @@ check_process_code_2(BIF_ALIST_2)
 	    ERTS_BIF_YIELD2(bif_export[BIF_check_process_code_2], BIF_P,
 			    BIF_ARG_1, BIF_ARG_2);
 	}
-	modp = erts_get_module(BIF_ARG_2);
 	res = check_process_code(rp, modp);
 #ifdef ERTS_SMP
-	if (BIF_P != rp)
+	if (BIF_P != rp) {
+	    erts_resume(rp, ERTS_PROC_LOCK_MAIN);
 	    erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
+	}
 #endif
 	BIF_RET(res);
     }
@@ -210,7 +229,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
 	goto badarg;
 
     erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-    erts_smp_block_system(0);
+    erts_smp_thr_progress_block();
 
     {
 	Module *modp = erts_get_module(BIF_ARG_1);
@@ -231,7 +250,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
 	}
     }
 
-    erts_smp_release_system();
+    erts_smp_thr_progress_unblock();
     erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
     if (res == am_badarg) {
@@ -323,7 +342,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
     }
 
     erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-    erts_smp_block_system(0);
+    erts_smp_thr_progress_block();
 
     if (BIF_ARG_2 == am_true) {
 	int i;
@@ -337,15 +356,14 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 		ep->code[0] == BIF_ARG_1 &&
 		ep->code[4] != 0) {
 		ep->address = (void *) ep->code[4];
-		ep->code[3] = 0;
 		ep->code[4] = 0;
 	    }
 	}
 	modp->code[MI_ON_LOAD_FUNCTION_PTR] = 0;
 	set_default_trace_pattern(BIF_ARG_1);
     } else if (BIF_ARG_2 == am_false) {
-	Eterm* code;
-	Eterm* end;
+	BeamInstr* code;
+	BeamInstr* end;
 
 	/*
 	 * The on_load function failed. Remove the loaded code.
@@ -354,7 +372,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	 */
 	erts_total_code_size -= modp->code_length;
 	code = modp->code;
-	end = (Eterm *)((char *)code + modp->code_length);
+	end = (BeamInstr *)((char *)code + modp->code_length);
 	erts_cleanup_funs_on_purge(code, end);
 	beam_catches_delmod(modp->catches, code, modp->code_length);
 	erts_free(ERTS_ALC_T_CODE, (void *) code);
@@ -363,12 +381,11 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	modp->catches = BEAM_CATCHES_NIL;
 	remove_from_address_table(code);
     }
-    erts_smp_release_system();
+    erts_smp_thr_progress_unblock();
     erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
     BIF_RET(am_true);
 }
 
-
 static void
 set_default_trace_pattern(Eterm module)
 {
@@ -397,28 +414,21 @@ set_default_trace_pattern(Eterm module)
 static Eterm
 check_process_code(Process* rp, Module* modp)
 {
-    Eterm* start;
+    BeamInstr* start;
     char* mod_start;
     Uint mod_size;
-    Eterm* end;
+    BeamInstr* end;
     Eterm* sp;
-#ifndef HYBRID /* FIND ME! */
-    ErlFunThing* funp;
+    struct erl_off_heap_header* oh;
     int done_gc = 0;
-#endif
 
 #define INSIDE(a) (start <= (a) && (a) < end)
-    if (modp == NULL) {		/* Doesn't exist. */
-	return am_false;
-    } else if (modp->old_code == NULL) { /* No old code. */
-	return am_false;
-    }
 
     /*
      * Pick up limits for the module.
      */
     start = modp->old_code;
-    end = (Eterm *)((char *)start + modp->old_code_length);
+    end = (BeamInstr *)((char *)start + modp->old_code_length);
     mod_start = (char *) start;
     mod_size = modp->old_code_length;
 
@@ -469,33 +479,31 @@ check_process_code(Process* rp, Module* modp)
      * See if there are funs that refer to the old version of the module.
      */
 
-#ifndef HYBRID /* FIND ME! */
  rescan:
-    for (funp = MSO(rp).funs; funp; funp = funp->next) {
-	Eterm* fun_code;
-
-	fun_code = funp->fe->address;
-
-	if (INSIDE((Eterm *) funp->fe->address)) {
-	    if (done_gc) {
-		return am_true;
-	    } else {
-		/*
-		 * Try to get rid of this fun by garbage collecting.
-		 * Clear both fvalue and ftrace to make sure they
-		 * don't hold any funs.
-		 */
-		rp->freason = EXC_NULL;
-		rp->fvalue = NIL;
-		rp->ftrace = NIL;
-		done_gc = 1;
-                FLAGS(rp) |= F_NEED_FULLSWEEP;
-		(void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
-		goto rescan;
+    for (oh = MSO(rp).first; oh; oh = oh->next) {
+	if (thing_subtag(oh->thing_word) == FUN_SUBTAG) {
+	    ErlFunThing* funp = (ErlFunThing*) oh;
+
+	    if (INSIDE((BeamInstr *) funp->fe->address)) {
+		if (done_gc) {
+		    return am_true;
+		} else {
+		    /*
+		    * Try to get rid of this fun by garbage collecting.
+		    * Clear both fvalue and ftrace to make sure they
+		    * don't hold any funs.
+		    */
+		    rp->freason = EXC_NULL;
+		    rp->fvalue = NIL;
+		    rp->ftrace = NIL;
+		    done_gc = 1;
+		    FLAGS(rp) |= F_NEED_FULLSWEEP;
+		    (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
+		    goto rescan;
+		}
 	    }
 	}
     }
-#endif
 
     /*
      * See if there are constants inside the module referenced by the process.
@@ -542,6 +550,7 @@ check_process_code(Process* rp, Module* modp)
 	} else {
 	    Eterm* literals;
 	    Uint lit_size;
+	    struct erl_off_heap_header* oh;
 
 	    /*
 	     * Try to get rid of constants by by garbage collecting.
@@ -555,7 +564,9 @@ check_process_code(Process* rp, Module* modp)
 	    (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
 	    literals = (Eterm *) modp->old_code[MI_LITERALS_START];
 	    lit_size = (Eterm *) modp->old_code[MI_LITERALS_END] - literals;
-	    erts_garbage_collect_literals(rp, literals, lit_size);
+	    oh = (struct erl_off_heap_header *)
+		modp->old_code[MI_LITERALS_OFF_HEAP];
+	    erts_garbage_collect_literals(rp, literals, lit_size, oh);
 	}
     }
     return am_false;
@@ -563,7 +574,7 @@ check_process_code(Process* rp, Module* modp)
 }
 
 #define in_area(ptr,start,nbytes) \
-    ((unsigned long)((char*)(ptr) - (char*)(start)) < (nbytes))
+    ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
 
 static int
 any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
@@ -576,7 +587,7 @@ any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
 	switch (primary_tag(val)) {
 	case TAG_PRIMARY_BOXED:
 	case TAG_PRIMARY_LIST:
-	    if (in_area(val, mod_start, mod_size)) {
+	    if (in_area(EXPAND_POINTER(val), mod_start, mod_size)) {
 		return 1;
 	    }
 	    break;
@@ -596,7 +607,7 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
 	switch (primary_tag(val)) {
 	case TAG_PRIMARY_BOXED:
 	case TAG_PRIMARY_LIST:
-	    if (in_area(val, mod_start, mod_size)) {
+	    if (in_area(EXPAND_POINTER(val), mod_start, mod_size)) {
 		return 1;
 	    }
 	    break;
@@ -617,8 +628,8 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size)
 static int
 purge_module(int module)
 {
-    Eterm* code;
-    Eterm* end;
+    BeamInstr* code;
+    BeamInstr* end;
     Module* modp;
 
     /*
@@ -633,9 +644,6 @@ purge_module(int module)
      * Any code to purge?
      */
     if (modp->old_code == 0) {
-	if (display_loads) {
-	    erts_printf("No code to purge for %T\n", make_atom(module));
-	}
 	return -1;
     }
 
@@ -653,9 +661,10 @@ purge_module(int module)
     ASSERT(erts_total_code_size >= modp->old_code_length);
     erts_total_code_size -= modp->old_code_length;
     code = modp->old_code;
-    end = (Eterm *)((char *)code + modp->old_code_length);
+    end = (BeamInstr *)((char *)code + modp->old_code_length);
     erts_cleanup_funs_on_purge(code, end);
     beam_catches_delmod(modp->old_catches, code, modp->old_code_length);
+    decrement_refc(code);
     erts_free(ERTS_ALC_T_CODE, (void *) code);
     modp->old_code = NULL;
     modp->old_code_length = 0;
@@ -665,7 +674,24 @@ purge_module(int module)
 }
 
 static void
-remove_from_address_table(Eterm* code)
+decrement_refc(BeamInstr* code)
+{
+    struct erl_off_heap_header* oh =
+	(struct erl_off_heap_header *) code[MI_LITERALS_OFF_HEAP];
+
+    while (oh) {
+	Binary* bptr;
+	ASSERT(thing_subtag(oh->thing_word) == REFC_BINARY_SUBTAG);
+	bptr = ((ProcBin*)oh)->val;
+	if (erts_refc_dectest(&bptr->refc, 0) == 0) {
+	    erts_bin_free(bptr);
+	}
+	oh = oh->next;
+    }
+}
+
+static void
+remove_from_address_table(BeamInstr* code)
 {
     int i;
 
@@ -706,10 +732,10 @@ delete_code(Process *c_p, ErtsProcLocks c_p_locks, Module* modp)
     if (modp->code != NULL && modp->code[MI_NUM_BREAKPOINTS] > 0) {
 	if (c_p && c_p_locks)
 	    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
-	erts_smp_block_system(0);
+	erts_smp_thr_progress_block();
 	erts_clear_module_break(modp);
 	modp->code[MI_NUM_BREAKPOINTS] = 0;
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 	if (c_p && c_p_locks)
 	    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
     }
@@ -738,11 +764,11 @@ delete_export_references(Eterm module)
 	Export *ep = export_list(i);
         if (ep != NULL && (ep->code[0] == module)) {
 	    if (ep->address == ep->code+3 &&
-		(ep->code[3] == (Eterm) em_apply_bif)) {
+		(ep->code[3] == (BeamInstr) em_apply_bif)) {
 		continue;
 	    }
 	    ep->address = ep->code+3;
-	    ep->code[3] = (Uint) em_call_error_handler;
+	    ep->code[3] = (BeamInstr) em_call_error_handler;
 	    ep->code[4] = 0;
 	    MatchSetUnref(ep->match_prog_set);
 	    ep->match_prog_set = NULL;
@@ -751,7 +777,7 @@ delete_export_references(Eterm module)
 }
 
 
-int
+Eterm
 beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module)
 {
     Module* modp = erts_put_module(module);
@@ -762,19 +788,16 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module)
      */
 
     if (modp->code != NULL && modp->old_code != NULL)  {
-	return -3;
+	return am_not_purged;
     } else if (modp->old_code == NULL) { /* Make the current version old. */
-	if (display_loads) {
-	    erts_printf("saving old code\n");
-	}
 	delete_code(c_p, c_p_locks, modp);
 	delete_export_references(module);
     }
-    return 0;
+    return NIL;
 }
 
 static int
-is_native(Eterm* code)
+is_native(BeamInstr* code)
 {
     return ((Eterm *)code[MI_FUNCTIONS])[1] != 0;
 }
diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 1abf1dc10c..d772bea02f 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2000-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2000-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
  
@@ -30,6 +30,7 @@
 #include "error.h"
 #include "erl_binary.h"
 #include "beam_bp.h"
+#include "erl_term.h"
 
 /* *************************************************************************
 ** Macros
@@ -100,6 +101,11 @@ do {                               \
     (b)->prev = (a);               \
 } while (0)
 
+
+#define BREAK_IS_BIF (1)
+#define BREAK_IS_ERL (0)
+
+
 /* *************************************************************************
 ** Local prototypes
 */
@@ -109,24 +115,42 @@ do {                               \
 */
 
 static int set_break(Eterm mfa[3], int specified,
-		     Binary *match_spec, Uint break_op, 
+		     Binary *match_spec, BeamInstr break_op,
 		     enum erts_break_op count_op, Eterm tracer_pid);
 static int set_module_break(Module *modp, Eterm mfa[3], int specified,
-			    Binary *match_spec, Uint break_op,
+			    Binary *match_spec, BeamInstr break_op,
 			    enum erts_break_op count_op, Eterm tracer_pid);
-static int set_function_break(Module *modp, Uint *pc,
-			      Binary *match_spec, Uint break_op,
+static int set_function_break(Module *modp, BeamInstr *pc, int bif,
+			      Binary *match_spec, BeamInstr break_op,
 			      enum erts_break_op count_op, Eterm tracer_pid); 
 
 static int clear_break(Eterm mfa[3], int specified, 
-		       Uint break_op);
+		       BeamInstr break_op);
 static int clear_module_break(Module *modp, Eterm mfa[3], int specified, 
-			      Uint break_op);
-static int clear_function_break(Module *modp, Uint *pc, 
-				Uint break_op);
-
-static BpData *is_break(Uint *pc, Uint break_op);
-
+			      BeamInstr break_op);
+static int clear_function_break(Module *modp, BeamInstr *pc, int bif,
+				BeamInstr break_op);
+
+static BpData *is_break(BeamInstr *pc, BeamInstr break_op);
+static BpData *get_break(Process *p, BeamInstr *pc, BeamInstr break_op);
+
+/* bp_hash */
+#define BP_TIME_ADD(pi0, pi1)                       \
+    do {                                            \
+	Uint r;                                     \
+	(pi0)->count   += (pi1)->count;             \
+	(pi0)->s_time  += (pi1)->s_time;            \
+	(pi0)->us_time += (pi1)->us_time;           \
+	r = (pi0)->us_time / 1000000;               \
+	(pi0)->s_time  += r;                        \
+	(pi0)->us_time  = (pi0)->us_time % 1000000; \
+    } while(0)
+
+static void bp_hash_init(bp_time_hash_t *hash, Uint n);
+static void bp_hash_rehash(bp_time_hash_t *hash, Uint n);
+static ERTS_INLINE bp_data_time_item_t * bp_hash_get(bp_time_hash_t *hash, bp_data_time_item_t *sitem);
+static ERTS_INLINE bp_data_time_item_t * bp_hash_put(bp_time_hash_t *hash, bp_data_time_item_t *sitem);
+static void bp_hash_delete(bp_time_hash_t *hash);
 
 
 /* *************************************************************************
@@ -143,117 +167,114 @@ erts_bp_init(void) {
 int 
 erts_set_trace_break(Eterm mfa[3], int specified, Binary *match_spec,
 		     Eterm tracer_pid) {
-    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
     return set_break(mfa, specified, match_spec,
-		     (Uint) BeamOp(op_i_trace_breakpoint), 0, tracer_pid);
+		     (BeamInstr) BeamOp(op_i_trace_breakpoint), 0, tracer_pid);
 }
 
 int 
 erts_set_mtrace_break(Eterm mfa[3], int specified, Binary *match_spec,
 		      Eterm tracer_pid) {
-    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
     return set_break(mfa, specified, match_spec,
-		     (Uint) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid);
+		     (BeamInstr) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid);
 }
 
+/* set breakpoint data for on exported bif entry */
+
 void
-erts_set_mtrace_bif(Uint *pc, Binary *match_spec, Eterm tracer_pid) {
-    BpDataTrace *bdt;
-    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
+erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, Eterm tracer_pid) {
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+    set_function_break(NULL, pc, BREAK_IS_BIF, match_spec, (BeamInstr) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid);
+}
 
-    bdt = (BpDataTrace *) pc[-4];
-    if (bdt) {
-	MatchSetUnref(bdt->match_spec);
-	MatchSetRef(match_spec);
-	bdt->match_spec = match_spec;
-	bdt->tracer_pid = tracer_pid;
-    } else {
-	bdt = Alloc(sizeof(BpDataTrace));
-	BpInit((BpData *) bdt, 0);
-	MatchSetRef(match_spec);
-	bdt->match_spec = match_spec;
-	bdt->tracer_pid = tracer_pid;
-	pc[-4] = (Uint) bdt;
-    }
+void erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op count_op) {
+    set_function_break(NULL, pc, BREAK_IS_BIF, NULL, (BeamInstr) BeamOp(op_i_time_breakpoint), count_op, NIL);
+}
+
+void erts_clear_time_trace_bif(BeamInstr *pc) {
+    clear_function_break(NULL, pc, BREAK_IS_BIF, (BeamInstr) BeamOp(op_i_time_breakpoint));
 }
 
 int 
 erts_set_debug_break(Eterm mfa[3], int specified) {
-    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
     return set_break(mfa, specified, NULL, 
-		     (Uint) BeamOp(op_i_debug_breakpoint), 0, NIL);
+		     (BeamInstr) BeamOp(op_i_debug_breakpoint), 0, NIL);
 }
 
 int 
 erts_set_count_break(Eterm mfa[3], int specified, enum erts_break_op count_op) {
-    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
     return set_break(mfa, specified, NULL, 
-		     (Uint) BeamOp(op_i_count_breakpoint), count_op, NIL);
+		     (BeamInstr) BeamOp(op_i_count_breakpoint), count_op, NIL);
 }
 
-
+int
+erts_set_time_break(Eterm mfa[3], int specified, enum erts_break_op count_op) {
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+    return set_break(mfa, specified, NULL,
+		     (BeamInstr) BeamOp(op_i_time_breakpoint), count_op, NIL);
+}
 
 int
 erts_clear_trace_break(Eterm mfa[3], int specified) {
-    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
     return clear_break(mfa, specified, 
-		       (Uint) BeamOp(op_i_trace_breakpoint));
+		       (BeamInstr) BeamOp(op_i_trace_breakpoint));
 }
 
 int
 erts_clear_mtrace_break(Eterm mfa[3], int specified) {
-    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
     return clear_break(mfa, specified, 
-		       (Uint) BeamOp(op_i_mtrace_breakpoint));
+		       (BeamInstr) BeamOp(op_i_mtrace_breakpoint));
 }
 
 void
-erts_clear_mtrace_bif(Uint *pc) {
-    BpDataTrace *bdt;
-    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
-    
-    bdt = (BpDataTrace *) pc[-4];
-    if (bdt) {
-	if (bdt->match_spec) {
-	    MatchSetUnref(bdt->match_spec);
-	}
-	Free(bdt);
-    }
-    pc[-4] = (Uint) NULL;
+erts_clear_mtrace_bif(BeamInstr *pc) {
+    clear_function_break(NULL, pc, BREAK_IS_BIF, (BeamInstr) BeamOp(op_i_mtrace_breakpoint));
 }
 
 int
 erts_clear_debug_break(Eterm mfa[3], int specified) {
-    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
     return clear_break(mfa, specified, 
-		       (Uint) BeamOp(op_i_debug_breakpoint));
+		       (BeamInstr) BeamOp(op_i_debug_breakpoint));
 }
 
 int
 erts_clear_count_break(Eterm mfa[3], int specified) {
-    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
     return clear_break(mfa, specified, 
-		       (Uint) BeamOp(op_i_count_breakpoint));
+		       (BeamInstr) BeamOp(op_i_count_breakpoint));
+}
+
+int
+erts_clear_time_break(Eterm mfa[3], int specified) {
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+    return clear_break(mfa, specified,
+		       (BeamInstr) BeamOp(op_i_time_breakpoint));
 }
 
 int
 erts_clear_break(Eterm mfa[3], int specified) {
-    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
     return clear_break(mfa, specified, 0);
 }
 
 int 
 erts_clear_module_break(Module *modp) {
-    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
     ASSERT(modp);
     return clear_module_break(modp, NULL, 0, 0);
 }
 
 int
-erts_clear_function_break(Module *modp, Uint *pc) {
-    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
+erts_clear_function_break(Module *modp, BeamInstr *pc) {
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
     ASSERT(modp);
-    return clear_function_break(modp, pc, 0);
+    return clear_function_break(modp, pc, BREAK_IS_ERL, 0);
 }
 
 
@@ -261,13 +282,16 @@ erts_clear_function_break(Module *modp, Uint *pc) {
 /*
  * SMP NOTE: Process p may have become exiting on return!
  */
-Uint 
-erts_trace_break(Process *p, Uint *pc, Eterm *args, 
+BeamInstr
+erts_trace_break(Process *p, BeamInstr *pc, Eterm *args,
 		 Uint32 *ret_flags, Eterm *tracer_pid) {
     Eterm tpid1, tpid2;
-    BpDataTrace *bdt = (BpDataTrace *) pc[-4];
-    
-    ASSERT(pc[-5] == (Uint) BeamOp(op_i_func_info_IaaI));
+    BpData **bds = (BpData **) (pc)[-4];
+    BpDataTrace *bdt = NULL;
+
+    ASSERT(bds);
+    ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+    bdt = (BpDataTrace *) bds[bp_sched2ix_proc(p)];
     ASSERT(bdt);
     bdt = (BpDataTrace *) bdt->next;
     ASSERT(bdt);
@@ -286,7 +310,7 @@ erts_trace_break(Process *p, Uint *pc, Eterm *args,
 	bdt->tracer_pid = tpid2;
 	ErtsSmpBPUnlock(bdt);
     }
-    pc[-4] = (Uint) bdt;
+    bds[bp_sched2ix_proc(p)] = (BpData *) bdt;
     return bdt->orig_instr;
 }
 
@@ -296,14 +320,17 @@ erts_trace_break(Process *p, Uint *pc, Eterm *args,
  * SMP NOTE: Process p may have become exiting on return!
  */
 Uint32
-erts_bif_mtrace(Process *p, Uint *pc, Eterm *args, int local, 
+erts_bif_mtrace(Process *p, BeamInstr *pc, Eterm *args, int local,
 		Eterm *tracer_pid) {
-    BpDataTrace *bdt = (BpDataTrace *) pc[-4];
-    
+    BpData **bds = (BpData **) (pc)[-4];
+    BpDataTrace *bdt = NULL;
+
+
     ASSERT(tracer_pid);
-    if (bdt) {
+    if (bds) {
 	Eterm tpid1, tpid2;
 	Uint32 flags;
+	bdt = (BpDataTrace *)bds[bp_sched2ix_proc(p)];
 
 	ErtsSmpBPLock(bdt);
 	tpid1 = tpid2 = bdt->tracer_pid;
@@ -326,9 +353,9 @@ erts_bif_mtrace(Process *p, Uint *pc, Eterm *args, int local,
 
 
 int 
-erts_is_trace_break(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) {
+erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) {
     BpDataTrace *bdt = 
-	(BpDataTrace *) is_break(pc, (Uint) BeamOp(op_i_trace_breakpoint));
+	(BpDataTrace *) is_break(pc, (BeamInstr) BeamOp(op_i_trace_breakpoint));
     
     if (bdt) {
 	if (match_spec_ret) {
@@ -345,9 +372,9 @@ erts_is_trace_break(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) {
 }
 
 int 
-erts_is_mtrace_break(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) {
+erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) {
     BpDataTrace *bdt = 
-	(BpDataTrace *) is_break(pc, (Uint) BeamOp(op_i_mtrace_breakpoint));
+	(BpDataTrace *) is_break(pc, (BeamInstr) BeamOp(op_i_mtrace_breakpoint));
     
     if (bdt) {
 	if (match_spec_ret) {
@@ -364,74 +391,451 @@ erts_is_mtrace_break(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) {
 }
 
 int
-erts_is_mtrace_bif(Uint *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) {
-    BpDataTrace *bdt = (BpDataTrace *) pc[-4];
-    
-    if (bdt) {
-	if (match_spec_ret) {
-	    *match_spec_ret = bdt->match_spec;
-	}
-	if (tracer_pid_ret) {
-	    ErtsSmpBPLock(bdt);
-	    *tracer_pid_ret = bdt->tracer_pid;
-	    ErtsSmpBPUnlock(bdt);
-	}
-	return !0;
-    }
-    return 0;
-}
-
-int
-erts_is_native_break(Uint *pc) {
+erts_is_native_break(BeamInstr *pc) {
 #ifdef HIPE
-    ASSERT(pc[-5] == (Uint) BeamOp(op_i_func_info_IaaI));
-    return pc[0]  == (Uint) BeamOp(op_hipe_trap_call) 
-	|| pc[0]  == (Uint) BeamOp(op_hipe_trap_call_closure);
+    ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+    return pc[0]  == (BeamInstr) BeamOp(op_hipe_trap_call)
+	|| pc[0]  == (BeamInstr) BeamOp(op_hipe_trap_call_closure);
 #else
     return 0;
 #endif
 }
 
 int 
-erts_is_count_break(Uint *pc, Sint *count_ret) {
+erts_is_count_break(BeamInstr *pc, Sint *count_ret) {
     BpDataCount *bdc = 
-	(BpDataCount *) is_break(pc, (Uint) BeamOp(op_i_count_breakpoint));
+	(BpDataCount *) is_break(pc, (BeamInstr) BeamOp(op_i_count_breakpoint));
     
     if (bdc) {
 	if (count_ret) {
-	    ErtsSmpBPLock(bdc);
-	    *count_ret = bdc->count;
-	    ErtsSmpBPUnlock(bdc);
+	    *count_ret = (Sint) erts_smp_atomic_read_nob(&bdc->acount);
 	}
 	return !0;
     }
     return 0;
 }
 
-Uint *
+int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) {
+    Uint i, ix;
+    bp_time_hash_t hash;
+    Uint size;
+    Eterm *hp, t;
+    bp_data_time_item_t *item = NULL;
+    BpDataTime *bdt = (BpDataTime *) is_break(pc, (BeamInstr) BeamOp(op_i_time_breakpoint));
+
+    if (bdt) {
+	if (retval) {
+	    /* collect all hashes to one hash */
+	    bp_hash_init(&hash, 64);
+	    /* foreach threadspecific hash */
+	    for (i = 0; i < bdt->n; i++) {
+		bp_data_time_item_t *sitem;
+
+	        /* foreach hash bucket not NIL*/
+		for(ix = 0; ix < bdt->hash[i].n; ix++) {
+		    item = &(bdt->hash[i].item[ix]);
+		    if (item->pid != NIL) {
+			sitem = bp_hash_get(&hash, item);
+			if (sitem) {
+			    BP_TIME_ADD(sitem, item);
+			} else {
+			    bp_hash_put(&hash, item);
+			}
+		    }
+		}
+	    }
+	    /* *retval should be NIL or term from previous bif in export entry */
+
+	    if (hash.used > 0) {
+		size = (5 + 2)*hash.used;
+		hp   = HAlloc(p, size);
+
+		for(ix = 0; ix < hash.n; ix++) {
+		    item = &(hash.item[ix]);
+		    if (item->pid != NIL) {
+			t = TUPLE4(hp, item->pid,
+				make_small(item->count),
+				make_small(item->s_time),
+				make_small(item->us_time));
+			hp += 5;
+			*retval = CONS(hp, t, *retval); hp += 2;
+		    }
+		}
+	    }
+	    bp_hash_delete(&hash);
+	}
+	return !0;
+    }
+
+    return 0;
+}
+
+
+BeamInstr *
 erts_find_local_func(Eterm mfa[3]) {
     Module *modp;
-    Uint** code_base;
-    Uint* code_ptr;
+    BeamInstr** code_base;
+    BeamInstr* code_ptr;
     Uint i,n;
 
     if ((modp = erts_get_module(mfa[0])) == NULL)
 	return NULL;
-    if ((code_base = (Uint **) modp->code) == NULL)
+    if ((code_base = (BeamInstr **) modp->code) == NULL)
 	return NULL;
-    n = (Uint) code_base[MI_NUM_FUNCTIONS];
+    n = (BeamInstr) code_base[MI_NUM_FUNCTIONS];
     for (i = 0; i < n; ++i) {
 	code_ptr = code_base[MI_FUNCTIONS+i];
-	ASSERT(((Uint) BeamOp(op_i_func_info_IaaI)) == code_ptr[0]);
+	ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == code_ptr[0]);
 	ASSERT(mfa[0] == ((Eterm) code_ptr[2]));
 	if (mfa[1] == ((Eterm) code_ptr[3]) &&
-	    ((Uint) mfa[2]) == code_ptr[4]) {
+	    ((BeamInstr) mfa[2]) == code_ptr[4]) {
 	    return code_ptr + 5;
 	}
     }
     return NULL;
 }
 
+static void bp_hash_init(bp_time_hash_t *hash, Uint n) {
+    Uint size = sizeof(bp_data_time_item_t)*n;
+    Uint i;
+
+    hash->n    = n;
+    hash->used = 0;
+
+    hash->item = (bp_data_time_item_t *)Alloc(size);
+    sys_memzero(hash->item, size);
+
+    for(i = 0; i < n; ++i) {
+	hash->item[i].pid = NIL;
+    }
+}
+
+static void bp_hash_rehash(bp_time_hash_t *hash, Uint n) {
+    bp_data_time_item_t *item = NULL;
+    Uint size = sizeof(bp_data_time_item_t)*n;
+    Uint ix;
+    Uint hval;
+
+    item = (bp_data_time_item_t *)Alloc(size);
+    sys_memzero(item, size);
+
+    for( ix = 0; ix < n; ++ix) {
+	item[ix].pid = NIL;
+    }
+
+    /* rehash, old hash -> new hash */
+
+    for( ix = 0; ix < hash->n; ix++) {
+	if (hash->item[ix].pid != NIL) {
+
+	    hval = ((hash->item[ix].pid) >> 4) % n; /* new n */
+
+	    while (item[hval].pid != NIL) {
+		hval = (hval + 1) % n;
+	    }
+	    item[hval].pid     = hash->item[ix].pid;
+	    item[hval].count   = hash->item[ix].count;
+	    item[hval].s_time  = hash->item[ix].s_time;
+	    item[hval].us_time = hash->item[ix].us_time;
+	}
+    }
+
+    Free(hash->item);
+    hash->n = n;
+    hash->item = item;
+}
+static ERTS_INLINE bp_data_time_item_t * bp_hash_get(bp_time_hash_t *hash, bp_data_time_item_t *sitem) {
+    Eterm pid = sitem->pid;
+    Uint hval = (pid >> 4) % hash->n;
+    bp_data_time_item_t *item = NULL;
+
+    item = hash->item;
+
+    while (item[hval].pid != pid) {
+	if (item[hval].pid == NIL) return NULL;
+	hval = (hval + 1) % hash->n;
+    }
+
+    return &(item[hval]);
+}
+
+static ERTS_INLINE bp_data_time_item_t * bp_hash_put(bp_time_hash_t *hash, bp_data_time_item_t* sitem) {
+    Uint hval;
+    float r = 0.0;
+    bp_data_time_item_t *item;
+
+    /* make sure that the hash is not saturated */
+    /* if saturated, rehash it */
+
+    r = hash->used / (float) hash->n;
+
+    if (r > 0.7f) {
+	bp_hash_rehash(hash, hash->n * 2);
+    }
+    /* Do hval after rehash */
+    hval = (sitem->pid >> 4) % hash->n;
+
+    /* find free slot */
+    item = hash->item;
+
+    while (item[hval].pid != NIL) {
+	hval = (hval + 1) % hash->n;
+    }
+    item = &(hash->item[hval]);
+
+    item->pid     = sitem->pid;
+    item->s_time  = sitem->s_time;
+    item->us_time = sitem->us_time;
+    item->count   = sitem->count;
+    hash->used++;
+
+    return item;
+}
+
+static void bp_hash_delete(bp_time_hash_t *hash) {
+    hash->n = 0;
+    hash->used = 0;
+    Free(hash->item);
+    hash->item = NULL;
+}
+
+static void bp_time_diff(bp_data_time_item_t *item, /* out */
+	process_breakpoint_time_t *pbt,             /* in  */
+	Uint ms, Uint s, Uint us) {
+    int ds,dus;
+#ifdef DEBUG
+    int dms;
+
+
+    dms = ms - pbt->ms;
+#endif
+    ds  = s  - pbt->s;
+    dus = us - pbt->us;
+
+    /* get_sys_now may return zero difftime,
+     * this is ok.
+     */
+
+#ifdef DEBUG
+    ASSERT(dms >= 0 || ds >= 0 || dus >= 0);
+#endif
+
+    if (dus < 0) {
+	dus += 1000000;
+	ds  -= 1;
+    }
+    if (ds < 0) {
+	ds += 1000000;
+    }
+
+    item->s_time  = ds;
+    item->us_time = dus;
+}
+
+void erts_schedule_time_break(Process *p, Uint schedule) {
+    Uint ms, s, us;
+    process_breakpoint_time_t *pbt = NULL;
+    bp_data_time_item_t sitem, *item = NULL;
+    bp_time_hash_t *h = NULL;
+    BpDataTime *pbdt = NULL;
+
+    ASSERT(p);
+
+    pbt = ERTS_PROC_GET_CALL_TIME(p);
+
+    if (pbt) {
+
+	switch(schedule) {
+	case ERTS_BP_CALL_TIME_SCHEDULE_EXITING :
+	    break;
+	case ERTS_BP_CALL_TIME_SCHEDULE_OUT :
+	    /* When a process is scheduled _out_,
+	     * timestamp it and add its delta to
+	     * the previous breakpoint.
+	     */
+
+	    pbdt = (BpDataTime *) get_break(p, pbt->pc, (BeamInstr) BeamOp(op_i_time_breakpoint));
+	    if (pbdt) {
+		get_sys_now(&ms,&s,&us);
+		bp_time_diff(&sitem, pbt, ms, s, us);
+		sitem.pid   = p->id;
+		sitem.count = 0;
+
+		h = &(pbdt->hash[bp_sched2ix_proc(p)]);
+
+		ASSERT(h);
+		ASSERT(h->item);
+
+		item = bp_hash_get(h, &sitem);
+		if (!item) {
+		    item = bp_hash_put(h, &sitem);
+		} else {
+		    BP_TIME_ADD(item, &sitem);
+		}
+	    }
+	    break;
+	case ERTS_BP_CALL_TIME_SCHEDULE_IN :
+	    /* When a process is scheduled _in_,
+	     * timestamp it and remove the previous
+	     * timestamp in the psd.
+	     */
+	    get_sys_now(&ms,&s,&us);
+	    pbt->ms = ms;
+	    pbt->s  = s;
+	    pbt->us = us;
+	    break;
+	default :
+	    ASSERT(0);
+		/* will never happen */
+	    break;
+	}
+    } /* pbt */
+}
+
+/* call_time breakpoint
+ * Accumulated times are added to the previous bp,
+ * not the current one. The current one is saved
+ * for future reference.
+ * The previous breakpoint is stored in the process it self, the psd.
+ * We do not need to store in a stack frame.
+ * There is no need for locking, each thread has its own
+ * area in each bp to save data.
+ * Since we need to diffrentiate between processes for each bp,
+ * every bp has a hash (per thread) to process-bp statistics.
+ * - egil
+ */
+
+void erts_trace_time_break(Process *p, BeamInstr *pc, BpDataTime *bdt, Uint type) {
+    Uint ms,s,us;
+    process_breakpoint_time_t *pbt = NULL;
+    bp_data_time_item_t sitem, *item = NULL;
+    bp_time_hash_t *h = NULL;
+    BpDataTime *pbdt = NULL;
+
+    ASSERT(p);
+    ASSERT(p->status == P_RUNNING);
+
+    /* get previous timestamp and breakpoint
+     * from the process psd  */
+
+    pbt = ERTS_PROC_GET_CALL_TIME(p);
+    get_sys_now(&ms,&s,&us);
+
+    switch(type) {
+	    /* get pbt
+	     * timestamp = t0
+	     * lookup bdt from code
+	     * set ts0 to pbt
+	     * add call count here?
+	     */
+	case ERTS_BP_CALL_TIME_CALL:
+	case ERTS_BP_CALL_TIME_TAIL_CALL:
+
+	    if (pbt) {
+		ASSERT(pbt->pc);
+		/* add time to previous code */
+		bp_time_diff(&sitem, pbt, ms, s, us);
+		sitem.pid   = p->id;
+		sitem.count = 0;
+
+		/* previous breakpoint */
+		pbdt = (BpDataTime *) get_break(p, pbt->pc, (BeamInstr) BeamOp(op_i_time_breakpoint));
+
+		/* if null then the breakpoint was removed */
+		if (pbdt) {
+		    h = &(pbdt->hash[bp_sched2ix_proc(p)]);
+
+		    ASSERT(h);
+		    ASSERT(h->item);
+
+		    item = bp_hash_get(h, &sitem);
+		    if (!item) {
+			item = bp_hash_put(h, &sitem);
+		    } else {
+			BP_TIME_ADD(item, &sitem);
+		    }
+		}
+
+	    } else {
+		/* first call of process to instrumented function */
+		pbt = Alloc(sizeof(process_breakpoint_time_t));
+		(void *) ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCK_MAIN, pbt);
+	    }
+	    /* add count to this code */
+	    sitem.pid     = p->id;
+	    sitem.count   = 1;
+	    sitem.s_time  = 0;
+	    sitem.us_time = 0;
+
+	    /* this breakpoint */
+	    ASSERT(bdt);
+	    h = &(bdt->hash[bp_sched2ix_proc(p)]);
+
+	    ASSERT(h);
+	    ASSERT(h->item);
+
+	    item = bp_hash_get(h, &sitem);
+	    if (!item) {
+		item = bp_hash_put(h, &sitem);
+	    } else {
+		BP_TIME_ADD(item, &sitem);
+	    }
+
+	    pbt->pc = pc;
+	    pbt->ms = ms;
+	    pbt->s  = s;
+	    pbt->us = us;
+	    break;
+
+	case ERTS_BP_CALL_TIME_RETURN:
+	    /* get pbt
+	     * lookup bdt from code
+	     * timestamp = t1
+	     * get ts0 from pbt
+	     * get item from bdt->hash[bp_hash(p->id)]
+	     * ack diff (t1, t0) to item
+	     */
+
+	    if(pbt) {
+		/* might have been removed due to
+		 * trace_pattern(false)
+		 */
+		ASSERT(pbt->pc);
+
+		bp_time_diff(&sitem, pbt, ms, s, us);
+		sitem.pid   = p->id;
+		sitem.count = 0;
+
+		/* previous breakpoint */
+		pbdt = (BpDataTime *) get_break(p, pbt->pc, (BeamInstr) BeamOp(op_i_time_breakpoint));
+
+		/* beware, the trace_pattern might have been removed */
+		if (pbdt) {
+		    h = &(pbdt->hash[bp_sched2ix_proc(p)]);
+
+		    ASSERT(h);
+		    ASSERT(h->item);
+
+		    item = bp_hash_get(h, &sitem);
+		    if (!item) {
+			item = bp_hash_put(h, &sitem);
+		    } else {
+			BP_TIME_ADD(item, &sitem);
+		    }
+		}
+
+		pbt->pc = pc;
+		pbt->ms = ms;
+		pbt->s  = s;
+		pbt->us = us;
+	    }
+	    break;
+	default :
+	    ASSERT(0);
+		/* will never happen */
+	    break;
+    }
+}
 
 
 /* *************************************************************************
@@ -440,7 +844,7 @@ erts_find_local_func(Eterm mfa[3]) {
 
 
 static int set_break(Eterm mfa[3], int specified, 
-		     Binary *match_spec, Eterm break_op, 
+		     Binary *match_spec, BeamInstr break_op,
 		     enum erts_break_op count_op, Eterm tracer_pid)
 {
     Module *modp;
@@ -470,45 +874,54 @@ static int set_break(Eterm mfa[3], int specified,
 }
 
 static int set_module_break(Module *modp, Eterm mfa[3], int specified,
-			    Binary *match_spec, Uint break_op, 
+			    Binary *match_spec, BeamInstr break_op,
 			    enum erts_break_op count_op, Eterm tracer_pid) {
-    Uint** code_base;
-    Uint* code_ptr;
+    BeamInstr** code_base;
+    BeamInstr* code_ptr;
     int num_processed = 0;
     Uint i,n;
 
     ASSERT(break_op);
     ASSERT(modp);
-    code_base = (Uint **) modp->code;
+    code_base = (BeamInstr **) modp->code;
     if (code_base == NULL) {
 	return 0;
     }
-    n = (Uint) code_base[MI_NUM_FUNCTIONS];
+    n = (BeamInstr) code_base[MI_NUM_FUNCTIONS];
     for (i = 0; i < n; ++i) {
 	code_ptr = code_base[MI_FUNCTIONS+i];
-	ASSERT(code_ptr[0] == (Uint) BeamOp(op_i_func_info_IaaI));
+	ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
 	if ((specified < 2 || mfa[1] == ((Eterm) code_ptr[3])) &&
 	    (specified < 3 || ((int) mfa[2]) == ((int) code_ptr[4]))) {
-	    Uint   *pc = code_ptr+5;
+	    BeamInstr *pc = code_ptr+5;
 	    
 	    num_processed +=
-		set_function_break(modp, pc, match_spec, 
+		set_function_break(modp, pc, BREAK_IS_ERL, match_spec,
 				   break_op, count_op, tracer_pid);
 	}
     }
     return num_processed;
 }
 
-static int set_function_break(Module *modp, Uint *pc, 
-			      Binary *match_spec, Uint break_op, 
+static int set_function_break(Module *modp, BeamInstr *pc, int bif,
+			      Binary *match_spec, BeamInstr break_op,
 			      enum erts_break_op count_op, Eterm tracer_pid) {
-    BpData *bd, **r;
+
+    BeamInstr **code_base = NULL;
+    BpData *bd, **r, ***rs;
     size_t size;
-    Uint **code_base = (Uint **)modp->code;
+    Uint ix = 0;
     
-    ASSERT(code_base);
-    ASSERT(code_base <= (Uint **)pc);
-    ASSERT((Uint **)pc < code_base + (modp->code_length/sizeof(Uint *)));
+    if (bif == BREAK_IS_ERL) {
+	code_base = (BeamInstr **)modp->code;
+	ASSERT(code_base);
+	ASSERT(code_base <= (BeamInstr **)pc);
+	ASSERT((BeamInstr **)pc < code_base + (modp->code_length/sizeof(BeamInstr *)));
+    } else {
+	ASSERT(*pc == (BeamInstr) em_apply_bif);
+	ASSERT(modp == NULL);
+    }
+
     /*
      * Currently no trace support for native code.
      */
@@ -517,8 +930,9 @@ static int set_function_break(Module *modp, Uint *pc,
     }
     /* Do not allow two breakpoints of the same kind */
     if ( (bd = is_break(pc, break_op))) {
-	if (break_op == (Uint) BeamOp(op_i_trace_breakpoint) 
-	    || break_op == (Uint) BeamOp(op_i_mtrace_breakpoint)) {
+	if (break_op == (BeamInstr) BeamOp(op_i_trace_breakpoint)
+	    || break_op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) {
+
 	    BpDataTrace *bdt = (BpDataTrace *) bd;
 	    Binary *old_match_spec;
 	    
@@ -531,71 +945,115 @@ static int set_function_break(Module *modp, Uint *pc,
 	    ErtsSmpBPUnlock(bdt);
 	    MatchSetUnref(old_match_spec);
 	} else {
+	    BpDataCount *bdc = (BpDataCount *) bd;
+	    erts_aint_t count = 0;
+	    erts_aint_t res   = 0;
+
 	    ASSERT(! match_spec);
 	    ASSERT(is_nil(tracer_pid));
-	    if (break_op == (Uint) BeamOp(op_i_count_breakpoint)) {
-		BpDataCount *bdc = (BpDataCount *) bd;
 		
-		ErtsSmpBPLock(bdc);
+	    if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) {
 		if (count_op == erts_break_stop) {
-		    if (bdc->count >= 0) {
-			bdc->count = -bdc->count-1; /* Stop call counter */
+		    count = erts_smp_atomic_read_nob(&bdc->acount);
+		    if (count >= 0) {
+			while(1) {
+			    res = erts_smp_atomic_cmpxchg_nob(&bdc->acount, -count - 1, count);
+			    if ((res == count) || count < 0) break;
+			    count = res;
+			}
 		    }
 		} else {
-		    bdc->count = 0; /* Reset call counter */
+		    /* Reset call counter */
+		    erts_smp_atomic_set_nob(&bdc->acount, 0);
+		}
+
+	    } else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint)) {
+		BpDataTime *bdt = (BpDataTime *) bd;
+		Uint i = 0;
+
+		ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+		if (count_op == erts_break_stop) {
+		    bdt->pause = 1;
+		} else {
+		    bdt->pause = 0;
+		    for (i = 0; i < bdt->n; i++) {
+			bp_hash_delete(&(bdt->hash[i]));
+			bp_hash_init(&(bdt->hash[i]), 32);
+		    }
 		}
-		ErtsSmpBPUnlock(bdc);
+
 	    } else {
 		ASSERT (! count_op);
 	    }
 	}
 	return 1;
     }
-    if (break_op == (Uint) BeamOp(op_i_trace_breakpoint) ||
-	break_op == (Uint) BeamOp(op_i_mtrace_breakpoint)) {
+    if (break_op == (BeamInstr) BeamOp(op_i_trace_breakpoint) ||
+	break_op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) {
 	size = sizeof(BpDataTrace);
     } else {
 	ASSERT(! match_spec);
 	ASSERT(is_nil(tracer_pid));
-	if (break_op == (Uint) BeamOp(op_i_count_breakpoint)) {
-	    if (count_op == erts_break_reset
-		|| count_op == erts_break_stop) {
+	if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) {
+	    if (count_op == erts_break_reset || count_op == erts_break_stop) {
 		/* Do not insert a new breakpoint */
 		return 1;
 	    }
 	    size = sizeof(BpDataCount);
+	} else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint))  {
+	    if (count_op == erts_break_reset || count_op == erts_break_stop) {
+		/* Do not insert a new breakpoint */
+		return 1;
+	    }
+	    size = sizeof(BpDataTime);
 	} else {
 	    ASSERT(! count_op);
-	    ASSERT(break_op == (Uint) BeamOp(op_i_debug_breakpoint));
+	    ASSERT(break_op == (BeamInstr) BeamOp(op_i_debug_breakpoint));
 	    size = sizeof(BpDataDebug);
 	}
     }
-    r = (BpData **) (pc-4);
+    rs = (BpData ***) (pc-4);
+    if (! *rs) {
+	size_t ssize = sizeof(BeamInstr) * erts_no_schedulers;
+	*rs = (BpData **) Alloc(ssize);
+	sys_memzero(*rs, ssize);
+    }
+
+    r = &((*rs)[0]);
+
     if (! *r) {
-	ASSERT(*pc != (Uint) BeamOp(op_i_trace_breakpoint));
-	ASSERT(*pc != (Uint) BeamOp(op_i_mtrace_breakpoint));
-	ASSERT(*pc != (Uint) BeamOp(op_i_debug_breakpoint));
-	ASSERT(*pc != (Uint) BeamOp(op_i_count_breakpoint));
+	ASSERT(*pc != (BeamInstr) BeamOp(op_i_trace_breakpoint));
+	ASSERT(*pc != (BeamInstr) BeamOp(op_i_mtrace_breakpoint));
+	ASSERT(*pc != (BeamInstr) BeamOp(op_i_debug_breakpoint));
+	ASSERT(*pc != (BeamInstr) BeamOp(op_i_count_breakpoint));
+	ASSERT(*pc != (BeamInstr) BeamOp(op_i_time_breakpoint));
 	/* First breakpoint; create singleton ring */
 	bd = Alloc(size);
 	BpInit(bd, *pc);
-	*pc = break_op;
 	*r = bd;
+	if (bif == BREAK_IS_ERL) {
+	    *pc = break_op;
+	}
     } else {
-	ASSERT(*pc == (Uint) BeamOp(op_i_trace_breakpoint) ||
-	       *pc == (Uint) BeamOp(op_i_mtrace_breakpoint) ||
-	       *pc == (Uint) BeamOp(op_i_debug_breakpoint) ||
-	       *pc == (Uint) BeamOp(op_i_count_breakpoint));
-	if (*pc == (Uint) BeamOp(op_i_debug_breakpoint)) {
+	ASSERT(*pc == (BeamInstr) BeamOp(op_i_trace_breakpoint) ||
+	       *pc == (BeamInstr) BeamOp(op_i_mtrace_breakpoint) ||
+	       *pc == (BeamInstr) BeamOp(op_i_debug_breakpoint) ||
+	       *pc == (BeamInstr) BeamOp(op_i_time_breakpoint) ||
+	       *pc == (BeamInstr) BeamOp(op_i_count_breakpoint) ||
+	       *pc == (BeamInstr) em_apply_bif);
+	if (*pc == (BeamInstr) BeamOp(op_i_debug_breakpoint)) {
 	    /* Debug bp must be last, so if it is also first; 
 	     * it must be singleton. */
-	    ASSERT(BpSingleton(*r)); 
+	    ASSERT(BpSingleton(*r));
 	    /* Insert new bp first in the ring, i.e second to last. */
 	    bd = Alloc(size);
 	    BpInitAndSpliceNext(bd, *pc, *r);
-	    *pc = break_op;
-	} else if ((*r)->prev->orig_instr 
-		   == (Uint) BeamOp(op_i_debug_breakpoint)) {
+	    if (bif == BREAK_IS_ERL) {
+		*pc = break_op;
+	    }
+	} else if ((*r)->prev->orig_instr
+		   == (BeamInstr) BeamOp(op_i_debug_breakpoint)) {
 	    /* Debug bp last in the ring; insert new second to last. */
 	    bd = Alloc(size);
 	    BpInitAndSplicePrev(bd, (*r)->prev->orig_instr, *r);
@@ -608,25 +1066,43 @@ static int set_function_break(Module *modp, Uint *pc,
 	    *r = bd;
 	}
     }
+    for (ix = 1; ix < erts_no_schedulers; ++ix) {
+	(*rs)[ix] = (*rs)[0];
+    }
+
+    bd->this_instr = break_op;
     /* Init the bp type specific data */
-    if (break_op == (Uint) BeamOp(op_i_trace_breakpoint) ||
-	break_op == (Uint) BeamOp(op_i_mtrace_breakpoint)) {
+    if (break_op == (BeamInstr) BeamOp(op_i_trace_breakpoint) ||
+	break_op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) {
 		
 	BpDataTrace *bdt = (BpDataTrace *) bd;
 		
 	MatchSetRef(match_spec);
 	bdt->match_spec = match_spec;
 	bdt->tracer_pid = tracer_pid;
-    } else if (break_op == (Uint) BeamOp(op_i_count_breakpoint)) {
+    } else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint)) {
+	BpDataTime *bdt = (BpDataTime *) bd;
+	Uint i = 0;
+
+	bdt->pause = 0;
+	bdt->n     = erts_no_schedulers;
+	bdt->hash  = Alloc(sizeof(bp_time_hash_t)*(bdt->n));
+
+	for (i = 0; i < bdt->n; i++) {
+	    bp_hash_init(&(bdt->hash[i]), 32);
+	}
+    } else if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) {
 	BpDataCount *bdc = (BpDataCount *) bd;
+	erts_smp_atomic_init_nob(&bdc->acount, 0);
+    }
 
-	bdc->count = 0;
+    if (bif == BREAK_IS_ERL) {
+	++(*(BeamInstr*)&code_base[MI_NUM_BREAKPOINTS]);
     }
-    ++(*(Uint*)&code_base[MI_NUM_BREAKPOINTS]);
     return 1;
 }
 
-static int clear_break(Eterm mfa[3], int specified, Uint break_op)
+static int clear_break(Eterm mfa[3], int specified, BeamInstr break_op)
 {
     int num_processed = 0;
     Module *modp;
@@ -652,52 +1128,71 @@ static int clear_break(Eterm mfa[3], int specified, Uint break_op)
 }
 
 static int clear_module_break(Module *m, Eterm mfa[3], int specified, 
-			      Uint break_op) {
-    Uint** code_base;
-    Uint* code_ptr;
+			      BeamInstr break_op) {
+    BeamInstr** code_base;
+    BeamInstr* code_ptr;
     int num_processed = 0;
-    Uint i,n;
+    Uint i;
+    BeamInstr n;
     
     ASSERT(m);
-    code_base = (Uint **) m->code;
+    code_base = (BeamInstr **) m->code;
     if (code_base == NULL) {
 	return 0;
     }
-    n = (Uint) code_base[MI_NUM_FUNCTIONS];
+    n = (BeamInstr) code_base[MI_NUM_FUNCTIONS];
     for (i = 0; i < n; ++i) {
 	code_ptr = code_base[MI_FUNCTIONS+i];
 	if ((specified < 2 || mfa[1] == ((Eterm) code_ptr[3])) &&
 	    (specified < 3 || ((int) mfa[2]) == ((int) code_ptr[4]))) {
-	    Uint *pc = code_ptr + 5;
+	    BeamInstr *pc = code_ptr + 5;
 	    
 	    num_processed += 
-		clear_function_break(m, pc, break_op);
+		clear_function_break(m, pc, BREAK_IS_ERL, break_op);
 	}
     }
     return num_processed;
 }
 
-static int clear_function_break(Module *m, Uint *pc, Uint break_op) {
+static int clear_function_break(Module *m, BeamInstr *pc, int bif, BeamInstr break_op) {
     BpData *bd;
-    Uint **code_base = (Uint **)m->code;
-    
-    ASSERT(code_base);
-    ASSERT(code_base <= (Uint **)pc);
-    ASSERT((Uint **)pc < code_base + (m->code_length/sizeof(Uint *)));
+    Uint ix = 0;
+    BeamInstr **code_base = NULL;
+
+    if (bif == BREAK_IS_ERL) {
+	code_base = (BeamInstr **)m->code;
+	ASSERT(code_base);
+	ASSERT(code_base <= (BeamInstr **)pc);
+	ASSERT((BeamInstr **)pc < code_base + (m->code_length/sizeof(BeamInstr *)));
+    } else {
+	ASSERT(*pc == (BeamInstr) em_apply_bif);
+	ASSERT(m == NULL);
+    }
+
     /*
      * Currently no trace support for native code.
      */
     if (erts_is_native_break(pc)) {
 	return 0;
     }
+
     while ( (bd = is_break(pc, break_op))) {
 	/* Remove all breakpoints of this type.
 	 * There should be only one of each type, 
 	 * but break_op may be 0 which matches any type. 
 	 */
-	Uint op;
-	BpData **r = (BpData **) (pc-4);
+	BeamInstr op;
+	BpData ***rs = (BpData ***) (pc - 4);
+	BpData   **r = NULL;
+
+#ifdef DEBUG
+	for (ix = 1; ix < erts_no_schedulers; ++ix) {
+	    ASSERT((*rs)[ix] == (*rs)[0]);
+	}
+#endif
 	
+	r = &((*rs)[0]);
+
 	ASSERT(*r);
 	/* Find opcode for this breakpoint */
 	if (break_op) {
@@ -713,8 +1208,11 @@ static int clear_function_break(Module *m, Uint *pc, Uint break_op) {
 	if (BpSingleton(bd)) {
 	    ASSERT(*r == bd);
 	    /* Only one breakpoint to remove */
-	    *r  = NULL;
-	    *pc = bd->orig_instr;
+	    if (bif == BREAK_IS_ERL) {
+		*pc = bd->orig_instr;
+	    }
+	    Free(*rs);
+	    *rs = NULL;
 	} else {
 	    BpData *bd_prev = bd->prev;
 	    
@@ -726,22 +1224,64 @@ static int clear_function_break(Module *m, Uint *pc, Uint break_op) {
 		bd_prev->orig_instr = bd->orig_instr;
 	    } else if (bd_prev == *r) {
 		/* We removed the first breakpoint in the ring */
-		*pc = bd->orig_instr;
+		if (bif == BREAK_IS_ERL) {
+		    *pc = bd->orig_instr;
+		}
 	    } else {
 		bd_prev->orig_instr = bd->orig_instr;
 	    }
 	}
-	if (op == (Uint) BeamOp(op_i_trace_breakpoint) ||
-	    op == (Uint) BeamOp(op_i_mtrace_breakpoint)) {
+	if (op == (BeamInstr) BeamOp(op_i_trace_breakpoint) ||
+	    op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) {
 	    
 	    BpDataTrace *bdt = (BpDataTrace *) bd;
-	    
 	    MatchSetUnref(bdt->match_spec);
 	}
+	if (op == (BeamInstr) BeamOp(op_i_time_breakpoint)) {
+	    BpDataTime *bdt = (BpDataTime *) bd;
+	    Uint i = 0;
+	    Uint j = 0;
+	    Process *h_p = NULL;
+	    bp_data_time_item_t *item = NULL;
+	    process_breakpoint_time_t *pbt = NULL;
+
+	    /* remove all psd associated with the hash
+	     * and then delete the hash.
+	     * ... sigh ...
+	     */
+
+	    for( i = 0; i < bdt->n; ++i) {
+		if (bdt->hash[i].used) {
+		    for (j = 0; j < bdt->hash[i].n; ++j) {
+			item = &(bdt->hash[i].item[j]);
+			if (item->pid != NIL) {
+			    h_p = process_tab[internal_pid_index(item->pid)];
+			    if (h_p) {
+				pbt = ERTS_PROC_SET_CALL_TIME(h_p, ERTS_PROC_LOCK_MAIN, NULL);
+				if (pbt) {
+				    Free(pbt);
+				}
+			    }
+			}
+		    }
+		}
+		bp_hash_delete(&(bdt->hash[i]));
+	    }
+	    Free(bdt->hash);
+	    bdt->hash = NULL;
+	    bdt->n = 0;
+	}
 	Free(bd);
-	ASSERT(((Uint) code_base[MI_NUM_BREAKPOINTS]) > 0);
-	--(*(Uint*)&code_base[MI_NUM_BREAKPOINTS]);
-    }
+	if (bif == BREAK_IS_ERL) {
+	    ASSERT(((BeamInstr) code_base[MI_NUM_BREAKPOINTS]) > 0);
+	    --(*(BeamInstr*)&code_base[MI_NUM_BREAKPOINTS]);
+	}
+	if (*rs) {
+	    for (ix = 1; ix < erts_no_schedulers; ++ix) {
+		(*rs)[ix] = (*rs)[0];
+	    }
+	}
+    } /* while bd != NULL */
     return 1;
 }
 
@@ -754,32 +1294,67 @@ static int clear_function_break(Module *m, Uint *pc, Uint break_op) {
 ** returned. The program counter must point to the first executable
 ** (breakpoint) instruction of the function.
 */
-static BpData *is_break(Uint *pc, Uint break_op) {
-    ASSERT(pc[-5] == (Uint) BeamOp(op_i_func_info_IaaI));
+
+BpData *erts_get_time_break(Process *p, BeamInstr *pc) {
+    return get_break(p, pc, (BeamInstr) BeamOp(op_i_time_breakpoint));
+}
+
+static BpData *get_break(Process *p, BeamInstr *pc, BeamInstr break_op) {
+    ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
     if (! erts_is_native_break(pc)) {
-	BpData *bd = (BpData *) pc[-4];
-	
-	if (break_op == 0) {
-	    return bd;
-	}
-	if (*pc == break_op) {
-	    ASSERT(bd);
-	    return bd->next;
-	}
-	if (! bd){
+	BpData **rs = (BpData **) pc[-4];
+	BpData  *bd = NULL, *ebd = NULL;
+
+	if (! rs) {
 	    return NULL;
 	}
+
+	bd = ebd = rs[bp_sched2ix_proc(p)];
+	ASSERT(bd);
+	if (bd->this_instr == break_op) {
+	    return bd;
+	}
+
 	bd = bd->next;
-	while (bd != (BpData *) pc[-4]) {
+	while (bd != ebd) {
 	    ASSERT(bd);
-	    if (bd->orig_instr == break_op) {
-		bd = bd->next;
+	    if (bd->this_instr == break_op) {
 		ASSERT(bd);
 		return bd;
-	    } else {
-		bd = bd->next;
 	    }
+	    bd = bd->next;
 	}
     }
     return NULL;
 }
+
+static BpData *is_break(BeamInstr *pc, BeamInstr break_op) {
+    BpData **rs;
+    BpData  *bd = NULL, *ebd = NULL;
+    ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+
+    if (erts_is_native_break(pc)) {
+	return NULL;
+    }
+    rs = (BpData **) pc[-4];
+    if (! rs) {
+	return NULL;
+    }
+
+    bd = ebd = rs[erts_bp_sched2ix()];
+    ASSERT(bd);
+    if ( (break_op == 0) || (bd->this_instr == break_op)) {
+	return bd;
+    }
+
+    bd = bd->next;
+    while (bd != ebd) {
+	ASSERT(bd);
+	if (bd->this_instr == break_op) {
+	    ASSERT(bd);
+	    return bd;
+	}
+	bd = bd->next;
+    }
+    return NULL;
+}
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index 44e6b294d8..167069552f 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2000-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2000-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -27,28 +27,46 @@
 
 
 
-/*
-** Common struct to all bp_data_*
-**
-** Two gotchas: 
-**
-** 1) The type of bp_data structure in the ring is deduced from the 
-**    orig_instr field of the structure _before_ in the ring, except for 
-**    the first structure in the ring that has its instruction in
-**    pc[0] of the code to execute.
-**
-** 2) pc[-4] points to the _last_ structure in the ring before the
-**    breakpoints are being executed.
-**
-** So, as an example, when a breakpointed function starts to execute,
-** the first instruction that is a breakpoint instruction at pc[0] finds
-** its data at ((BpData *) pc[-4])->next and has to cast that pointer 
-** to the correct bp_data type.
+/* A couple of gotchas:
+ *
+ * The breakpoint structure from BeamInstr,
+ * In beam_emu where the instruction counter pointer, I (or pc),
+ * points to the *current* instruction. At that time, if the instruction
+ * is a breakpoint instruction the pc looks like the following,
+ *
+ * I[-5]   | op_i_func_info_IaaI |        scheduler specific entries
+ * I[-4]   |    BpData** bpa     |  --> | BpData * bdas1 | ... | BpData * bdasN |
+ * I[-3]   |  Tagged Module      |            |                    |
+ * I[-2]   |  Tagged Function    |            V                    V
+ * I[-1]   |  Arity              |          BpData -> BpData -> BpData -> BpData
+ * I[0]    | The bp instruction  |            ^       * the bp wheel *      |
+ *                                            |------------------------------
+ *
+ * Common struct to all bp_data_*
+ *
+ * 1) The type of bp_data structure in the ring is deduced from the
+ *    orig_instr field of the structure _before_ in the ring, except for
+ *    the first structure in the ring that has its instruction in
+ *    pc[0] of the code to execute.
+ *    This is valid as long as you don't search for the function while it is
+ *    being executed by something else. Or is in the middle of its rotation for
+ *    any other reason.
+ *    A key, the bp beam instruction, is included for this reason.
+ *
+ * 2) pc[-4][sched_id - 1] points to the _last_ structure in the ring before the
+ *    breakpoints are being executed.
+ *
+ * So, as an example, when a breakpointed function starts to execute,
+ * the first instruction that is a breakpoint instruction at pc[0] finds
+ * its data at ((BpData **) pc[-4][sched_id - 1])->next and has to cast that pointer
+ * to the correct bp_data type.
 */
+
 typedef struct bp_data {
     struct bp_data *next; /* Doubly linked ring pointers */
     struct bp_data *prev; /* -"-                         */
-    Uint orig_instr;      /* The original instruction to execute */
+    BeamInstr orig_instr;      /* The original instruction to execute */
+    BeamInstr this_instr;      /* key */
 } BpData;
 /*
 ** All the following bp_data_.. structs must begin the same way
@@ -57,7 +75,8 @@ typedef struct bp_data {
 typedef struct bp_data_trace {
     struct bp_data *next;
     struct bp_data *prev;
-    Uint            orig_instr;
+    BeamInstr       orig_instr;
+    BeamInstr       this_instr;   /* key */
     Binary         *match_spec;
     Eterm           tracer_pid;
 } BpDataTrace;
@@ -65,18 +84,58 @@ typedef struct bp_data_trace {
 typedef struct bp_data_debug {
     struct bp_data *next;
     struct bp_data *prev;
-    Uint            orig_instr;
+    BeamInstr       orig_instr;
+    BeamInstr       this_instr;   /* key */
 } BpDataDebug;
 
-typedef struct bp_data_count { /* Call count */
+typedef struct bp_data_count {    /* Call count */
     struct bp_data *next;
     struct bp_data *prev;
-    Uint            orig_instr;
-    Sint            count;
+    BeamInstr       orig_instr;
+    BeamInstr       this_instr;   /* key */
+    erts_smp_atomic_t acount;
 } BpDataCount;
 
+typedef struct {
+    Eterm pid;
+    Sint  count;
+    Uint  s_time;
+    Uint  us_time;
+} bp_data_time_item_t;
+
+typedef struct {
+    Uint n;
+    Uint used;
+    bp_data_time_item_t *item;
+} bp_time_hash_t;
+
+typedef struct bp_data_time {     /* Call time */
+    struct bp_data *next;
+    struct bp_data *prev;
+    BeamInstr       orig_instr;
+    BeamInstr       this_instr;   /* key */
+    Uint	    pause;
+    Uint	    n;
+    bp_time_hash_t  *hash;
+} BpDataTime;
+
+typedef struct {
+    Uint ms;
+    Uint s;
+    Uint us;
+    BeamInstr *pc;
+} process_breakpoint_time_t; /* used within psd */
+
 extern erts_smp_spinlock_t erts_bp_lock;
 
+#define ERTS_BP_CALL_TIME_SCHEDULE_IN      (0)
+#define ERTS_BP_CALL_TIME_SCHEDULE_OUT     (1)
+#define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2)
+
+#define ERTS_BP_CALL_TIME_CALL      (0)
+#define ERTS_BP_CALL_TIME_RETURN    (1)
+#define ERTS_BP_CALL_TIME_TAIL_CALL (2)
+
 #ifdef ERTS_SMP
 #define ErtsSmpBPLock(BDC) erts_smp_spin_lock(&erts_bp_lock)
 #define ErtsSmpBPUnlock(BDC) erts_smp_spin_unlock(&erts_bp_lock)
@@ -85,31 +144,44 @@ extern erts_smp_spinlock_t erts_bp_lock;
 #define ErtsSmpBPUnlock(BDC)
 #endif
 
-#define ErtsCountBreak(pc,instr_result)                     \
-do {                                                        \
-    BpDataCount *bdc = (BpDataCount *) (pc)[-4];            \
-                                                            \
-    ASSERT((pc)[-5] == (Uint) BeamOp(op_i_func_info_IaaI)); \
-    ASSERT(bdc);                                            \
-    bdc = (BpDataCount *) bdc->next;                        \
-    ASSERT(bdc);                                            \
-    (pc)[-4] = (Uint) bdc;                                  \
-    ErtsSmpBPLock(bdc);                                     \
-    if (bdc->count >= 0) bdc->count++;                      \
-    ErtsSmpBPUnlock(bdc);                                   \
-    *(instr_result) = bdc->orig_instr;                      \
+#ifdef ERTS_SMP
+#define bp_sched2ix_proc(p) ((p)->scheduler_data->no - 1)
+#else
+#define bp_sched2ix_proc(p) (0)
+#endif
+
+#define ErtsCountBreak(p, pc,instr_result)                       \
+do {                                                             \
+    BpData **bds = (BpData **) (pc)[-4];                         \
+    BpDataCount *bdc = NULL;                                     \
+    Uint ix = bp_sched2ix_proc( (p) );                           \
+    erts_aint_t count = 0;                                       \
+                                                                 \
+    ASSERT((pc)[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); \
+    ASSERT(bds);                                                 \
+    bdc = (BpDataCount *) bds[ix];                               \
+    bdc = (BpDataCount *) bdc->next;                             \
+    ASSERT(bdc);                                                 \
+    bds[ix] = (BpData *) bdc;                                    \
+    count = erts_smp_atomic_read_nob(&bdc->acount);                  \
+    if (count >= 0)  erts_smp_atomic_inc_nob(&bdc->acount);      \
+    *(instr_result) = bdc->orig_instr;                           \
 } while (0)
 
-#define ErtsBreakSkip(pc,instr_result)                      \
-do {                                                        \
-    BpData *bd = (BpData *) (pc)[-4];                       \
-                                                            \
-    ASSERT((pc)[-5] == (Uint) BeamOp(op_i_func_info_IaaI)); \
-    ASSERT(bd);                                             \
-    bd = bd->next;                                          \
-    ASSERT(bd);                                             \
-    (pc)[-4] = (Uint) bd;                                   \
-    *(instr_result) = bd->orig_instr;                       \
+#define ErtsBreakSkip(p, pc,instr_result)                        \
+do {                                                             \
+    BpData **bds = (BpData **) (pc)[-4];                         \
+    BpData *bd = NULL;                                           \
+    Uint ix = bp_sched2ix_proc( (p) );                           \
+                                                                 \
+    ASSERT((pc)[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); \
+    ASSERT(bds);                                                 \
+    bd = bds[ix];                                                \
+    ASSERT(bd);                                                  \
+    bd = bd->next;                                               \
+    ASSERT(bd);                                                  \
+    bds[ix] = bd;                                                \
+    *(instr_result) = bd->orig_instr;                            \
 } while (0)
 
 enum erts_break_op{
@@ -133,9 +205,9 @@ int erts_clear_trace_break(Eterm mfa[3], int specified);
 int erts_set_mtrace_break(Eterm mfa[3], int specified, Binary *match_spec,
 			  Eterm tracer_pid);
 int erts_clear_mtrace_break(Eterm mfa[3], int specified);
-void erts_set_mtrace_bif(Uint *pc, Binary *match_spec, 
+void erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec,
 			 Eterm tracer_pid);
-void erts_clear_mtrace_bif(Uint *pc);
+void erts_clear_mtrace_bif(BeamInstr *pc);
 int erts_set_debug_break(Eterm mfa[3], int specified);
 int erts_clear_debug_break(Eterm mfa[3], int specified);
 int erts_set_count_break(Eterm mfa[3], int specified, enum erts_break_op);
@@ -144,22 +216,48 @@ int erts_clear_count_break(Eterm mfa[3], int specified);
 
 int erts_clear_break(Eterm mfa[3], int specified);
 int erts_clear_module_break(Module *modp);
-int erts_clear_function_break(Module *modp, Uint *pc);
+int erts_clear_function_break(Module *modp, BeamInstr *pc);
 
-Uint erts_trace_break(Process *p, Uint *pc, Eterm *args, 
+BeamInstr erts_trace_break(Process *p, BeamInstr *pc, Eterm *args,
 		      Uint32 *ret_flags, Eterm *tracer_pid);
-Uint32 erts_bif_mtrace(Process *p, Uint *pc, Eterm *args, 
+Uint32 erts_bif_mtrace(Process *p, BeamInstr *pc, Eterm *args,
 		       int local, Eterm *tracer_pid);
 
-int erts_is_trace_break(Uint *pc, Binary **match_spec_ret, 
+int erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret,
 			Eterm *tracer_pid_ret);
-int erts_is_mtrace_break(Uint *pc, Binary **match_spec_ret, 
+int erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret,
 			 Eterm *tracer_pid_rte);
-int erts_is_mtrace_bif(Uint *pc, Binary **match_spec_ret, 
+int erts_is_mtrace_bif(BeamInstr *pc, Binary **match_spec_ret,
 		       Eterm *tracer_pid_ret);
-int erts_is_native_break(Uint *pc);
-int erts_is_count_break(Uint *pc, Sint *count_ret);
+int erts_is_native_break(BeamInstr *pc);
+int erts_is_count_break(BeamInstr *pc, Sint *count_ret);
+int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *call_time);
+
+void erts_trace_time_break(Process *p, BeamInstr *pc, BpDataTime *bdt, Uint type);
+void erts_schedule_time_break(Process *p, Uint out);
+int erts_set_time_break(Eterm mfa[3], int specified, enum erts_break_op);
+int erts_clear_time_break(Eterm mfa[3], int specified);
+
+int erts_is_time_trace_bif(Process *p, BeamInstr *pc, Eterm *call_time);
+void erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op);
+void erts_clear_time_trace_bif(BeamInstr *pc);
+BpData *erts_get_time_break(Process *p, BeamInstr *pc);
+
+BeamInstr *erts_find_local_func(Eterm mfa[3]);
 
-Uint *erts_find_local_func(Eterm mfa[3]);
+ERTS_GLB_INLINE Uint erts_bp_sched2ix(void);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE Uint erts_bp_sched2ix(void)
+{
+#ifdef ERTS_SMP
+    ErtsSchedulerData *esdp;
+    esdp = erts_get_scheduler_data();
+    return esdp->no - 1;
+#else
+    return 0;
+#endif
+}
+#endif
 
 #endif /* _BEAM_BP_H */
diff --git a/erts/emulator/beam/beam_catches.c b/erts/emulator/beam/beam_catches.c
index d5cef1cad2..406ef1db5f 100644
--- a/erts/emulator/beam/beam_catches.c
+++ b/erts/emulator/beam/beam_catches.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2000-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2000-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -22,24 +22,30 @@
 #endif
 #include "sys.h"
 #include "beam_catches.h"
+#include "global.h"
 
-/* XXX: should use dynamic reallocation */
-#define TABSIZ (16*1024)
-static struct {
-    Eterm *cp;
+/* R14B04 has about 380 catches when starting erlang */
+#define DEFAULT_TABSIZE (1024)
+typedef struct {
+    BeamInstr *cp;
     unsigned cdr;
-} beam_catches[TABSIZ];
+} beam_catch_t;
 
 static int free_list;
 static unsigned high_mark;
+static unsigned tabsize;
+static beam_catch_t *beam_catches;
 
 void beam_catches_init(void)
 {
+    tabsize   = DEFAULT_TABSIZE;
     free_list = -1;
     high_mark = 0;
+
+    beam_catches = erts_alloc(ERTS_ALC_T_CODE, sizeof(beam_catch_t)*DEFAULT_TABSIZE);
 }
 
-unsigned beam_catches_cons(Eterm *cp, unsigned cdr)
+unsigned beam_catches_cons(BeamInstr *cp, unsigned cdr)
 {
     int i;
 
@@ -50,48 +56,48 @@ unsigned beam_catches_cons(Eterm *cp, unsigned cdr)
      * This avoids the need to initialise the free list in
      * beam_catches_init(), which would cost O(TABSIZ) time.
      */
-    if( (i = free_list) >= 0 ) {
+    if( free_list >= 0 ) {
+	i = free_list;
 	free_list = beam_catches[i].cdr;
-    } else if( (i = high_mark) < TABSIZ ) {
-	high_mark = i + 1;
+    } else if( high_mark < tabsize ) {
+	i = high_mark;
+	high_mark++;
     } else {
-	fprintf(stderr, "beam_catches_cons: no free slots :-(\r\n");
-	exit(1);
+	/* No free slots and table is full: realloc table */
+	tabsize      = 2*tabsize;
+	beam_catches = erts_realloc(ERTS_ALC_T_CODE, beam_catches, sizeof(beam_catch_t)*tabsize);
+	i = high_mark;
+	high_mark++;
     }
 
-    beam_catches[i].cp = cp;
+    beam_catches[i].cp  = cp;
     beam_catches[i].cdr = cdr;
 
     return i;
 }
 
-Eterm *beam_catches_car(unsigned i)
+BeamInstr *beam_catches_car(unsigned i)
 {
-    if( i >= TABSIZ ) {
-	fprintf(stderr,
-		"beam_catches_car: index %#x is out of range\r\n", i);
-	abort();
+    if( i >= tabsize ) {
+	erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i);
     }
     return beam_catches[i].cp;
 }
 
-void beam_catches_delmod(unsigned head, Eterm *code, unsigned code_bytes)
+void beam_catches_delmod(unsigned head, BeamInstr *code, unsigned code_bytes)
 {
     unsigned i, cdr;
 
     for(i = head; i != (unsigned)-1;) {
-	if( i >= TABSIZ ) {
-	    fprintf(stderr,
-		    "beam_catches_delmod: index %#x is out of range\r\n", i);
-	    abort();
+	if( i >= tabsize ) {
+	    erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i);
 	}
 	if( (char*)beam_catches[i].cp - (char*)code >= code_bytes ) {
-	    fprintf(stderr,
+	    erl_exit(1,
 		    "beam_catches_delmod: item %#x has cp %#lx which is not "
 		    "in module's range [%#lx,%#lx[\r\n",
 		    i, (long)beam_catches[i].cp,
 		    (long)code, (long)((char*)code + code_bytes));
-	    abort();
 	}
 	beam_catches[i].cp = 0;
 	cdr = beam_catches[i].cdr;
diff --git a/erts/emulator/beam/beam_catches.h b/erts/emulator/beam/beam_catches.h
index ccf33d5e86..6223427f0d 100644
--- a/erts/emulator/beam/beam_catches.h
+++ b/erts/emulator/beam/beam_catches.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2000-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2000-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -23,9 +23,9 @@
 #define BEAM_CATCHES_NIL	(-1)
 
 void beam_catches_init(void);
-unsigned beam_catches_cons(Eterm* cp, unsigned cdr);
-Eterm *beam_catches_car(unsigned i);
-void beam_catches_delmod(unsigned head, Eterm* code, unsigned code_bytes);
+unsigned beam_catches_cons(BeamInstr* cp, unsigned cdr);
+BeamInstr *beam_catches_car(unsigned i);
+void beam_catches_delmod(unsigned head, BeamInstr* code, unsigned code_bytes);
 
 #define catch_pc(x)	beam_catches_car(catch_val((x)))
 
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 4242a4161e..8041c92162 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -37,27 +37,31 @@
 #include "beam_load.h"
 #include "beam_bp.h"
 #include "erl_binary.h"
+#include "erl_thr_progress.h"
 
 #ifdef ARCH_64
 # define HEXF "%016bpX"
 #else
 # define HEXF "%08bpX"
 #endif
+#define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm))))
 
 void dbg_bt(Process* p, Eterm* sp);
-void dbg_where(Eterm* addr, Eterm x0, Eterm* reg);
+void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg);
 
-static void print_big(int to, void *to_arg, Eterm* addr);
-static int print_op(int to, void *to_arg, int op, int size, Eterm* addr);
-Eterm
-erts_debug_same_2(Process* p, Eterm term1, Eterm term2)
+static int print_op(int to, void *to_arg, int op, int size, BeamInstr* addr);
+
+BIF_RETTYPE
+erts_debug_same_2(BIF_ALIST_2)
 {
-    return (term1 == term2) ? am_true : am_false;
+    return (BIF_ARG_1 == BIF_ARG_2) ? am_true : am_false;
 }
 
-Eterm
-erts_debug_flat_size_1(Process* p, Eterm term)
+BIF_RETTYPE
+erts_debug_flat_size_1(BIF_ALIST_1)
 {
+    Process* p = BIF_P;
+    Eterm term = BIF_ARG_1;
     Uint size = size_object(term);
 
     if (IS_USMALL(0, size)) {
@@ -68,9 +72,13 @@ erts_debug_flat_size_1(Process* p, Eterm term)
     }
 }
 
-Eterm
-erts_debug_breakpoint_2(Process* p, Eterm MFA, Eterm bool)
+
+BIF_RETTYPE
+erts_debug_breakpoint_2(BIF_ALIST_2)
 {
+    Process* p = BIF_P;
+    Eterm MFA = BIF_ARG_1;
+    Eterm bool = BIF_ARG_2;
     Eterm* tp;
     Eterm mfa[3];
     int i;
@@ -107,7 +115,7 @@ erts_debug_breakpoint_2(Process* p, Eterm MFA, Eterm bool)
     }
 
     erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
-    erts_smp_block_system(0);
+    erts_smp_thr_progress_block();
 
     if (bool == am_true) {
 	res = make_small(erts_set_debug_break(mfa, specified));
@@ -115,7 +123,7 @@ erts_debug_breakpoint_2(Process* p, Eterm MFA, Eterm bool)
 	res = make_small(erts_clear_debug_break(mfa, specified));
     }
 
-    erts_smp_release_system();
+    erts_smp_thr_progress_unblock();
     erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
 
     return res;
@@ -124,24 +132,77 @@ erts_debug_breakpoint_2(Process* p, Eterm MFA, Eterm bool)
     BIF_ERROR(p, BADARG);
 }
 
-Eterm
-erts_debug_disassemble_1(Process* p, Eterm addr)
+#if 0 /* Kept for conveninence when hard debugging. */
+void debug_dump_code(BeamInstr *I, int num)
 {
+    BeamInstr *code_ptr = I;
+    BeamInstr *end = code_ptr + num;
+    erts_dsprintf_buf_t *dsbufp;
+    BeamInstr instr;
+    int i;
+
+    dsbufp = erts_create_tmp_dsbuf(0);
+    while (code_ptr < end) {
+	erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr);
+	instr = (BeamInstr) code_ptr[0];
+	for (i = 0; i < NUM_SPECIFIC_OPS; i++) {
+	    if (instr == (BeamInstr) BeamOp(i) && opc[i].name[0] != '\0') {
+		code_ptr += print_op(ERTS_PRINT_DSBUF, (void *) dsbufp,
+				     i, opc[i].sz-1, code_ptr+1) + 1;
+		break;
+	    }
+	}
+	if (i >= NUM_SPECIFIC_OPS) {
+	    erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp,
+		       "unknown " HEXF "\n", instr);
+	    code_ptr++;
+	}
+    }
+    dsbufp->str[dsbufp->str_len] = 0;
+    erts_fprintf(stderr,"%s", dsbufp->str);
+    erts_destroy_tmp_dsbuf(dsbufp);
+}
+#endif
+
+BIF_RETTYPE
+erts_debug_instructions_0(BIF_ALIST_0)
+{
+    int i = 0;
+    Uint needed = num_instructions * 2;
+    Eterm* hp;
+    Eterm res = NIL;
+
+    for (i = 0; i < num_instructions; i++) {
+	needed += 2*strlen(opc[i].name);
+    }
+    hp = HAlloc(BIF_P, needed);
+    for (i = num_instructions-1; i >= 0; i--) {
+	Eterm s = erts_bld_string_n(&hp, 0, opc[i].name, strlen(opc[i].name));
+	res = erts_bld_cons(&hp, 0, s, res);
+    }
+    return res;
+}
+
+BIF_RETTYPE
+erts_debug_disassemble_1(BIF_ALIST_1)
+{
+    Process* p = BIF_P;
+    Eterm addr = BIF_ARG_1;
     erts_dsprintf_buf_t *dsbufp;
     Eterm* hp;
     Eterm* tp;
     Eterm bin;
     Eterm mfa;
-    Eterm* funcinfo = NULL;	/* Initialized to eliminate warning. */
-    Uint* code_base;
-    Uint* code_ptr = NULL;	/* Initialized to eliminate warning. */
-    Uint instr;
-    Uint uaddr;
+    BeamInstr* funcinfo = NULL;	/* Initialized to eliminate warning. */
+    BeamInstr* code_base;
+    BeamInstr* code_ptr = NULL;	/* Initialized to eliminate warning. */
+    BeamInstr instr;
+    BeamInstr uaddr;
     Uint hsz;
     int i;
 
-    if (term_to_Uint(addr, &uaddr)) {
-	code_ptr = (Uint *) uaddr;
+    if (term_to_UWord(addr, &uaddr)) {
+	code_ptr = (BeamInstr *) uaddr;
 	if ((funcinfo = find_function_from_pc(code_ptr)) == NULL) {
 	    BIF_RET(am_false);
 	}
@@ -180,14 +241,14 @@ erts_debug_disassemble_1(Process* p, Eterm addr)
 	     * But this code_ptr will point to the start of the Export,
 	     * not the function's func_info instruction. BOOM !?
 	     */
-	    code_ptr = ((Eterm *) ep->address) - 5;
+	    code_ptr = ((BeamInstr *) ep->address) - 5;
 	    funcinfo = code_ptr+2;
 	} else if (modp == NULL || (code_base = modp->code) == NULL) {
 	    BIF_RET(am_undef);
 	} else {
 	    n = code_base[MI_NUM_FUNCTIONS];
 	    for (i = 0; i < n; i++) {
-		code_ptr = (Uint *) code_base[MI_FUNCTIONS+i];
+		code_ptr = (BeamInstr *) code_base[MI_FUNCTIONS+i];
 		if (code_ptr[3] == name && code_ptr[4] == arity) {
 		    funcinfo = code_ptr+2;
 		    break;
@@ -203,9 +264,9 @@ erts_debug_disassemble_1(Process* p, Eterm addr)
 
     dsbufp = erts_create_tmp_dsbuf(0);
     erts_print(ERTS_PRINT_DSBUF, (void *) dsbufp, HEXF ": ", code_ptr);
-    instr = (Uint) code_ptr[0];
+    instr = (BeamInstr) code_ptr[0];
     for (i = 0; i < NUM_SPECIFIC_OPS; i++) {
-	if (instr == (Uint) BeamOp(i) && opc[i].name[0] != '\0') {
+	if (instr == (BeamInstr) BeamOp(i) && opc[i].name[0] != '\0') {
 	    code_ptr += print_op(ERTS_PRINT_DSBUF, (void *) dsbufp,
 				 i, opc[i].sz-1, code_ptr+1) + 1;
 	    break;
@@ -216,15 +277,15 @@ erts_debug_disassemble_1(Process* p, Eterm addr)
 		   "unknown " HEXF "\n", instr);
 	code_ptr++;
     }
-    bin = new_binary(p, (byte *) dsbufp->str, (int) dsbufp->str_len);
+    bin = new_binary(p, (byte *) dsbufp->str, dsbufp->str_len);
     erts_destroy_tmp_dsbuf(dsbufp);
     hsz = 4+4;
-    (void) erts_bld_uint(NULL, &hsz, (Uint) code_ptr);
+    (void) erts_bld_uword(NULL, &hsz, (BeamInstr) code_ptr);
     hp = HAlloc(p, hsz);
-    addr = erts_bld_uint(&hp, NULL, (Uint) code_ptr);
+    addr = erts_bld_uword(&hp, NULL, (BeamInstr) code_ptr);
     ASSERT(is_atom(funcinfo[0]));
     ASSERT(is_atom(funcinfo[1]));
-    mfa = TUPLE3(hp, funcinfo[0], funcinfo[1], make_small(funcinfo[2]));
+    mfa = TUPLE3(hp, (Eterm) funcinfo[0], (Eterm) funcinfo[1], make_small((Eterm) funcinfo[2]));
     hp += 4;
     return TUPLE3(hp, addr, bin, mfa);
 }
@@ -236,20 +297,20 @@ dbg_bt(Process* p, Eterm* sp)
 
     while (sp < stack) {
 	if (is_CP(*sp)) {
-	    Eterm* addr = find_function_from_pc(cp_val(*sp));
+	    BeamInstr* addr = find_function_from_pc(cp_val(*sp));
 	    if (addr)
 		erts_fprintf(stderr,
 			     HEXF ": %T:%T/%bpu\n",
-			     addr, addr[0], addr[1], addr[2]);
+			     addr, (Eterm) addr[0], (Eterm) addr[1], addr[2]);
 	}
 	sp++;
     }
 }
 
 void
-dbg_where(Eterm* addr, Eterm x0, Eterm* reg)
+dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg)
 {
-    Eterm* f = find_function_from_pc(addr);
+    BeamInstr* f = find_function_from_pc(addr);
 
     if (f == NULL) {
 	erts_fprintf(stderr, "???\n");
@@ -259,7 +320,7 @@ dbg_where(Eterm* addr, Eterm x0, Eterm* reg)
 
 	addr = f;
 	arity = addr[2];
-	erts_fprintf(stderr, HEXF ": %T:%T(", addr, addr[0], addr[1]);
+	erts_fprintf(stderr, HEXF ": %T:%T(", addr, (Eterm) addr[0], (Eterm) addr[1]);
 	for (i = 0; i < arity; i++)
 	    erts_fprintf(stderr, i ? ", %T" : "%T", i ? reg[i] : x0);
 	erts_fprintf(stderr, ")\n");
@@ -267,18 +328,19 @@ dbg_where(Eterm* addr, Eterm x0, Eterm* reg)
 }
 
 static int
-print_op(int to, void *to_arg, int op, int size, Eterm* addr)
+print_op(int to, void *to_arg, int op, int size, BeamInstr* addr)
 {
     int i;
-    Uint tag;
+    BeamInstr tag;
     char* sign;
     char* start_prog;		/* Start of program for packer. */
     char* prog;			/* Current position in packer program. */
-    Uint stack[8];		/* Stack for packer. */
-    Uint* sp = stack;		/* Points to next free position. */
-    Uint packed = 0;		/* Accumulator for packed operations. */
-    Uint args[8];		/* Arguments for this instruction. */
-    Uint* ap;			/* Pointer to arguments. */
+    BeamInstr stack[8];		/* Stack for packer. */
+    BeamInstr* sp = stack;		/* Points to next free position. */
+    BeamInstr packed = 0;		/* Accumulator for packed operations. */
+    BeamInstr args[8];		/* Arguments for this instruction. */
+    BeamInstr* ap;			/* Pointer to arguments. */
+    BeamInstr* unpacked;		/* Unpacked arguments */
 
     start_prog = opc[op].pack;
 
@@ -288,7 +350,7 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr)
 	 * Avoid copying because instructions containing bignum operands
 	 * are bigger than actually declared.
 	 */
-	ap = (Uint *) addr;
+	ap = (BeamInstr *) addr;
     } else {
 	/*
 	 * Copy all arguments to a local buffer for the unpacking.
@@ -324,9 +386,15 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr)
 		packed >>= BEAM_TIGHT_SHIFT;
 		break;
 	    case '6':		/* Shift 16 steps */
-		*ap++ = packed & 0xffff;
-		packed >>= 16;
+		*ap++ = packed & BEAM_LOOSE_MASK;
+		packed >>= BEAM_LOOSE_SHIFT;
 		break;
+#ifdef ARCH_64
+	    case 'w':		/* Shift 32 steps */
+		*ap++ = packed & BEAM_WIDE_MASK;
+		packed >>= BEAM_WIDE_SHIFT;
+		break;
+#endif
 	    case 'p':
 		*sp++ = *--ap;
 		break;
@@ -353,7 +421,7 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr)
 	    break;
 	case 'x':		/* x(N) */
 	    if (reg_index(ap[0]) == 0) {
-		erts_print(to, to_arg, "X[0]");
+		erts_print(to, to_arg, "x[0]");
 	    } else {
 		erts_print(to, to_arg, "x(%d)", reg_index(ap[0]));
 	    }
@@ -390,11 +458,11 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr)
 	case 'i':		/* Tagged integer */
 	case 'c':		/* Tagged constant */
 	case 'q':		/* Tagged literal */
-	    erts_print(to, to_arg, "%T", *ap);
+	    erts_print(to, to_arg, "%T", (Eterm) *ap);
 	    ap++;
 	    break;
 	case 'A':
-	    erts_print(to, to_arg, "%d", arityval(ap[0]));
+	    erts_print(to, to_arg, "%d", arityval( (Eterm) ap[0]));
 	    ap++;
 	    break;
 	case 'd':		/* Destination (x(0), x(N), y(N)) */
@@ -421,30 +489,36 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr)
 	    ap++;
 	    break;
 	case 'f':		/* Destination label */
-	    erts_print(to, to_arg, "f(%X)", *ap);
-	    ap++;
+	    {
+		BeamInstr* f = find_function_from_pc((BeamInstr *)*ap);
+		if (f+3 != (BeamInstr *) *ap) {
+		    erts_print(to, to_arg, "f(" HEXF ")", *ap);
+		} else {
+		    erts_print(to, to_arg, "%T:%T/%bpu", (Eterm) f[0], (Eterm) f[1], f[2]);
+		}
+		ap++;
+	    }
 	    break;
 	case 'p':		/* Pointer (to label) */
 	    {
-		Eterm* f = find_function_from_pc((Eterm *)*ap);
-
-		if (f+3 != (Eterm *) *ap) {
-		    erts_print(to, to_arg, "p(%X)", *ap);
+		BeamInstr* f = find_function_from_pc((BeamInstr *)*ap);
+		if (f+3 != (BeamInstr *) *ap) {
+		    erts_print(to, to_arg, "p(" HEXF ")", *ap);
 		} else {
-		    erts_print(to, to_arg, "%T:%T/%bpu", f[0], f[1], f[2]);
+		    erts_print(to, to_arg, "%T:%T/%bpu", (Eterm) f[0], (Eterm) f[1], f[2]);
 		}
 		ap++;
 	    }
 	    break;
 	case 'j':		/* Pointer (to label) */
-	    erts_print(to, to_arg, "j(%X)", *ap);
+	    erts_print(to, to_arg, "j(" HEXF ")", *ap);
 	    ap++;
 	    break;
 	case 'e':		/* Export entry */
 	    {
 		Export* ex = (Export *) *ap;
 		erts_print(to, to_arg,
-			   "%T:%T/%bpu", ex->code[0], ex->code[1], ex->code[2]);
+			   "%T:%T/%bpu", (Eterm) ex->code[0], (Eterm) ex->code[1], ex->code[2]);
 		ap++;
 	    }
 	    break;
@@ -467,7 +541,8 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr)
 	    ap++;
 	    break;
 	case 'P':	/* Byte offset into tuple (see beam_load.c) */
-	    erts_print(to, to_arg, "%d", (*ap / sizeof(Eterm*)) - 1);
+	case 'Q':	/* Like 'P', but packable */
+	    erts_print(to, to_arg, "%d", (*ap / sizeof(Eterm)) - 1);
 	    ap++;
 	    break;
 	case 'l':		/* fr(N) */
@@ -487,62 +562,90 @@ print_op(int to, void *to_arg, int op, int size, Eterm* addr)
      * Print more information about certain instructions.
      */
 
+    unpacked = ap;
     ap = addr + size;
     switch (op) {
-    case op_i_select_val_sfI:
+    case op_i_select_val_rfI:
+    case op_i_select_val_xfI:
+    case op_i_select_val_yfI:
+	{
+	    int n = ap[-1];
+
+	    while (n > 0) {
+		erts_print(to, to_arg, "%T f(" HEXF ") ", (Eterm) ap[0], ap[1]);
+		ap += 2;
+		size += 2;
+		n--;
+	    }
+	}
+	break;
+    case op_i_select_tuple_arity_rfI:
+    case op_i_select_tuple_arity_xfI:
+    case op_i_select_tuple_arity_yfI:
 	{
 	    int n = ap[-1];
 
 	    while (n > 0) {
-		erts_print(to, to_arg, "%T f(%X) ", ap[0], ap[1]);
+		Uint arity = arityval(ap[0]);
+		erts_print(to, to_arg, " {%d} f(" HEXF ")", arity, ap[1]);
 		ap += 2;
 		size += 2;
 		n--;
 	    }
 	}
 	break;
-    case op_i_jump_on_val_sfII:
+    case op_i_jump_on_val_rfII:
+    case op_i_jump_on_val_xfII:
+    case op_i_jump_on_val_yfII:
 	{
 	    int n;
 	    for (n = ap[-2]; n > 0; n--) {
-		erts_print(to, to_arg, "f(%X) ", ap[0]);
+		erts_print(to, to_arg, "f(" HEXF ") ", ap[0]);
 		ap++;
 		size++;
 	    }
 	}
 	break;
-    case op_i_select_big_sf:
-	while (ap[0]) {
-	    int arity = thing_arityval(ap[0]);
-	    print_big(to, to_arg, ap);
-	    size += arity+1;
-	    ap += arity+1;
-	    erts_print(to, to_arg, " f(%X) ", ap[0]);
-	    ap++;
-	    size++;
+    case op_i_jump_on_val_zero_rfI:
+    case op_i_jump_on_val_zero_xfI:
+    case op_i_jump_on_val_zero_yfI:
+	{
+	    int n;
+	    for (n = ap[-1]; n > 0; n--) {
+		erts_print(to, to_arg, "f(" HEXF ") ", ap[0]);
+		ap++;
+		size++;
+	    }
+	}
+	break;
+    case op_i_put_tuple_rI:
+    case op_i_put_tuple_xI:
+    case op_i_put_tuple_yI:
+	{
+	    int n = unpacked[-1];
+
+	    while (n > 0) {
+		if (!is_header(ap[0])) {
+		    erts_print(to, to_arg, " %T", (Eterm) ap[0]);
+		} else {
+		    switch ((ap[0] >> 2) & 0x03) {
+		    case R_REG_DEF:
+			erts_print(to, to_arg, " x(0)");
+			break;
+		    case X_REG_DEF:
+			erts_print(to, to_arg, " x(%d)", ap[0] >> 4);
+			break;
+		    case Y_REG_DEF:
+			erts_print(to, to_arg, " y(%d)", ap[0] >> 4);
+			break;
+		    }
+		}
+		ap++, size++, n--;
+	    }
 	}
-	ap++;
-	size++;
 	break;
     }
     erts_print(to, to_arg, "\n");
 
     return size;
 }
-
-static void
-print_big(int to, void *to_arg, Eterm* addr)
-{
-    int i;
-    int k;
-
-    i = BIG_SIZE(addr);
-    if (BIG_SIGN(addr))
-	erts_print(to, to_arg, "-#integer(%d) = {", i);
-    else
-	erts_print(to, to_arg, "#integer(%d) = {", i);
-    erts_print(to, to_arg, "%d", BIG_DIGIT(addr, 0));
-    for (k = 1; k < i; k++)
-	erts_print(to, to_arg, ",%d", BIG_DIGIT(addr, k));
-    erts_print(to, to_arg, "}");
-}
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 2f7f48193d..6d3b15cd46 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -26,7 +26,6 @@
 #include "erl_vm.h"
 #include "global.h"
 #include "erl_process.h"
-#include "erl_nmgc.h"
 #include "error.h"
 #include "bif.h"
 #include "big.h"
@@ -36,15 +35,17 @@
 #include "dist.h"
 #include "beam_bp.h"
 #include "beam_catches.h"
+#include "erl_thr_progress.h"
 #ifdef HIPE
 #include "hipe_mode_switch.h"
 #include "hipe_bif1.h"
 #endif
+#include "dtrace-wrapper.h"
 
 /* #define HARDDEBUG 1 */
 
 #if defined(NO_JUMP_TABLE)
-#  define OpCase(OpCode)    case op_##OpCode: lb_##OpCode
+#  define OpCase(OpCode)    case op_##OpCode
 #  define CountCase(OpCode) case op_count_##OpCode
 #  define OpCode(OpCode)    ((Uint*)op_##OpCode)
 #  define Goto(Rel) {Go = (int)(Rel); goto emulator_loop;}
@@ -52,7 +53,7 @@
 #else
 #  define OpCase(OpCode)    lb_##OpCode
 #  define CountCase(OpCode) lb_count_##OpCode
-#  define Goto(Rel) goto *(Rel)
+#  define Goto(Rel) goto *((void *)Rel)
 #  define LabelAddr(Label) &&Label
 #  define OpCode(OpCode)  (&&lb_##OpCode)
 #endif
@@ -70,7 +71,7 @@ do {									\
     }									\
     else								\
 	erts_lc_check_exact(NULL, 0);					\
-    ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING);				\
+    	ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());		\
 } while (0)
 #    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
         if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN)
@@ -117,6 +118,7 @@ do {                                     \
 #endif
 
 #define GET_BIF_ADDRESS(p) ((BifFunction) (((Export *) p)->code[4]))
+#define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm))))
 
 
 /*
@@ -138,8 +140,8 @@ do {                                     \
 #define VALID_INSTR(IP) (0 <= (int)(IP) && ((int)(IP) < (NUMBER_OF_OPCODES*2+10)))
 #else
 #define VALID_INSTR(IP) \
-   ((Sint)LabelAddr(emulator_loop) <= (Sint)(IP) && \
-    (Sint)(IP) < (Sint)LabelAddr(end_emulator_loop))
+   ((SWord)LabelAddr(emulator_loop) <= (SWord)(IP) && \
+    (SWord)(IP) < (SWord)LabelAddr(end_emulator_loop))
 #endif /* NO_JUMP_TABLE */
 
 #define SET_CP(p, ip)           \
@@ -181,11 +183,11 @@ do {                                     \
  
 #define StoreBifResult(Dst, Result)                          \
   do {                                                       \
-    Eterm* stb_next;                                         \
+    BeamInstr* stb_next;                                         \
     Eterm stb_reg;                                           \
     stb_reg = Arg(Dst);                                      \
     I += (Dst) + 2;                                          \
-    stb_next = (Eterm *) *I;                                 \
+    stb_next = (BeamInstr *) *I;                                 \
     CHECK_TERM(Result);                                      \
     switch (beam_reg_tag(stb_reg)) {                         \
     case R_REG_DEF:                                          \
@@ -197,7 +199,7 @@ do {                                     \
     }                                                        \
   } while (0)
 
-#define ClauseFail() goto lb_jump_f
+#define ClauseFail() goto jump_f
 
 #define SAVE_CP(X)				\
    do {						\
@@ -205,7 +207,7 @@ do {                                     \
       c_p->cp = 0;				\
    } while(0)
 
-#define RESTORE_CP(X)		SET_CP(c_p, cp_val(*(X)))
+#define RESTORE_CP(X)		SET_CP(c_p, (BeamInstr *) cp_val(*(X)))
 
 #define ISCATCHEND(instr) ((Eterm *) *(instr) == OpCode(catch_end_y))
 
@@ -213,13 +215,13 @@ do {                                     \
  * Special Beam instructions.
  */
 
-Eterm beam_apply[2];
-Eterm beam_exit[1];
-Eterm beam_continue_exit[1];
+BeamInstr beam_apply[2];
+BeamInstr beam_exit[1];
+BeamInstr beam_continue_exit[1];
 
-Eterm* em_call_error_handler;
-Eterm* em_apply_bif;
-Eterm* em_call_traced_function;
+BeamInstr* em_call_error_handler;
+BeamInstr* em_apply_bif;
+BeamInstr* em_call_traced_function;
 
 
 /* NOTE These should be the only variables containing trace instructions.
@@ -227,9 +229,16 @@ Eterm* em_call_traced_function;
 **      for the refering variable (one of these), and rouge references
 **      will most likely cause chaos.
 */
-Eterm beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */
-Eterm beam_return_trace[1];    /* OpCode(i_return_trace) */
-Eterm beam_exception_trace[1]; /* UGLY also OpCode(i_return_trace) */
+BeamInstr beam_return_to_trace[1];   /* OpCode(i_return_to_trace) */
+BeamInstr beam_return_trace[1];      /* OpCode(i_return_trace) */
+BeamInstr beam_exception_trace[1];   /* UGLY also OpCode(i_return_trace) */
+BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */
+
+
+/*
+ * We should warn only once for tuple funs.
+ */
+static erts_smp_atomic_t warned_for_tuple_funs;
 
 /*
  * All Beam instructions in numerical order.
@@ -243,20 +252,6 @@ void** beam_ops;
 extern int count_instructions;
 #endif
 
-#if defined(HYBRID)
-#define SWAPIN             \
-    g_htop = global_htop;  \
-    g_hend = global_hend;  \
-    HTOP = HEAP_TOP(c_p);  \
-    E = c_p->stop
-
-#define SWAPOUT            \
-    global_htop = g_htop;  \
-    global_hend = g_hend;  \
-    HEAP_TOP(c_p) = HTOP;  \
-    c_p->stop = E
-
-#else
 #define SWAPIN             \
     HTOP = HEAP_TOP(c_p);  \
     E = c_p->stop
@@ -284,8 +279,6 @@ extern int count_instructions;
 
 #define LIGHT_SWAPIN HTOP = HEAP_TOP(c_p)
 
-#endif
-
 #ifdef FORCE_HEAP_FRAGS
 #  define HEAP_SPACE_VERIFIED(Words) do { \
       c_p->space_verified = (Words);	  \
@@ -301,47 +294,13 @@ extern int count_instructions;
      PROCESS_MAIN_CHK_LOCKS((P));					\
      ERTS_SMP_UNREQ_PROC_MAIN_LOCK((P))
 
-#if defined(HYBRID)
-#  define POST_BIF_GC_SWAPIN_0(_p, _res)				\
-     if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) {	\
-       _res = erts_gc_after_bif_call((_p), (_res), NULL, 0);		\
-     }									\
-     SWAPIN
-
-#  define POST_BIF_GC_SWAPIN(_p, _res, _regs, _arity)			\
-     if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) {	\
-       _regs[0] = r(0);							\
-       _res = erts_gc_after_bif_call((_p), (_res), _regs, (_arity));	\
-       r(0) = _regs[0];							\
-     }									\
-     SWAPIN
-#else
-#  define POST_BIF_GC_SWAPIN_0(_p, _res)				\
-     ERTS_SMP_REQ_PROC_MAIN_LOCK((_p));					\
-     PROCESS_MAIN_CHK_LOCKS((_p));					\
-     if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) {	\
-       _res = erts_gc_after_bif_call((_p), (_res), NULL, 0);		\
-       E = (_p)->stop;							\
-     }									\
-     HTOP = HEAP_TOP((_p))
-
-#  define POST_BIF_GC_SWAPIN(_p, _res, _regs, _arity)			\
-     ERTS_SMP_REQ_PROC_MAIN_LOCK((_p));					\
-     PROCESS_MAIN_CHK_LOCKS((_p));					\
-     if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) {	\
-       _regs[0] = r(0);							\
-       _res = erts_gc_after_bif_call((_p), (_res), _regs, (_arity));	\
-       r(0) = _regs[0];							\
-       E = (_p)->stop;							\
-     }									\
-     HTOP = HEAP_TOP((_p))
-#endif
-
 #define db(N) (N)
 #define tb(N) (N)
 #define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N)))
 #define yb(N) (*(Eterm *) (((unsigned char *)E) + (N)))
 #define fb(N) (*(double *) (((unsigned char *)&(freg[0].fd)) + (N)))
+#define Qb(N) (N)
+#define Ib(N) (N)
 #define x(N) reg[N]
 #define y(N) E[N]
 #define r(N) x##N
@@ -363,6 +322,7 @@ extern int count_instructions;
            reg[0] = r(0); \
            PROCESS_MAIN_CHK_LOCKS(c_p); \
            FCALLS -= erts_garbage_collect(c_p, needed + (HeapNeed), reg, (M)); \
+           ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
            PROCESS_MAIN_CHK_LOCKS(c_p); \
            r(0) = reg[0]; \
            SWAPIN; \
@@ -416,6 +376,7 @@ extern int count_instructions;
        reg[0] = r(0);                                           		\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
        FCALLS -= erts_garbage_collect(c_p, need, reg, (Live));  		\
+       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);					\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
        r(0) = reg[0];                                           		\
        SWAPIN;                                                  		\
@@ -438,6 +399,7 @@ extern int count_instructions;
        reg[0] = r(0);                                           \
        PROCESS_MAIN_CHK_LOCKS(c_p);                             \
        FCALLS -= erts_garbage_collect(c_p, need, reg, (Live));  \
+       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);			\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             \
        r(0) = reg[0];                                           \
        SWAPIN;                                                  \
@@ -460,6 +422,7 @@ extern int count_instructions;
        reg[Live] = Extra;						\
        PROCESS_MAIN_CHK_LOCKS(c_p);					\
        FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)+1);	\
+       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);				\
        PROCESS_MAIN_CHK_LOCKS(c_p);					\
        if (Live > 0) {							\
 	   r(0) = reg[0];						\
@@ -470,35 +433,12 @@ extern int count_instructions;
     HEAP_SPACE_VERIFIED(need);                                      \
   } while (0)
 
-#ifdef HYBRID
-#ifdef INCREMENTAL
-#define TestGlobalHeap(Nh, Live, hp)                                    \
-  do {                                                                  \
-    unsigned need = (Nh);                                               \
-    ASSERT(global_heap <= g_htop && g_htop <= global_hend);             \
-    SWAPOUT;                                                            \
-    reg[0] = r(0);                                                      \
-    FCALLS -= need;                                                     \
-    (hp) = IncAlloc(c_p,need,reg,(Live));                               \
-    r(0) = reg[0];                                                      \
-    SWAPIN;                                                             \
-  } while (0)
-#else
-#define TestGlobalHeap(Nh, Live, hp)                                    \
-  do {                                                                  \
-    unsigned need = (Nh);                                               \
-    ASSERT(global_heap <= g_htop && g_htop <= global_hend);             \
-    if (g_hend - g_htop < need) {                                       \
-       SWAPOUT;                                                         \
-       reg[0] = r(0);                                                   \
-       FCALLS -= erts_global_garbage_collect(c_p, need, reg, (Live));   \
-       r(0) = reg[0];                                                   \
-       SWAPIN;                                                          \
-    }                                                                   \
-    (hp) = global_htop;                                                 \
+#define TestHeapPutList(Need, Reg)		\
+  do {						\
+     TestHeap((Need), 1);			\
+     PutList(Reg, r(0), r(0), StoreSimpleDest);	\
+     CHECK_TERM(r(0));				\
   } while (0)
-#endif
-#endif /* HYBRID */
 
 #define Init(N) make_blank(yb(N))
 
@@ -514,6 +454,11 @@ extern int count_instructions;
      SWAPIN;							\
   } while (0)
 
+#define PutTuple(Dst, Arity)			\
+ do {						\
+   Dst = make_tuple(HTOP);			\
+   pt_arity = (Arity);				\
+ } while (0)
 
 /*
  * Check that we haven't used the reductions and jump to function pointed to by
@@ -522,8 +467,8 @@ extern int count_instructions;
 
 #define DispatchMacro()				\
   do {						\
-     Eterm* dis_next;				\
-     dis_next = (Eterm *) *I;			\
+     BeamInstr* dis_next;				\
+     dis_next = (BeamInstr *) *I;			\
      CHECK_ARGS(I);				\
      if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
         FCALLS--;				\
@@ -535,8 +480,8 @@ extern int count_instructions;
 
 #define DispatchMacroFun()			\
   do {						\
-     Eterm* dis_next;				\
-     dis_next = (Eterm *) *I;			\
+     BeamInstr* dis_next;				\
+     dis_next = (BeamInstr *) *I;			\
      CHECK_ARGS(I);				\
      if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
         FCALLS--;				\
@@ -590,7 +535,7 @@ extern int count_instructions;
     ASSERT(VALID_INSTR(*I));   \
     Goto(*I)
 
-#define PreFetch(N, Dst) do { Dst = (Eterm *) *(I + N + 1); } while (0)
+#define PreFetch(N, Dst) do { Dst = (BeamInstr *) *(I + N + 1); } while (0)
 #define NextPF(N, Dst)         \
     I += N + 1;                \
     ASSERT(VALID_INSTR(Dst));  \
@@ -644,7 +589,7 @@ extern int count_instructions;
 #define DeallocateReturn(Deallocate)       \
   do {                                     \
     int words_to_pop = (Deallocate);       \
-    SET_I(cp_val(*E));                     \
+    SET_I((BeamInstr *) cp_val(*E));                     \
     E = ADD_BYTE_OFFSET(E, words_to_pop);  \
     CHECK_TERM(r(0));                      \
     Goto(*I);                              \
@@ -657,79 +602,77 @@ extern int count_instructions;
 #define MoveCall(Src, Dest, CallDest, Size)	\
     (Dest) = (Src);				\
     SET_CP(c_p, I+Size+1);			\
-    SET_I((Eterm *) CallDest);			\
+    SET_I((BeamInstr *) CallDest);			\
     Dispatch();
 
 #define MoveCallLast(Src, Dest, CallDest, Deallocate)	\
     (Dest) = (Src);					\
     RESTORE_CP(E);					\
     E = ADD_BYTE_OFFSET(E, (Deallocate));		\
-    SET_I((Eterm *) CallDest);				\
+    SET_I((BeamInstr *) CallDest);				\
     Dispatch();
 
 #define MoveCallOnly(Src, Dest, CallDest)	\
     (Dest) = (Src);				\
-    SET_I((Eterm *) CallDest);			\
+    SET_I((BeamInstr *) CallDest);			\
     Dispatch();
 
+#define MoveJump(Src)				\
+     r(0) = (Src);				\
+     SET_I((BeamInstr *) Arg(0));		\
+     Goto(*I);
+
 #define GetList(Src, H, T) do {			\
    Eterm* tmp_ptr = list_val(Src);		\
    H = CAR(tmp_ptr);				\
    T = CDR(tmp_ptr); } while (0)
 
-#define GetTupleElement(Src, Element, Dest)				  \
-  do {									  \
-    tmp_arg1 = (Eterm) (((unsigned char *) tuple_val(Src)) + (Element));  \
-    (Dest) = (*(Eterm *)tmp_arg1);					  \
+#define GetTupleElement(Src, Element, Dest)				\
+  do {									\
+    tmp_arg1 = (Eterm) COMPRESS_POINTER(((unsigned char *) tuple_val(Src)) + 	\
+				(Element));				\
+    (Dest) = (*(Eterm *) EXPAND_POINTER(tmp_arg1));			\
   } while (0)
 
-#define ExtractNextElement(Dest)				\
-    tmp_arg1 += sizeof(Eterm);					\
-    (Dest) = (* (Eterm *) (((unsigned char *) tmp_arg1)))
+#define ExtractNextElement(Dest)					  \
+    tmp_arg1 += sizeof(Eterm);						  \
+    (Dest) = (* (Eterm *) (((unsigned char *) EXPAND_POINTER(tmp_arg1))))
 
-#define ExtractNextElement2(Dest)		\
-  do {						\
-    Eterm* ene_dstp = &(Dest);			\
-    ene_dstp[0] = ((Eterm *) tmp_arg1)[1];	\
-    ene_dstp[1] = ((Eterm *) tmp_arg1)[2];	\
-    tmp_arg1 += sizeof(Eterm) + sizeof(Eterm);	\
+#define ExtractNextElement2(Dest)				\
+  do {								\
+    Eterm* ene_dstp = &(Dest);					\
+    ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1];	\
+    ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2];	\
+    tmp_arg1 += sizeof(Eterm) + sizeof(Eterm);			\
   } while (0)
 
 #define ExtractNextElement3(Dest)		\
   do {						\
     Eterm* ene_dstp = &(Dest);			\
-    ene_dstp[0] = ((Eterm *) tmp_arg1)[1];	\
-    ene_dstp[1] = ((Eterm *) tmp_arg1)[2];	\
-    ene_dstp[2] = ((Eterm *) tmp_arg1)[3];	\
+    ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1];	\
+    ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2];	\
+    ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3];	\
     tmp_arg1 += 3*sizeof(Eterm);		\
   } while (0)
 
 #define ExtractNextElement4(Dest)		\
   do {						\
     Eterm* ene_dstp = &(Dest);			\
-    ene_dstp[0] = ((Eterm *) tmp_arg1)[1];	\
-    ene_dstp[1] = ((Eterm *) tmp_arg1)[2];	\
-    ene_dstp[2] = ((Eterm *) tmp_arg1)[3];	\
-    ene_dstp[3] = ((Eterm *) tmp_arg1)[4];	\
+    ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1];	\
+    ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2];	\
+    ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3];	\
+    ene_dstp[3] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[4];	\
     tmp_arg1 += 4*sizeof(Eterm);		\
   } while (0)
 
 #define ExtractElement(Element, Dest)		\
   do {						\
      tmp_arg1 += (Element);			\
-     (Dest) = (* (Eterm *) tmp_arg1);		\
+     (Dest) = (* (Eterm *) EXPAND_POINTER(tmp_arg1));		\
   } while (0)
 
-#define PutTuple(Arity, Src, Dest)		\
-     ASSERT(is_arity_value(Arity));		\
-     Dest = make_tuple(HTOP);			\
-     HTOP[0] = (Arity);				\
-     HTOP[1] = (Src);				\
-     HTOP += 2
-
-#define Put(Word) *HTOP++ = (Word)
-
 #define EqualImmed(X, Y, Action) if (X != Y) { Action; }
+#define NotEqualImmed(X, Y, Action) if (X == Y) { Action; }
 
 #define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; }
 
@@ -759,8 +702,13 @@ extern int count_instructions;
 
 #define IsTuple(X, Action) if (is_not_tuple(X)) Action
 
-#define IsArity(Pointer, Arity, Fail) \
-    if (*(Eterm *)(tmp_arg1 = (Eterm)tuple_val(Pointer)) != (Arity)) { Fail; }
+#define IsArity(Pointer, Arity, Fail)					  \
+    if (*(Eterm *)							  \
+	EXPAND_POINTER(tmp_arg1 = (Eterm) 				  \
+		       COMPRESS_POINTER(tuple_val(Pointer))) != (Arity))  \
+    { 									  \
+        Fail; 								  \
+    }
 
 #define IsFunction(X, Action)			\
   do {						\
@@ -769,18 +717,21 @@ extern int count_instructions;
      }						\
   } while (0)
 
-#define IsFunction2(F, A, Action)		\
-  do {						\
-     if (is_function_2(c_p, F, A) != am_true ) {\
-          Action;				\
-     }						\
+#define IsFunction2(F, A, Action)			\
+  do {							\
+     if (erl_is_function(c_p, F, A) != am_true ) {	\
+          Action;					\
+     }							\
   } while (0)
 
-#define IsTupleOfArity(Src, Arity, Fail) \
-  do { \
-    if (is_not_tuple(Src) || *(Eterm *)(tmp_arg1 = (Eterm) tuple_val(Src)) != Arity) { \
-        Fail; \
-    } \
+#define IsTupleOfArity(Src, Arity, Fail)				      \
+  do {									      \
+    if (is_not_tuple(Src) || 						      \
+	*(Eterm *)							      \
+	EXPAND_POINTER(tmp_arg1 = 					      \
+		       (Eterm) COMPRESS_POINTER(tuple_val(Src))) != Arity) { \
+        Fail;								      \
+    }									      \
   } while (0)
 
 #define IsBoolean(X, Fail) if ((X) != am_true && (X) != am_false) { Fail; }
@@ -791,7 +742,7 @@ extern int count_instructions;
 #define IsBitstring(Src, Fail) \
   if (is_not_binary(Src)) { Fail; }
 
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 #define BsSafeMul(A, B, Fail, Target)		\
    do { Uint64 _res = (A) * (B);		\
       if (_res / B != A) { Fail; }		\
@@ -973,36 +924,58 @@ extern int count_instructions;
 #define IsPid(Src, Fail) if (is_not_pid(Src)) { Fail; }
 #define IsRef(Src, Fail) if (is_not_ref(Src)) { Fail; }
 
-static BifFunction translate_gc_bif(void* gcf);
-static Eterm* handle_error(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf);
-static Eterm* next_catch(Process* c_p, Eterm *reg);
+/*
+ * process_main() is already huge, so we want to avoid inlining
+ * into it. Especially functions that are seldom used.
+ */
+#ifdef __GNUC__
+#  define NOINLINE __attribute__((__noinline__))
+#else
+#  define NOINLINE
+#endif
+
+/*
+ * The following functions are called directly by process_main().
+ * Don't inline them.
+ */
+static BifFunction translate_gc_bif(void* gcf) NOINLINE;
+static BeamInstr* handle_error(Process* c_p, BeamInstr* pc,
+			       Eterm* reg, BifFunction bf) NOINLINE;
+static BeamInstr* call_error_handler(Process* p, BeamInstr* ip,
+				     Eterm* reg, Eterm func) NOINLINE;
+static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) NOINLINE;
+static BeamInstr* apply(Process* p, Eterm module, Eterm function,
+			Eterm args, Eterm* reg) NOINLINE;
+static BeamInstr* call_fun(Process* p, int arity,
+			   Eterm* reg, Eterm args) NOINLINE;
+static BeamInstr* apply_fun(Process* p, Eterm fun,
+			    Eterm args, Eterm* reg) NOINLINE;
+static Eterm new_fun(Process* p, Eterm* reg,
+		     ErlFunEntry* fe, int num_free) NOINLINE;
+
+
+/*
+ * Functions not directly called by process_main(). OK to inline.
+ */
+static BeamInstr* next_catch(Process* c_p, Eterm *reg);
 static void terminate_proc(Process* c_p, Eterm Value);
 static Eterm add_stacktrace(Process* c_p, Eterm Value, Eterm exc);
-static void save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg,
+static void save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg,
 			     BifFunction bf, Eterm args);
 static struct StackTrace * get_trace_from_exc(Eterm exc);
 static Eterm make_arglist(Process* c_p, Eterm* reg, int a);
-static Eterm call_error_handler(Process* p, Eterm* ip, Eterm* reg);
-static Eterm call_breakpoint_handler(Process* p, Eterm* fi, Eterm* reg);
-static Uint* fixed_apply(Process* p, Eterm* reg, Uint arity);
-static Eterm* apply(Process* p, Eterm module, Eterm function,
-		     Eterm args, Eterm* reg);
-static int hibernate(Process* c_p, Eterm module, Eterm function,
-		     Eterm args, Eterm* reg);
-static Eterm* call_fun(Process* p, int arity, Eterm* reg, Eterm args);
-static Eterm* apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg);
-static Eterm new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free);
-
-#if defined(_OSE_) || defined(VXWORKS)
+
+#if defined(VXWORKS)
 static int init_done;
 #endif
 
 void
 init_emulator(void)
 {
-#if defined(_OSE_) || defined(VXWORKS)
+#if defined(VXWORKS)
     init_done = 0;
 #endif
+    erts_smp_atomic_init_nob(&warned_for_tuple_funs, (erts_aint_t) 0);
     process_main();
 }
 
@@ -1031,6 +1004,101 @@ init_emulator(void)
 #  define REG_tmp_arg2
 #endif
 
+#ifdef USE_VM_PROBES
+#  define USE_VM_CALL_PROBES
+#endif
+
+#ifdef USE_VM_CALL_PROBES
+
+#define DTRACE_LOCAL_CALL(p, m, f, a)					\
+    if (DTRACE_ENABLED(local_function_entry)) {				\
+        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);		\
+        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);			\
+        int depth = STACK_START(p) - STACK_TOP(p);			\
+        dtrace_fun_decode(p, m, f, a,					\
+                          process_name, mfa);				\
+        DTRACE3(local_function_entry, process_name, mfa, depth);	\
+    }
+
+#define DTRACE_GLOBAL_CALL(p, m, f, a)					\
+    if (DTRACE_ENABLED(global_function_entry)) {			\
+        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);		\
+        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);			\
+        int depth = STACK_START(p) - STACK_TOP(p);			\
+        dtrace_fun_decode(p, m, f, a,					\
+                          process_name, mfa);				\
+        DTRACE3(global_function_entry, process_name, mfa, depth);	\
+    }
+
+#define DTRACE_RETURN(p, m, f, a)                               \
+    if (DTRACE_ENABLED(function_return)) {                      \
+        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);     \
+        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);              \
+        int depth = STACK_START(p) - STACK_TOP(p);              \
+        dtrace_fun_decode(p, m, f, a,                           \
+                          process_name, mfa);                   \
+        DTRACE3(function_return, process_name, mfa, depth);     \
+    }
+
+#define DTRACE_BIF_ENTRY(p, m, f, a)                            \
+    if (DTRACE_ENABLED(bif_entry)) {                            \
+        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);     \
+        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);              \
+        dtrace_fun_decode(p, m, f, a,                           \
+                          process_name, mfa);                   \
+        DTRACE2(bif_entry, process_name, mfa);                  \
+    }
+
+#define DTRACE_BIF_RETURN(p, m, f, a)                           \
+    if (DTRACE_ENABLED(bif_return)) {                           \
+        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);     \
+        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);              \
+        dtrace_fun_decode(p, m, f, a,                           \
+                          process_name, mfa);                   \
+        DTRACE2(bif_return, process_name, mfa);                 \
+    }
+
+#define DTRACE_NIF_ENTRY(p, m, f, a)                            \
+    if (DTRACE_ENABLED(nif_entry)) {                            \
+        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);     \
+        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);              \
+        dtrace_fun_decode(p, m, f, a,                           \
+                          process_name, mfa);                   \
+        DTRACE2(nif_entry, process_name, mfa);                  \
+    }
+
+#define DTRACE_NIF_RETURN(p, m, f, a)                           \
+    if (DTRACE_ENABLED(nif_return)) {                           \
+        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);     \
+        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);              \
+        dtrace_fun_decode(p, m, f, a,                           \
+                          process_name, mfa);                   \
+        DTRACE2(nif_return, process_name, mfa);                 \
+    }
+
+#else /* USE_VM_PROBES */
+
+#define DTRACE_LOCAL_CALL(p, m, f, a)  do {} while (0)
+#define DTRACE_GLOBAL_CALL(p, m, f, a) do {} while (0)
+#define DTRACE_RETURN(p, m, f, a)      do {} while (0)
+#define DTRACE_BIF_ENTRY(p, m, f, a)   do {} while (0)
+#define DTRACE_BIF_RETURN(p, m, f, a)  do {} while (0)
+#define DTRACE_NIF_ENTRY(p, m, f, a)   do {} while (0)
+#define DTRACE_NIF_RETURN(p, m, f, a)  do {} while (0)
+
+#endif /* USE_VM_PROBES */
+
+#ifdef USE_VM_PROBES
+void
+dtrace_drvport_str(ErlDrvPort drvport, char *port_buf)
+{
+    Port *port = erts_drvport2port(drvport);
+
+    erts_snprintf(port_buf, DTRACE_TERM_BUF_SIZE, "#Port<%lu.%lu>",
+                  port_channel_no(port->id),
+                  port_number(port->id));
+}
+#endif
 /*
  * process_main() is called twice:
  * The first call performs some initialisation, including exporting
@@ -1039,13 +1107,13 @@ init_emulator(void)
  */
 void process_main(void)
 {
-#if !defined(_OSE_) && !defined(VXWORKS)
+#if !defined(VXWORKS)
     static int init_done = 0;
 #endif
     Process* c_p = NULL;
     int reds_used;
 #ifdef DEBUG
-    Eterm pid;
+    ERTS_DECLARE_DUMMY(Eterm pid);
 #endif
 
     /*
@@ -1063,12 +1131,6 @@ void process_main(void)
      */
     register Eterm* HTOP REG_htop = NULL;
 
-
-#ifdef HYBRID
-     Eterm *g_htop;
-     Eterm *g_hend;
-#endif
-
     /* Stack pointer.  Grows downwards; points
      * to last item pushed (normally a saved
      * continuation pointer).
@@ -1078,7 +1140,7 @@ void process_main(void)
     /*
      * Pointer to next threaded instruction.
      */
-    register Eterm *I REG_I = NULL;
+    register BeamInstr *I REG_I = NULL;
 
     /* Number of reductions left.  This function
      * returns to the scheduler when FCALLS reaches zero.
@@ -1090,27 +1152,17 @@ void process_main(void)
      */
     register Eterm tmp_arg1 REG_tmp_arg1 = NIL;
     register Eterm tmp_arg2 REG_tmp_arg2 = NIL;
-    Eterm tmp_big[2];		/* Temporary buffer for small bignums. */
-
-#ifndef ERTS_SMP
-    static Eterm save_reg[ERTS_X_REGS_ALLOCATED];
-    /* X registers -- not used directly, but
-     * through 'reg', because using it directly
-     * needs two instructions on a SPARC,
-     * while using it through reg needs only
-     * one.
-     */
+#if HEAP_ON_C_STACK
+    Eterm tmp_big[2];           /* Temporary buffer for small bignums if HEAP_ON_C_STACK. */
+#else
+    Eterm *tmp_big;		/* Temporary buffer for small bignums if !HEAP_ON_C_STACK. */
+#endif
 
     /*
-     * Floating point registers.
-     */
-    static FloatDef freg[MAX_REG];
-#else
-    /* X regisers and floating point registers are located in
+     * X registers and floating point registers are located in
      * scheduler specific data.
      */
     register FloatDef *freg;
-#endif
 
     /*
      * For keeping the negative old value of 'reds' when call saving is active.
@@ -1130,6 +1182,8 @@ void process_main(void)
 
     Uint temp_bits; /* Temporary used by BsSkipBits2 & BsGetInteger2 */
 
+    Eterm pt_arity;		/* Used by do_put_tuple */
+
     ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */
 
 
@@ -1141,14 +1195,10 @@ void process_main(void)
      * Note: c_p->arity must be set to reflect the number of useful terms in
      * c_p->arg_reg before calling the scheduler.
      */
-
     if (!init_done) {
 	init_done = 1;
 	goto init_emulator;
     }
-#ifndef ERTS_SMP
-    reg = save_reg;	/* XXX: probably wastes a register on x86 */
-#endif
     c_p = NULL;
     reds_used = 0;
     goto do_schedule1;
@@ -1158,21 +1208,27 @@ void process_main(void)
  do_schedule1:
     PROCESS_MAIN_CHK_LOCKS(c_p);
     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+#if HALFWORD_HEAP
+    ASSERT(erts_get_scheduler_data()->num_tmp_heap_used == 0);
+#endif
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
     c_p = schedule(c_p, reds_used);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 #ifdef DEBUG
-    pid = c_p->id;
+    pid = c_p->id; /* Save for debugging purpouses */
 #endif
     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
     PROCESS_MAIN_CHK_LOCKS(c_p);
-#ifdef ERTS_SMP
-    reg = c_p->scheduler_data->save_reg;
-    freg = c_p->scheduler_data->freg;
+    reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array;
+    freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array;
+#if !HEAP_ON_C_STACK
+    tmp_big = ERTS_PROC_GET_SCHDATA(c_p)->beam_emu_tmp_heap;
 #endif
     ERL_BITS_RELOAD_STATEP(c_p);
     {
 	int reds;
 	Eterm* argp;
-	Eterm* next;
+	BeamInstr *next;
 	int i;
 
 	argp = c_p->arg_reg;
@@ -1199,7 +1255,7 @@ void process_main(void)
 	    FCALLS = REDS_IN(c_p) = reds;
 	}
 
-	next = (Eterm *) *I;
+	next = (BeamInstr *) *I;
 	r(0) = c_p->arg_reg[0];
 #ifdef HARDDEBUG
 	if (c_p->arity > 0) {
@@ -1208,6 +1264,30 @@ void process_main(void)
 #endif
 	SWAPIN;
 	ASSERT(VALID_INSTR(next));
+
+#ifdef USE_VM_PROBES
+        if (DTRACE_ENABLED(process_scheduled)) {
+            DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE);
+            DTRACE_CHARBUF(fun_buf, DTRACE_TERM_BUF_SIZE);
+            dtrace_proc_str(c_p, process_buf);
+
+            if (ERTS_PROC_IS_EXITING(c_p)) {
+                strcpy(fun_buf, "<exiting>");
+            } else {
+                BeamInstr *fptr = find_function_from_pc(c_p->i);
+                if (fptr) {
+                    dtrace_fun_decode(c_p, (Eterm)fptr[0],
+                                      (Eterm)fptr[1], (Uint)fptr[2],
+                                      NULL, fun_buf);
+                } else {
+                    erts_snprintf(fun_buf, sizeof(fun_buf),
+                                  "<unknown/%p>", next);
+                }
+            }
+
+            DTRACE2(process_scheduled, process_buf, fun_buf);
+        }
+#endif
 	Goto(next);
     }
 
@@ -1223,6 +1303,52 @@ void process_main(void)
 #define STORE_ARITH_RESULT(res) StoreBifResult(2, (res));
 #define ARITH_FUNC(name) erts_gc_##name
 
+	{
+	    Eterm increment_reg_val;
+	    Eterm increment_val;
+	    Uint live;
+	    Eterm result;
+
+	OpCase(i_increment_yIId):
+	    increment_reg_val = yb(Arg(0));
+	    goto do_increment;
+
+	OpCase(i_increment_xIId):
+	    increment_reg_val = xb(Arg(0));
+	    goto do_increment;
+
+	OpCase(i_increment_rIId):
+	    increment_reg_val = r(0);
+	    I--;
+
+	do_increment:
+	    increment_val = Arg(1);
+	    if (is_small(increment_reg_val)) {
+		Sint i = signed_val(increment_reg_val) + increment_val;
+		ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
+		if (MY_IS_SSMALL(i)) {
+		    result = make_small(i);
+		store_result:
+		    StoreBifResult(3, result);
+		}
+	    }
+
+	    live = Arg(2);
+	    SWAPOUT;
+	    reg[0] = r(0);
+	    reg[live] = increment_reg_val;
+	    reg[live+1] = make_small(increment_val);
+	    result = erts_gc_mixed_plus(c_p, reg, live);
+	    r(0) = reg[0];
+	    SWAPIN;
+	    ERTS_HOLE_CHECK(c_p);
+	    if (is_value(result)) {
+		goto store_result;
+	    }
+	    ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue));
+	    goto find_func_info;
+	}
+	    
  OpCase(i_plus_jId):
  {
      Eterm result;
@@ -1286,12 +1412,59 @@ void process_main(void)
     }
     Next(1);
 
+    {
+	Eterm is_eq_exact_lit_val;
+
+    OpCase(i_is_eq_exact_literal_xfc):
+	is_eq_exact_lit_val = xb(Arg(0));
+	I++;
+	goto do_is_eq_exact_literal;
+
+    OpCase(i_is_eq_exact_literal_yfc):
+	is_eq_exact_lit_val = yb(Arg(0));
+	I++;
+	goto do_is_eq_exact_literal;
+
+    OpCase(i_is_eq_exact_literal_rfc):
+	is_eq_exact_lit_val = r(0);
+
+    do_is_eq_exact_literal:
+	if (!eq(Arg(1), is_eq_exact_lit_val)) {
+	    ClauseFail();
+	}
+	Next(2);
+    }
+
+    {
+	Eterm is_ne_exact_lit_val;
+
+    OpCase(i_is_ne_exact_literal_xfc):
+	is_ne_exact_lit_val = xb(Arg(0));
+	I++;
+	goto do_is_ne_exact_literal;
+
+    OpCase(i_is_ne_exact_literal_yfc):
+	is_ne_exact_lit_val = yb(Arg(0));
+	I++;
+	goto do_is_ne_exact_literal;
+
+    OpCase(i_is_ne_exact_literal_rfc):
+	is_ne_exact_lit_val = r(0);
+
+    do_is_ne_exact_literal:
+	if (eq(Arg(1), is_ne_exact_lit_val)) {
+	    ClauseFail();
+	}
+	Next(2);
+    }
+
  OpCase(i_move_call_only_fcr): {
      r(0) = Arg(1);
  }
  /* FALL THROUGH */
  OpCase(i_call_only_f): {
-     SET_I((Eterm *) Arg(0));
+     SET_I((BeamInstr *) Arg(0));
+     DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]);
      Dispatch();
  }
 
@@ -1302,7 +1475,8 @@ void process_main(void)
  OpCase(i_call_last_fP): {
      RESTORE_CP(E);
      E = ADD_BYTE_OFFSET(E, Arg(1));
-     SET_I((Eterm *) Arg(0));
+     SET_I((BeamInstr *) Arg(0));
+     DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]);
      Dispatch();
  }
 
@@ -1313,7 +1487,8 @@ void process_main(void)
  /* FALL THROUGH */
  OpCase(i_call_f): {
      SET_CP(c_p, I+2);
-     SET_I((Eterm *) Arg(0));
+     SET_I((BeamInstr *) Arg(0));
+     DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]);
      Dispatch();
  }
 
@@ -1330,6 +1505,12 @@ void process_main(void)
      * is not loaded, it points to code which will invoke the error handler
      * (see lb_call_error_handler below).
      */
+#ifdef USE_VM_CALL_PROBES
+    if (DTRACE_ENABLED(global_function_entry)) {
+	BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->address);
+	DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]);
+    }
+#endif
     Dispatchx();
 
  OpCase(i_move_call_ext_cre): {
@@ -1339,6 +1520,12 @@ void process_main(void)
  /* FALL THROUGH */
  OpCase(i_call_ext_e):
     SET_CP(c_p, I+2);
+#ifdef USE_VM_CALL_PROBES
+    if (DTRACE_ENABLED(global_function_entry)) {
+	BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->address);
+	DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]);
+    }
+#endif
     Dispatchx();
 
  OpCase(i_move_call_ext_only_ecr): {
@@ -1346,10 +1533,16 @@ void process_main(void)
  }
  /* FALL THROUGH */
  OpCase(i_call_ext_only_e):
+#ifdef USE_VM_CALL_PROBES
+    if (DTRACE_ENABLED(global_function_entry)) {
+	BeamInstr* fp = (BeamInstr *) (((Export *) Arg(0))->address);
+	DTRACE_GLOBAL_CALL(c_p, (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]);
+    }
+#endif
     Dispatchx();
 
  OpCase(init_y): {
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(1, next);
      make_blank(yb(Arg(0)));
@@ -1357,7 +1550,7 @@ void process_main(void)
  }
 
  OpCase(i_trim_I): {
-     Eterm* next;
+     BeamInstr *next;
      Uint words;
      Uint cp;
 
@@ -1369,8 +1562,28 @@ void process_main(void)
      NextPF(1, next);
  }
 
+ OpCase(move_x1_c): {
+	x(1) = Arg(0);
+	Next(1);
+    }
+
+ OpCase(move_x2_c): {
+	x(2) = Arg(0);
+	Next(1);
+    }
+
+
  OpCase(return): {
+#ifdef USE_VM_CALL_PROBES
+    BeamInstr* fptr;
+#endif
     SET_I(c_p->cp);
+
+#ifdef USE_VM_CALL_PROBES
+    if (DTRACE_ENABLED(function_return) && (fptr = find_function_from_pc(c_p->cp))) {
+        DTRACE_RETURN(c_p, (Eterm)fptr[0], (Eterm)fptr[1], (Uint)fptr[2]);
+    }
+#endif
     /*
      * We must clear the CP to make sure that a stale value do not
      * create a false module dependcy preventing code upgrading.
@@ -1382,30 +1595,6 @@ void process_main(void)
     Goto(*I);
  }
 
- OpCase(test_heap_1_put_list_Iy): {
-     Eterm* next;
-
-     PreFetch(2, next);
-     TestHeap(Arg(0), 1);
-     PutList(yb(Arg(1)), r(0), r(0), StoreSimpleDest);
-     CHECK_TERM(r(0));
-     NextPF(2, next);
- }
-
- OpCase(put_string_IId):
-    {
-      unsigned char* s;
-      int len;
-      Eterm result;
-
-      len = Arg(0);		/* Length. */
-      result = NIL;
-      for (s = (unsigned char *) Arg(1); len > 0; s--, len--) {
-	  PutList(make_small(*s), result, result, StoreSimpleDest);
-      }
-      StoreBifResult(2, result);
-    }
-
     /*
      * Send is almost a standard call-BIF with two arguments, except for:
      *    1) It cannot be traced.
@@ -1414,14 +1603,22 @@ void process_main(void)
      */
 
  OpCase(send): {
-     Eterm* next;
+     BeamInstr *next;
      Eterm result;
 
      PRE_BIF_SWAPOUT(c_p);
      c_p->fcalls = FCALLS - 1;
-     result = send_2(c_p, r(0), x(1));
+     reg[0] = r(0);
+     result = erl_send(c_p, r(0), x(1));
      PreFetch(0, next);
-     POST_BIF_GC_SWAPIN(c_p, result, reg, 2);
+     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+     PROCESS_MAIN_CHK_LOCKS(c_p);
+     if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) {
+	 result = erts_gc_after_bif_call(c_p, result, reg, 2);
+	 r(0) = reg[0];
+	 E = c_p->stop;
+     }
+     HTOP = HEAP_TOP(c_p);
      FCALLS = c_p->fcalls;
      if (is_value(result)) {
 	 r(0) = result;
@@ -1429,33 +1626,44 @@ void process_main(void)
 	 NextPF(0, next);
      } else if (c_p->freason == TRAP) {
 	 SET_CP(c_p, I+1);
-	 SET_I((Eterm *) c_p->def_arg_reg[3]);
+	 SET_I(c_p->i);
 	 SWAPIN;
-	 r(0) = c_p->def_arg_reg[0];
-	 x(1) = c_p->def_arg_reg[1];
+	 r(0) = reg[0];
 	 Dispatch();
      }
      goto find_func_info;
  }
 
- OpCase(i_element_jssd): {
-     Eterm index;
-     Eterm tuple;
-
-     /*
-      * Inlined version of element/2 for speed.
-      */
-     GetArg2(1, index, tuple);
-     if (is_small(index) && is_tuple(tuple)) {
-	 Eterm* tp = tuple_val(tuple);
-
-	 if ((signed_val(index) >= 1) &&
-	     (signed_val(index) <= arityval(*tp))) {
-	     Eterm result = tp[signed_val(index)];
-	     StoreBifResult(3, result);
-	 }
-     }
- }
+    {
+	Eterm element_index;
+	Eterm element_tuple;
+
+    OpCase(i_element_xjsd):
+	element_tuple = xb(Arg(0));
+	I++;
+	goto do_element;
+
+    OpCase(i_element_yjsd):
+	element_tuple = yb(Arg(0));
+	I++;
+	goto do_element;
+
+    OpCase(i_element_rjsd):
+	element_tuple = r(0);
+	/* Fall through */
+
+    do_element:
+	GetArg1(1, element_index);
+	if (is_small(element_index) && is_tuple(element_tuple)) {
+	    Eterm* tp = tuple_val(element_tuple);
+
+	    if ((signed_val(element_index) >= 1) &&
+		(signed_val(element_index) <= arityval(*tp))) {
+		Eterm result = tp[signed_val(element_index)];
+		StoreBifResult(2, result);
+	    }
+	}
+    }
  /* Fall through */
 
  OpCase(badarg_j):
@@ -1463,24 +1671,32 @@ void process_main(void)
     c_p->freason = BADARG;
     goto lb_Cl_error;
 
- OpCase(i_fast_element_jIsd): {
-     Eterm tuple;
-
-     /*
-      * Inlined version of element/2 for even more speed.
-      * The first argument is an untagged integer >= 1.
-      * The second argument is guaranteed to be a register operand.
-      */
-     GetArg1(2, tuple);
-     if (is_tuple(tuple)) {
-	 Eterm* tp = tuple_val(tuple);
-	 tmp_arg2 = Arg(1);
-	 if (tmp_arg2 <= arityval(*tp)) {
-	     Eterm result = tp[tmp_arg2];
-	     StoreBifResult(3, result);
-	 }
-     }
+    {
+	Eterm fast_element_tuple;
+
+    OpCase(i_fast_element_rjId):
+	fast_element_tuple = r(0);
+
+    do_fast_element:
+	if (is_tuple(fast_element_tuple)) {
+	    Eterm* tp = tuple_val(fast_element_tuple);
+	    Eterm pos = Arg(1);	/* Untagged integer >= 1 */
+	    if (pos <= arityval(*tp)) {
+		Eterm result = tp[pos];
+		StoreBifResult(2, result);
+	    }
+	}
      goto badarg;
+
+    OpCase(i_fast_element_xjId):
+     fast_element_tuple = xb(Arg(0));
+     I++;
+     goto do_fast_element;
+
+    OpCase(i_fast_element_yjId):
+     fast_element_tuple = yb(Arg(0));
+     I++;
+     goto do_fast_element;
  }
 
  OpCase(catch_yf):
@@ -1506,6 +1722,7 @@ void process_main(void)
 		 SWAPOUT;
 		 PROCESS_MAIN_CHK_LOCKS(c_p);
 		 FCALLS -= erts_garbage_collect(c_p, 3, reg+2, 1);
+		 ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 		 PROCESS_MAIN_CHK_LOCKS(c_p);
 		 SWAPIN;
 	     }
@@ -1531,6 +1748,10 @@ void process_main(void)
  /*
   * Skeleton for receive statement:
   *
+  *             recv_mark L1                     Optional
+  *             call make_ref/monitor            Optional
+  *             ...
+  *             recv_set L1                      Optional
   *      L1:          <-------------------+
   *                   <-----------+       |
   *     	     	       	  |   	  |
@@ -1549,13 +1770,41 @@ void process_main(void)
   *
   */
 
+ OpCase(recv_mark_f): {
+     /*
+      * Save the current position in message buffer and the
+      * the label for the loop_rec/2 instruction for the
+      * the receive statement.
+      */
+     c_p->msg.mark = (BeamInstr *) Arg(0);
+     c_p->msg.saved_last = c_p->msg.last;
+     Next(1);
+ }
+
+ OpCase(i_recv_set): {
+     /*
+      * If the mark is valid (points to the loop_rec/2
+      * instruction that follows), we know that the saved
+      * position points to the first message that could
+      * possibly be matched out.
+      *
+      * If the mark is invalid, we do nothing, meaning that
+      * we will look through all messages in the message queue.
+      */
+     if (c_p->msg.mark == (BeamInstr *) (I+1)) {
+	 c_p->msg.save = c_p->msg.saved_last;
+     }
+     I++;
+     /* Fall through to the loop_rec/2 instruction */
+ }
+
     /*
      * Pick up the next message and place it in x(0).
      * If no message, jump to a wait or wait_timeout instruction.
      */
  OpCase(i_loop_rec_fr):
  {
-     Eterm* next;
+     BeamInstr *next;
      ErlMessage* msgp;
 
  loop_rec__:
@@ -1579,7 +1828,7 @@ void process_main(void)
 	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 	 else {
 #endif
-	     SET_I((Eterm *) Arg(0));
+	     SET_I((BeamInstr *) Arg(0));
 	     Goto(*I);		/* Jump to a wait or wait_timeout instruction */
 #ifdef ERTS_SMP
 	 }
@@ -1592,6 +1841,7 @@ void process_main(void)
 				       PROCESS_MAIN_CHK_LOCKS(c_p);
 				   },
 				   {
+				       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 				       PROCESS_MAIN_CHK_LOCKS(c_p);
 				       r(0) = reg[0];
 				       SWAPIN;
@@ -1602,6 +1852,7 @@ void process_main(void)
 	  * remove it...
 	  */
 	 ASSERT(!msgp->data.attached);
+         /* TODO: Add DTrace probe for this bad message situation? */
 	 UNLINK_MESSAGE(c_p, msgp);
 	 free_message(msgp);
 	 goto loop_rec__;
@@ -1615,7 +1866,7 @@ void process_main(void)
   * Remove a (matched) message from the message queue.
   */
  OpCase(remove_message): {
-     Eterm* next;
+     BeamInstr *next;
      ErlMessage* msgp;
 
      PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -1627,29 +1878,94 @@ void process_main(void)
 	 save_calls(c_p, &exp_receive);
      }
      if (ERL_MESSAGE_TOKEN(msgp) == NIL) {
-	 SEQ_TRACE_TOKEN(c_p) = NIL;
+#ifdef USE_VM_PROBES
+	 if (DT_UTAG(c_p) != NIL) {
+	     if (DT_UTAG_FLAGS(c_p) & DT_UTAG_PERMANENT) {
+		 SEQ_TRACE_TOKEN(c_p) = am_have_dt_utag;
+#ifdef DTRACE_TAG_HARDDEBUG
+		 if (DT_UTAG_FLAGS(c_p) & DT_UTAG_SPREADING) 
+		     erts_fprintf(stderr,
+				  "Dtrace -> (%T) stop spreading "
+				  "tag %T with message %T\r\n",
+				  c_p->id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp));
+#endif
+	     } else {
+#ifdef DTRACE_TAG_HARDDEBUG
+		 erts_fprintf(stderr,
+			      "Dtrace -> (%T) kill tag %T with "
+			      "message %T\r\n",
+			      c_p->id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp));
+#endif
+		 DT_UTAG(c_p) = NIL;
+		 SEQ_TRACE_TOKEN(c_p) = NIL;
+	     }
+	 } else {
+#endif
+	     SEQ_TRACE_TOKEN(c_p) = NIL;
+#ifdef USE_VM_PROBES
+	 }
+	 DT_UTAG_FLAGS(c_p) &= ~DT_UTAG_SPREADING;
+#endif
      } else if (ERL_MESSAGE_TOKEN(msgp) != am_undefined) {
 	 Eterm msg;
 	 SEQ_TRACE_TOKEN(c_p) = ERL_MESSAGE_TOKEN(msgp);
-	 ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p)));
-	 ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5);
-	 ASSERT(is_small(SEQ_TRACE_TOKEN_SERIAL(c_p)));
-	 ASSERT(is_small(SEQ_TRACE_TOKEN_LASTCNT(c_p)));
-	 ASSERT(is_small(SEQ_TRACE_TOKEN_FLAGS(c_p)));
-	 ASSERT(is_pid(SEQ_TRACE_TOKEN_SENDER(c_p)));
-	 c_p->seq_trace_lastcnt = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p));
-	 if (c_p->seq_trace_clock < unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p))) {
-	     c_p->seq_trace_clock = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p));
+#ifdef USE_VM_PROBES
+	 if (ERL_MESSAGE_TOKEN(msgp) == am_have_dt_utag) {
+	     if (DT_UTAG(c_p) == NIL) {
+		 DT_UTAG(c_p) = ERL_MESSAGE_DT_UTAG(msgp);
+	     }
+	     DT_UTAG_FLAGS(c_p) |= DT_UTAG_SPREADING;
+#ifdef DTRACE_TAG_HARDDEBUG
+	     erts_fprintf(stderr,
+			  "Dtrace -> (%T) receive tag (%T) "
+			  "with message %T\r\n",
+			  c_p->id, DT_UTAG(c_p), ERL_MESSAGE_TERM(msgp));
+#endif
+	 } else {
+#endif
+	     ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p)));
+	     ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5);
+	     ASSERT(is_small(SEQ_TRACE_TOKEN_SERIAL(c_p)));
+	     ASSERT(is_small(SEQ_TRACE_TOKEN_LASTCNT(c_p)));
+	     ASSERT(is_small(SEQ_TRACE_TOKEN_FLAGS(c_p)));
+	     ASSERT(is_pid(SEQ_TRACE_TOKEN_SENDER(c_p)));
+	     c_p->seq_trace_lastcnt = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p));
+	     if (c_p->seq_trace_clock < unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p))) {
+		 c_p->seq_trace_clock = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p));
+	     }
+	     msg = ERL_MESSAGE_TERM(msgp);
+	     seq_trace_output(SEQ_TRACE_TOKEN(c_p), msg, SEQ_TRACE_RECEIVE, 
+			      c_p->id, c_p);
+#ifdef USE_VM_PROBES
 	 }
-	 msg = ERL_MESSAGE_TERM(msgp);
-	 seq_trace_output(SEQ_TRACE_TOKEN(c_p), msg, SEQ_TRACE_RECEIVE, 
-			  c_p->id, c_p);
+#endif
+     }
+#ifdef USE_VM_PROBES
+     if (DTRACE_ENABLED(message_receive)) {
+         Eterm token2 = NIL;
+         DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
+         Sint tok_label = 0;
+         Sint tok_lastcnt = 0;
+         Sint tok_serial = 0;
+
+         dtrace_proc_str(c_p, receiver_name);
+         token2 = SEQ_TRACE_TOKEN(c_p);
+         if (token2 != NIL && token2 != am_have_dt_utag) {
+             tok_label = signed_val(SEQ_TRACE_T_LABEL(token2));
+             tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2));
+             tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2));
+         }
+         DTRACE6(message_receive,
+                 receiver_name, size_object(ERL_MESSAGE_TERM(msgp)),
+                 c_p->msg.len - 1, tok_label, tok_lastcnt, tok_serial);
      }
+#endif
      UNLINK_MESSAGE(c_p, msgp);
      JOIN_MESSAGE(c_p);
      CANCEL_TIMER(c_p);
      free_message(msgp);
 
+     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
      PROCESS_MAIN_CHK_LOCKS(c_p);
 
      NextPF(0, next);
@@ -1660,7 +1976,7 @@ void process_main(void)
      * message didn't match), then jump to the loop_rec instruction.
      */
  OpCase(loop_rec_end_f): {
-     SET_I((Eterm *) Arg(0));
+     SET_I((BeamInstr *) Arg(0));
      SAVE_MESSAGE(c_p);
      goto loop_rec__;
  }
@@ -1690,12 +2006,12 @@ void process_main(void)
      }
      GetArg1(1, timeout_value);
      if (timeout_value != make_small(0)) {
-#if !defined(ARCH_64)
+#if !defined(ARCH_64) || HALFWORD_HEAP
 	 Uint time_val;
 #endif
 
 	 if (is_small(timeout_value) && signed_val(timeout_value) > 0 &&
-#if defined(ARCH_64)
+#if defined(ARCH_64) && !HALFWORD_HEAP
 	     ((unsigned_val(timeout_value) >> 32) == 0)
 #else
 	     1
@@ -1706,14 +2022,18 @@ void process_main(void)
 	      * c_p->def_arg_reg[0].  Note that it is safe to use this
 	      * location because there are no living x registers in
 	      * a receive statement.
+	      * Note that for the halfword emulator, the two first elements
+	      * of the array are used.
 	      */
-	     c_p->def_arg_reg[0] = (Eterm) (I+3);
+	     BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg;
+	     *pi = I+3;
 	     set_timer(c_p, unsigned_val(timeout_value));
 	 } else if (timeout_value == am_infinity) {
 	     c_p->flags |= F_TIMO;
-#if !defined(ARCH_64)
+#if !defined(ARCH_64) || HALFWORD_HEAP
 	 } else if (term_to_Uint(timeout_value, &time_val)) {
-	     c_p->def_arg_reg[0] = (Eterm) (I+3);
+	     BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg;
+	     *pi = I+3;
 	     set_timer(c_p, time_val);
 #endif
 	 } else {		/* Wrong time */
@@ -1742,7 +2062,7 @@ void process_main(void)
 
 	 wait2: {
 	     ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	     c_p->i = (Eterm *) Arg(0); /* L1 */
+	     c_p->i = (BeamInstr *) Arg(0); /* L1 */
 	     SWAPOUT;
 	     c_p->arity = 0;
 	     c_p->status = P_WAITING;
@@ -1770,7 +2090,8 @@ void process_main(void)
       * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag.
       */
      if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) {
-	 c_p->def_arg_reg[0] = (Eterm) (I+3);
+	 BeamInstr** p = (BeamInstr **) c_p->def_arg_reg;
+	 *p = I+3;
 	 set_timer(c_p, Arg(1));
      }
      goto wait2;
@@ -1785,7 +2106,7 @@ void process_main(void)
  }
 
  OpCase(timeout): {
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(0, next);
      if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) {
@@ -1799,14 +2120,93 @@ void process_main(void)
      NextPF(0, next);
  }
 
- OpCase(i_select_val_sfI):
-     GetArg1(0, tmp_arg1);
+
+ {
+     Eterm select_val2;
+
+ OpCase(i_select_tuple_arity2_yfAfAf):
+     select_val2 = yb(Arg(0));
+     goto do_select_tuple_arity2;
+
+ OpCase(i_select_tuple_arity2_xfAfAf):
+     select_val2 = xb(Arg(0));
+     goto do_select_tuple_arity2;
+
+ OpCase(i_select_tuple_arity2_rfAfAf):
+     select_val2 = r(0);
+     I--;
+
+ do_select_tuple_arity2:
+     if (is_not_tuple(select_val2)) {
+	 goto select_val2_fail;
+     }
+     select_val2 = *tuple_val(select_val2);
+     goto do_select_val2;
+
+ OpCase(i_select_val2_yfcfcf):
+     select_val2 = yb(Arg(0));
+     goto do_select_val2;
+
+ OpCase(i_select_val2_xfcfcf):
+     select_val2 = xb(Arg(0));
+     goto do_select_val2;
+
+ OpCase(i_select_val2_rfcfcf):
+     select_val2 = r(0);
+     I--;
+
+ do_select_val2:
+     if (select_val2 == Arg(2)) {
+	 I += 2;
+     } else if (select_val2 == Arg(4)) {
+	 I += 4;
+     }
+
+ select_val2_fail:
+     SET_I((BeamInstr *) Arg(1));
+     Goto(*I);
+ }
+
+ {
+     Eterm select_val;
+
+ OpCase(i_select_tuple_arity_xfI):
+     select_val = xb(Arg(0));
+     goto do_select_tuple_arity;
+
+ OpCase(i_select_tuple_arity_yfI):
+     select_val = yb(Arg(0));
+     goto do_select_tuple_arity;
+
+ OpCase(i_select_tuple_arity_rfI):
+     select_val = r(0);
+     I--;
+
+ do_select_tuple_arity:
+     if (is_tuple(select_val)) {
+	 select_val = *tuple_val(select_val);
+	 goto do_binary_search;
+     }
+     SET_I((BeamInstr *) Arg(1));
+     Goto(*I);
+
+ OpCase(i_select_val_xfI):
+     select_val = xb(Arg(0));
+     goto do_binary_search;
+
+ OpCase(i_select_val_yfI):
+     select_val = yb(Arg(0));
+     goto do_binary_search;
+     
+ OpCase(i_select_val_rfI):
+     select_val = r(0);
+     I--;
 
  do_binary_search:
  {
      struct Pairs {
-	 Eterm val;
-	 Eterm* addr;
+	 BeamInstr val;
+	 BeamInstr* addr;
      };
      struct Pairs* low;
      struct Pairs* high;
@@ -1837,48 +2237,98 @@ void process_main(void)
 	 unsigned int boffset = ((unsigned int)bdiff >> 1) & ~(sizeof(struct Pairs)-1);
 
 	 mid = (struct Pairs*)((char*)low + boffset);
-	 if (tmp_arg1 < mid->val) {
+	 if (select_val < mid->val) {
 	     high = mid;
-	 } else if (tmp_arg1 > mid->val) {
+	 } else if (select_val > mid->val) {
 	     low = mid + 1;
 	 } else {
 	     SET_I(mid->addr);
 	     Goto(*I);
 	 }
      }
-     SET_I((Eterm *) Arg(1));
+     SET_I((BeamInstr *) Arg(1));
      Goto(*I);
  }
+ }
 
- OpCase(i_jump_on_val_zero_sfI):
  {
-     Eterm index;
-
-     GetArg1(0, index);
-     if (is_small(index)) {
-	 index = signed_val(index);
-	 if (index < Arg(2)) {
-	     SET_I((Eterm *) (&Arg(3))[index]);
+     Eterm jump_on_val_zero_index;
+     
+ OpCase(i_jump_on_val_zero_yfI):
+     jump_on_val_zero_index = yb(Arg(0));
+     goto do_jump_on_val_zero_index;
+
+ OpCase(i_jump_on_val_zero_xfI):
+     jump_on_val_zero_index = xb(Arg(0));
+     goto do_jump_on_val_zero_index;
+
+ OpCase(i_jump_on_val_zero_rfI):
+     jump_on_val_zero_index = r(0);
+     I--;
+
+ do_jump_on_val_zero_index:
+     if (is_small(jump_on_val_zero_index)) {
+	 jump_on_val_zero_index = signed_val(jump_on_val_zero_index);
+	 if (jump_on_val_zero_index < Arg(2)) {
+	     SET_I((BeamInstr *) (&Arg(3))[jump_on_val_zero_index]);
 	     Goto(*I);
 	 }
      }
-     SET_I((Eterm *) Arg(1));
+     SET_I((BeamInstr *) Arg(1));
      Goto(*I);
  }
 
- OpCase(i_jump_on_val_sfII):
  {
-     Eterm index;
+     Eterm jump_on_val_index;
 
-     GetArg1(0, index);
-     if (is_small(index)) {
-	 index = (Uint) (signed_val(index) - Arg(3));
-	 if (index < Arg(2)) {
-	     SET_I((Eterm *) (&Arg(4))[index]);
+ 
+ OpCase(i_jump_on_val_yfII):
+     jump_on_val_index = yb(Arg(0));
+     goto do_jump_on_val_index;
+
+ OpCase(i_jump_on_val_xfII):
+     jump_on_val_index = xb(Arg(0));
+     goto do_jump_on_val_index;
+
+ OpCase(i_jump_on_val_rfII):
+     jump_on_val_index = r(0);
+     I--;
+
+ do_jump_on_val_index:
+     if (is_small(jump_on_val_index)) {
+	 jump_on_val_index = (Uint) (signed_val(jump_on_val_index) - Arg(3));
+	 if (jump_on_val_index < Arg(2)) {
+	     SET_I((BeamInstr *) (&Arg(4))[jump_on_val_index]);
 	     Goto(*I);
 	 }
      }
-     SET_I((Eterm *) Arg(1));
+     SET_I((BeamInstr *) Arg(1));
+     Goto(*I);
+ }
+
+ do_put_tuple: {
+     Eterm* hp = HTOP;
+
+     *hp++ = make_arityval(pt_arity);
+
+     do {
+	 Eterm term = *I++;
+	 switch (term & _TAG_IMMED1_MASK) {
+	 case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:
+	     *hp++ = r(0);
+	     break;
+	 case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:
+	     *hp++ = x(term >> _TAG_IMMED1_SIZE);
+	     break;
+	 case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:
+	     *hp++ = y(term >> _TAG_IMMED1_SIZE);
+	     break;
+	 default:
+	     *hp++ = term;
+	     break;
+	 }
+     } while (--pt_arity != 0);
+     HTOP = hp;
      Goto(*I);
  }
 
@@ -1898,24 +2348,25 @@ void process_main(void)
 
  OpCase(bif1_fbsd):
     {
-	Eterm (*bf)(Process*, Eterm);
-	Eterm arg;
+	Eterm (*bf)(Process*, Eterm*);
+	Eterm tmp_reg[1];
 	Eterm result;
 
-	GetArg1(2, arg);
+	GetArg1(2, tmp_reg[0]);
 	bf = (BifFunction) Arg(1);
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	result = (*bf)(c_p, arg);
+	result = (*bf)(c_p, tmp_reg);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
 	if (is_value(result)) {
 	    StoreBifResult(3, result);
 	}
-	SET_I((Eterm *) Arg(0));
+	SET_I((BeamInstr *) Arg(0));
 	Goto(*I);
     }
 
@@ -1925,25 +2376,26 @@ void process_main(void)
 
  OpCase(bif1_body_bsd):
     {
-	Eterm (*bf)(Process*, Eterm);
+	Eterm (*bf)(Process*, Eterm*);
 
-	Eterm arg;
+	Eterm tmp_reg[1];
 	Eterm result;
 
-	GetArg1(1, arg);
+	GetArg1(1, tmp_reg[0]);
 	bf = (BifFunction) Arg(0);
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	result = (*bf)(c_p, arg);
+	result = (*bf)(c_p, tmp_reg);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
 	if (is_value(result)) {
 	    StoreBifResult(2, result);
 	}
-	reg[0] = arg;
+	reg[0] = tmp_reg[0];
 	SWAPOUT;
 	I = handle_error(c_p, I, reg, bf);
 	goto post_error_handling;
@@ -1955,7 +2407,7 @@ void process_main(void)
 	GcBifFunction bf;
 	Eterm arg;
 	Eterm result;
-	Uint live = Arg(3);
+	Uint live = (Uint) Arg(3);
 
 	GetArg1(2, arg);
 	reg[0] = r(0);
@@ -1966,6 +2418,7 @@ void process_main(void)
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
 	result = (*bf)(c_p, reg, live);
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	SWAPIN;
@@ -1976,7 +2429,7 @@ void process_main(void)
 	    StoreBifResult(4, result);
 	}
 	if (Arg(0) != 0) {
-	    SET_I((Eterm *) Arg(0));
+	    SET_I((BeamInstr *) Arg(0));
 	    Goto(*I);
 	}
 	reg[0] = arg;
@@ -1984,27 +2437,106 @@ void process_main(void)
 	goto post_error_handling;
     }
 
+ OpCase(i_gc_bif2_jIId): /* Note, one less parameter than the i_gc_bif1
+			    and i_gc_bif3 */
+    {
+	typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
+	GcBifFunction bf;
+	Eterm result;
+	Uint live = (Uint) Arg(2);
+
+	reg[0] = r(0);
+	reg[live++] = tmp_arg1;
+	reg[live] = tmp_arg2;
+	bf = (GcBifFunction) Arg(1);
+	c_p->fcalls = FCALLS;
+	SWAPOUT;
+	PROCESS_MAIN_CHK_LOCKS(c_p);
+	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	result = (*bf)(c_p, reg, live);
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	PROCESS_MAIN_CHK_LOCKS(c_p);
+	SWAPIN;
+	r(0) = reg[0];
+	ERTS_HOLE_CHECK(c_p);
+	FCALLS = c_p->fcalls;
+	if (is_value(result)) {
+	    StoreBifResult(3, result);
+	}
+	if (Arg(0) != 0) {
+	    SET_I((BeamInstr *) Arg(0));
+	    Goto(*I);
+	}
+	reg[0] = tmp_arg1;
+	reg[1] = tmp_arg2;
+	I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
+	goto post_error_handling;
+    }
+
+ OpCase(i_gc_bif3_jIsId):
+    {
+	typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
+	GcBifFunction bf;
+	Eterm arg;
+	Eterm result;
+	Uint live = (Uint) Arg(3);
+
+	GetArg1(2, arg);
+	reg[0] = r(0);
+	reg[live++] = arg;
+	reg[live++] = tmp_arg1;
+	reg[live] = tmp_arg2;
+	bf = (GcBifFunction) Arg(1);
+	c_p->fcalls = FCALLS;
+	SWAPOUT;
+	PROCESS_MAIN_CHK_LOCKS(c_p);
+	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	result = (*bf)(c_p, reg, live);
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	PROCESS_MAIN_CHK_LOCKS(c_p);
+	SWAPIN;
+	r(0) = reg[0];
+	ERTS_HOLE_CHECK(c_p);
+	FCALLS = c_p->fcalls;
+	if (is_value(result)) {
+	    StoreBifResult(4, result);
+	}
+	if (Arg(0) != 0) {
+	    SET_I((BeamInstr *) Arg(0));
+	    Goto(*I);
+	}
+	reg[0] = arg;
+	reg[1] = tmp_arg1;
+	reg[2] = tmp_arg2;
+	I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
+	goto post_error_handling;
+    }
+
  /*
   * Guards bifs and, or, xor in guards.
   */
  OpCase(i_bif2_fbd):
     {
-	Eterm (*bf)(Process*, Eterm, Eterm);
+	Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2};
+	Eterm (*bf)(Process*, Eterm*);
 	Eterm result;
 
 	bf = (BifFunction) Arg(1);
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	result = (*bf)(c_p, tmp_arg1, tmp_arg2);
+	result = (*bf)(c_p, tmp_reg);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
 	if (is_value(result)) {
 	    StoreBifResult(2, result);
 	}
-	SET_I((Eterm *) Arg(0));
+	SET_I((BeamInstr *) Arg(0));
 	Goto(*I);
     }
 
@@ -2013,14 +2545,16 @@ void process_main(void)
   */
  OpCase(i_bif2_body_bd):
     {
-	Eterm (*bf)(Process*, Eterm, Eterm);
+	Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2};
+	Eterm (*bf)(Process*, Eterm*);
 	Eterm result;
 
 	bf = (BifFunction) Arg(0);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	result = (*bf)(c_p, tmp_arg1, tmp_arg2);
+	result = (*bf)(c_p, tmp_reg);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_HOLE_CHECK(c_p);
 	if (is_value(result)) {
@@ -2038,79 +2572,11 @@ void process_main(void)
      * The most general BIF call.  The BIF may build any amount of data
      * on the heap.  The result is always returned in r(0).
      */
- OpCase(call_bif0_e):
-    {
-	Eterm (*bf)(Process*, Uint*) = GET_BIF_ADDRESS(Arg(0));
-
-	PRE_BIF_SWAPOUT(c_p);
-	c_p->fcalls = FCALLS - 1;
-	if (FCALLS <= 0) {
-	    save_calls(c_p, (Export *) Arg(0));
-	}
-
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	r(0) = (*bf)(c_p, I);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(r(0)));
-	ERTS_HOLE_CHECK(c_p);
-	POST_BIF_GC_SWAPIN_0(c_p, r(0));
-	FCALLS = c_p->fcalls;
-	if (is_value(r(0))) {
-	    CHECK_TERM(r(0));
-	    Next(1);
-	}
-	else if (c_p->freason == TRAP) {
-	    goto call_bif_trap3;
-	}
-
-	/*
-	 * Error handling.  SWAPOUT is not needed because it was done above.
-	 */
-	ASSERT(c_p->stop == E);
-	reg[0] = r(0);
-	I = handle_error(c_p, I, reg, bf);
-	goto post_error_handling;
-    }
-
- OpCase(call_bif1_e):
-    {
-	Eterm (*bf)(Process*, Eterm, Uint*) = GET_BIF_ADDRESS(Arg(0));
-	Eterm result;
-	Eterm* next;
-
-	c_p->fcalls = FCALLS - 1;
-	if (FCALLS <= 0) {
-	    save_calls(c_p, (Export *) Arg(0));
-	}
-	PreFetch(1, next);
-	PRE_BIF_SWAPOUT(c_p);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	result = (*bf)(c_p, r(0), I);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
-	ERTS_HOLE_CHECK(c_p);
-	POST_BIF_GC_SWAPIN(c_p, result, reg, 1);
-	FCALLS = c_p->fcalls;
-        if (is_value(result)) {
-	    r(0) = result;
-	    CHECK_TERM(r(0));
-	    NextPF(1, next);
-	} else if (c_p->freason == TRAP) {
-	    goto call_bif_trap3;
-	}
-
-	/*
-	 * Error handling.  SWAPOUT is not needed because it was done above.
-	 */
-	ASSERT(c_p->stop == E);
-	reg[0] = r(0);
-	I = handle_error(c_p, I, reg, bf);
-	goto post_error_handling;
-    }
-
- OpCase(call_bif2_e):
+ OpCase(call_bif_e):
     {
-	Eterm (*bf)(Process*, Eterm, Eterm, Uint*) = GET_BIF_ADDRESS(Arg(0));
+	Eterm (*bf)(Process*, Eterm*, BeamInstr*) = GET_BIF_ADDRESS(Arg(0));
 	Eterm result;
-	Eterm* next;
+	BeamInstr *next;
 
 	PRE_BIF_SWAPOUT(c_p);
 	c_p->fcalls = FCALLS - 1;
@@ -2118,61 +2584,29 @@ void process_main(void)
 	   save_calls(c_p, (Export *) Arg(0));
 	}
 	PreFetch(1, next);
-	CHECK_TERM(r(0));
-	CHECK_TERM(x(1));
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	result = (*bf)(c_p, r(0), x(1), I);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
-	ERTS_HOLE_CHECK(c_p);
-	POST_BIF_GC_SWAPIN(c_p, result, reg, 2);
-	FCALLS = c_p->fcalls;
-	if (is_value(result)) {
-	    r(0) = result;
-	    CHECK_TERM(r(0));
-	    NextPF(1, next);
-	} else if (c_p->freason == TRAP) {
-	    goto call_bif_trap3;
-	}
-
-	/*
-	 * Error handling.  SWAPOUT is not needed because it was done above.
-	 */
-	ASSERT(c_p->stop == E);
 	reg[0] = r(0);
-	I = handle_error(c_p, I, reg, bf);
-	goto post_error_handling;
-    }
-
- OpCase(call_bif3_e):
-    {
-	Eterm (*bf)(Process*, Eterm, Eterm, Eterm, Uint*) = GET_BIF_ADDRESS(Arg(0));
-	Eterm result;
-	Eterm* next;
-
-	PRE_BIF_SWAPOUT(c_p);
-	c_p->fcalls = FCALLS - 1;
-	if (FCALLS <= 0) {
-	   save_calls(c_p, (Export *) Arg(0));
-	}
-	PreFetch(1, next);
-	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	result = (*bf)(c_p, r(0), x(1), x(2), I);
+	result = (*bf)(c_p, reg, I);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_HOLE_CHECK(c_p);
-	POST_BIF_GC_SWAPIN(c_p, result, reg, 3);
+	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	PROCESS_MAIN_CHK_LOCKS(c_p);
+	if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) {
+	    Uint arity = ((Export *)Arg(0))->code[2];
+	    result = erts_gc_after_bif_call(c_p, result, reg, arity);
+	    E = c_p->stop;
+	}
+	HTOP = HEAP_TOP(c_p);
 	FCALLS = c_p->fcalls;
 	if (is_value(result)) {
 	    r(0) = result;
 	    CHECK_TERM(r(0));
 	    NextPF(1, next);
 	} else if (c_p->freason == TRAP) {
-	call_bif_trap3:
 	    SET_CP(c_p, I+2);
-	    SET_I((Eterm *)c_p->def_arg_reg[3]);
+	    SET_I(c_p->i);
 	    SWAPIN;
-	    r(0) = c_p->def_arg_reg[0];
-	    x(1) = c_p->def_arg_reg[1];
-	    x(2) = c_p->def_arg_reg[2];
+	    r(0) = reg[0];
 	    Dispatch();
 	}
 
@@ -2180,7 +2614,6 @@ void process_main(void)
 	 * Error handling.  SWAPOUT is not needed because it was done above.
 	 */
 	ASSERT(c_p->stop == E);
-	reg[0] = r(0);
 	I = handle_error(c_p, I, reg, bf);
 	goto post_error_handling;
     }
@@ -2276,7 +2709,8 @@ void process_main(void)
  lb_Cl_error: {
      if (Arg(0) != 0) {
 	 OpCase(jump_f): {
-	     SET_I((Eterm *) Arg(0));
+	 jump_f:
+	     SET_I((BeamInstr *) Arg(0));
 	     Goto(*I);
 	 }
      }
@@ -2444,23 +2878,25 @@ void process_main(void)
 
  OpCase(i_int_bnot_jsId):
  {
-     GetArg1(1, tmp_arg1);
-     if (is_small(tmp_arg1)) {
-	 tmp_arg1 = make_small(~signed_val(tmp_arg1));
+     Eterm bnot_val;
+
+     GetArg1(1, bnot_val);
+     if (is_small(bnot_val)) {
+	 bnot_val = make_small(~signed_val(bnot_val));
      } else {
 	 Uint live = Arg(2);
 	 SWAPOUT;
 	 reg[0] = r(0);
-	 reg[live] = tmp_arg1;
-	 tmp_arg1 = erts_gc_bnot(c_p, reg, live);
+	 reg[live] = bnot_val;
+	 bnot_val = erts_gc_bnot(c_p, reg, live);
 	 r(0) = reg[0];
 	 SWAPIN;
 	 ERTS_HOLE_CHECK(c_p);
-	 if (is_nil(tmp_arg1)) {
+	 if (is_nil(bnot_val)) {
 	     goto lb_Cl_error;
 	 }
      }
-     StoreBifResult(3, tmp_arg1);
+     StoreBifResult(3, bnot_val);
  }
 
  badarith:
@@ -2468,7 +2904,7 @@ void process_main(void)
     goto lb_Cl_error;
 
  OpCase(i_apply): {
-     Eterm* next;
+     BeamInstr *next;
      SWAPOUT;
      next = apply(c_p, r(0), x(1), x(2), reg);
      SWAPIN;
@@ -2483,13 +2919,13 @@ void process_main(void)
  }
 
  OpCase(i_apply_last_P): {
-     Eterm* next;
+     BeamInstr *next;
      SWAPOUT;
      next = apply(c_p, r(0), x(1), x(2), reg);
      SWAPIN;
      if (next != NULL) {
 	 r(0) = reg[0];
-	 SET_CP(c_p, (Eterm *) E[0]);
+	 SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
 	 E = ADD_BYTE_OFFSET(E, Arg(0));
 	 SET_I(next);
 	 Dispatch();
@@ -2499,7 +2935,7 @@ void process_main(void)
  }
 
  OpCase(i_apply_only): {
-     Eterm* next;
+     BeamInstr *next;
      SWAPOUT;
      next = apply(c_p, r(0), x(1), x(2), reg);
      SWAPIN;
@@ -2513,7 +2949,7 @@ void process_main(void)
  }
 
  OpCase(apply_I): {
-     Eterm* next;
+     BeamInstr *next;
 
      reg[0] = r(0);
      SWAPOUT;
@@ -2530,7 +2966,7 @@ void process_main(void)
  }
 
  OpCase(apply_last_IP): {
-     Eterm* next;
+     BeamInstr *next;
 
      reg[0] = r(0);
      SWAPOUT;
@@ -2538,7 +2974,7 @@ void process_main(void)
      SWAPIN;
      if (next != NULL) {
 	 r(0) = reg[0];
-	 SET_CP(c_p, (Eterm *) E[0]);
+	 SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
 	 E = ADD_BYTE_OFFSET(E, Arg(1));
 	 SET_I(next);
 	 Dispatch();
@@ -2548,7 +2984,7 @@ void process_main(void)
  }
 
  OpCase(i_apply_fun): {
-     Eterm* next;
+     BeamInstr *next;
 
      SWAPOUT;
      next = apply_fun(c_p, r(0), x(1), reg);
@@ -2563,14 +2999,14 @@ void process_main(void)
  }
 
  OpCase(i_apply_fun_last_P): {
-     Eterm* next;
+     BeamInstr *next;
 
      SWAPOUT;
      next = apply_fun(c_p, r(0), x(1), reg);
      SWAPIN;
      if (next != NULL) {
 	 r(0) = reg[0];
-	 SET_CP(c_p, (Eterm *) E[0]);
+	 SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
 	 E = ADD_BYTE_OFFSET(E, Arg(0));
 	 SET_I(next);
 	 Dispatchfun();
@@ -2579,7 +3015,7 @@ void process_main(void)
  }
 
  OpCase(i_apply_fun_only): {
-     Eterm* next;
+     BeamInstr *next;
 
      SWAPOUT;
      next = apply_fun(c_p, r(0), x(1), reg);
@@ -2593,10 +3029,11 @@ void process_main(void)
  }
 
  OpCase(i_call_fun_I): {
-     Eterm* next;
+     BeamInstr *next;
 
      SWAPOUT;
      reg[0] = r(0);
+
      next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE);
      SWAPIN;
      if (next != NULL) {
@@ -2609,7 +3046,7 @@ void process_main(void)
  }
 
  OpCase(i_call_fun_last_IP): {
-     Eterm* next;
+     BeamInstr *next;
 
      SWAPOUT;
      reg[0] = r(0);
@@ -2617,7 +3054,7 @@ void process_main(void)
      SWAPIN;
      if (next != NULL) {
 	r(0) = reg[0];
-	SET_CP(c_p, (Eterm *) E[0]);
+	SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
 	E = ADD_BYTE_OFFSET(E, Arg(1));
 	SET_I(next);
 	Dispatchfun();
@@ -2714,123 +3151,10 @@ void process_main(void)
      goto do_schedule1;
  }
 
- OpCase(i_select_tuple_arity_sfI):
- {
-     GetArg1(0, tmp_arg1);
-
-     if (is_tuple(tmp_arg1)) {
-	 tmp_arg1 = *tuple_val(tmp_arg1);
-	 goto do_binary_search;
-     }
-     SET_I((Eterm *) Arg(1));
-     Goto(*I);
- }     
-
- OpCase(i_select_big_sf):
-    {
-	Eterm* bigp;
-	Uint arity;
-	Eterm* given;
-	Uint given_arity;
-	Uint given_size;
-
-	GetArg1(0, tmp_arg1);
-	if (is_big(tmp_arg1)) {
-
-	    /*
-	     * The loader has sorted the bignumbers in descending order
-	     * on the arity word.  Therefore, we know that the search
-	     * has failed as soon as we encounter an arity word less than
-	     * the arity word of the given number.  There is a zero word
-	     * (less than any valid arity word) stored after the last bignumber.
-	     */
-
- 	    given = big_val(tmp_arg1);
-	    given_arity = given[0];
-	    given_size = thing_arityval(given_arity);
-	    bigp = &Arg(2);
-	    while ((arity = bigp[0]) > given_arity) {
-		bigp += thing_arityval(arity) + 2;
-	    }
-	    while (bigp[0] == given_arity) {
-		if (memcmp(bigp+1, given+1, sizeof(Eterm)*given_size) == 0) {
-		    SET_I((Eterm *) bigp[given_size+1]);
-		    Goto(*I);
-		}
-		bigp += thing_arityval(arity) + 2;
-	    }
-	}
-
-	/*
-	 * Failed.
-	 */
-
-	SET_I((Eterm *) Arg(1));
-	Goto(*I);
-    }
-
-#ifdef ARCH_64
- OpCase(i_select_float_sfI):
- {
-     Uint f;
-     int n;
-     struct ValLabel {
-	 Uint f;
-	 Eterm* addr;
-     };
-     struct ValLabel* ptr;
-
-     GetArg1(0, tmp_arg1);
-     ASSERT(is_float(tmp_arg1));
-     f = float_val(tmp_arg1)[1];
-     n = Arg(2);
-     ptr = (struct ValLabel *) &Arg(3);
-     while (n-- > 0) {
-	 if (ptr->f == f) {
-	     SET_I(ptr->addr);
-	     Goto(*I);
-	 }
-	 ptr++;
-     }
-     SET_I((Eterm *) Arg(1));
-     Goto(*I);
- }
-#else
- OpCase(i_select_float_sfI):
- {
-     Uint fpart1;
-     Uint fpart2;
-     int n;
-     struct ValLabel {
-	 Uint fpart1;
-	 Uint fpart2;
-	 Eterm* addr;
-     };
-     struct ValLabel* ptr;
-
-     GetArg1(0, tmp_arg1);
-     ASSERT(is_float(tmp_arg1));
-     fpart1 = float_val(tmp_arg1)[1];
-     fpart2 = float_val(tmp_arg1)[2];
-
-     n = Arg(2);
-     ptr = (struct ValLabel *) &Arg(3);
-     while (n-- > 0) {
-	 if (ptr->fpart1 == fpart1 && ptr->fpart2 == fpart2) {
-	     SET_I(ptr->addr);
-	     Goto(*I);
-	 }
-	 ptr++;
-     }
-     SET_I((Eterm *) Arg(1));
-     Goto(*I);
- }
-#endif
-
  OpCase(set_tuple_element_sdP): {
      Eterm element;
      Eterm tuple;
-     Eterm* next;
+     BeamInstr *next;
      Eterm* p;
      
      PreFetch(3, next);
@@ -2872,15 +3196,17 @@ void process_main(void)
 	the first argument. We also handle atom tags in the first
 	argument for backwards compatibility.
      */
-     GetArg2(0, tmp_arg1, tmp_arg2);
-     c_p->fvalue = tmp_arg2;
+     Eterm raise_val1;
+     Eterm raise_val2;
+     GetArg2(0, raise_val1, raise_val2);
+     c_p->fvalue = raise_val2;
      if (c_p->freason == EXC_NULL) {
        /* a safety check for the R10-0 case; should not happen */
        c_p->ftrace = NIL;
        c_p->freason = EXC_ERROR;
      }
      /* for R10-0 code, keep existing c_p->ftrace and hope it's correct */
-     switch (tmp_arg1) {
+     switch (raise_val1) {
      case am_throw:
        c_p->freason = EXC_THROWN & ~EXF_SAVETRACE;
        break;
@@ -2896,8 +3222,8 @@ void process_main(void)
 	   passed from a user! Currently only expecting generated calls.
 	*/
 	 struct StackTrace *s;
-	 c_p->ftrace = tmp_arg1;
-	 s = get_trace_from_exc(tmp_arg1);
+	 c_p->ftrace = raise_val1;
+	 s = get_trace_from_exc(raise_val1);
 	 if (s == NULL) {
 	   c_p->freason = EXC_ERROR;
 	 } else {
@@ -2908,11 +3234,24 @@ void process_main(void)
      goto find_func_info;
  }
 
- OpCase(badmatch_s): {
-     GetArg1(0, tmp_arg1);
-     c_p->fvalue = tmp_arg1;
-     c_p->freason = BADMATCH;
- }
+    {
+	Eterm badmatch_val;
+
+    OpCase(badmatch_y):
+	badmatch_val = yb(Arg(0));
+	goto do_badmatch;
+
+    OpCase(badmatch_x):
+	badmatch_val = xb(Arg(0));
+	goto do_badmatch;
+
+    OpCase(badmatch_r):
+	badmatch_val = r(0);
+
+    do_badmatch:
+	c_p->fvalue = badmatch_val;
+	c_p->freason = BADMATCH;
+    }
  /* Fall through here */
 
  find_func_info: {
@@ -2935,17 +3274,16 @@ void process_main(void)
      */
     SWAPOUT;
     reg[0] = r(0);
-    tmp_arg1 = call_error_handler(c_p, I-3, reg);
+    I = call_error_handler(c_p, I-3, reg, am_undefined_function);
     r(0) = reg[0];
     SWAPIN;
-    if (tmp_arg1) {
-	SET_I(c_p->i);
-	Dispatch();
+    if (I) {
+	Goto(*I);
     }
 
  /* Fall through */
  OpCase(error_action_code): {
- no_error_handler:
+    handle_error:
      reg[0] = r(0);
      SWAPOUT;
      I = handle_error(c_p, NULL, reg, NULL);
@@ -2963,128 +3301,117 @@ void process_main(void)
      }
  }
 
- OpCase(call_nif):
-     {
-          /*
-	   * call_nif is always first instruction in function:
-	   *
-	   * I[-3]: Module
-	   * I[-2]: Function
-	   * I[-1]: Arity
-	   * I[0]: &&call_nif
-	   * I[1]: Function pointer to NIF function
-	   * I[2]: Pointer to erl_module_nif
-	   */
-     	 BifFunction vbf;
-
-	 c_p->current = I-3; /* current and vbf set to please handle_error */ 
-	 SWAPOUT;
-	 c_p->fcalls = FCALLS - 1;
-	 PROCESS_MAIN_CHK_LOCKS(c_p);
-	 tmp_arg2 = I[-1];
-	 ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
-
-	 ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	 {
-	     typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
-	     NifF* fp = vbf = (NifF*) I[1];
-	     struct enif_environment_t env;
-	     erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]);
-	     reg[0] = r(0);
-	     tmp_arg1 = (*fp)(&env, tmp_arg2, reg);
-	     erts_post_nif(&env);
-	 }
-	 ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1));
-	 PROCESS_MAIN_CHK_LOCKS(c_p);
-	 goto apply_bif_or_nif_epilogue;
-	 
- OpCase(apply_bif):
-	/*
-	 * At this point, I points to the code[3] in the export entry for
-	 * the BIF:
-	 *
-	 * code[0]: Module
-	 * code[1]: Function
-	 * code[2]: Arity
-	 * code[3]: &&apply_bif
-	 * code[4]: Function pointer to BIF function
-	 */
+    {
+	Eterm nif_bif_result;
+	Eterm bif_nif_arity;
 
-	c_p->current = I-3;	/* In case we apply process_info/1,2 or load_nif/1 */
-	c_p->i = I;		/* In case we apply check_process_code/2. */
-	c_p->arity = 0;		/* To allow garbage collection on ourselves
-				 * (check_process_code/2).
-				 */
-	SWAPOUT;
-	c_p->fcalls = FCALLS - 1;
-	vbf = (BifFunction) Arg(0);
-	PROCESS_MAIN_CHK_LOCKS(c_p);
-	tmp_arg2 = I[-1];
-	ASSERT(tmp_arg2 <= 3);
-	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
-	switch (tmp_arg2) {
-	case 3:
+    OpCase(call_nif):
+	{
+	    /*
+	     * call_nif is always first instruction in function:
+	     *
+	     * I[-3]: Module
+	     * I[-2]: Function
+	     * I[-1]: Arity
+	     * I[0]: &&call_nif
+	     * I[1]: Function pointer to NIF function
+	     * I[2]: Pointer to erl_module_nif
+	     */
+	    BifFunction vbf;
+
+	    DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
+	    c_p->current = I-3; /* current and vbf set to please handle_error */ 
+	    SWAPOUT;
+	    c_p->fcalls = FCALLS - 1;
+	    PROCESS_MAIN_CHK_LOCKS(c_p);
+	    bif_nif_arity = I[-1];
+	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+
+	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	    {
-		Eterm (*bf)(Process*, Eterm, Eterm, Eterm, Uint*) = vbf;
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-		tmp_arg1 = (*bf)(c_p, r(0), x(1), x(2), I);
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1));
-		PROCESS_MAIN_CHK_LOCKS(c_p);
+		typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
+		NifF* fp = vbf = (NifF*) I[1];
+		struct enif_environment_t env;
+		erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]);
+		reg[0] = r(0);
+		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
+		erts_post_nif(&env);
 	    }
-	    break;
-	case 2:
+	    ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result));
+	    PROCESS_MAIN_CHK_LOCKS(c_p);
+	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+
+	    DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
+	    goto apply_bif_or_nif_epilogue;
+	 
+	OpCase(apply_bif):
+	    /*
+	     * At this point, I points to the code[3] in the export entry for
+	     * the BIF:
+	     *
+	     * code[0]: Module
+	     * code[1]: Function
+	     * code[2]: Arity
+	     * code[3]: &&apply_bif
+	     * code[4]: Function pointer to BIF function
+	     */
+
+	    c_p->current = I-3;	/* In case we apply process_info/1,2 or load_nif/1 */
+	    c_p->i = I;		/* In case we apply check_process_code/2. */
+	    c_p->arity = 0;		/* To allow garbage collection on ourselves
+					 * (check_process_code/2).
+					 */
+	    DTRACE_BIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
+
+	    SWAPOUT;
+	    c_p->fcalls = FCALLS - 1;
+	    vbf = (BifFunction) Arg(0);
+	    PROCESS_MAIN_CHK_LOCKS(c_p);
+	    bif_nif_arity = I[-1];
+	    ASSERT(bif_nif_arity <= 3);
+	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+	    reg[0] = r(0);
 	    {
-		Eterm (*bf)(Process*, Eterm, Eterm, Uint*) = vbf;
+		Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf;
 		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-		tmp_arg1 = (*bf)(c_p, r(0), x(1), I);
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1));
+		nif_bif_result = (*bf)(c_p, reg, I);
+		ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
+		       is_non_value(nif_bif_result));
+		ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 		PROCESS_MAIN_CHK_LOCKS(c_p);
 	    }
-	    break;
-	case 1:
-	    {
-		Eterm (*bf)(Process*, Eterm, Uint*) = vbf;
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-		tmp_arg1 = (*bf)(c_p, r(0), I);
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1));
-		PROCESS_MAIN_CHK_LOCKS(c_p);
+
+	    DTRACE_BIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
+
+	apply_bif_or_nif_epilogue:
+	    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	    ERTS_HOLE_CHECK(c_p);
+	    if (c_p->mbuf) {
+		nif_bif_result = erts_gc_after_bif_call(c_p, nif_bif_result,
+						  reg, bif_nif_arity);
 	    }
-	    break;
-	case 0:
-	    {
-		Eterm (*bf)(Process*, Uint*) = vbf;
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-		tmp_arg1 = (*bf)(c_p, I);
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1));
-		PROCESS_MAIN_CHK_LOCKS(c_p);
-		break;
+	    SWAPIN;  /* There might have been a garbage collection. */
+	    FCALLS = c_p->fcalls;
+	    if (is_value(nif_bif_result)) {
+		r(0) = nif_bif_result;
+		CHECK_TERM(r(0));
+		SET_I(c_p->cp);
+		c_p->cp = 0;
+		Goto(*I);
+	    } else if (c_p->freason == TRAP) {
+		SET_I(c_p->i);
+		r(0) = reg[0];
+		if (c_p->flags & F_HIBERNATE_SCHED) {
+		    c_p->flags &= ~F_HIBERNATE_SCHED;
+		    goto do_schedule;
+		}
+		Dispatch();
 	    }
+	    I = handle_error(c_p, c_p->cp, reg, vbf);
+	    goto post_error_handling;
 	}
-apply_bif_or_nif_epilogue:
-	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
-	ERTS_HOLE_CHECK(c_p);
-	if (c_p->mbuf) {
-	    reg[0] = r(0);
-	    tmp_arg1 = erts_gc_after_bif_call(c_p, tmp_arg1, reg, tmp_arg2);
-	    r(0) = reg[0];
-	}
-	SWAPIN;			/* There might have been a garbage collection. */
-	FCALLS = c_p->fcalls;
-	if (is_value(tmp_arg1)) {
-	    r(0) = tmp_arg1;
-	    CHECK_TERM(r(0));
-	    SET_I(c_p->cp);
-	    Goto(*I);
-	} else if (c_p->freason == TRAP) {
-	    SET_I((Eterm *)c_p->def_arg_reg[3]);
-	    r(0) = c_p->def_arg_reg[0];
-	    x(1) = c_p->def_arg_reg[1];
-	    x(2) = c_p->def_arg_reg[2];
-	    Dispatch();
-	}
-	reg[0] = r(0);
-	I = handle_error(c_p, c_p->cp, reg, vbf);
-	goto post_error_handling;
     }
 
  OpCase(i_get_sd):
@@ -3097,17 +3424,25 @@ apply_bif_or_nif_epilogue:
 	StoreBifResult(1, result);
     }
 
- OpCase(i_put_tuple_only_Ad): {
-     tmp_arg1 = make_tuple(HTOP);
-     *HTOP++ = Arg(0);
-     StoreBifResult(1, tmp_arg1);
- }
+    {
+	Eterm case_end_val;
 
- OpCase(case_end_s):
-    GetArg1(0, tmp_arg1);
-    c_p->fvalue = tmp_arg1;
-    c_p->freason = EXC_CASE_CLAUSE;
-    goto find_func_info;
+    OpCase(case_end_x):
+	case_end_val = xb(Arg(0));
+	goto do_case_end;
+
+    OpCase(case_end_y):
+	case_end_val = yb(Arg(0));
+	goto do_case_end;
+
+    OpCase(case_end_r):
+	case_end_val = r(0);
+
+    do_case_end:
+	c_p->fvalue = case_end_val;
+	c_p->freason = EXC_CASE_CLAUSE;
+	goto find_func_info;
+    }
 
  OpCase(if_end):
     c_p->freason = EXC_IF_CLAUSE;
@@ -3116,14 +3451,17 @@ apply_bif_or_nif_epilogue:
  OpCase(i_func_info_IaaI): {
      c_p->freason = EXC_FUNCTION_CLAUSE;
      c_p->current = I + 2;
-     goto lb_error_action_code;
+     goto handle_error;
  }
 
  OpCase(try_case_end_s):
-    GetArg1(0, tmp_arg1);
-    c_p->fvalue = tmp_arg1;
-    c_p->freason = EXC_TRY_CLAUSE;
-    goto find_func_info;
+    {
+	Eterm try_case_end_val;
+	GetArg1(0, try_case_end_val);
+	c_p->fvalue = try_case_end_val;
+	c_p->freason = EXC_TRY_CLAUSE;
+	goto find_func_info;
+    }
 
  /*
   * Construction of binaries using new instructions.
@@ -3202,7 +3540,7 @@ apply_bif_or_nif_epilogue:
       * Operands: NotUsed Live Dst
       */
  do_bs_init_bits_known:
-     num_bytes = (num_bits+7) >> 3;
+     num_bytes = ((Uint64)num_bits+(Uint64)7) >> 3;
      if (num_bits & 7) {
 	 alloc += ERL_SUB_BIN_SIZE;
      }
@@ -3270,12 +3608,12 @@ apply_bif_or_nif_epilogue:
 	 HTOP += PROC_BIN_SIZE;
 	 pb->thing_word = HEADER_PROC_BIN;
 	 pb->size = num_bytes;
-	 pb->next = MSO(c_p).mso;
-	 MSO(c_p).mso = pb;
+	 pb->next = MSO(c_p).first;
+	 MSO(c_p).first = (struct erl_off_heap_header*) pb;
 	 pb->val = bptr;
 	 pb->bytes = (byte*) bptr->orig_bytes;
 	 pb->flags = 0;
-	 MSO(c_p).overhead += pb->size / sizeof(Eterm);
+	 OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm));
 	 new_binary = make_binary(pb);
 	 goto do_bits_sub_bin;
      }
@@ -3371,13 +3709,13 @@ apply_bif_or_nif_epilogue:
 	 HTOP += PROC_BIN_SIZE;
 	 pb->thing_word = HEADER_PROC_BIN;
 	 pb->size = tmp_arg1;
-	 pb->next = MSO(c_p).mso;
-	 MSO(c_p).mso = pb;
+	 pb->next = MSO(c_p).first;
+	 MSO(c_p).first = (struct erl_off_heap_header*) pb;
 	 pb->val = bptr;
 	 pb->bytes = (byte*) bptr->orig_bytes;
 	 pb->flags = 0;
 	 
-	 MSO(c_p).overhead += tmp_arg1 / sizeof(Eterm);
+	 OH_OVERHEAD(&(MSO(c_p)), tmp_arg1 / sizeof(Eterm));
 
 	 StoreBifResult(2, make_binary(pb));
      }
@@ -3413,42 +3751,6 @@ apply_bif_or_nif_epilogue:
 	 }
  }
 
- OpCase(i_bs_bits_to_bytes_rjd): {
-     tmp_arg1 = r(0);
-     goto do_bits_to_bytes;
- }
-
- OpCase(i_bs_bits_to_bytes_yjd): {
-     tmp_arg1 = yb(Arg(0));
-     I++;
-     goto do_bits_to_bytes;
-
- OpCase(i_bs_bits_to_bytes_xjd): {
-     tmp_arg1 = xb(Arg(0));
-     I++;
- }
-
- do_bits_to_bytes:
-     {
-	 if (is_valid_bit_size(tmp_arg1)) {
-	     tmp_arg1 = make_small(unsigned_val(tmp_arg1) >> 3);
-	 } else {
-	     Uint bytes;
-	     if (!term_to_Uint(tmp_arg1, &bytes)) {
-		 goto badarg;
-	     }
-	     tmp_arg1 = bytes;
-	     if ((tmp_arg1 & 0x07) != 0) {
-		 goto badarg;
-	     }
-	     SWAPOUT;
-	     tmp_arg1 = erts_make_integer(tmp_arg1 >> 3, c_p);
-	     HTOP = HEAP_TOP(c_p);
-	 }
-	 StoreBifResult(1, tmp_arg1);
-     }
- }
-
  OpCase(i_bs_add_jId): {
      Uint Unit = Arg(1);
      if (is_both_small(tmp_arg1, tmp_arg2)) {
@@ -3486,7 +3788,7 @@ apply_bif_or_nif_epilogue:
 
 	 /*
 	  * Now we know that one of the arguments is
-	  * not at small. We must convert both arguments
+	  * not a small. We must convert both arguments
 	  * to Uints and check for errors at the same time.
 	  *
 	  * Error checking is tricky.
@@ -3534,7 +3836,7 @@ apply_bif_or_nif_epilogue:
 
  OpCase(bs_put_string_II):
     {
-	Eterm* next;
+	BeamInstr *next;
 	PreFetch(2, next);
 	erts_new_bs_put_string(ERL_BITS_ARGS_2((byte *) Arg(1), Arg(0)));
 	NextPF(2, next);
@@ -3669,8 +3971,7 @@ apply_bif_or_nif_epilogue:
       * too big numbers).
       */
      if (is_not_small(val) || val > make_small(0x10FFFFUL) ||
-	 (make_small(0xD800UL) <= val && val <= make_small(0xDFFFUL)) ||
-	 val == make_small(0xFFFEUL) || val == make_small(0xFFFFUL)) {
+	 (make_small(0xD800UL) <= val && val <= make_small(0xDFFFUL))) {
 	 goto badarg;
      }
      Next(2);
@@ -3689,8 +3990,8 @@ apply_bif_or_nif_epilogue:
       * the valid range).
       */
      if (is_not_small(tmp_arg1) || tmp_arg1 > make_small(0x10FFFFUL) ||
-	 (make_small(0xD800UL) <= tmp_arg1 && tmp_arg1 <= make_small(0xDFFFUL)) ||
-	 tmp_arg1 == make_small(0xFFFEUL) || tmp_arg1 == make_small(0xFFFFUL)) {
+	 (make_small(0xD800UL) <= tmp_arg1 &&
+	  tmp_arg1 <= make_small(0xDFFFUL))) {
 	 ErlBinMatchBuffer *mb = ms_matchbuffer(tmp_arg2);
 
 	 mb->offset -= 32;
@@ -3705,21 +4006,22 @@ apply_bif_or_nif_epilogue:
 
  {
      Eterm header;
-     Eterm* next;
+     BeamInstr *next;
      Uint slots;
+     Eterm context;
 
      OpCase(i_bs_start_match2_rfIId): {
-	 tmp_arg1 = r(0);
+	 context = r(0);
 
      do_start_match:
 	 slots = Arg(2);
-	 if (!is_boxed(tmp_arg1)) {
+	 if (!is_boxed(context)) {
 	     ClauseFail();
 	 }
 	 PreFetch(4, next);
-	 header = *boxed_val(tmp_arg1);
+	 header = *boxed_val(context);
 	 if (header_is_bin_matchstate(header)) {
-	     ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(tmp_arg1);
+	     ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(context);
 	     Uint actual_slots = HEADER_NUM_SLOTS(header);
 	     ms->save_offset[0] = ms->mb.offset;
 	     if (actual_slots < slots) {
@@ -3727,8 +4029,8 @@ apply_bif_or_nif_epilogue:
 		 Uint live = Arg(1);
 		 Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots);
 
-		 TestHeapPreserve(wordsneeded, live, tmp_arg1);
-		 ms = (ErlBinMatchState *) boxed_val(tmp_arg1);
+		 TestHeapPreserve(wordsneeded, live, context);
+		 ms = (ErlBinMatchState *) boxed_val(context);
 		 dst = (ErlBinMatchState *) HTOP;
 		 *dst = *ms;
 		 *HTOP = HEADER_BIN_MATCHSTATE(slots);
@@ -3740,12 +4042,12 @@ apply_bif_or_nif_epilogue:
 	     Eterm result;
 	     Uint live = Arg(1);
 	     Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots);
-	     TestHeapPreserve(wordsneeded, live, tmp_arg1);
+	     TestHeapPreserve(wordsneeded, live, context);
 	     HEAP_TOP(c_p) = HTOP;
 #ifdef DEBUG
 	     c_p->stop = E;	/* Needed for checking in HeapOnlyAlloc(). */
 #endif
-	     result = erts_bs_start_match_2(c_p, tmp_arg1, slots);
+	     result = erts_bs_start_match_2(c_p, context, slots);
 	     HTOP = HEAP_TOP(c_p);
 	     HEAP_SPACE_VERIFIED(0);
 	     if (is_non_value(result)) {
@@ -3759,19 +4061,19 @@ apply_bif_or_nif_epilogue:
 	 NextPF(4, next);
      }
      OpCase(i_bs_start_match2_xfIId): {
-	 tmp_arg1 = xb(Arg(0));
+	 context = xb(Arg(0));
 	 I++;
 	 goto do_start_match;
      }
      OpCase(i_bs_start_match2_yfIId): {
-	 tmp_arg1 = yb(Arg(0));
+	 context = yb(Arg(0));
 	 I++;
 	 goto do_start_match;
      }
  }
 
  OpCase(bs_test_zero_tail2_fr): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchBuffer *_mb;
      
      PreFetch(1, next);
@@ -3783,7 +4085,7 @@ apply_bif_or_nif_epilogue:
  }
 
  OpCase(bs_test_zero_tail2_fx): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchBuffer *_mb;
      
      PreFetch(2, next);
@@ -3795,7 +4097,7 @@ apply_bif_or_nif_epilogue:
  }
 
  OpCase(bs_test_tail_imm2_frI): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchBuffer *_mb;
      PreFetch(2, next);
      _mb = ms_matchbuffer(r(0));
@@ -3805,7 +4107,7 @@ apply_bif_or_nif_epilogue:
      NextPF(2, next);
  }
  OpCase(bs_test_tail_imm2_fxI): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchBuffer *_mb;
      PreFetch(3, next);
      _mb = ms_matchbuffer(xb(Arg(1)));
@@ -3816,7 +4118,7 @@ apply_bif_or_nif_epilogue:
  }
 
  OpCase(bs_test_unit_frI): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchBuffer *_mb;
      PreFetch(2, next);
      _mb = ms_matchbuffer(r(0));
@@ -3826,7 +4128,7 @@ apply_bif_or_nif_epilogue:
      NextPF(2, next);
  }
  OpCase(bs_test_unit_fxI): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchBuffer *_mb;
      PreFetch(3, next);
      _mb = ms_matchbuffer(xb(Arg(1)));
@@ -3837,7 +4139,7 @@ apply_bif_or_nif_epilogue:
  }
 
  OpCase(bs_test_unit8_fr): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchBuffer *_mb;
      PreFetch(1, next);
      _mb = ms_matchbuffer(r(0));
@@ -3847,7 +4149,7 @@ apply_bif_or_nif_epilogue:
      NextPF(1, next);
  }
  OpCase(bs_test_unit8_fx): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchBuffer *_mb;
      PreFetch(2, next);
      _mb = ms_matchbuffer(xb(Arg(1)));
@@ -3857,93 +4159,105 @@ apply_bif_or_nif_epilogue:
      NextPF(2, next);
  }
 
+ {
+     Eterm bs_get_integer8_context;
+
  OpCase(i_bs_get_integer_8_rfd): {
-     tmp_arg1 = r(0);
-     goto do_bs_get_integer_8;
- }
+	 bs_get_integer8_context = r(0);
+	 goto do_bs_get_integer_8;
+     }
 
  OpCase(i_bs_get_integer_8_xfd): {
-     tmp_arg1 = xb(Arg(0));
-     I++;
- }
+	 bs_get_integer8_context = xb(Arg(0));
+	 I++;
+     }
 
  do_bs_get_integer_8: {
-     ErlBinMatchBuffer *_mb;
-     Eterm _result;
-     _mb = ms_matchbuffer(tmp_arg1);
-     if (_mb->size - _mb->offset < 8) {
-	 ClauseFail();
-     }
-     if (BIT_OFFSET(_mb->offset) != 0) {
-	 _result = erts_bs_get_integer_2(c_p, 8, 0, _mb);
-     } else {
-	 _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]);
-	 _mb->offset += 8;
+	 ErlBinMatchBuffer *_mb;
+	 Eterm _result;
+	 _mb = ms_matchbuffer(bs_get_integer8_context);
+	 if (_mb->size - _mb->offset < 8) {
+	     ClauseFail();
+	 }
+	 if (BIT_OFFSET(_mb->offset) != 0) {
+	     _result = erts_bs_get_integer_2(c_p, 8, 0, _mb);
+	 } else {
+	     _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]);
+	     _mb->offset += 8;
+	 }
+	 StoreBifResult(1, _result);
      }
-     StoreBifResult(1, _result);
  }
 
- OpCase(i_bs_get_integer_16_rfd): {
-     tmp_arg1 = r(0);
+ {
+     Eterm bs_get_integer_16_context;
+
+ OpCase(i_bs_get_integer_16_rfd):
+     bs_get_integer_16_context = r(0);
      goto do_bs_get_integer_16;
- }
 
- OpCase(i_bs_get_integer_16_xfd): {
-     tmp_arg1 = xb(Arg(0));
+ OpCase(i_bs_get_integer_16_xfd):
+     bs_get_integer_16_context = xb(Arg(0));
      I++;
- }
 
- do_bs_get_integer_16: {
-     ErlBinMatchBuffer *_mb;
-     Eterm _result;
-     _mb = ms_matchbuffer(tmp_arg1);
-     if (_mb->size - _mb->offset < 16) {
-	 ClauseFail();
-     }
-     if (BIT_OFFSET(_mb->offset) != 0) {
-	 _result = erts_bs_get_integer_2(c_p, 16, 0, _mb);
-     } else {
-	 _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset)));
-	 _mb->offset += 16;
+ do_bs_get_integer_16:
+     {
+	 ErlBinMatchBuffer *_mb;
+	 Eterm _result;
+	 _mb = ms_matchbuffer(bs_get_integer_16_context);
+	 if (_mb->size - _mb->offset < 16) {
+	     ClauseFail(); 
+	 }
+	 if (BIT_OFFSET(_mb->offset) != 0) {
+	     _result = erts_bs_get_integer_2(c_p, 16, 0, _mb);
+	 } else {
+	     _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset)));
+	     _mb->offset += 16;
+	 }
+	 StoreBifResult(1, _result);
      }
-     StoreBifResult(1, _result);
  }
 
- OpCase(i_bs_get_integer_32_rfId): {
-     tmp_arg1 = r(0);
+ {
+     Eterm bs_get_integer_32_context;
+
+ OpCase(i_bs_get_integer_32_rfId):
+     bs_get_integer_32_context = r(0);
      goto do_bs_get_integer_32;
- }
+
      
- OpCase(i_bs_get_integer_32_xfId): {
-     tmp_arg1 = xb(Arg(0));
+ OpCase(i_bs_get_integer_32_xfId):
+     bs_get_integer_32_context = xb(Arg(0));
      I++;
- }
 
- do_bs_get_integer_32: {
-     ErlBinMatchBuffer *_mb;
-     Uint32 _integer;
-     Eterm _result;
-     _mb = ms_matchbuffer(tmp_arg1);
-     if (_mb->size - _mb->offset < 32) { ClauseFail(); }
-     if (BIT_OFFSET(_mb->offset) != 0) {
-	 _integer = erts_bs_get_unaligned_uint32(_mb);
-     } else {
-	 _integer = get_int32(_mb->base + _mb->offset/8);
-     }
-     _mb->offset += 32;
-#ifndef ARCH_64
-     if (IS_USMALL(0, _integer)) {
+
+ do_bs_get_integer_32:
+     {
+	 ErlBinMatchBuffer *_mb;
+	 Uint32 _integer;
+	 Eterm _result;
+	 _mb = ms_matchbuffer(bs_get_integer_32_context);
+	 if (_mb->size - _mb->offset < 32) { ClauseFail(); }
+	 if (BIT_OFFSET(_mb->offset) != 0) {
+	     _integer = erts_bs_get_unaligned_uint32(_mb);
+	 } else {
+	     _integer = get_int32(_mb->base + _mb->offset/8);
+	 }
+	 _mb->offset += 32;
+#if !defined(ARCH_64) || HALFWORD_HEAP
+	 if (IS_USMALL(0, _integer)) {
 #endif
-	 _result = make_small(_integer);
-#ifndef ARCH_64
-     } else {
-	 TestHeap(BIG_UINT_HEAP_SIZE, Arg(1));
-	 _result = uint_to_big((Uint) _integer, HTOP);
-	 HTOP += BIG_UINT_HEAP_SIZE;
-	 HEAP_SPACE_VERIFIED(0);
-     }
+	     _result = make_small(_integer);
+#if !defined(ARCH_64) || HALFWORD_HEAP
+	 } else {
+	     TestHeap(BIG_UINT_HEAP_SIZE, Arg(1));
+	     _result = uint_to_big((Uint) _integer, HTOP);
+	     HTOP += BIG_UINT_HEAP_SIZE;
+	     HEAP_SPACE_VERIFIED(0);
+	 }
 #endif
-     StoreBifResult(2, _result);
+	 StoreBifResult(2, _result);
+     }
  }
 
  /* Operands: Size Live Fail Flags Dst */
@@ -4041,54 +4355,64 @@ apply_bif_or_nif_epilogue:
      StoreBifResult(3, result);
  }
 
- /* Operands: MatchContext Fail Dst */
+ {
+     Eterm get_utf8_context;
+
+     /* Operands: MatchContext Fail Dst */
  OpCase(i_bs_get_utf8_rfd): {
-     tmp_arg1 = r(0);
-     goto do_bs_get_utf8;
- }
+	 get_utf8_context = r(0);
+	 goto do_bs_get_utf8;
+     }
 
  OpCase(i_bs_get_utf8_xfd): {
-     tmp_arg1 = xb(Arg(0));
-     I++;
- }
+	 get_utf8_context = xb(Arg(0));
+	 I++;
+     }
 
- /*
-  * tmp_arg1 = match_context
-  * Operands: Fail Dst
-  */
+     /*
+      * get_utf8_context = match_context
+      * Operands: Fail Dst
+      */
 
-    do_bs_get_utf8: {
-     Eterm result = erts_bs_get_utf8(ms_matchbuffer(tmp_arg1));
-     if (is_non_value(result)) {
-	 ClauseFail();
+ do_bs_get_utf8: {
+	 Eterm result = erts_bs_get_utf8(ms_matchbuffer(get_utf8_context));
+	 if (is_non_value(result)) {
+	     ClauseFail();
+	 }
+	 StoreBifResult(1, result);
      }
-     StoreBifResult(1, result);
  }
 
- /* Operands: MatchContext Fail Flags Dst */
+ {
+     Eterm get_utf16_context;
+
+     /* Operands: MatchContext Fail Flags Dst */
  OpCase(i_bs_get_utf16_rfId): {
-     tmp_arg1 = r(0);
-     goto do_bs_get_utf16;
- }
+	 get_utf16_context = r(0);
+	 goto do_bs_get_utf16;
+     }
 
  OpCase(i_bs_get_utf16_xfId): {
-     tmp_arg1 = xb(Arg(0));
-     I++;
- }
+	 get_utf16_context = xb(Arg(0));
+	 I++;
+     }
 
- /*
-  * tmp_arg1 = match_context
-  * Operands: Fail Flags Dst
-  */
-    do_bs_get_utf16: {
-     Eterm result = erts_bs_get_utf16(ms_matchbuffer(tmp_arg1), Arg(1));
-     if (is_non_value(result)) {
-	 ClauseFail();
+     /*
+      * get_utf16_context = match_context
+      * Operands: Fail Flags Dst
+      */
+ do_bs_get_utf16: {
+	 Eterm result = erts_bs_get_utf16(ms_matchbuffer(get_utf16_context),
+					  Arg(1));
+	 if (is_non_value(result)) {
+	     ClauseFail();
+	 }
+	 StoreBifResult(2, result);
      }
-     StoreBifResult(2, result);
  }
 
  {
+     Eterm context_to_binary_context;
      ErlBinMatchBuffer* mb;
      ErlSubBin* sb;
      Uint size;
@@ -4097,27 +4421,29 @@ apply_bif_or_nif_epilogue:
      Uint hole_size;
 
      OpCase(bs_context_to_binary_r): {
-	 tmp_arg1 = x0;
+	 context_to_binary_context = x0;
 	 I -= 2;
 	 goto do_context_to_binary;
      }
 
      /* Unfortunately, inlining can generate this instruction. */
      OpCase(bs_context_to_binary_y): {
-	 tmp_arg1 = yb(Arg(0));
+	 context_to_binary_context = yb(Arg(0));
 	 goto do_context_to_binary0;
      }
 
      OpCase(bs_context_to_binary_x): {
-	 tmp_arg1 = xb(Arg(0));
+	 context_to_binary_context = xb(Arg(0));
      
      do_context_to_binary0:
 	 I--;
      }
 
  do_context_to_binary:
-     if (is_boxed(tmp_arg1) && header_is_bin_matchstate(*boxed_val(tmp_arg1))) {
-	 ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(tmp_arg1);
+     if (is_boxed(context_to_binary_context) &&
+	 header_is_bin_matchstate(*boxed_val(context_to_binary_context))) {
+	 ErlBinMatchState* ms;
+	 ms = (ErlBinMatchState *) boxed_val(context_to_binary_context);
 	 mb = &ms->mb;
 	 offs = ms->save_offset[0];
 	 size = mb->size - offs;
@@ -4126,17 +4452,17 @@ apply_bif_or_nif_epilogue:
      Next(2);
 
      OpCase(i_bs_get_binary_all_reuse_rfI): {
-	 tmp_arg1 = x0;
+	 context_to_binary_context = x0;
 	 goto do_bs_get_binary_all_reuse;
      }
 
      OpCase(i_bs_get_binary_all_reuse_xfI): {
-	 tmp_arg1 = xb(Arg(0));
+	 context_to_binary_context = xb(Arg(0));
 	 I++;
      }
 
  do_bs_get_binary_all_reuse:
-     mb = ms_matchbuffer(tmp_arg1);
+     mb = ms_matchbuffer(context_to_binary_context);
      size = mb->size - mb->offset;
      if (size % Arg(1) != 0) {
 	 ClauseFail();
@@ -4145,7 +4471,7 @@ apply_bif_or_nif_epilogue:
 
  do_bs_get_binary_all_reuse_common:
      orig = mb->orig;
-     sb = (ErlSubBin *) boxed_val(tmp_arg1);
+     sb = (ErlSubBin *) boxed_val(context_to_binary_context);
      hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE;
      sb->thing_word = HEADER_SUB_BIN;
      sb->size = BYTE_OFFSET(size);
@@ -4161,18 +4487,20 @@ apply_bif_or_nif_epilogue:
  }
 
  {
+     Eterm match_string_context;
+
      OpCase(i_bs_match_string_rfII): {
-	 tmp_arg1 = r(0);
+	 match_string_context = r(0);
 	 goto do_bs_match_string;
      }
      OpCase(i_bs_match_string_xfII): {
-	 tmp_arg1 = xb(Arg(0));
+	 match_string_context = xb(Arg(0));
 	 I++;
      }
 
  do_bs_match_string:
      {
-	 Eterm* next;
+	 BeamInstr *next;
 	 byte* bytes;
 	 Uint bits;
 	 ErlBinMatchBuffer* mb;
@@ -4181,7 +4509,7 @@ apply_bif_or_nif_epilogue:
 	 PreFetch(3, next);
 	 bits = Arg(1);
 	 bytes = (byte *) Arg(2);
-	 mb = ms_matchbuffer(tmp_arg1);
+	 mb = ms_matchbuffer(match_string_context);
 	 if (mb->size - mb->offset < bits) {
 	     ClauseFail();
 	 }
@@ -4199,7 +4527,7 @@ apply_bif_or_nif_epilogue:
  }
 
  OpCase(i_bs_save2_rI): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchState *_ms;
      PreFetch(1, next);
      _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0));
@@ -4207,7 +4535,7 @@ apply_bif_or_nif_epilogue:
      NextPF(1, next);
  }
  OpCase(i_bs_save2_xI): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchState *_ms;
      PreFetch(2, next);
      _ms = (ErlBinMatchState*) boxed_val((Eterm) xb(Arg(0)));
@@ -4216,7 +4544,7 @@ apply_bif_or_nif_epilogue:
  }
 
  OpCase(i_bs_restore2_rI): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchState *_ms;
      PreFetch(1, next);
      _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0));
@@ -4224,7 +4552,7 @@ apply_bif_or_nif_epilogue:
      NextPF(1, next);
  }
  OpCase(i_bs_restore2_xI): {
-     Eterm* next;
+     BeamInstr *next;
      ErlBinMatchState *_ms;
      PreFetch(2, next);
      _ms = (ErlBinMatchState*) boxed_val((Eterm) xb(Arg(0)));
@@ -4241,7 +4569,7 @@ apply_bif_or_nif_epilogue:
   * deallocate not followed by a return, and that should work.
   */
  OpCase(deallocate_I): {
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(1, next);
      D(Arg(0));
@@ -4264,7 +4592,7 @@ apply_bif_or_nif_epilogue:
      */
  OpCase(call_traced_function): {
      if (IS_TRACED_FL(c_p, F_TRACE_CALLS)) {
-	 unsigned offset = offsetof(Export, code) + 3*sizeof(Eterm);
+	 unsigned offset = offsetof(Export, code) + 3*sizeof(BeamInstr);
 	 Export* ep = (Export *) (((char *)I)-offset);
 	 Uint32 flags;
 
@@ -4274,6 +4602,7 @@ apply_bif_or_nif_epilogue:
 	 ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
 	 flags = erts_call_trace(c_p, ep->code, ep->match_prog_set, reg,
 				 0, &c_p->tracer_proc);
+	 ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	 ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
 	 PROCESS_MAIN_CHK_LOCKS(c_p);
 	 ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -4285,32 +4614,32 @@ apply_bif_or_nif_epilogue:
 		 /* SWAPOUT, SWAPIN was done and r(0) was saved above */
 		 PROCESS_MAIN_CHK_LOCKS(c_p);
 		 FCALLS -= erts_garbage_collect(c_p, 3, reg, ep->code[2]);
+		 ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 		 PROCESS_MAIN_CHK_LOCKS(c_p);
 		 r(0) = reg[0];
 		 SWAPIN;
 	     }
 	     E -= 3;
 	     ASSERT(c_p->htop <= E && E <= c_p->hend);
-	     ASSERT(is_CP((Eterm)(ep->code)));
+	     ASSERT(is_CP((BeamInstr)(ep->code)));
 	     ASSERT(is_internal_pid(c_p->tracer_proc) || 
 		    is_internal_port(c_p->tracer_proc));
-	     E[2] = make_cp(c_p->cp);
+	     E[2] = make_cp(c_p->cp); /* Code in lower range on halfword */
 	     E[1] = am_true; /* Process tracer */
 	     E[0] = make_cp(ep->code);
-	     c_p->cp = (Eterm*)
-		 make_cp(flags & MATCH_SET_EXCEPTION_TRACE
-			 ? beam_exception_trace : beam_return_trace);
+	     c_p->cp = (flags & MATCH_SET_EXCEPTION_TRACE)
+		 ? beam_exception_trace : beam_return_trace;
 	     erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 	     c_p->trace_flags |= F_EXCEPTION_TRACE;
 	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 	 }
      }
-     SET_I((Uint *) Arg(0));
+     SET_I((BeamInstr *)Arg(0));
      Dispatch();
  }
 
  OpCase(return_trace): {
-     Uint* code = (Uint *) E[0];
+     BeamInstr* code = (BeamInstr *) (UWord) E[0];
      
      SWAPOUT;		/* Needed for shared heap */
      ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
@@ -4318,44 +4647,118 @@ apply_bif_or_nif_epilogue:
      ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
      SWAPIN;
      c_p->cp = NULL;
-     SET_I((Eterm *) E[2]);
+     SET_I((BeamInstr *) cp_val(E[2]));
      E += 3;
      Goto(*I);
  }
 
  OpCase(i_count_breakpoint): {
-     Uint real_I;
+     BeamInstr real_I;
      
-     ErtsCountBreak((Uint *) I, &real_I);
+     ErtsCountBreak(c_p, (BeamInstr *) I, &real_I);
+     ASSERT(VALID_INSTR(real_I));
+     Goto(real_I);
+ }
+
+ /* need to send mfa instead of bdt pointer
+  * the pointer might be deallocated.
+  */
+
+ OpCase(i_time_breakpoint): {
+     BeamInstr real_I;
+     BpData **bds = (BpData **) (I)[-4];
+     BpDataTime *bdt = NULL;
+     Uint ix = 0;
+#ifdef ERTS_SMP
+     ix = c_p->scheduler_data->no - 1;
+#else
+     ix = 0;
+#endif
+     bdt = (BpDataTime *)bds[ix];
+
+     ASSERT((I)[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
+     ASSERT(bdt);
+     bdt = (BpDataTime *) bdt->next;
+     ASSERT(bdt);
+     bds[ix] = (BpData *) bdt;
+     real_I = bdt->orig_instr;
      ASSERT(VALID_INSTR(real_I));
+
+     if (IS_TRACED_FL(c_p, F_TRACE_CALLS) && !(bdt->pause)) {
+	 if (	(*(c_p->cp) == (BeamInstr) OpCode(i_return_time_trace)) ||
+		(*(c_p->cp) == (BeamInstr) OpCode(return_trace)) ||
+		(*(c_p->cp) == (BeamInstr) OpCode(i_return_to_trace))) {
+	     /* This _IS_ a tail recursive call */
+	     SWAPOUT;
+	     erts_trace_time_break(c_p, I, bdt, ERTS_BP_CALL_TIME_TAIL_CALL);
+	     SWAPIN;
+	 } else {
+	     SWAPOUT;
+	     erts_trace_time_break(c_p, I, bdt, ERTS_BP_CALL_TIME_CALL);
+
+	     /* r register needs to be copied to the array
+	      * for the garbage collector
+	      */
+	     ASSERT(c_p->htop <= E && E <= c_p->hend);
+	     if (E - 2 < HTOP) {
+		 reg[0] = r(0);
+		 PROCESS_MAIN_CHK_LOCKS(c_p);
+		 FCALLS -= erts_garbage_collect(c_p, 2, reg, I[-1]);
+		 ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+		 PROCESS_MAIN_CHK_LOCKS(c_p);
+		 r(0) = reg[0];
+	     }
+	     SWAPIN;
+
+	     ASSERT(c_p->htop <= E && E <= c_p->hend);
+
+	     E -= 2;
+	     E[0] = make_cp(I);
+	     E[1] = make_cp(c_p->cp);     /* original return address */
+	     c_p->cp = beam_return_time_trace;
+	 }
+     }
+
      Goto(real_I);
  }
 
+ OpCase(i_return_time_trace): {
+     BeamInstr *pc = (BeamInstr *) (UWord) E[0];
+     SWAPOUT;
+     erts_trace_time_break(c_p, pc, NULL, ERTS_BP_CALL_TIME_RETURN);
+     SWAPIN;
+     c_p->cp = NULL;
+     SET_I((BeamInstr *) cp_val(E[1]));
+     E += 2;
+     Goto(*I);
+ }
+
  OpCase(i_trace_breakpoint):
      if (! IS_TRACED_FL(c_p, F_TRACE_CALLS)) {
-	 Uint real_I;
+	 BeamInstr real_I;
 	 
-	 ErtsBreakSkip((Uint *) I, &real_I);
+	 ErtsBreakSkip(c_p, (BeamInstr *) I, &real_I);
 	 Goto(real_I);
      }
  /* Fall through to next case */
  OpCase(i_mtrace_breakpoint): {
-     Uint real_I;
+     BeamInstr real_I;
      Uint32 flags;
      Eterm tracer_pid;
-     Uint *cpp;
+     Uint* cpp;
      int return_to_trace = 0, need = 0;
      flags = 0;
      SWAPOUT;
      reg[0] = r(0);
 
-     if (*cp_val((Eterm)c_p->cp) 
-	 == (Uint) OpCode(return_trace)) {
-	 cpp = (Uint*)&E[2];
-     } else if (*cp_val((Eterm)c_p->cp) 
-		== (Uint) OpCode(i_return_to_trace)) {
+     if (*(c_p->cp) == (BeamInstr) OpCode(return_trace)) {
+	 cpp = &E[2];
+     } else if (*(c_p->cp) == (BeamInstr) OpCode(i_return_to_trace)) {
+	 return_to_trace = !0;
+	 cpp = &E[0];
+     } else if (*(c_p->cp) == (BeamInstr) OpCode(i_return_time_trace)) {
 	 return_to_trace = !0;
-	 cpp = (Uint*)&E[0];
+	 cpp = &E[0];
      } else {
 	 cpp = NULL;
      }
@@ -4364,19 +4767,21 @@ apply_bif_or_nif_epilogue:
 	  * return_trace and/or i_return_to_trace stackframes
 	  * on the stack, they are not intermixed with y registers
 	  */
-	 Eterm *cp_save = c_p->cp;
+	 BeamInstr *cp_save = c_p->cp;
 	 for (;;) {
 	     ASSERT(is_CP(*cpp));
-	     if (*cp_val(*cpp) == (Uint) OpCode(return_trace)) {
+	     if (*cp_val(*cpp) == (BeamInstr) OpCode(return_trace)) {
 		 cpp += 3;
-	     } else if (*cp_val(*cpp) == (Uint) OpCode(i_return_to_trace)) {
+	     } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_to_trace)) {
 		 return_to_trace = !0;
 		 cpp += 1;
+	     } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_time_trace)) {
+		 cpp += 2;
 	     } else
 		 break;
 	 }
-	 c_p->cp = (Eterm *) *cpp;
-	 ASSERT(is_CP((Eterm)c_p->cp));
+	 c_p->cp = (BeamInstr *) cp_val(*cpp);
+	 ASSERT(is_CP(*cpp));
 	 ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
 	 real_I = erts_trace_break(c_p, I, reg, &flags, &tracer_pid);
 	 ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
@@ -4403,6 +4808,7 @@ apply_bif_or_nif_epilogue:
 	     /* SWAPOUT was done and r(0) was saved above */
 	     PROCESS_MAIN_CHK_LOCKS(c_p);
 	     FCALLS -= erts_garbage_collect(c_p, need, reg, I[-1]);
+	     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	     PROCESS_MAIN_CHK_LOCKS(c_p);
 	     r(0) = reg[0];
 	     SWAPIN;
@@ -4412,12 +4818,12 @@ apply_bif_or_nif_epilogue:
 	 E -= 1;
 	 ASSERT(c_p->htop <= E && E <= c_p->hend);
 	 E[0] = make_cp(c_p->cp);
-	 c_p->cp = (Eterm *) make_cp(beam_return_to_trace);
+	 c_p->cp = (BeamInstr *) beam_return_to_trace;
      }
      if (flags & MATCH_SET_RX_TRACE) {
 	 E -= 3;
 	 ASSERT(c_p->htop <= E && E <= c_p->hend);
-	 ASSERT(is_CP((Eterm) (I - 3)));
+	 ASSERT(is_CP((Eterm) (UWord) (I - 3)));
 	 ASSERT(am_true == tracer_pid || 
 		is_internal_pid(tracer_pid) || is_internal_port(tracer_pid));
 	 E[2] = make_cp(c_p->cp);
@@ -4425,9 +4831,9 @@ apply_bif_or_nif_epilogue:
 	 E[0] = make_cp(I - 3); /* We ARE at the beginning of an 
 				   instruction,
 				   the funcinfo is above i. */
-	 c_p->cp = (Eterm*)
-	     make_cp(flags & MATCH_SET_EXCEPTION_TRACE
-		     ? beam_exception_trace : beam_return_trace);
+	 c_p->cp =
+	     (flags & MATCH_SET_EXCEPTION_TRACE)
+		     ? beam_exception_trace : beam_return_trace;
 	 erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
 	 c_p->trace_flags |= F_EXCEPTION_TRACE;
 	 erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR);
@@ -4440,10 +4846,10 @@ apply_bif_or_nif_epilogue:
 	 Uint *cpp = (Uint*) E;
 	 for(;;) {
 	     ASSERT(is_CP(*cpp));
-	     if (*cp_val(*cpp) == (Uint) OpCode(return_trace)) {
+	     if (*cp_val(*cpp) == (BeamInstr) OpCode(return_trace)) {
 		 do ++cpp; while(is_not_CP(*cpp));
 		 cpp += 2;
-	     } else if (*cp_val(*cpp) == (Uint) OpCode(i_return_to_trace)) {
+	     } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_to_trace)) {
 		 do ++cpp; while(is_not_CP(*cpp));
 	     } else break;
 	 }
@@ -4454,104 +4860,18 @@ apply_bif_or_nif_epilogue:
 	 SWAPIN;
      }
      c_p->cp = NULL;
-     SET_I((Eterm *) E[0]);
+     SET_I((BeamInstr *) cp_val(E[0]));
      E += 1;
      Goto(*I);
  }
 
  /*
-  * Instructions for allocating on the message area.
-  */
-
- OpCase(i_global_cons):
- {
-     Eterm *next;
-#ifdef HYBRID
-     Eterm *hp;
-
-     PreFetch(0,next);
-     TestGlobalHeap(2,2,hp);
-     hp[0] = r(0);
-     hp[1] = x(1);
-     r(0) = make_list(hp);
-#ifndef INCREMENTAL
-     global_htop += 2;
-#endif
-     NextPF(0,next);
-#else
-     PreFetch(0,next);
-     c_p->freason = EXC_INTERNAL_ERROR;
-     goto find_func_info;
-#endif
- }
-
- OpCase(i_global_tuple):
- {
-     Eterm *next;
-     int len;
-#ifdef HYBRID
-     Eterm list;
-     Eterm *hp;
-#endif
-
-     if ((len = list_length(r(0))) < 0) {
-         goto badarg;
-     }
-
-     PreFetch(0,next);
-#ifdef HYBRID
-     TestGlobalHeap(len + 1,1,hp);
-     list = r(0);
-     r(0) = make_tuple(hp);
-     *hp++ = make_arityval(len);
-     while(is_list(list))
-     {
-         Eterm* cons = list_val(list);
-         *hp++ = CAR(cons);
-         list = CDR(cons);
-     }
-#ifndef INCREMENTAL
-     global_htop += len + 1;
-#endif
-     NextPF(0,next);
-#else
-     c_p->freason = EXC_INTERNAL_ERROR;
-     goto find_func_info;
-#endif
- }
-
- OpCase(i_global_copy):
- {
-     Eterm *next;
-     PreFetch(0,next);
-#ifdef HYBRID
-     if (!IS_CONST(r(0)))
-     {
-         BM_SWAP_TIMER(system,copy);
-         SWAPOUT;
-         reg[0] = r(0);
-         reg[1] = NIL;
-         r(0) = copy_struct_lazy(c_p,r(0),0);
-         ASSERT(ma_src_top == 0);
-         ASSERT(ma_dst_top == 0);
-         ASSERT(ma_offset_top == 0);
-         SWAPIN;
-         BM_SWAP_TIMER(copy,system);
-     }
-     NextPF(0,next);
-#else
-     c_p->freason = EXC_INTERNAL_ERROR;
-     goto find_func_info;
-#endif
- }
-
- /*
   * New floating point instructions.
   */
 
  OpCase(fmove_ql): {
      Eterm fr = Arg(1);
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(2, next);
      GET_DOUBLE(Arg(0), *(FloatDef*)ADD_BYTE_OFFSET(freg, fr));
@@ -4561,7 +4881,7 @@ apply_bif_or_nif_epilogue:
  OpCase(fmove_dl): {
      Eterm targ1;
      Eterm fr = Arg(1);
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(2, next);
      GetR(0, targ1);
@@ -4570,7 +4890,7 @@ apply_bif_or_nif_epilogue:
      NextPF(2, next);
  }
 
- OpCase(fmove_new_ld): {
+ OpCase(fmove_ld): {
      Eterm fr = Arg(0);
      Eterm dest = make_float(HTOP);
 
@@ -4582,7 +4902,7 @@ apply_bif_or_nif_epilogue:
  OpCase(fconv_dl): {
      Eterm targ1;
      Eterm fr = Arg(1);
-     Eterm* next;
+     BeamInstr *next;
 
      GetR(0, targ1);
      PreFetch(2, next);
@@ -4600,18 +4920,18 @@ apply_bif_or_nif_epilogue:
      NextPF(2, next);
  }
 
- /*
-  * Old allocating fmove.
-  */
-
-
 #ifdef NO_FPE_SIGNALS
      OpCase(fclearerror):
      OpCase(i_fcheckerror):
 	 erl_exit(1, "fclearerror/i_fcheckerror without fpe signals (beam_emu)");
+#  define ERTS_NO_FPE_CHECK_INIT ERTS_FP_CHECK_INIT
+#  define ERTS_NO_FPE_ERROR ERTS_FP_ERROR
 #else
+#  define ERTS_NO_FPE_CHECK_INIT(p)
+#  define ERTS_NO_FPE_ERROR(p, a, b)
+
      OpCase(fclearerror): {
-	 Eterm* next;
+	 BeamInstr *next;
 
 	 PreFetch(0, next);
 	 ERTS_FP_CHECK_INIT(c_p);
@@ -4619,62 +4939,58 @@ apply_bif_or_nif_epilogue:
      }
 
      OpCase(i_fcheckerror): {
-	 Eterm* next;
+	 BeamInstr *next;
 
 	 PreFetch(0, next);
 	 ERTS_FP_ERROR(c_p, freg[0].fd, goto fbadarith);
 	 NextPF(0, next);
      }
-#  undef ERTS_FP_CHECK_INIT
-#  undef ERTS_FP_ERROR
-#  define ERTS_FP_CHECK_INIT(p)
-#  define ERTS_FP_ERROR(p, a, b)
 #endif
 
 
  OpCase(i_fadd_lll): {
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(3, next);
-     ERTS_FP_CHECK_INIT(c_p);
+     ERTS_NO_FPE_CHECK_INIT(c_p);
      fb(Arg(2)) = fb(Arg(0)) + fb(Arg(1));
-     ERTS_FP_ERROR(c_p, fb(Arg(2)), goto fbadarith);
+     ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith);
      NextPF(3, next);
  }
  OpCase(i_fsub_lll): {
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(3, next);
-     ERTS_FP_CHECK_INIT(c_p);
+     ERTS_NO_FPE_CHECK_INIT(c_p);
      fb(Arg(2)) = fb(Arg(0)) - fb(Arg(1));
-     ERTS_FP_ERROR(c_p, fb(Arg(2)), goto fbadarith);
+     ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith);
      NextPF(3, next);
  }
  OpCase(i_fmul_lll): {
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(3, next);
-     ERTS_FP_CHECK_INIT(c_p);
+     ERTS_NO_FPE_CHECK_INIT(c_p);
      fb(Arg(2)) = fb(Arg(0)) * fb(Arg(1));
-     ERTS_FP_ERROR(c_p, fb(Arg(2)), goto fbadarith);
+     ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith);
      NextPF(3, next);
  }
  OpCase(i_fdiv_lll): {
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(3, next);
-     ERTS_FP_CHECK_INIT(c_p);
+     ERTS_NO_FPE_CHECK_INIT(c_p);
      fb(Arg(2)) = fb(Arg(0)) / fb(Arg(1));
-     ERTS_FP_ERROR(c_p, fb(Arg(2)), goto fbadarith);
+     ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith);
      NextPF(3, next);
  }
  OpCase(i_fnegate_ll): {
-     Eterm* next;
+     BeamInstr *next;
 
      PreFetch(2, next);
-     ERTS_FP_CHECK_INIT(c_p);
+     ERTS_NO_FPE_CHECK_INIT(c_p);
      fb(Arg(1)) = -fb(Arg(0));
-     ERTS_FP_ERROR(c_p, fb(Arg(1)), goto fbadarith);
+     ERTS_NO_FPE_ERROR(c_p, fb(Arg(1)), goto fbadarith);
      NextPF(2, next);
 
  fbadarith:
@@ -4728,15 +5044,13 @@ apply_bif_or_nif_epilogue:
      c_p->def_arg_reg[4] = -neg_o_reds;
      reg[0] = r(0);
      c_p = hipe_mode_switch(c_p, cmd, reg);
-#ifdef ERTS_SMP
-     reg = c_p->scheduler_data->save_reg;
-     freg = c_p->scheduler_data->freg;
-#endif
+     reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array;
+     freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array;
      ERL_BITS_RELOAD_STATEP(c_p);
      neg_o_reds = -c_p->def_arg_reg[4];
      FCALLS = c_p->fcalls;
      SWAPIN;
-     switch( c_p->def_arg_reg[3] ) {
+     switch( c_p->def_arg_reg[3] ) { /* Halfword wont work with hipe yet! */
        case HIPE_MODE_SWITCH_RES_RETURN:
 	 ASSERT(is_value(reg[0]));
 	 MoveReturn(reg[0], r(0));
@@ -4748,7 +5062,7 @@ apply_bif_or_nif_epilogue:
 	 /* This can be used to call any function value, but currently it's
 	    only used to call closures referring to unloaded modules. */
 	 {
-	     Eterm *next;
+	     BeamInstr *next;
 
 	     next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE);
 	     SWAPIN;
@@ -4805,7 +5119,8 @@ apply_bif_or_nif_epilogue:
 
  OpCase(i_hibernate): {
      SWAPOUT;
-     if (hibernate(c_p, r(0), x(1), x(2), reg)) {
+     if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) {
+	 c_p->flags &= ~F_HIBERNATE_SCHED;
 	 goto do_schedule;
      } else {
 	 I = handle_error(c_p, I, reg, hibernate_3);
@@ -4816,14 +5131,13 @@ apply_bif_or_nif_epilogue:
  OpCase(i_debug_breakpoint): {
      SWAPOUT;
      reg[0] = r(0);
-     tmp_arg1 = call_breakpoint_handler(c_p, I-3, reg);
+     I = call_error_handler(c_p, I-3, reg, am_breakpoint);
      r(0) = reg[0];
      SWAPIN;
-     if (tmp_arg1) {
-	 SET_I(c_p->i);
-	 Dispatch();
+     if (I) {
+	 Goto(*I);
      }
-     goto no_error_handler;
+     goto handle_error;
  }
 
 
@@ -4845,8 +5159,8 @@ apply_bif_or_nif_epilogue:
 
  OpCase(int_code_end):
  OpCase(label_L):
- OpCase(too_old_compiler):
  OpCase(on_load):
+ OpCase(line_I):
     erl_exit(1, "meta op\n");
 
     /*
@@ -4881,13 +5195,15 @@ apply_bif_or_nif_epilogue:
      em_call_error_handler = OpCode(call_error_handler);
      em_call_traced_function = OpCode(call_traced_function);
      em_apply_bif = OpCode(apply_bif);
-     beam_apply[0] = (Eterm) OpCode(i_apply);
-     beam_apply[1] = (Eterm) OpCode(normal_exit);
-     beam_exit[0] = (Eterm) OpCode(error_action_code);
-     beam_continue_exit[0] = (Eterm) OpCode(continue_exit);
-     beam_return_to_trace[0] = (Eterm) OpCode(i_return_to_trace);
-     beam_return_trace[0] = (Eterm) OpCode(return_trace);
-     beam_exception_trace[0] = (Eterm) OpCode(return_trace); /* UGLY */
+
+     beam_apply[0]             = (BeamInstr) OpCode(i_apply);
+     beam_apply[1]             = (BeamInstr) OpCode(normal_exit);
+     beam_exit[0]              = (BeamInstr) OpCode(error_action_code);
+     beam_continue_exit[0]     = (BeamInstr) OpCode(continue_exit);
+     beam_return_to_trace[0]   = (BeamInstr) OpCode(i_return_to_trace);
+     beam_return_trace[0]      = (BeamInstr) OpCode(return_trace);
+     beam_exception_trace[0]   = (BeamInstr) OpCode(return_trace); /* UGLY */
+     beam_return_time_trace[0] = (BeamInstr) OpCode(i_return_time_trace);
 
      /*
       * Enter all BIFs into the export table.
@@ -4897,8 +5213,10 @@ apply_bif_or_nif_epilogue:
 			      bif_table[i].name,
 			      bif_table[i].arity);
 	 bif_export[i] = ep;
-	 ep->code[3] = (Eterm) OpCode(apply_bif);
-	 ep->code[4] = (Eterm) bif_table[i].f;
+	 ep->code[3] = (BeamInstr) OpCode(apply_bif);
+	 ep->code[4] = (BeamInstr) bif_table[i].f;
+	 /* XXX: set func info for bifs */
+	 ep->fake_op_func_info_for_hipe[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI);
      }
 
      return;
@@ -4943,6 +5261,10 @@ translate_gc_bif(void* gcf)
 	return round_1;
     } else if (gcf == erts_gc_trunc_1) {
 	return round_1;
+    } else if (gcf == erts_gc_binary_part_2) {
+	return binary_part_2;
+    } else if (gcf == erts_gc_binary_part_3) {
+	return binary_part_3;
     } else {
 	erl_exit(1, "bad gc bif");
     }
@@ -5001,8 +5323,8 @@ Eterm error_atom[NUMBER_EXIT_CODES] = {
  * at the point of the original exception.
  */
 
-static Eterm*
-handle_error(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf)
+static BeamInstr*
+handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf)
 {
     Eterm* hp;
     Eterm Value = c_p->fvalue;
@@ -5056,7 +5378,7 @@ handle_error(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf)
     /* Find a handler or die */
     if ((c_p->catches > 0 || IS_TRACED_FL(c_p, F_EXCEPTION_TRACE))
 	&& !(c_p->freason & EXF_PANIC)) {
-	Eterm *new_pc;
+	BeamInstr *new_pc;
         /* The Beam handler code (catch_end or try_end) checks reg[0]
 	   for THE_NON_VALUE to see if the previous code finished
 	   abnormally. If so, reg[1], reg[2] and reg[3] should hold the
@@ -5082,30 +5404,37 @@ handle_error(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf)
 /*
  * Find the nearest catch handler
  */
-static Eterm*
+static BeamInstr*
 next_catch(Process* c_p, Eterm *reg) {
     int active_catches = c_p->catches > 0;
     int have_return_to_trace = 0;
     Eterm *ptr, *prev, *return_to_trace_ptr = NULL;
-    Uint i_return_trace = beam_return_trace[0];
-    Uint i_return_to_trace = beam_return_to_trace[0];
+
+    BeamInstr i_return_trace      = beam_return_trace[0];
+    BeamInstr i_return_to_trace   = beam_return_to_trace[0];
+    BeamInstr i_return_time_trace = beam_return_time_trace[0];
+
     ptr = prev = c_p->stop;
     ASSERT(is_CP(*ptr));
     ASSERT(ptr <= STACK_START(c_p));
     if (ptr == STACK_START(c_p)) return NULL;
     if ((is_not_CP(*ptr) || (*cp_val(*ptr) != i_return_trace &&
-			     *cp_val(*ptr) != i_return_to_trace))
+			     *cp_val(*ptr) != i_return_to_trace &&
+			     *cp_val(*ptr) != i_return_time_trace ))
 	&& c_p->cp) {
 	/* Can not follow cp here - code may be unloaded */
-	Uint *cpp = cp_val((Eterm) c_p->cp);
+	BeamInstr *cpp = c_p->cp;
 	if (cpp == beam_exception_trace) {
-	    erts_trace_exception(c_p, (Eterm*) ptr[0],
+	    erts_trace_exception(c_p, cp_val(ptr[0]),
 				 reg[1], reg[2], ptr+1);
 	    /* Skip return_trace parameters */
 	    ptr += 2;
 	} else if (cpp == beam_return_trace) {
 	    /* Skip return_trace parameters */
 	    ptr += 2;
+	} else if (cpp == beam_return_time_trace) {
+	    /* Skip return_trace parameters */
+	    ptr += 1;
 	} else if (cpp == beam_return_to_trace) {
 	    have_return_to_trace = !0; /* Record next cp */
 	}
@@ -5123,7 +5452,7 @@ next_catch(Process* c_p, Eterm *reg) {
 		    if (is_catch(*ptr) && active_catches) goto found_catch;
 		}
 		if (cp_val(*prev) == beam_exception_trace) {
-		    erts_trace_exception(c_p, (Eterm*) ptr[0],
+		    erts_trace_exception(c_p, cp_val(ptr[0]),
 					 reg[1], reg[2], ptr+1);
 		}
 		/* Skip return_trace parameters */
@@ -5135,6 +5464,13 @@ next_catch(Process* c_p, Eterm *reg) {
 		}
 		have_return_to_trace = !0; /* Record next cp */
 		return_to_trace_ptr = NULL;
+	    } else if (*cp_val(*prev) == i_return_time_trace) {
+		/* Skip stack frame variables */
+		while (++ptr, ptr < STACK_START(c_p) && is_not_CP(*ptr)) {
+		    if (is_catch(*ptr) && active_catches) goto found_catch;
+		}
+		/* Skip return_trace parameters */
+		ptr += 1;
 	    } else {
 		if (have_return_to_trace) {
 		    /* Record this cp as possible return_to trace cp */
@@ -5241,6 +5577,25 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) {
  * that c_p->ftrace will point to a cons cell which holds the given args
  * and the saved data (encoded as a bignum).
  *
+ * There is an issue with line number information. Line number
+ * information is associated with the address *before* an operation
+ * that may fail or be stored stored on the stack. But continuation
+ * pointers point after its call instruction, not before. To avoid
+ * finding the wrong line number, we'll need to adjust them so that
+ * they point at the beginning of the call instruction or inside the
+ * call instruction. Since its impractical to point at the beginning,
+ * we'll do the simplest thing and decrement the continuation pointers
+ * by one.
+ *
+ * Here is an example of what can go wrong. Without the adjustment
+ * of continuation pointers, the call at line 42 below would seem to
+ * be at line 43:
+ *
+ * line 42
+ * call ...
+ * line 43
+ * gc_bif ...
+ *
  * (It would be much better to put the arglist - when it exists - in the
  * error value instead of in the actual trace; e.g. '{badarg, Args}'
  * instead of using 'badarg' with Args in the trace. The arglist may
@@ -5252,7 +5607,7 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) {
  */
 
 static void
-save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf,
+save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 		Eterm args) {
     struct StackTrace* s;
     int sz;
@@ -5263,7 +5618,7 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf,
     }
 
     /* Create a container for the exception data */
-    sz = (offsetof(struct StackTrace, trace) + sizeof(Eterm)*depth
+    sz = (offsetof(struct StackTrace, trace) + sizeof(BeamInstr *)*depth
           + sizeof(Eterm) - 1) / sizeof(Eterm);
     s = (struct StackTrace *) HAlloc(c_p, 1 + sz);
     /* The following fields are inside the bignum */
@@ -5298,7 +5653,6 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf,
 	    ASSERT(c_p->current);
 	    s->current = c_p->current;
 	    a = s->current[2];
-	    ASSERT(s->current[2] <= 3);
 	}
 	/* Save first stack entry */
 	ASSERT(pc);
@@ -5308,7 +5662,7 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf,
 	}
 	/* Save second stack entry if CP is valid and different from pc */
 	if (depth > 0 && c_p->cp != 0 && c_p->cp != pc) {
-	    s->trace[s->depth++] = c_p->cp;
+	    s->trace[s->depth++] = c_p->cp - 1;
 	    depth--;
 	}
 	s->pc = NULL;
@@ -5328,13 +5682,13 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf,
 	    /* Save first stack entry */
 	    ASSERT(c_p->cp);
 	    if (depth > 0) {
-		s->trace[s->depth++] = c_p->cp;
+		s->trace[s->depth++] = c_p->cp - 1;
 		depth--;
 	    }
 	    s->pc = NULL; /* Ignore pc */
 	} else {
 	    if (depth > 0 && c_p->cp != 0 && c_p->cp != pc) {
-		s->trace[s->depth++] = c_p->cp;
+		s->trace[s->depth++] = c_p->cp - 1;
 		depth--;
 	    }
 	    s->pc = pc;
@@ -5349,23 +5703,31 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf,
     }
 
     /* Save the actual stack trace */
+    erts_save_stacktrace(c_p, s, depth);
+}
+
+void
+erts_save_stacktrace(Process* p, struct StackTrace* s, int depth)
+{
     if (depth > 0) {
-	Eterm *ptr, *prev = s->depth ? s->trace[s->depth-1] : NULL;
-	Uint i_return_trace = beam_return_trace[0];
-	Uint i_return_to_trace = beam_return_to_trace[0];
+	Eterm *ptr;
+	BeamInstr *prev = s->depth ? s->trace[s->depth-1] : NULL;
+	BeamInstr i_return_trace = beam_return_trace[0];
+	BeamInstr i_return_to_trace = beam_return_to_trace[0];
+
 	/*
 	 * Traverse the stack backwards and add all unique continuation
 	 * pointers to the buffer, up to the maximum stack trace size.
 	 * 
 	 * Skip trace stack frames.
 	 */
-	ptr = c_p->stop;
-	if (ptr < STACK_START(c_p) 
-	    && (is_not_CP(*ptr)|| (*cp_val(*ptr) != i_return_trace &&
-				   *cp_val(*ptr) != i_return_to_trace))
-	    && c_p->cp) {
-	    /* Can not follow cp here - code may be unloaded */
-	    Uint *cpp = cp_val((Eterm) c_p->cp);
+	ptr = p->stop;
+	if (ptr < STACK_START(p) &&
+	    (is_not_CP(*ptr)|| (*cp_val(*ptr) != i_return_trace &&
+				*cp_val(*ptr) != i_return_to_trace)) &&
+	    p->cp) {
+	    /* Cannot follow cp here - code may be unloaded */
+	    BeamInstr *cpp = p->cp;
 	    if (cpp == beam_exception_trace || cpp == beam_return_trace) {
 		/* Skip return_trace parameters */
 		ptr += 2;
@@ -5374,7 +5736,7 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf,
 		ptr += 1;
 	    }
 	}
-	while (ptr < STACK_START(c_p) && depth > 0) {
+	while (ptr < STACK_START(p) && depth > 0) {
 	    if (is_CP(*ptr)) {
 		if (*cp_val(*ptr) == i_return_trace) {
 		    /* Skip stack frame variables */
@@ -5385,11 +5747,11 @@ save_stacktrace(Process* c_p, Eterm* pc, Eterm* reg, BifFunction bf,
 		    /* Skip stack frame variables */
 		    do ++ptr; while (is_not_CP(*ptr));
 		} else {
-		    Eterm *cp = (Eterm *)(*ptr);
+		    BeamInstr *cp = cp_val(*ptr);
 		    if (cp != prev) {
 			/* Record non-duplicates only */
 			prev = cp;
-			s->trace[s->depth++] = cp;
+			s->trace[s->depth++] = cp - 1;
 			depth--;
 		    }
 		    ptr++;
@@ -5457,9 +5819,14 @@ build_stacktrace(Process* c_p, Eterm exc) {
     struct StackTrace* s;
     Eterm  args;
     int    depth;
-    Eterm* current;
-    Eterm  Where = NIL;
-    Eterm* next_p = &Where;
+    FunctionInfo fi;
+    FunctionInfo* stk;
+    FunctionInfo* stkp;
+    Eterm res = NIL;
+    Uint heap_size;
+    Eterm* hp;
+    Eterm mfa;
+    int i;
 
     if (! (s = get_trace_from_exc(exc))) {
         return NIL;
@@ -5478,117 +5845,61 @@ build_stacktrace(Process* c_p, Eterm exc) {
      * saved s->current should already contain the proper value.
      */
     if (s->pc != NULL) {
-	current = find_function_from_pc(s->pc);
+	erts_lookup_function_info(&fi, s->pc, 1);
+    } else if (GET_EXC_INDEX(s->freason) ==
+	       GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) {
+	erts_lookup_function_info(&fi, s->current, 1);
     } else {
-	current = s->current;
+	erts_set_current_function(&fi, s->current);
     }
+
     /*
-     * If current is still NULL, default to the initial function
+     * If fi.current is still NULL, default to the initial function
      * (e.g. spawn_link(erlang, abs, [1])).
      */
-    if (current == NULL) {
-	current = c_p->initial;
+    if (fi.current == NULL) {
+	erts_set_current_function(&fi, c_p->initial);
 	args = am_true; /* Just in case */
     } else {
 	args = get_args_from_exc(exc);
     }
 
-    depth = s->depth;
-    
     /*
-     * Add the {M,F,A} for the current function 
-     * (where A is arity or [Argument]).
+     * Look up all saved continuation pointers and calculate
+     * needed heap space.
      */
-    {
-	int i;
-	Eterm mfa;
-	Uint heap_size = 6*(depth+1);
-	Eterm* hp = HAlloc(c_p, heap_size);
-	Eterm* hp_end = hp + heap_size;
-
-	if (args != am_true) {
-	    /* We have an arglist - use it */
-	    mfa = TUPLE3(hp, current[0], current[1], args);
-	} else {
-	    Eterm arity = make_small(current[2]);
-	    mfa = TUPLE3(hp, current[0], current[1], arity);
-	}
-	hp += 4;
-	ASSERT(*next_p == NIL);
-	*next_p = CONS(hp, mfa, NIL);
-	next_p = &CDR(list_val(*next_p));
-	hp += 2;
-
-	/* 
-	 * Finally, we go through the saved continuation pointers.
-	 */
-	for (i = 0; i < depth; i++) {
-	    Eterm *fi = find_function_from_pc((Eterm *) s->trace[i]);
-	    if (fi == NULL) continue;
-	    mfa = TUPLE3(hp, fi[0], fi[1], make_small(fi[2]));
-	    hp += 4;
-	    ASSERT(*next_p == NIL);
-	    *next_p = CONS(hp, mfa, NIL);
-	    next_p = &CDR(list_val(*next_p));
-	    hp += 2;
+    depth = s->depth;
+    stk = stkp = (FunctionInfo *) erts_alloc(ERTS_ALC_T_TMP,
+				      depth*sizeof(FunctionInfo));
+    heap_size = fi.needed + 2;
+    for (i = 0; i < depth; i++) {
+	erts_lookup_function_info(stkp, s->trace[i], 1);
+	if (stkp->current) {
+	    heap_size += stkp->needed + 2;
+	    stkp++;
 	}
-	ASSERT(hp <= hp_end);
-	HRelease(c_p, hp_end, hp);
-    }
-    return Where;
-}
-
-
-static Eterm
-call_error_handler(Process* p, Eterm* fi, Eterm* reg)
-{
-    Eterm* hp;
-    Export* ep;
-    int arity;
-    Eterm args;
-    Uint sz;
-    int i;
-
-    /*
-     * Search for the error_handler module.
-     */
-    ep = erts_find_function(erts_proc_get_error_handler(p),
-			    am_undefined_function, 3);
-    if (ep == NULL) {		/* No error handler */
-	p->current = fi;
-	p->freason = EXC_UNDEF;
-	return 0;
     }
-    p->i = ep->address;
 
     /*
-     * Create a list with all arguments in the x registers.
+     * Allocate heap space and build the stacktrace.
      */
-
-    arity = fi[2];
-    sz = 2 * arity;
-    if (HeapWordsLeft(p) < sz) {
-	erts_garbage_collect(p, sz, reg, arity);
-    }
-    hp = HEAP_TOP(p);
-    HEAP_TOP(p) += sz;
-    args = NIL;
-    for (i = arity-1; i >= 0; i--) {
-	args = CONS(hp, reg[i], args);
+    hp = HAlloc(c_p, heap_size);
+    while (stkp > stk) {
+	stkp--;
+	hp = erts_build_mfa_item(stkp, hp, am_true, &mfa);
+	res = CONS(hp, mfa, res);
 	hp += 2;
     }
+    hp = erts_build_mfa_item(&fi, hp, args, &mfa);
+    res = CONS(hp, mfa, res);
 
-    /*
-     * Set up registers for call to error_handler:undefined_function/3.
-     */
-    reg[0] = fi[0];
-    reg[1] = fi[1];
-    reg[2] = args;
-    return 1;
+    erts_free(ERTS_ALC_T_TMP, (void *) stk);
+    return res;
 }
 
-static Eterm
-call_breakpoint_handler(Process* p, Eterm* fi, Eterm* reg)
+
+static BeamInstr*
+call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func)
 {
     Eterm* hp;
     Export* ep;
@@ -5598,16 +5909,14 @@ call_breakpoint_handler(Process* p, Eterm* fi, Eterm* reg)
     int i;
 
     /*
-     * Search for error handler module.
+     * Search for the error_handler module.
      */
-    ep = erts_find_function(erts_proc_get_error_handler(p),
-			    am_breakpoint, 3);
+    ep = erts_find_function(erts_proc_get_error_handler(p), func, 3);
     if (ep == NULL) {		/* No error handler */
 	p->current = fi;
 	p->freason = EXC_UNDEF;
 	return 0;
     }
-    p->i = ep->address;
 
     /*
      * Create a list with all arguments in the x registers.
@@ -5627,15 +5936,14 @@ call_breakpoint_handler(Process* p, Eterm* fi, Eterm* reg)
     }
 
     /*
-     * Set up registers for call to error_handler:breakpoint/3.
+     * Set up registers for call to error_handler:<func>/3.
      */
     reg[0] = fi[0];
     reg[1] = fi[1];
     reg[2] = args;
-    return 1;
+    return ep->address;
 }
 
-
 
 static Export*
 apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity, Eterm* reg)
@@ -5681,7 +5989,7 @@ apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity,
     return ep;
 }
 
-static Uint*
+static BeamInstr*
 apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg)
 {
     int arity;
@@ -5760,10 +6068,16 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg)
 	save_calls(p, ep);
     }
 
+#ifdef USE_VM_CALL_PROBES
+    if (DTRACE_ENABLED(global_function_entry)) {
+        BeamInstr *fptr = (BeamInstr *) ep->address;
+	DTRACE_GLOBAL_CALL(p, (Eterm)fptr[-3], (Eterm)fptr[-2], (Uint)fptr[-1]);
+    }
+#endif
     return ep->address;
 }
 
-static Uint*
+static BeamInstr*
 fixed_apply(Process* p, Eterm* reg, Uint arity)
 {
     Export* ep;
@@ -5809,11 +6123,17 @@ fixed_apply(Process* p, Eterm* reg, Uint arity)
 	save_calls(p, ep);
     }
 
+#ifdef USE_VM_CALL_PROBES
+    if (DTRACE_ENABLED(global_function_entry)) {
+        BeamInstr *fptr = (BeamInstr *) ep->address;
+	DTRACE_GLOBAL_CALL(p, (Eterm)fptr[-3], (Eterm)fptr[-2], (Uint)fptr[-1]);
+    }
+#endif
     return ep->address;
 }
 
-static int
-hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg)
+int
+erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg)
 {
     int arity;
     Eterm tmp;
@@ -5858,6 +6178,15 @@ hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg)
 	c_p->max_arg_reg = sizeof(c_p->def_arg_reg)/sizeof(c_p->def_arg_reg[0]);
     }
 
+#ifdef USE_VM_PROBES
+    if (DTRACE_ENABLED(process_hibernate)) {
+        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);
+        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);
+        dtrace_fun_decode(c_p, module, function, arity,
+                          process_name, mfa);
+        DTRACE2(process_hibernate, process_name, mfa);
+    }
+#endif
     /*
      * Arrange for the process to be resumed at the given MFA with
      * the stack cleared.
@@ -5869,7 +6198,7 @@ hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg)
     c_p->stop = STACK_START(c_p);
     c_p->catches = 0;
     c_p->i = beam_apply;
-    c_p->cp = (Eterm *) beam_apply+1;
+    c_p->cp = (BeamInstr *) beam_apply+1;
 
     /*
      * If there are no waiting messages, garbage collect and
@@ -5884,22 +6213,25 @@ hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg)
 	c_p->fvalue = NIL;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	erts_garbage_collect_hibernate(c_p);
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	c_p->status = P_WAITING;
 #ifdef ERTS_SMP
 	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
 	if (c_p->msg.len > 0)
 	    erts_add_to_runq(c_p);
+	else
 #endif
+	    c_p->status = P_WAITING;
     }
     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
     c_p->current = bif_export[BIF_hibernate_3]->code;
+    c_p->flags |= F_HIBERNATE_SCHED; /* Needed also when woken! */
     return 1;
 }
 
-static Uint*
+static BeamInstr*
 call_fun(Process* p,		/* Current process. */
 	 int arity,		/* Number of arguments for Fun. */
 	 Eterm* reg,		/* Contents of registers. */
@@ -5919,7 +6251,7 @@ call_fun(Process* p,		/* Current process. */
     if (is_fun_header(hdr)) {
 	ErlFunThing* funp = (ErlFunThing *) fun_val(fun);
 	ErlFunEntry* fe;
-	Eterm* code_ptr;
+	BeamInstr* code_ptr;
 	Eterm* var_ptr;
 	int actual_arity;
 	unsigned num_free;
@@ -5930,6 +6262,9 @@ call_fun(Process* p,		/* Current process. */
 	actual_arity = (int) code_ptr[-1];
 
 	if (actual_arity == arity+num_free) {
+	    DTRACE_LOCAL_CALL(p, (Eterm)code_ptr[-3],
+			(Eterm)code_ptr[-2],
+			code_ptr[-1]);
 	    if (num_free == 0) {
 		return code_ptr;
 	    } else {
@@ -5947,7 +6282,7 @@ call_fun(Process* p,		/* Current process. */
 	} else {
 	    /*
 	     * Something wrong here. First build a list of the arguments.
-	     */  
+	     */
 
 	    if (is_non_value(args)) {
 		Uint sz = 2 * arity;
@@ -6010,13 +6345,19 @@ call_fun(Process* p,		/* Current process. */
 		reg[0] = module;
 		reg[1] = fun;
 		reg[2] = args;
+		reg[3] = NIL;
 		return ep->address;
 	    }
 	}
     } else if (is_export_header(hdr)) {
-	Export* ep = (Export *) (export_val(fun))[1];
-	int actual_arity = (int) ep->code[2];
+	Export *ep;
+	int actual_arity;
+
+	ep = *((Export **) (export_val(fun) + 1));
+	actual_arity = (int) ep->code[2];
+
 	if (arity == actual_arity) {
+	    DTRACE_GLOBAL_CALL(p, ep->code[0], ep->code[1], (Uint)ep->code[2]);
 	    return ep->address;
 	} else {
 	    /*
@@ -6048,6 +6389,26 @@ call_fun(Process* p,		/* Current process. */
 	if (!is_atom(module) || !is_atom(function)) {
 	    goto badfun;
 	}
+
+	/*
+	 * If this is the first time a tuple fun is used,
+	 * send a warning to the logger.
+	 */
+	if (erts_smp_atomic_xchg_nob(&warned_for_tuple_funs,
+				     (erts_aint_t) 1) == 0) {
+	    erts_dsprintf_buf_t* dsbufp;
+
+	    dsbufp = erts_create_logger_dsbuf();
+	    erts_dsprintf(dsbufp, "Call to tuple fun {%T,%T}.\n\n"
+			  "Tuple funs are deprecated and will be removed "
+			  "in R16. Use \"fun M:F/A\" instead, for example "
+			  "\"fun %T:%T/%d\".\n\n"
+			  "(This warning will only be shown the first time "
+			  "a tuple fun is called.)\n",
+			  module, function, module, function, arity);
+	    erts_send_warning_to_logger(p->group_leader, dsbufp);
+	}
+
 	if ((ep = erts_find_export_entry(module, function, arity)) == NULL) {
 	    ep = erts_find_export_entry(erts_proc_get_error_handler(p),
 					am_undefined_function, 3);
@@ -6072,6 +6433,7 @@ call_fun(Process* p,		/* Current process. */
 	    reg[1] = function;
 	    reg[2] = args;
 	}
+	DTRACE_GLOBAL_CALL(p, module, function, arity);
 	return ep->address;
     } else {
     badfun:
@@ -6082,7 +6444,7 @@ call_fun(Process* p,		/* Current process. */
     }
 }
 
-static Eterm*
+static BeamInstr*
 apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg)
 {
     int arity;
@@ -6125,6 +6487,7 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free)
     if (HEAP_LIMIT(p) - HEAP_TOP(p) <= needed) {
 	PROCESS_MAIN_CHK_LOCKS(p);
 	erts_garbage_collect(p, needed, reg, num_free);
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
 	PROCESS_MAIN_CHK_LOCKS(p);
     }
     hp = p->htop;
@@ -6133,10 +6496,8 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free)
     hp = funp->env;
     erts_refc_inc(&fe->refc, 2);
     funp->thing_word = HEADER_FUN;
-#ifndef HYBRID /* FIND ME! */
-    funp->next = MSO(p).funs;
-    MSO(p).funs = funp;
-#endif
+    funp->next = MSO(p).first;
+    MSO(p).first = (struct erl_off_heap_header*) funp;
     funp->fe = fe;
     funp->num_free = num_free;
     funp->creator = p->id;
@@ -6176,7 +6537,7 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity)
     if ((ep = export_get(&e)) == NULL) {
 	return 0;
     }
-    return ep->address == ep->code+3 && (ep->code[3] == (Uint) em_apply_bif);
+    return ep->address == ep->code+3 && (ep->code[3] == (BeamInstr) em_apply_bif);
 }
 
 
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 99fab28dce..dd788df6e4 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -51,9 +51,9 @@ ErlDrvBinary* erts_gzinflate_buffer(char*, int);
 #define EXPORTED  2
 
 #ifdef NO_JUMP_TABLE
-#  define BeamOpCode(Op) ((Uint)(Op))
+#  define BeamOpCode(Op) ((BeamInstr)(Op))
 #else
-#  define BeamOpCode(Op) ((Eterm)beam_ops[Op])
+#  define BeamOpCode(Op) ((BeamInstr)beam_ops[Op])
 #endif
 
 #if defined(WORDS_BIGENDIAN)
@@ -89,13 +89,12 @@ typedef struct {
 } Label;
 
 /*
- * Type for a operand for a generic instruction.
+ * Type for an operand for a generic instruction.
  */
 
 typedef struct {
     unsigned type;		/* Type of operand. */
-    Uint val;			/* Value of operand. */
-    Uint bigarity;		/* Arity for bignumbers (only). */
+    BeamInstr val;		/* Value of operand. */
 } GenOpArg;
 
 /*
@@ -142,7 +141,7 @@ typedef struct {
 typedef struct {
     Eterm function;		/* Tagged atom for function. */
     int arity;			/* Arity. */
-    Eterm* address;		/* Address to function in code. */
+    BeamInstr* address;		/* Address to function in code. */
 } ExportEntry;
 
 #define MakeIffId(a, b, c, d) \
@@ -159,6 +158,7 @@ typedef struct {
 #define LITERAL_CHUNK 6
 #define ATTR_CHUNK 7
 #define COMPILE_CHUNK 8
+#define LINE_CHUNK 9
 
 #define NUM_CHUNK_TYPES (sizeof(chunk_types)/sizeof(chunk_types[0]))
 
@@ -183,6 +183,7 @@ static Uint chunk_types[] = {
     MakeIffId('L', 'i', 't', 'T'), /* 6 */
     MakeIffId('A', 't', 't', 'r'), /* 7 */
     MakeIffId('C', 'I', 'n', 'f'), /* 8 */
+    MakeIffId('L', 'i', 'n', 'e'), /* 9 */
 };
 
 /*
@@ -205,6 +206,7 @@ typedef struct {
     Eterm term;			/* The tagged term (in the heap). */
     Uint heap_size;		/* (Exact) size on the heap. */
     Uint offset;		/* Offset from temporary location to final. */
+    ErlOffHeap off_heap;	/* Start of linked list of ProcBins. */
     Eterm* heap;		/* Heap for term. */
 } Literal;
 
@@ -232,10 +234,19 @@ struct string_patch {
 };
 
 /*
+ * This structure associates a code offset with a source code location.
+ */
+
+typedef struct {
+    int pos;			/* Position in code */
+    Uint32 loc;			/* Location in source code */
+} LineInstr;
+
+/*
  * This structure contains all information about the module being loaded.
  */  
 
-typedef struct {
+typedef struct LoaderState {
     /*
      * The current logical file within the binary.
      */
@@ -243,6 +254,7 @@ typedef struct {
     char* file_name;		/* Name of file we are reading (usually chunk name). */
     byte* file_p;		/* Current pointer within file. */
     unsigned file_left;		/* Number of bytes left in file. */
+    ErlDrvBinary* bin;		/* Binary holding BEAM file (or NULL) */
 
     /*
      * The following are used mainly for diagnostics.
@@ -274,13 +286,11 @@ typedef struct {
     int num_functions;		/* Number of functions in module. */
     int num_labels;		/* Number of labels. */
     int code_buffer_size;	/* Size of code buffer in words.  */
-    Eterm* code;		/* Loaded code. */
+    BeamInstr* code;		/* Loaded code. */
     int ci;			/* Current index into loaded code. */
     Label* labels;
-    Uint put_strings;		/* Linked list of put_string instructions. */
-    Uint new_bs_put_strings;	/* Linked list of i_new_bs_put_string instructions. */
     StringPatch* string_patches; /* Linked list of position into string table to patch. */
-    Uint catches;		/* Linked list of catch_yf instructions. */
+    BeamInstr catches;		/* Linked list of catch_yf instructions. */
     unsigned loaded_size;	/* Final size of code when loaded. */
     byte mod_md5[16];		/* MD5 for module code. */
     int may_load_nif;           /* true if NIFs may later be loaded for this module */  
@@ -329,30 +339,56 @@ typedef struct {
     Uint total_literal_size;	/* Total heap size for all literals. */
 
     /*
-     * Floating point.
+     * Line table.
      */
-    int new_float_instructions;	/* New allocation scheme for floating point. */
+    BeamInstr* line_item;	/* Line items from the BEAM file. */
+    int num_line_items;		/* Number of line items. */
+    LineInstr* line_instr;	/* Line instructions */
+    int num_line_instrs;	/* Maximum number of line instructions */
+    int current_li;		/* Current line instruction */
+    int* func_line;		/* Mapping from function to first line instr */
+    Eterm* fname;		/* List of file names */
+    int num_fnames;		/* Number of filenames in fname table */
+    int loc_size;		/* Size of location info in bytes (2/4) */
 } LoaderState;
 
-typedef struct {
-    unsigned num_functions;	/* Number of functions. */
-    Eterm* func_tab[1];		/* Pointers to each function. */
-} LoadedCode;
-
-#define GetTagAndValue(Stp, Tag, Val) \
-   do { \
-      Uint __w; \
-      GetByte(Stp, __w); \
-      Tag = __w & 0x07; \
-      if ((__w & 0x08) == 0) { \
-	 Val = __w >> 4; \
-      } else if ((__w & 0x10) == 0) { \
-	 Val = ((__w >> 5) << 8); \
-	 GetByte(Stp, __w); \
-	 Val |= __w; \
-      } else { \
-	if (!get_int_val(Stp, __w, &(Val))) goto load_error; \
-      } \
+/*
+ * Layout of the line table.
+ */
+
+#define MI_LINE_FNAME_PTR 0
+#define MI_LINE_LOC_TAB 1
+#define MI_LINE_LOC_SIZE 2
+#define MI_LINE_FUNC_TAB 3
+
+#define LINE_INVALID_LOCATION (0)
+
+/*
+ * Macros for manipulating locations.
+ */
+
+#define IS_VALID_LOCATION(File, Line) \
+    ((unsigned) (File) < 255 && (unsigned) (Line) < ((1 << 24) - 1))
+#define MAKE_LOCATION(File, Line) (((File) << 24) | (Line))
+#define LOC_FILE(Loc) ((Loc) >> 24)
+#define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1))
+
+#define GetTagAndValue(Stp, Tag, Val)					\
+   do {									\
+      BeamInstr __w;							\
+      GetByte(Stp, __w);						\
+      Tag = __w & 0x07;							\
+      if ((__w & 0x08) == 0) {						\
+	 Val = __w >> 4;						\
+      } else if ((__w & 0x10) == 0) {					\
+	 Val = ((__w >> 5) << 8);					\
+	 GetByte(Stp, __w);						\
+	 Val |= __w;							\
+      } else {								\
+	 int __res = get_tag_and_value(Stp, __w, (Tag), &(Val));	\
+         if (__res < 0) goto load_error;				\
+         Tag = (unsigned) __res;					\
+      }									\
    } while (0)
 
 
@@ -388,7 +424,7 @@ typedef struct {
        goto load_error; \
     } else { \
        int __n = (N); \
-       Uint __result = 0; \
+       BeamInstr __result = 0; \
        Stp->file_left -= (unsigned) __n; \
        while (__n-- > 0) { \
           __result = __result << 8 | *Stp->file_p++; \
@@ -460,36 +496,34 @@ typedef struct {
   } while (0)
 
 
-static int bin_load(Process *c_p, ErtsProcLocks c_p_locks,
-		    Eterm group_leader, Eterm* modp, byte* bytes, int unloaded_size);
-static void init_state(LoaderState* stp);
-static int insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
-			   Eterm group_leader, Eterm module,
-			   Eterm* code, Uint size, Uint catches);
+static void free_state(LoaderState* stp);
+static Eterm insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
+			     Eterm group_leader, Eterm module,
+			     BeamInstr* code, Uint size);
+static int init_iff_file(LoaderState* stp, byte* code, Uint size);
 static int scan_iff_file(LoaderState* stp, Uint* chunk_types,
 			 Uint num_types, Uint num_mandatory);
+static int verify_chunks(LoaderState* stp);
 static int load_atom_table(LoaderState* stp);
 static int load_import_table(LoaderState* stp);
 static int read_export_table(LoaderState* stp);
 static int read_lambda_table(LoaderState* stp);
 static int read_literal_table(LoaderState* stp);
+static int read_line_table(LoaderState* stp);
 static int read_code_header(LoaderState* stp);
 static int load_code(LoaderState* stp);
 static GenOp* gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index,
 			  GenOpArg Tuple, GenOpArg Dst);
-static GenOp* gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail,
+static GenOp* gen_split_values(LoaderState* stp, GenOpArg S,
+			       GenOpArg TypeFail, GenOpArg Fail,
 			       GenOpArg Size, GenOpArg* Rest);
 static GenOp* gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 			     GenOpArg Size, GenOpArg* Rest);
-static GenOp* gen_select_big(LoaderState* stp, GenOpArg S, GenOpArg Fail,
-			     GenOpArg Size, GenOpArg* Rest);
+static GenOp* gen_select_literals(LoaderState* stp, GenOpArg S,
+				  GenOpArg Fail, GenOpArg Size,
+				  GenOpArg* Rest);
 static GenOp* const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 			       GenOpArg Size, GenOpArg* Rest);
-static GenOp* gen_func_info(LoaderState* stp, GenOpArg mod, GenOpArg Func,
-			    GenOpArg arity, GenOpArg label);
-static GenOp*
-gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
-	      GenOpArg Src, GenOpArg Dst);
 
 static int freeze_code(LoaderState* stp);
 
@@ -499,8 +533,8 @@ static void load_printf(int line, LoaderState* context, char *fmt, ...);
 static int transform_engine(LoaderState* st);
 static void id_to_string(Uint id, char* s);
 static void new_genop(LoaderState* stp);
-static int get_int_val(LoaderState* stp, Uint len_code, Uint* result);
-static int get_erlang_integer(LoaderState* stp, Uint len_code, Uint* result);
+static int get_tag_and_value(LoaderState* stp, Uint len_code,
+			     unsigned tag, BeamInstr* result);
 static int new_label(LoaderState* stp);
 static void new_literal_patch(LoaderState* stp, int pos);
 static void new_string_patch(LoaderState* stp, int pos);
@@ -513,7 +547,9 @@ static Eterm compilation_info_for_module(Process* p, Eterm mod);
 static Eterm native_addresses(Process* p, Eterm mod);
 int patch_funentries(Eterm Patchlist);
 int patch(Eterm Addresses, Uint fe);
-static int safe_mul(Uint a, Uint b, Uint* resp);
+static int safe_mul(UWord a, UWord b, UWord* resp);
+static void lookup_loc(FunctionInfo* fi, BeamInstr* pc,
+		       BeamInstr* modp, int idx);
 
 
 static int must_swap_floats;
@@ -558,7 +594,7 @@ define_file(LoaderState* stp, char* name, int idx)
     stp->file_left = stp->chunks[idx].size;
 }
 
-int
+Eterm
 erts_load_module(Process *c_p,
 		 ErtsProcLocks c_p_locks,
 		 Eterm group_leader, /* Group leader or NIL if none. */
@@ -567,51 +603,53 @@ erts_load_module(Process *c_p,
 				 * On return, contains the actual module name.
 				 */
 		 byte* code,	/* Points to the code to load */
-		 int size)	/* Size of code to load. */
+		 Uint size)	/* Size of code to load. */
 {
-    ErlDrvBinary* bin;
-    int result;
+    LoaderState* stp = erts_alloc_loader_state();
+    Eterm retval;
 
-    if (size >= 4 && code[0] == 'F' && code[1] == 'O' &&
-	code[2] == 'R' && code[3] == '1') {
-	/*
-	 * The BEAM module is not compressed.
-	 */
-	result = bin_load(c_p, c_p_locks, group_leader, modp, code, size);
-    } else {
-	/*
-	 * The BEAM module is compressed (or possibly invalid/corrupted).
-	 */
-	if ((bin = (ErlDrvBinary *) erts_gzinflate_buffer((char*)code, size)) == NULL) {
-	    return -1;
-	}
-	result = bin_load(c_p, c_p_locks, group_leader, modp,
-			  (byte*)bin->orig_bytes, bin->orig_size);
-	driver_free_binary(bin);
+    retval = erts_prepare_loading(stp, c_p, group_leader, modp,
+				  code, size);
+    if (retval != NIL) {
+	return retval;
     }
-    return result;
+    return erts_finish_loading(stp, c_p, c_p_locks, modp);
 }
+/* #define LOAD_MEMORY_HARD_DEBUG 1*/
+
+#if defined(LOAD_MEMORY_HARD_DEBUG) && defined(DEBUG)
+/* Requires allocators ERTS_ALLOC_UTIL_HARD_DEBUG also set in erl_alloc_util.h */
+extern void check_allocators(void);
+extern void check_allocated_block(Uint type, void *blk);
+#define CHKALLOC() check_allocators()
+#define CHKBLK(TYPE,BLK) if ((BLK) != NULL) check_allocated_block((TYPE),(BLK))
+#else
+#define CHKALLOC() /* nothing */
+#define CHKBLK(TYPE,BLK) /* nothing */
+#endif
 
-
-static int
-bin_load(Process *c_p, ErtsProcLocks c_p_locks,
-	 Eterm group_leader, Eterm* modp, byte* bytes, int unloaded_size)
+Eterm
+erts_prepare_loading(LoaderState* stp, Process *c_p, Eterm group_leader,
+		     Eterm* modp, byte* code, Uint unloaded_size)
 {
-    LoaderState state;
-    int rval = -1;
+    Eterm retval = am_badfile;
+
+    stp->module = *modp;
+    stp->group_leader = group_leader;
 
-    init_state(&state);
-    state.module = *modp;
-    state.group_leader = group_leader;
+#if defined(LOAD_MEMORY_HARD_DEBUG) && defined(DEBUG)
+    erts_fprintf(stderr,"Loading a module\n");
+#endif
 
     /*
      * Scan the IFF file.
      */
 
-    state.file_name = "IFF header for Beam file";
-    state.file_p = bytes;
-    state.file_left = unloaded_size;
-    if (!scan_iff_file(&state, chunk_types, NUM_CHUNK_TYPES, NUM_MANDATORY)) {
+    CHKALLOC();
+    CHKBLK(ERTS_ALC_T_CODE,stp->code);
+    if (!init_iff_file(stp, code, unloaded_size) ||
+	!scan_iff_file(stp, chunk_types, NUM_CHUNK_TYPES, NUM_MANDATORY) ||
+	!verify_chunks(stp)) {
 	goto load_error;
     }
 
@@ -619,17 +657,38 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * Read the header for the code chunk.
      */
 
-    define_file(&state, "code chunk header", CODE_CHUNK);
-    if (!read_code_header(&state)) {
+    CHKBLK(ERTS_ALC_T_CODE,stp->code);
+    define_file(stp, "code chunk header", CODE_CHUNK);
+    if (!read_code_header(stp)) {
 	goto load_error;
     }
 
     /*
+     * Initialize code area.
+     */
+    stp->code_buffer_size = erts_next_heap_size(2048 + stp->num_functions, 0);
+    stp->code = (BeamInstr *) erts_alloc(ERTS_ALC_T_CODE,
+				    sizeof(BeamInstr) * stp->code_buffer_size);
+
+    stp->code[MI_NUM_FUNCTIONS] = stp->num_functions;
+    stp->ci = MI_FUNCTIONS + stp->num_functions + 1;
+
+    stp->code[MI_ATTR_PTR] = 0;
+    stp->code[MI_ATTR_SIZE] = 0;
+    stp->code[MI_ATTR_SIZE_ON_HEAP] = 0;
+    stp->code[MI_COMPILE_PTR] = 0;
+    stp->code[MI_COMPILE_SIZE] = 0;
+    stp->code[MI_COMPILE_SIZE_ON_HEAP] = 0;
+    stp->code[MI_NUM_BREAKPOINTS] = 0;
+
+
+    /*
      * Read the atom table.
      */
 
-    define_file(&state, "atom table", ATOM_CHUNK);
-    if (!load_atom_table(&state)) {
+    CHKBLK(ERTS_ALC_T_CODE,stp->code);
+    define_file(stp, "atom table", ATOM_CHUNK);
+    if (!load_atom_table(stp)) {
 	goto load_error;
     }
 
@@ -637,8 +696,9 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * Read the import table.
      */
 
-    define_file(&state, "import table", IMP_CHUNK);
-    if (!load_import_table(&state)) {
+    CHKBLK(ERTS_ALC_T_CODE,stp->code);
+    define_file(stp, "import table", IMP_CHUNK);
+    if (!load_import_table(stp)) {
 	goto load_error;
     }
 
@@ -646,9 +706,10 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * Read the lambda (fun) table.
      */
 
-    if (state.chunks[LAMBDA_CHUNK].size > 0) {
-	define_file(&state, "lambda (fun) table", LAMBDA_CHUNK);
-	if (!read_lambda_table(&state)) {
+    CHKBLK(ERTS_ALC_T_CODE,stp->code);
+    if (stp->chunks[LAMBDA_CHUNK].size > 0) {
+	define_file(stp, "lambda (fun) table", LAMBDA_CHUNK);
+	if (!read_lambda_table(stp)) {
 	    goto load_error;
 	}
     }
@@ -657,9 +718,22 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * Read the literal table.
      */
 
-    if (state.chunks[LITERAL_CHUNK].size > 0) {
-	define_file(&state, "literals table (constant pool)", LITERAL_CHUNK);
-	if (!read_literal_table(&state)) {
+    CHKBLK(ERTS_ALC_T_CODE,stp->code);
+    if (stp->chunks[LITERAL_CHUNK].size > 0) {
+	define_file(stp, "literals table (constant pool)", LITERAL_CHUNK);
+	if (!read_literal_table(stp)) {
+	    goto load_error;
+	}
+    }
+
+    /*
+     * Read the line table (if present).
+     */
+
+    CHKBLK(ERTS_ALC_T_CODE,stp->code);
+    if (stp->chunks[LINE_CHUNK].size > 0) {
+	define_file(stp, "line table", LINE_CHUNK);
+	if (!read_line_table(stp)) {
 	    goto load_error;
 	}
     }
@@ -668,20 +742,67 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * Load the code chunk.
      */
 
-    state.file_name = "code chunk";
-    state.file_p = state.code_start;
-    state.file_left = state.code_size;
-    if (!load_code(&state) || !freeze_code(&state)) {
+    CHKBLK(ERTS_ALC_T_CODE,stp->code);
+    stp->file_name = "code chunk";
+    stp->file_p = stp->code_start;
+    stp->file_left = stp->code_size;
+    if (!load_code(stp)) {
+	goto load_error;
+    }
+    CHKBLK(ERTS_ALC_T_CODE,stp->code);
+    if (!freeze_code(stp)) {
 	goto load_error;
     }
 
+
     /*
      * Read and validate the export table.  (This must be done after
      * loading the code, because it contains labels.)
      */
     
-    define_file(&state, "export table", EXP_CHUNK);
-    if (!read_export_table(&state)) {
+    CHKBLK(ERTS_ALC_T_CODE,stp->code);
+    define_file(stp, "export table", EXP_CHUNK);
+    if (!read_export_table(stp)) {
+	goto load_error;
+    }
+
+    /*
+     * Good so far.
+     */
+
+    retval = NIL;
+
+ load_error:
+    if (retval != NIL) {
+	free_state(stp);
+    }
+    return retval;
+}
+
+Eterm
+erts_finish_loading(LoaderState* stp, Process* c_p,
+		    ErtsProcLocks c_p_locks, Eterm* modp)
+{
+    Eterm retval;
+
+    /*
+     * No other process may run since we will update the export
+     * table which is not protected by any locks.
+     */
+
+    ERTS_SMP_LC_ASSERT(erts_initialized == 0 ||
+		       erts_smp_thr_progress_is_blocking());
+
+    /*
+     * Make current code for the module old and insert the new code
+     * as current.  This will fail if there already exists old code
+     * for the module.
+     */
+
+    CHKBLK(ERTS_ALC_T_CODE,stp->code);
+    retval = insert_new_code(c_p, c_p_locks, stp->group_leader, stp->module,
+			     stp->code, stp->loaded_size);
+    if (retval != NIL) {
 	goto load_error;
     }
 
@@ -690,79 +811,43 @@ bin_load(Process *c_p, ErtsProcLocks c_p_locks,
      * exported and imported functions.  This can't fail.
      */
     
-    rval = insert_new_code(c_p, c_p_locks, state.group_leader, state.module,
-			   state.code, state.loaded_size, state.catches);
-    if (rval < 0) {
-	goto load_error;
-    }
-    final_touch(&state);
+    erts_export_consolidate();
+    CHKBLK(ERTS_ALC_T_CODE,stp->code);
+    final_touch(stp);
 
     /*
      * Loading succeded.
      */
-    rval = 0;
-    state.code = NULL;		/* Prevent code from being freed. */
-    *modp = state.module;
+    CHKBLK(ERTS_ALC_T_CODE,stp->code);
+#if defined(LOAD_MEMORY_HARD_DEBUG) && defined(DEBUG)
+    erts_fprintf(stderr,"Loaded %T\n",*modp);
+#if 0
+    debug_dump_code(stp->code,stp->ci);
+#endif
+#endif
+    stp->code = NULL;		/* Prevent code from being freed. */
+    *modp = stp->module;
 
     /*
      * If there is an on_load function, signal an error to
      * indicate that the on_load function must be run.
      */
-    if (state.on_load) {
-	rval = -5;
+    if (stp->on_load) {
+	retval = am_on_load;
     }
 
  load_error:
-    if (state.code != 0) {
-	erts_free(ERTS_ALC_T_CODE, state.code);
-    }
-    if (state.labels != NULL) {
-	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.labels);
-    }
-    if (state.atom != NULL) {
-	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.atom);
-    }
-    if (state.import != NULL) {
-	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.import);
-    }
-    if (state.export != NULL) {
-	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.export);
-    }
-    if (state.lambdas != state.def_lambdas) {
-	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.lambdas);
-    }
-    if (state.literals != NULL) {
-	int i;
-	for (i = 0; i < state.num_literals; i++) {
-	    if (state.literals[i].heap != NULL) {
-		erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.literals[i].heap);
-	    }
-	}
-	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.literals);
-    }
-    while (state.literal_patches != NULL) {
-	LiteralPatch* next = state.literal_patches->next;
-	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.literal_patches);
-	state.literal_patches = next;
-    }
-    while (state.string_patches != NULL) {
-	StringPatch* next = state.string_patches->next;
-	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.string_patches);
-	state.string_patches = next;
-    }
-    while (state.genop_blocks) {
-	GenOpBlock* next = state.genop_blocks->next;
-	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.genop_blocks);
-	state.genop_blocks = next;
-    }
-
-    return rval;
+    free_state(stp);
+    return retval;
 }
 
-
-static void
-init_state(LoaderState* stp)
+LoaderState*
+erts_alloc_loader_state(void)
 {
+    LoaderState* stp;
+
+    stp = erts_alloc(ERTS_ALC_T_LOADER_TMP, sizeof(LoaderState));
+    stp->bin = NULL;
     stp->function = THE_NON_VALUE; /* Function not known yet */
     stp->arity = 0;
     stp->specific_op = -1;
@@ -784,26 +869,100 @@ init_state(LoaderState* stp)
     stp->total_literal_size = 0;
     stp->literal_patches = 0;
     stp->string_patches = 0;
-    stp->new_float_instructions = 0;
     stp->may_load_nif = 0;
     stp->on_load = 0;
+    stp->line_item = 0;
+    stp->line_instr = 0;
+    stp->func_line = 0;
+    stp->fname = 0;
+    return stp;
 }
 
-static int
+static void
+free_state(LoaderState* stp)
+{
+    if (stp->bin != 0) {
+	driver_free_binary(stp->bin);
+    }
+    if (stp->code != 0) {
+	erts_free(ERTS_ALC_T_CODE, stp->code);
+    }
+    if (stp->labels != NULL) {
+	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->labels);
+    }
+    if (stp->atom != NULL) {
+	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->atom);
+    }
+    if (stp->import != NULL) {
+	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->import);
+    }
+    if (stp->export != NULL) {
+	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->export);
+    }
+    if (stp->lambdas != stp->def_lambdas) {
+	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->lambdas);
+    }
+    if (stp->literals != NULL) {
+	int i;
+	for (i = 0; i < stp->num_literals; i++) {
+	    if (stp->literals[i].heap != NULL) {
+		erts_free(ERTS_ALC_T_LOADER_TMP,
+			  (void *) stp->literals[i].heap);
+	    }
+	}
+	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->literals);
+    }
+    while (stp->literal_patches != NULL) {
+	LiteralPatch* next = stp->literal_patches->next;
+	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->literal_patches);
+	stp->literal_patches = next;
+    }
+    while (stp->string_patches != NULL) {
+	StringPatch* next = stp->string_patches->next;
+	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->string_patches);
+	stp->string_patches = next;
+    }
+    while (stp->genop_blocks) {
+	GenOpBlock* next = stp->genop_blocks->next;
+	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) stp->genop_blocks);
+	stp->genop_blocks = next;
+    }
+
+    if (stp->line_item != 0) {
+	erts_free(ERTS_ALC_T_LOADER_TMP, stp->line_item);
+    }
+
+    if (stp->line_instr != 0) {
+	erts_free(ERTS_ALC_T_LOADER_TMP, stp->line_instr);
+    }
+
+    if (stp->func_line != 0) {
+	erts_free(ERTS_ALC_T_LOADER_TMP, stp->func_line);
+    }
+
+    if (stp->fname != 0) {
+	erts_free(ERTS_ALC_T_LOADER_TMP, stp->fname);
+    }
+
+    erts_free(ERTS_ALC_T_LOADER_TMP, stp);
+}
+
+static Eterm
 insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
-		Eterm group_leader, Eterm module, Eterm* code, Uint size, Uint catches)
+		Eterm group_leader, Eterm module, BeamInstr* code,
+		Uint size)
 {
     Module* modp;
-    int rval;
+    Eterm retval;
     int i;
 
-    if ((rval = beam_make_current_old(c_p, c_p_locks, module)) < 0) {
+    if ((retval = beam_make_current_old(c_p, c_p_locks, module)) != NIL) {
 	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
 	erts_dsprintf(dsbufp,
 		      "Module %T must be purged before loading\n",
 		      module);
 	erts_send_error_to_logger(group_leader, dsbufp);
-	return rval;
+	return retval;
     }
 
     /*
@@ -814,7 +973,7 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
     modp = erts_put_module(module);
     modp->code = code;
     modp->code_length = size;
-    modp->catches = catches;
+    modp->catches = BEAM_CATCHES_NIL; /* Will be filled in later. */
 
     /*
      * Update address table (used for finding a function from a PC value).
@@ -833,30 +992,54 @@ insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
 	modules[i] = modules[i-1];
     }
     modules[i].start = code;
-    modules[i].end = (Eterm *) (((byte *)code) + size);
+    modules[i].end = (BeamInstr *) (((byte *)code) + size);
     num_loaded_modules++;
     mid_module = &modules[num_loaded_modules/2];
-    return 0;
+    return NIL;
 }
 
 static int
-scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory)
+init_iff_file(LoaderState* stp, byte* code, Uint size)
 {
-    MD5_CTX context;
+    Uint form_id = MakeIffId('F', 'O', 'R', '1');
     Uint id;
     Uint count;
-    int i;
+
+    if (size < 4) {
+	goto load_error;
+    }
 
     /*
-     * The binary must start with an IFF 'FOR1' chunk.
+     * Check if the module is compressed (or possibly invalid/corrupted).
      */
+    if (MakeIffId(code[0], code[1], code[2], code[3]) != form_id) {
+	stp->bin = (ErlDrvBinary *) erts_gzinflate_buffer((char*)code, size);
+	if (stp->bin == NULL) {
+	    goto load_error;
+	}
+	code = (byte*)stp->bin->orig_bytes;
+	size = stp->bin->orig_size;
+	if (size < 4) {
+	    goto load_error;
+	}
+    }
 
-    GetInt(stp, 4, id);
-    if (id != MakeIffId('F', 'O', 'R', '1')) {
+    /*
+     * The binary must start with an IFF 'FOR1' chunk.
+     */
+    if (MakeIffId(code[0], code[1], code[2], code[3]) != form_id) {
 	LoadError0(stp, "not a BEAM file: no IFF 'FOR1' chunk");
     }
 
     /*
+     * Initialize our "virtual file system".
+     */
+
+    stp->file_name = "IFF header for Beam file";
+    stp->file_p = code + 4;
+    stp->file_left = size - 4;
+
+    /*
      * Retrieve the chunk size and verify it.  If the size is equal to
      * or less than the size of the binary, it is ok and we will use it
      * as the limit for the logical file size.
@@ -877,6 +1060,21 @@ scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mand
     if (id != MakeIffId('B', 'E', 'A', 'M')) {
 	LoadError0(stp, "not a BEAM file: IFF form type is not 'BEAM'");
     }
+    return 1;
+
+ load_error:
+    return 0;
+}
+
+/*
+ * Scan the IFF file. The header should have been verified by init_iff_file().
+ */
+static int
+scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory)
+{
+    Uint count;
+    Uint id;
+    int i;
 
     /*
      * Initialize the chunks[] array in the state.
@@ -933,17 +1131,25 @@ scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mand
 	stp->file_p += count;
 	stp->file_left -= count;
     }
+    return 1;
 
-    /*
-     * At this point, we have read the entire IFF file, and we
-     * know that it is syntactically correct.
-     *
-     * Now check that it contains all mandatory chunks. At the
-     * same time calculate the MD5 for the module.
-     */
+ load_error:
+    return 0;
+}
+
+/*
+ * Verify that all mandatory chunks are present and calculate
+ * MD5 for the module.
+ */
+
+static int
+verify_chunks(LoaderState* stp)
+{
+    int i;
+    MD5_CTX context;
 
     MD5Init(&context);
-    for (i = 0; i < num_mandatory; i++) {
+    for (i = 0; i < NUM_MANDATORY; i++) {
 	if (stp->chunks[i].start != NULL) {
 	    MD5Update(&context, stp->chunks[i].start, stp->chunks[i].size);
 	} else {
@@ -953,41 +1159,49 @@ scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mand
 	    LoadError1(stp, "mandatory chunk of type '%s' not found\n", sbuf);
 	}
     }
-    if (LITERAL_CHUNK < num_types) {
-	if (stp->chunks[LAMBDA_CHUNK].start != 0) {
-	    byte* start = stp->chunks[LAMBDA_CHUNK].start;
-	    Uint left = stp->chunks[LAMBDA_CHUNK].size;
 
-	    /*
-	     * The idea here is to ignore the OldUniq field for the fun; it is
-	     * based on the old broken hash function, which can be different
-	     * on little endian and big endian machines.
-	     */
-	    if (left >= 4) {
-		static byte zero[4];
-		MD5Update(&context, start, 4);
-		start += 4;
-		left -= 4;
+    /*
+     * If there is a lambda chunk, include parts of it in the MD5.
+     */
+    if (stp->chunks[LAMBDA_CHUNK].start != 0) {
+	byte* start = stp->chunks[LAMBDA_CHUNK].start;
+	Uint left = stp->chunks[LAMBDA_CHUNK].size;
+
+	/*
+	 * The idea here is to ignore the OldUniq field for the fun; it is
+	 * based on the old broken hash function, which can be different
+	 * on little endian and big endian machines.
+	 */
+	if (left >= 4) {
+	    static byte zero[4];
+	    MD5Update(&context, start, 4);
+	    start += 4;
+	    left -= 4;
 		
-		while (left >= 24) {
-		    /* Include: Function Arity Index NumFree */
-		    MD5Update(&context, start, 20);
-		    /* Set to zero: OldUniq */
-		    MD5Update(&context, zero, 4);
-		    start += 24;
-		    left -= 24;
-		}
-	    }
-	    /* Can't happen for a correct 'FunT' chunk */
-	    if (left > 0) {
-		MD5Update(&context, start, left);
+	    while (left >= 24) {
+		/* Include: Function Arity Index NumFree */
+		MD5Update(&context, start, 20);
+		/* Set to zero: OldUniq */
+		MD5Update(&context, zero, 4);
+		start += 24;
+		left -= 24;
 	    }
 	}
-	if (stp->chunks[LITERAL_CHUNK].start != 0) {
-	    MD5Update(&context, stp->chunks[LITERAL_CHUNK].start,
-		      stp->chunks[LITERAL_CHUNK].size);
+	/* Can't happen for a correct 'FunT' chunk */
+	if (left > 0) {
+	    MD5Update(&context, start, left);
 	}
     }
+
+
+    /*
+     * If there is a literal chunk, include it in the MD5.
+     */
+    if (stp->chunks[LITERAL_CHUNK].start != 0) {
+	MD5Update(&context, stp->chunks[LITERAL_CHUNK].start,
+		  stp->chunks[LITERAL_CHUNK].size);
+    }
+
     MD5Final(stp->mod_md5, &context);
     return 1;
 
@@ -1083,7 +1297,7 @@ load_import_table(LoaderState* stp)
 	 * the BIF function.
 	 */
 	if ((e = erts_find_export_entry(mod, func, arity)) != NULL) {
-	    if (e->code[3] == (Uint) em_apply_bif) {
+	    if (e->code[3] == (BeamInstr) em_apply_bif) {
 		stp->import[i].bf = (BifFunction) e->code[4];
 		if (func == am_load_nif && mod == am_erlang && arity == 2) {
 		    stp->may_load_nif = 1;
@@ -1151,7 +1365,7 @@ read_export_table(LoaderState* stp)
 	 * redefine).
 	 */
 	if ((e = erts_find_export_entry(stp->module, func, arity)) != NULL) {
-	    if (e->code[3] == (Uint) em_apply_bif) {
+	    if (e->code[3] == (BeamInstr) em_apply_bif) {
 		int j;
 
 		for (j = 0; j < sizeof(allow_redef)/sizeof(allow_redef[0]); j++) {
@@ -1220,7 +1434,7 @@ static int
 read_literal_table(LoaderState* stp)
 {
     int i;
-    Uint uncompressed_sz;
+    uLongf uncompressed_sz;
     byte* uncompressed = 0;
 
     GetInt(stp, 4, uncompressed_sz);
@@ -1230,7 +1444,7 @@ read_literal_table(LoaderState* stp)
 	LoadError0(stp, "failed to uncompress literal table (constant pool)");
     }
     stp->file_p = uncompressed;
-    stp->file_left = uncompressed_sz;
+    stp->file_left = (unsigned) uncompressed_sz;
     GetInt(stp, 4, stp->num_literals);
     stp->literals = (Literal *) erts_alloc(ERTS_ALC_T_LOADER_TMP,
 					   stp->num_literals * sizeof(Literal));
@@ -1249,12 +1463,14 @@ read_literal_table(LoaderState* stp)
 
 	GetInt(stp, 4, sz);	/* Size of external term format. */
 	GetString(stp, p, sz);
-	if ((heap_size = erts_decode_ext_size(p, sz, 1)) < 0) {
+	if ((heap_size = erts_decode_ext_size(p, sz)) < 0) {
 	    LoadError1(stp, "literal %d: bad external format", i);
 	}
 	hp = stp->literals[i].heap = erts_alloc(ERTS_ALC_T_LOADER_TMP,
 						heap_size*sizeof(Eterm));
-	val = erts_decode_ext(&hp, NULL, &p);
+	stp->literals[i].off_heap.first = 0;
+	stp->literals[i].off_heap.overhead = 0;
+	val = erts_decode_ext(&hp, &stp->literals[i].off_heap, &p);
 	stp->literals[i].heap_size = hp - stp->literals[i].heap;
 	if (stp->literals[i].heap_size > heap_size) {
 	    erl_exit(1, "overrun by %d word(s) for literal heap, term %d",
@@ -1276,6 +1492,138 @@ read_literal_table(LoaderState* stp)
     return 0;
 }
 
+static int
+read_line_table(LoaderState* stp)
+{
+    unsigned version;
+    ERTS_DECLARE_DUMMY(unsigned flags);
+    int num_line_items;
+    BeamInstr* lp;
+    int i;
+    BeamInstr fname_index;
+    BeamInstr tag;
+
+    /*
+     * If the emulator flag ignoring the line information was given,
+     * return immediately.
+     */
+
+    if (erts_no_line_info) {
+	return 1;
+    }
+
+    /*
+     * Check version of line table.
+     */
+
+    GetInt(stp, 4, version);
+    if (version != 0) {
+	/*
+	 * Wrong version. Silently ignore the line number chunk.
+	 */
+	return 1;
+    }
+
+    /*
+     * Read the remaining header words. The flag word is reserved
+     * for possible future use; for the moment we ignore it.
+     */
+    GetInt(stp, 4, flags);
+    GetInt(stp, 4, stp->num_line_instrs);
+    GetInt(stp, 4, num_line_items);
+    GetInt(stp, 4, stp->num_fnames);
+
+    /*
+     * Calculate space and allocate memory for the line item table.
+     */
+
+    num_line_items++;
+    lp = (BeamInstr *) erts_alloc(ERTS_ALC_T_LOADER_TMP,
+				  num_line_items * sizeof(BeamInstr));
+    stp->line_item = lp;
+    stp->num_line_items = num_line_items;
+
+    /*
+     * The zeroth entry in the line item table is special.
+     * It contains the undefined location.
+     */
+
+    *lp++ = LINE_INVALID_LOCATION;
+    num_line_items--;
+
+    /*
+     * Read all the line items.
+     */
+
+    stp->loc_size = stp->num_fnames ? 4 : 2;
+    fname_index = 0;
+    while (num_line_items-- > 0) {
+	BeamInstr val;
+	BeamInstr loc;
+
+	GetTagAndValue(stp, tag, val);
+	if (tag == TAG_i) {
+	    if (IS_VALID_LOCATION(fname_index, val)) {
+		loc = MAKE_LOCATION(fname_index, val);
+	    } else {
+		/*
+		 * Too many files or huge line number. Silently invalidate
+		 * the location.
+		 */
+		loc = LINE_INVALID_LOCATION;
+	    }
+	    *lp++ = loc;
+	    if (val > 0xFFFF) {
+		stp->loc_size = 4;
+	    }
+	} else if (tag == TAG_a) {
+	    if (val > stp->num_fnames) {
+		LoadError2(stp, "file index overflow (%d/%d)",
+			   val, stp->num_fnames);
+	    }
+	    fname_index = val;
+	    num_line_items++;
+	} else {
+	    LoadError1(stp, "bad tag '%c' (expected 'a' or 'i')",
+		       tag_to_letter[tag]);
+	}
+    }
+
+    /*
+     * Read all filenames.
+     */
+
+    if (stp->num_fnames != 0) {
+	stp->fname = (Eterm *) erts_alloc(ERTS_ALC_T_LOADER_TMP,
+					      stp->num_fnames *
+					      sizeof(Eterm));
+	for (i = 0; i < stp->num_fnames; i++) {
+	    byte* fname;
+	    Uint n;
+
+	    GetInt(stp, 2, n);
+	    GetString(stp, fname, n);
+	    stp->fname[i] = am_atom_put((char*)fname, n);
+	}
+    }
+
+    /*
+     * Allocate the arrays to be filled while code is being loaded.
+     */
+    stp->line_instr = (LineInstr *) erts_alloc(ERTS_ALC_T_LOADER_TMP,
+					       stp->num_line_instrs *
+					       sizeof(LineInstr));
+    stp->current_li = 0;
+    stp->func_line = (int *)  erts_alloc(ERTS_ALC_T_LOADER_TMP,
+					 stp->num_functions *
+					 sizeof(int));
+
+    return 1;
+
+ load_error:
+    return 0;
+}
+
 
 static int
 read_code_header(LoaderState* stp)
@@ -1310,10 +1658,15 @@ read_code_header(LoaderState* stp)
     /*
      * Verify the number of the highest opcode used.
      */
-
     GetInt(stp, 4, opcode_max);
     if (opcode_max > MAX_GENERIC_OPCODE) {
-	LoadError2(stp, "use of opcode %d; this emulator supports only up to %d",
+	LoadError2(stp,
+		   "This BEAM file was compiled for a later version"
+		   " of the run-time system than " ERLANG_OTP_RELEASE ".\n"
+		   "  To fix this, please recompile this module with an "
+		   ERLANG_OTP_RELEASE " compiler.\n"
+		   "  (Use of opcode %d; this emulator supports "
+		   "only up to %d.)",
 		   opcode_max, MAX_GENERIC_OPCODE);
     }
 
@@ -1334,24 +1687,6 @@ read_code_header(LoaderState* stp)
 #endif
     }
 
-    /*
-     * Initialize code area.
-     */
-    stp->code_buffer_size = erts_next_heap_size(2048 + stp->num_functions, 0);
-    stp->code = (Eterm*) erts_alloc(ERTS_ALC_T_CODE,
-				    sizeof(Eterm) * stp->code_buffer_size);
-
-    stp->code[MI_NUM_FUNCTIONS] = stp->num_functions;
-    stp->ci = MI_FUNCTIONS + stp->num_functions + 1;
-
-    stp->code[MI_ATTR_PTR] = 0;
-    stp->code[MI_ATTR_SIZE_ON_HEAP] = 0;
-    stp->code[MI_COMPILE_PTR] = 0;
-    stp->code[MI_COMPILE_SIZE_ON_HEAP] = 0;
-    stp->code[MI_NUM_BREAKPOINTS] = 0;
-
-    stp->put_strings = 0;
-    stp->new_bs_put_strings = 0;
     stp->catches = 0;
     return 1;
 
@@ -1365,29 +1700,30 @@ read_code_header(LoaderState* stp)
        LoadError2(Stp, "bad tag %d; expected %d", Actual, Expected); \
     } else {}
 
-#define Need(w) \
-    ASSERT(ci <= code_buffer_size); \
-    if (code_buffer_size < ci+(w)) { \
-        code_buffer_size = erts_next_heap_size(ci+(w), 0); \
-	stp->code = code \
-	    = (Eterm *) erts_realloc(ERTS_ALC_T_CODE, \
-				     (void *) code, \
-				     code_buffer_size * sizeof(Eterm)); \
-    }
+#define CodeNeed(w) do {						\
+    ASSERT(ci <= code_buffer_size);					\
+    if (code_buffer_size < ci+(w)) {					\
+        code_buffer_size = erts_next_heap_size(ci+(w), 0);		\
+	stp->code = code						\
+	    = (BeamInstr *) erts_realloc(ERTS_ALC_T_CODE,			\
+				     (void *) code,			\
+				     code_buffer_size * sizeof(BeamInstr));	\
+    } 									\
+} while (0)
     
+#define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm))))
 
 
 static int
 load_code(LoaderState* stp)
 {
     int i;
-    int tmp;
     int ci;
-    int last_func_start = 0;
+    int last_func_start = 0;	/* Needed by nif loading and line instructions */
     char* sign;
     int arg;			/* Number of current argument. */
     int num_specific;		/* Number of specific ops for current. */
-    Eterm* code;
+    BeamInstr* code;
     int code_buffer_size;
     int specific;
     Uint last_label = 0;	/* Number of last label. */
@@ -1396,6 +1732,14 @@ load_code(LoaderState* stp)
     GenOp** last_op_next = NULL;
     int arity;
 
+    /*
+     * The size of the loaded func_info instruction is needed
+     * by both the nif functionality and line instructions.
+     */
+    enum {
+	FUNC_INFO_SZ = 5
+    };
+
     code = stp->code;
     code_buffer_size = stp->code_buffer_size;
     ci = stp->ci;
@@ -1441,46 +1785,15 @@ load_code(LoaderState* stp)
 	last_op->arity = 0;
 	ASSERT(arity <= MAX_OPARGS);
 
-#define GetValue(Stp, First, Val) \
-   do { \
-      if (((First) & 0x08) == 0) { \
-	 Val = (First) >> 4; \
-      } else if (((First) & 0x10) == 0) { \
-         Uint __w; \
-	 GetByte(Stp, __w); \
-	 Val = (((First) >> 5) << 8) | __w; \
-      } else { \
-	if (!get_int_val(Stp, (First), &(Val))) goto load_error; \
-      } \
-   } while (0)
-
 	for (arg = 0; arg < arity; arg++) {
-	    Uint first;
-
-	    GetByte(stp, first);
-	    last_op->a[arg].type = first & 0x07;
+	    GetTagAndValue(stp, last_op->a[arg].type, last_op->a[arg].val);
 	    switch (last_op->a[arg].type) {
 	    case TAG_i:
-		if ((first & 0x08) == 0) {
-		    last_op->a[arg].val = first >> 4;
-		} else if ((first & 0x10) == 0) {
-		    Uint w;
-		    GetByte(stp, w);
-		    ASSERT(first < 0x800);
-		    last_op->a[arg].val = ((first >> 5) << 8) | w;
-		} else {
-		    int i = get_erlang_integer(stp, first, &(last_op->a[arg].val));
-		    if (i < 0) {
-			goto load_error;
-		    }
-		    last_op->a[arg].type = i;
-		}
-		break;
 	    case TAG_u:
-		GetValue(stp, first, last_op->a[arg].val);
+	    case TAG_q:
+	    case TAG_o:
 		break;
 	    case TAG_x:
-		GetValue(stp, first, last_op->a[arg].val);
 		if (last_op->a[arg].val == 0) {
 		    last_op->a[arg].type = TAG_r;
 		} else if (last_op->a[arg].val >= MAX_REG) {
@@ -1489,7 +1802,6 @@ load_code(LoaderState* stp)
 		}
 		break;
 	    case TAG_y:
-		GetValue(stp, first, last_op->a[arg].val);
 		if (last_op->a[arg].val >= MAX_REG) {
 		    LoadError1(stp, "invalid y register number: %u",
 			       last_op->a[arg].val);
@@ -1497,7 +1809,6 @@ load_code(LoaderState* stp)
 		last_op->a[arg].val += CP_SIZE;
 		break;
 	    case TAG_a:
-		GetValue(stp, first, last_op->a[arg].val);
 		if (last_op->a[arg].val == 0) {
 		    last_op->a[arg].type = TAG_n;
 		} else if (last_op->a[arg].val >= stp->num_atoms) {
@@ -1507,7 +1818,6 @@ load_code(LoaderState* stp)
 		}
 		break;
 	    case TAG_f:
-		GetValue(stp, first, last_op->a[arg].val);
 		if (last_op->a[arg].val == 0) {
 		    last_op->a[arg].type = TAG_p;
 		} else if (last_op->a[arg].val >= stp->num_labels) {
@@ -1515,7 +1825,6 @@ load_code(LoaderState* stp)
 		}
 		break;
 	    case TAG_h:
-		GetValue(stp, first, last_op->a[arg].val);
 		if (last_op->a[arg].val > 65535) {
 		    LoadError1(stp, "invalid range for character data type: %u",
 			       last_op->a[arg].val);
@@ -1523,22 +1832,21 @@ load_code(LoaderState* stp)
 		break;
 	    case TAG_z:
 		{
-		    Uint ext_tag;
 		    unsigned tag;
 
-		    GetValue(stp, first, ext_tag);
-		    switch (ext_tag) {
+		    switch (last_op->a[arg].val) {
 		    case 0:	/* Floating point number */
 			{
 			    Eterm* hp;
-# ifndef ARCH_64
+/* XXX:PaN - Halfword should use ARCH_64 variant instead */
+#if !defined(ARCH_64) || HALFWORD_HEAP
 			    Uint high, low;
 # endif
 			    last_op->a[arg].val = new_literal(stp, &hp,
 							      FLOAT_SIZE_OBJECT);
 			    hp[0] = HEADER_FLONUM;
 			    last_op->a[arg].type = TAG_q;
-# ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 			    GetInt(stp, 8, hp[1]);
 # else
 			    GetInt(stp, 4, high);
@@ -1575,12 +1883,11 @@ load_code(LoaderState* stp)
 			break;
 		    case 3: 	/* Allocation list. */
 			{
-			    Uint n;
-			    Uint type;
-			    Uint val;
-			    Uint words = 0;
+			    BeamInstr n;
+			    BeamInstr type;
+			    BeamInstr val;
+			    BeamInstr words = 0;
 			    
-			    stp->new_float_instructions = 1;
 			    GetTagAndValue(stp, tag, n);
 			    VerifyTag(stp, tag, TAG_u);
 			    while (n-- > 0) {
@@ -1607,7 +1914,7 @@ load_code(LoaderState* stp)
 			}
 		    case 4:	/* Literal. */
 			{
-			    Uint val;
+			    BeamInstr val;
 
 			    GetTagAndValue(stp, tag, val);
 			    VerifyTag(stp, tag, TAG_u);
@@ -1619,7 +1926,8 @@ load_code(LoaderState* stp)
 			    break;
 			}
 		    default:
-			LoadError1(stp, "invalid extended tag %d", ext_tag);
+			LoadError1(stp, "invalid extended tag %d",
+				   last_op->a[arg].val);
 			break;
 		    }
 		}
@@ -1630,7 +1938,6 @@ load_code(LoaderState* stp)
 	    }
 	    last_op->arity++;
 	}
-#undef GetValue
 
 	ASSERT(arity == last_op->arity);
 
@@ -1672,14 +1979,6 @@ load_code(LoaderState* stp)
 	}
 
 	/*
-	 * Special error message instruction.
-	 */
-	if (stp->genop->op == genop_too_old_compiler_0) {
-	    LoadError0(stp, "please re-compile this module with an " 
-		       ERLANG_OTP_RELEASE " compiler");
-	}
-
-	/*
 	 * From the collected generic instruction, find the specific
 	 * instruction.
 	 */
@@ -1730,11 +2029,31 @@ load_code(LoaderState* stp)
 			       ERLANG_OTP_RELEASE " compiler ");
 		}
 
-		LoadError0(stp, "no specific operation found");
+		/*
+		 * Some generic instructions should have a special
+		 * error message.
+		 */
+		switch (stp->genop->op) {
+		case genop_too_old_compiler_0:
+		    LoadError0(stp, "please re-compile this module with an "
+			       ERLANG_OTP_RELEASE " compiler");
+		case genop_unsupported_guard_bif_3:
+		    {
+			Eterm Mod = (Eterm) stp->genop->a[0].val;
+			Eterm Name = (Eterm) stp->genop->a[1].val;
+			Uint arity = (Uint) stp->genop->a[2].val;
+			FREE_GENOP(stp, stp->genop);
+			stp->genop = 0;
+			LoadError3(stp, "unsupported guard BIF: %T:%T/%d\n",
+				   Mod, Name, arity);
+		    }
+		default:
+		    LoadError0(stp, "no specific operation found");
+		}
 	    }
 
 	    stp->specific_op = specific;
-	    Need(opc[stp->specific_op].sz+2); /* Extra margin for packing */
+	    CodeNeed(opc[stp->specific_op].sz+16); /* Extra margin for packing */
 	    code[ci++] = BeamOpCode(stp->specific_op);
 	}
 	
@@ -1772,7 +2091,7 @@ load_code(LoaderState* stp)
 	    case 'c':		/* Tagged constant */
 		switch (tag) {
 		case TAG_i:
-		    code[ci++] = make_small(tmp_op->a[arg].val);
+		    code[ci++] = (BeamInstr) make_small((Uint) tmp_op->a[arg].val);
 		    break;
 		case TAG_a:
 		    code[ci++] = tmp_op->a[arg].val;
@@ -1802,7 +2121,7 @@ load_code(LoaderState* stp)
 		    code[ci++] = make_yreg(tmp_op->a[arg].val);
 		    break;
 		case TAG_i:
-		    code[ci++] = make_small(tmp_op->a[arg].val);
+		    code[ci++] = (BeamInstr) make_small((Uint)tmp_op->a[arg].val);
 		    break;
 		case TAG_a:
 		    code[ci++] = tmp_op->a[arg].val;
@@ -1896,12 +2215,12 @@ load_code(LoaderState* stp)
 		if (stp->import[i].bf == NULL) {
 		    LoadError1(stp, "not a BIF: import table index %d", i);
 		}
-		code[ci++] = (Eterm) stp->import[i].bf;
+		code[ci++] = (BeamInstr) stp->import[i].bf;
 		break;
-	    case 'P':		/* Byte offset into tuple */
+	    case 'P':		/* Byte offset into tuple or stack */
+	    case 'Q':		/* Like 'P', but packable */
 		VerifyTag(stp, tag, TAG_u);
-		tmp = tmp_op->a[arg].val;
-		code[ci++] = (Eterm) ((tmp_op->a[arg].val+1) * sizeof(Eterm *));
+		code[ci++] = (BeamInstr) ((tmp_op->a[arg].val+1) * sizeof(Eterm));
 		break;
 	    case 'l':		/* Floating point register. */
 		VerifyTag(stp, tag_to_letter[tag], *sign);
@@ -1919,75 +2238,14 @@ load_code(LoaderState* stp)
 	}
 
 	/*
-	 * Load any list arguments using the primitive tags.
-	 */
-
-	for ( ; arg < tmp_op->arity; arg++) {
-	    switch (tmp_op->a[arg].type) {
-	    case TAG_i:
-		Need(1);
-		code[ci++] = make_small(tmp_op->a[arg].val);
-		break;
-	    case TAG_u:
-	    case TAG_a:
-	    case TAG_v:
-		Need(1);
-		code[ci++] = tmp_op->a[arg].val;
-		break;
-	    case TAG_f:
-		Need(1);
-		code[ci] = stp->labels[tmp_op->a[arg].val].patches;
-		stp->labels[tmp_op->a[arg].val].patches = ci;
-		ci++;
-		break;
-	    case TAG_q:
-		{
-		    Eterm lit;
-
-		    lit = stp->literals[tmp_op->a[arg].val].term;
-		    if (is_big(lit)) {
-			Eterm* bigp;
-			Uint size;
-
-			bigp = big_val(lit);
-			size = bignum_header_arity(*bigp);
-			Need(size+1);
-			code[ci++] = *bigp++;
-			while (size-- > 0) {
-			    code[ci++] = *bigp++;
-			}
-		    } else if (is_float(lit)) {
-#ifdef ARCH_64
-			Need(1);
-			code[ci++] = float_val(stp->literals[tmp_op->a[arg].val].term)[1];
-#else
-			Eterm* fptr;
-
-			fptr = float_val(stp->literals[tmp_op->a[arg].val].term)+1;
-			Need(2);
-			code[ci++] = *fptr++;
-			code[ci++] = *fptr;
-#endif
-		    } else {
-			LoadError0(stp, "literal is neither float nor big");
-		    }
-		}
-		break;
-	    default:
-		LoadError1(stp, "unsupported primitive type '%c'",
-			   tag_to_letter[tmp_op->a[arg].type]);
-	    }
-	}
-
-	/*
 	 * The packing engine.
 	 */
 	if (opc[stp->specific_op].pack[0]) {
 	    char* prog;		/* Program for packing engine. */
-	    Uint stack[8];	/* Stack. */
-	    Uint* sp = stack;	/* Points to next free position. */
-	    Uint packed = 0;	/* Accumulator for packed operations. */
-	    
+	    BeamInstr stack[8];	/* Stack. */
+	    BeamInstr* sp = stack;	/* Points to next free position. */
+	    BeamInstr packed = 0;	/* Accumulator for packed operations. */
+
 	    for (prog = opc[stp->specific_op].pack; *prog; prog++) {
 		switch (*prog) {
 		case 'g':	/* Get instruction; push on stack. */
@@ -2000,8 +2258,13 @@ load_code(LoaderState* stp)
 		    packed = (packed << BEAM_TIGHT_SHIFT) | code[--ci];
 		    break;
 		case '6':	/* Shift 16 steps */
-		    packed = (packed << 16) | code[--ci];
+		    packed = (packed << BEAM_LOOSE_SHIFT) | code[--ci];
+		    break;
+#ifdef ARCH_64
+		case 'w':	/* Shift 32 steps */
+		    packed = (packed << BEAM_WIDE_SHIFT) | code[--ci];
 		    break;
+#endif
 		case 'p':	/* Put instruction (from stack). */
 		    code[ci++] = *--sp;
 		    break;
@@ -2017,13 +2280,64 @@ load_code(LoaderState* stp)
 	}
 
 	/*
+	 * Load any list arguments using the primitive tags.
+	 */
+
+	for ( ; arg < tmp_op->arity; arg++) {
+	    switch (tmp_op->a[arg].type) {
+	    case TAG_i:
+		CodeNeed(1);
+		code[ci++] = make_small(tmp_op->a[arg].val);
+		break;
+	    case TAG_u:
+	    case TAG_a:
+	    case TAG_v:
+		CodeNeed(1);
+		code[ci++] = tmp_op->a[arg].val;
+		break;
+	    case TAG_f:
+		CodeNeed(1);
+		code[ci] = stp->labels[tmp_op->a[arg].val].patches;
+		stp->labels[tmp_op->a[arg].val].patches = ci;
+		ci++;
+		break;
+	    case TAG_r:
+		CodeNeed(1);
+		code[ci++] = (R_REG_DEF << _TAG_PRIMARY_SIZE) |
+		    TAG_PRIMARY_HEADER;
+		break;
+	    case TAG_x:
+		CodeNeed(1);
+		code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) |
+		    (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER;
+		break;
+	    case TAG_y:
+		CodeNeed(1);
+		code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) |
+		    (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER;
+		break;
+	    case TAG_n:
+		CodeNeed(1);
+		code[ci++] = NIL;
+		break;
+	    case TAG_q:
+		CodeNeed(1);
+		new_literal_patch(stp, ci);
+		code[ci++] = tmp_op->a[arg].val;
+		break;
+	    default:
+		LoadError1(stp, "unsupported primitive type '%c'",
+			   tag_to_letter[tmp_op->a[arg].type]);
+	    }
+	}
+
+	/*
 	 * Handle a few special cases.
 	 */
 	switch (stp->specific_op) {
 	case op_i_func_info_IaaI:
 	    {
 		Uint offset;
-		enum { FINFO_SZ = 5 };
 
 		if (function_number >= stp->num_functions) {
 		    LoadError1(stp, "too many functions in module (header said %d)",
@@ -2031,26 +2345,37 @@ load_code(LoaderState* stp)
 		}
 
 		if (stp->may_load_nif) {
-		    const int finfo_ix = ci - FINFO_SZ;		    
+		    const int finfo_ix = ci - FUNC_INFO_SZ;
 		    enum { MIN_FUNC_SZ = 3 };		    
 		    if (finfo_ix - last_func_start < MIN_FUNC_SZ && last_func_start) {		   
 			/* Must make room for call_nif op */
 			int pad = MIN_FUNC_SZ - (finfo_ix - last_func_start);
 			ASSERT(pad > 0 && pad < MIN_FUNC_SZ);
-			Need(pad);
-			sys_memmove(&code[finfo_ix+pad], &code[finfo_ix], FINFO_SZ*sizeof(Eterm));
-			sys_memset(&code[finfo_ix], 0, pad*sizeof(Eterm));
+			CodeNeed(pad);
+			sys_memmove(&code[finfo_ix+pad], &code[finfo_ix],
+				    FUNC_INFO_SZ*sizeof(BeamInstr));
+			sys_memset(&code[finfo_ix], 0, pad*sizeof(BeamInstr));
 			ci += pad;
 			stp->labels[last_label].value += pad;
 		    }
 		}
 		last_func_start = ci;
+
+		/*
+		 * Save current offset of into the line instruction array.
+		 */
+
+		if (stp->func_line) {
+		    stp->func_line[function_number] = stp->current_li;
+		}
+
 		/*
 		 * Save context for error messages.
 		 */
 		stp->function = code[ci-2];
 		stp->arity = code[ci-1];
-		ASSERT(stp->labels[last_label].value == ci - FINFO_SZ);
+
+		ASSERT(stp->labels[last_label].value == ci - FUNC_INFO_SZ);
 		offset = MI_FUNCTIONS + function_number;
 		code[offset] = stp->labels[last_label].patches;
 		stp->labels[last_label].patches = offset;
@@ -2072,61 +2397,7 @@ load_code(LoaderState* stp)
 	    /* Remember offset for the on_load function. */
 	    stp->on_load = ci;
 	    break;
-	case op_put_string_IId:
-	    {
-		/*
-		 * At entry:
-		 *
-		 * code[ci-4]	&&lb_put_string_IId
-		 * code[ci-3]	length of string
-		 * code[ci-2]   offset into string table
-		 * code[ci-1]   destination register
-		 *
-		 * Since we don't know the address of the string table yet,
-		 * just check the offset and length for validity, and use
-		 * the instruction field as a link field to link all put_string
-		 * instructions into a single linked list.  At exit:
-		 *
-		 * code[ci-4]	pointer to next put_string instruction (or 0
-		 *		if this is the last)
-		 */
-		Uint offset = code[ci-2];
-		Uint len = code[ci-3];
-		unsigned strtab_size = stp->chunks[STR_CHUNK].size;
-		if (offset > strtab_size || offset + len > strtab_size) {
-		    LoadError2(stp, "invalid string reference %d, size %d", offset, len);
-		}
-		code[ci-4] = stp->put_strings;
-		stp->put_strings = ci - 4;
-	    }
-	    break;
 	case op_bs_put_string_II:
-	    {
-		/*
-		 * At entry:
-		 *
-		 * code[ci-3]	&&lb_i_new_bs_put_string_II
-		 * code[ci-2]	length of string
-		 * code[ci-1]   offset into string table
-		 *
-		 * Since we don't know the address of the string table yet,
-		 * just check the offset and length for validity, and use
-		 * the instruction field as a link field to link all put_string
-		 * instructions into a single linked list.  At exit:
-		 *
-		 * code[ci-3]	pointer to next i_new_bs_put_string instruction (or 0
-		 *		if this is the last)
-		 */
-		Uint offset = code[ci-1];
-		Uint len = code[ci-2];
-		unsigned strtab_size = stp->chunks[STR_CHUNK].size;
-		if (offset > strtab_size || offset + len > strtab_size) {
-		    LoadError2(stp, "invalid string reference %d, size %d", offset, len);
-		}
-		code[ci-3] = stp->new_bs_put_strings;
-		stp->new_bs_put_strings = ci - 3;
-	    }
-	    break;
 	case op_i_bs_match_string_rfII:
 	case op_i_bs_match_string_xfII:
 	    new_string_patch(stp, ci-1);
@@ -2141,6 +2412,45 @@ load_code(LoaderState* stp)
 	    stp->catches = ci-3;
 	    break;
 
+	case op_line_I:
+	    if (stp->line_item) {
+		BeamInstr item = code[ci-1];
+		BeamInstr loc;
+		int li;
+		if (item >= stp->num_line_items) {
+		    LoadError2(stp, "line instruction index overflow (%d/%d)",
+			       item, stp->num_line_items);
+		}
+		li = stp->current_li;
+		if (li >= stp->num_line_instrs) {
+		    LoadError2(stp, "line instruction table overflow (%d/%d)",
+			       li, stp->num_line_instrs);
+		}
+		loc = stp->line_item[item];
+
+		if (ci - 2 == last_func_start) {
+		    /*
+		     * This line instruction directly follows the func_info
+		     * instruction. Its address must be adjusted to point to
+		     * func_info instruction.
+		     */
+		    stp->line_instr[li].pos = last_func_start - FUNC_INFO_SZ;
+		    stp->line_instr[li].loc = stp->line_item[item];
+		    stp->current_li++;
+		} else if (li <= stp->func_line[function_number-1] ||
+			   stp->line_instr[li-1].loc != loc) {
+		    /*
+		     * Only store the location if it is different
+		     * from the previous location in the same function.
+		     */
+		    stp->line_instr[li].pos = ci - 2;
+		    stp->line_instr[li].loc = stp->line_item[item];
+		    stp->current_li++;
+		}
+	    }
+	    ci -= 2;		/* Get rid of the instruction */
+	    break;
+
 	    /*
 	     * End of code found.
 	     */
@@ -2161,7 +2471,6 @@ load_code(LoaderState* stp)
 	}
     }
     
-#undef Need
 
  load_error:
     return 0;
@@ -2178,6 +2487,8 @@ load_code(LoaderState* stp)
 #define no_fpe_signals(St) 0
 #endif
 
+#define never(St) 0
+
 /*
  * Predicate that tests whether a jump table can be used.
  */
@@ -2212,11 +2523,12 @@ use_jump_tab(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
 }
 
 /*
- * Predicate to test whether all values in a table are big numbers.
+ * Predicate to test whether all values in a table are either
+ * floats or bignums.
  */
 
 static int
-all_values_are_big(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
+floats_or_bignums(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
 {
     int i;
 
@@ -2228,9 +2540,6 @@ all_values_are_big(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
 	if (Rest[i].type != TAG_q) {
 	    return 0;
 	}
-	if (is_not_big(stp->literals[Rest[i].val].term)) {
-	    return 0;
-	}
 	if (Rest[i+1].type != TAG_f) {
 	    return 0;
 	}
@@ -2290,6 +2599,14 @@ mixed_types(LoaderState* stp, GenOpArg Size, GenOpArg* Rest)
     return 0;
 }
 
+static int
+same_label(LoaderState* stp, GenOpArg Target, GenOpArg Label)
+{
+    return Target.type = TAG_f && Label.type == TAG_u &&
+	Target.val == Label.val;
+}
+
+
 /*
  * Generate an instruction for element/2.
  */
@@ -2301,23 +2618,23 @@ gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index,
     GenOp* op;
 
     NEW_GENOP(stp, op);
-    op->op = genop_i_element_4;
     op->arity = 4;
-    op->a[0] = Fail;
-    op->a[1] = Index;
-    op->a[2] = Tuple;
-    op->a[3] = Dst;
     op->next = NULL;
 
-    /*
-     * If safe, generate a faster instruction.
-     */
-
     if (Index.type == TAG_i && Index.val > 0 &&
 	(Tuple.type == TAG_r || Tuple.type == TAG_x || Tuple.type == TAG_y)) {
 	op->op = genop_i_fast_element_4;
-	op->a[1].type = TAG_u;
-	op->a[1].val = Index.val;
+	op->a[0] = Tuple;
+	op->a[1] = Fail;
+	op->a[2].type = TAG_u;
+	op->a[2].val = Index.val;
+	op->a[3] = Dst;
+    } else {
+	op->op = genop_i_element_4;
+	op->a[0] = Tuple;
+	op->a[1] = Fail;
+	op->a[2] = Index;
+	op->a[3] = Dst;
     }
 
     return op;
@@ -2373,7 +2690,7 @@ gen_get_integer2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, GenOpArg Live,
 		 GenOpArg Flags, GenOpArg Dst)
 {
     GenOp* op;
-    Uint bits;
+    UWord bits;
 
     NEW_GENOP(stp, op);
 
@@ -2559,17 +2876,10 @@ should_gen_heap_bin(LoaderState* stp, GenOpArg Src)
 static int
 binary_too_big(LoaderState* stp, GenOpArg Size)
 {
-    return Size.type == TAG_u && ((Size.val >> (8*sizeof(Uint)-3)) != 0);
-}
-
-static int
-binary_too_big_bits(LoaderState* stp, GenOpArg Size)
-{
-    return Size.type == TAG_u && (((Size.val+7)/8) >> (8*sizeof(Uint)-3) != 0);
+    return Size.type == TAG_o ||
+	(Size.type == TAG_u && ((Size.val >> (8*sizeof(Uint)-3)) != 0));
 }
 
-#define new_float_allocation(Stp) ((Stp)->new_float_instructions)
-
 static GenOp*
 gen_put_binary(LoaderState* stp, GenOpArg Fail,GenOpArg Size,
 	       GenOpArg Unit, GenOpArg Flags, GenOpArg Src)
@@ -2782,6 +3092,52 @@ gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms,
     return op;
 }
 
+static GenOp*
+gen_increment(LoaderState* stp, GenOpArg Reg, GenOpArg Integer,
+	      GenOpArg Live, GenOpArg Dst)
+{
+    GenOp* op;
+
+    NEW_GENOP(stp, op);
+    op->op = genop_i_increment_4;
+    op->arity = 4;
+    op->next = NULL;
+    op->a[0] = Reg;
+    op->a[1].type = TAG_u;
+    op->a[1].val = Integer.val;
+    op->a[2] = Live;
+    op->a[3] = Dst;
+    return op;
+}
+
+static GenOp*
+gen_increment_from_minus(LoaderState* stp, GenOpArg Reg, GenOpArg Integer,
+			 GenOpArg Live, GenOpArg Dst)
+{
+    GenOp* op;
+
+    NEW_GENOP(stp, op);
+    op->op = genop_i_increment_4;
+    op->arity = 4;
+    op->next = NULL;
+    op->a[0] = Reg;
+    op->a[1].type = TAG_u;
+    op->a[1].val = -Integer.val;
+    op->a[2] = Live;
+    op->a[3] = Dst;
+    return op;
+}
+
+/*
+ * Test whether the negation of the given number is small.
+ */
+static int
+negation_is_small(LoaderState* stp, GenOpArg Int)
+{
+    return Int.type == TAG_i && IS_SSMALL(-Int.val);
+}
+
+
 static int
 smp(LoaderState* stp)
 {
@@ -2838,14 +3194,14 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
     op->a[1].type = TAG_u;
     
     if (Time.type == TAG_i && (timeout = Time.val) >= 0 &&
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 	(timeout >> 32) == 0
 #else
 	1
 #endif
 	) {
 	op->a[1].val = timeout;
-#if !defined(ARCH_64)
+#if !defined(ARCH_64) || HALFWORD_HEAP
     } else if (Time.type == TAG_q) {
 	Eterm big;
 
@@ -2856,11 +3212,13 @@ gen_literal_timeout(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
 	if (big_arity(big) > 1 || big_sign(big)) {
 	    goto error;
 	} else {
-	    (void) term_to_Uint(big, &op->a[1].val);
+	    Uint u;
+	    (void) term_to_Uint(big, &u);
+	    op->a[1].val = (BeamInstr) u;
 	}
 #endif
     } else {
-#if !defined(ARCH_64)
+#if !defined(ARCH_64) || HALFWORD_HEAP
     error:
 #endif
 	op->op = genop_i_wait_error_0;
@@ -2883,14 +3241,14 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
     op->a[1].type = TAG_u;
     
     if (Time.type == TAG_i && (timeout = Time.val) >= 0 &&
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 	(timeout >> 32) == 0
 #else
 	1
 #endif
 	) {
 	op->a[1].val = timeout;
-#ifndef ARCH_64
+#if !defined(ARCH_64) || HALFWORD_HEAP
     } else if (Time.type == TAG_q) {
 	Eterm big;
 
@@ -2901,11 +3259,13 @@ gen_literal_timeout_locked(LoaderState* stp, GenOpArg Fail, GenOpArg Time)
 	if (big_arity(big) > 1 || big_sign(big)) {
 	    goto error;
 	} else {
-	    (void) term_to_Uint(big, &op->a[1].val);
+	    Uint u;
+	    (void) term_to_Uint(big, &u);
+	    op->a[1].val = (BeamInstr) u;
 	}
 #endif
     } else {
-#ifndef ARCH_64
+#if !defined(ARCH_64) || HALFWORD_HEAP
     error:
 #endif
 	op->op = genop_i_wait_error_locked_0;
@@ -2969,6 +3329,21 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 	ASSERT(op->a[i].val < op->a[i+2].val);
     }
 #endif
+
+    /*
+     * Use a special-cased instruction if there are only two values.
+     */
+    if (size == 2) {
+	op->op = genop_i_select_tuple_arity2_6;
+	op->arity--;
+	op->a[2].type = TAG_u;
+	op->a[2].val = arityval(op->a[3].val);
+	op->a[3] = op->a[4];
+	op->a[4].type = TAG_u;
+	op->a[4].val = arityval(op->a[5].val);
+	op->a[5] = op->a[6];
+    }
+
     return op;
 }
 
@@ -2978,18 +3353,24 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail,
  */
 
 static GenOp*
-gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail,
-		 GenOpArg Size, GenOpArg* Rest)
+gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg TypeFail,
+		 GenOpArg Fail, GenOpArg Size, GenOpArg* Rest)
 
 {
     GenOp* op1;
     GenOp* op2;
     GenOp* label;
-    Uint type;
+    GenOp* is_integer;
     int i;
 
     ASSERT(Size.val >= 2 && Size.val % 2 == 0);
 
+    NEW_GENOP(stp, is_integer);
+    is_integer->op = genop_is_integer_2;
+    is_integer->arity = 2;
+    is_integer->a[0] = TypeFail;
+    is_integer->a[1] = S;
+
     NEW_GENOP(stp, label);
     label->op = genop_label_1;
     label->arity = 1;
@@ -3015,15 +3396,13 @@ gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail,
     op2->a[2].type = TAG_u;
     op2->a[2].val = 0;
 
-    op1->next = label;
-    label->next = op2;
-    op2->next = NULL;
-
-    type = Rest[0].type;
+    /*
+     * Split the list.
+     */
 
     ASSERT(Size.type == TAG_u);
     for (i = 0; i < Size.val; i += 2) {
-	GenOp* op = (Rest[i].type == type) ? op1 : op2;
+	GenOp* op = (Rest[i].type == TAG_q) ? op2 : op1;
 	int dst = 3 + op->a[2].val;
 
 	ASSERT(Rest[i+1].type == TAG_f);
@@ -3032,13 +3411,36 @@ gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 	op->arity += 2;
 	op->a[2].val += 2;
     }
+    ASSERT(op1->a[2].val > 0);
+    ASSERT(op2->a[2].val > 0);
 
     /*
-     * None of the instructions should have zero elements in the list.
+     * Order the instruction sequence appropriately.
      */
 
-    ASSERT(op1->a[2].val > 0);
-    ASSERT(op2->a[2].val > 0);
+    if (TypeFail.val == Fail.val) {
+	/*
+	 * select_val L1 S ... (small numbers)
+	 * label L1
+	 * is_integer Fail S
+	 * select_val Fail S ... (bignums)
+	 */
+	op1->next = label;
+	label->next = is_integer;
+	is_integer->next = op2;
+    } else {
+	/*
+	 * is_integer TypeFail S
+	 * select_val L1 S ... (small numbers)
+	 * label L1
+	 * select_val Fail S ... (bignums)
+	 */
+	is_integer->next = op1;
+	op1->next = label;
+	label->next = op2;
+	op1 = is_integer;
+    }
+    op2->next = NULL;
 
     return op1;
 }
@@ -3060,6 +3462,29 @@ gen_jump_tab(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpAr
     ASSERT(Size.val >= 2 && Size.val % 2 == 0);
 
     /*
+     * If there is only one choice, don't generate a jump table.
+     */
+    if (Size.val == 2) {
+	GenOp* jump;
+
+	NEW_GENOP(stp, op);
+	op->arity = 3;
+	op->op = genop_is_ne_exact_3;
+	op->a[0] = Rest[1];
+	op->a[1] = S;
+	op->a[2] = Rest[0];
+
+	NEW_GENOP(stp, jump);
+	jump->next = NULL;
+	jump->arity = 1;
+	jump->op = genop_jump_1;
+	jump->a[0] = Fail;
+
+	op->next = jump;
+	return op;
+    }
+
+    /*
      * Calculate the minimum and maximum values and size of jump table.
      */
 
@@ -3075,7 +3500,6 @@ gen_jump_tab(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpAr
     }
     size = max - min + 1;
 
-
     /*
      * Allocate structure and fill in the fixed fields.
      */
@@ -3107,7 +3531,7 @@ gen_jump_tab(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpAr
 	op->a[i] = Fail;
     }
     for (i = 0; i < Size.val; i += 2) {
-	int index;
+	Sint index;
 	index = fixed_args+Rest[i].val-min;
 	ASSERT(fixed_args <= index && index < arity);
 	op->a[index] = Rest[i+1];
@@ -3131,8 +3555,9 @@ genopargcompare(GenOpArg* a, GenOpArg* b)
 }
 
 /*
- * Generate a select_val instruction.  We know that a jump table is not suitable,
- * and that all values are of the same type (integer, atoms, floats; never bignums).
+ * Generate a select_val instruction.  We know that a jump table
+ * is not suitable, and that all values are of the same type
+ * (integer or atoms).
  */
 
 static GenOp*
@@ -3146,12 +3571,7 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 
     NEW_GENOP(stp, op);
     op->next = NULL;
-    if (Rest[0].type != TAG_q) {
-	op->op = genop_i_select_val_3;
-    } else {
-	ASSERT(is_float(stp->literals[Rest[0].val].term));
-	op->op = genop_i_select_float_3;
-    }
+    op->op = genop_i_select_val_3;
     GENOP_ARITY(op, arity);
     op->a[0] = S;
     op->a[1] = Fail;
@@ -3173,19 +3593,19 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
     }
 #endif
 
-    return op;
-}
-
-/* 
- *  Compare function for qsort().
- */
+    /*
+     * Use a special-cased instruction if there are only two values.
+     */
+    if (size == 2) {
+	op->op = genop_i_select_val2_6;
+	op->arity--;
+	op->a[2] = op->a[3];
+	op->a[3] = op->a[4];
+	op->a[4] = op->a[5];
+	op->a[5] = op->a[6];
+    }
 
-static int
-genbigcompare(GenOpArg* a, GenOpArg* b)
-{
-    int val = (int)(b->bigarity - a->bigarity);
-    
-    return val != 0 ? val : ((int) (a->val - b->val));
+    return op;
 }
 
 /*
@@ -3193,37 +3613,35 @@ genbigcompare(GenOpArg* a, GenOpArg* b)
  */
 
 static GenOp*
-gen_select_big(LoaderState* stp, GenOpArg S, GenOpArg Fail,
+gen_select_literals(LoaderState* stp, GenOpArg S, GenOpArg Fail,
 	       GenOpArg Size, GenOpArg* Rest)
 {
     GenOp* op;
-    int arity = Size.val + 2 + 1;
-    int size = Size.val / 2;
+    GenOp* jump;
+    GenOp** prev_next = &op;
+
     int i;
 
-    NEW_GENOP(stp, op);
-    op->next = NULL;
-    op->op = genop_i_select_big_2;
-    GENOP_ARITY(op, arity);
-    op->a[0] = S;
-    op->a[1] = Fail;
     for (i = 0; i < Size.val; i += 2) {
+	GenOp* op;
 	ASSERT(Rest[i].type == TAG_q);
-	op->a[i+2] = Rest[i];
-	op->a[i+2].bigarity = *big_val(stp->literals[op->a[i+2].val].term);
-	op->a[i+3] = Rest[i+1];
-    }
-    ASSERT(i+2 == arity-1);
-    op->a[arity-1].type = TAG_u;
-    op->a[arity-1].val = 0;
-
-    /*
-     * Sort the values in descending arity order.
-     */
-
-    qsort(op->a+2, size, 2*sizeof(GenOpArg), 
-	  (int (*)(const void *, const void *)) genbigcompare);
 
+	NEW_GENOP(stp, op);
+	op->op = genop_is_ne_exact_3;
+	op->arity = 3;
+	op->a[0] = Rest[i+1];
+	op->a[1] = S;
+	op->a[2] = Rest[i];
+	*prev_next = op;
+	prev_next = &op->next;
+    }
+
+    NEW_GENOP(stp, jump);
+    jump->next = NULL;
+    jump->op = genop_jump_1;
+    jump->arity = 1;
+    jump->a[0] = Fail;
+    *prev_next = jump;
     return op;
 }
 
@@ -3241,7 +3659,6 @@ const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
     int i;
 
     ASSERT(Size.type == TAG_u);
-    ASSERT(S.type == TAG_q);
 
     NEW_GENOP(stp, op);
     op->next = NULL;
@@ -3252,18 +3669,32 @@ const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
      * Search for a literal matching the controlling expression.
      */
 
-    if (S.type == TAG_q) {
-	Eterm expr = stp->literals[S.val].term;
-	for (i = 0; i < Size.val; i += 2) {
-	    if (Rest[i].type == TAG_q) {
-		Eterm term = stp->literals[Rest[i].val].term;
-		if (eq(term, expr)) {
-		    ASSERT(Rest[i+1].type == TAG_f);
-		    op->a[0] = Rest[i+1];
-		    return op;
+    switch (S.type) {
+    case TAG_q:
+	{
+	    Eterm expr = stp->literals[S.val].term;
+	    for (i = 0; i < Size.val; i += 2) {
+		if (Rest[i].type == TAG_q) {
+		    Eterm term = stp->literals[Rest[i].val].term;
+		    if (eq(term, expr)) {
+			ASSERT(Rest[i+1].type == TAG_f);
+			op->a[0] = Rest[i+1];
+			return op;
+		    }
 		}
 	    }
 	}
+	break;
+    case TAG_i:
+    case TAG_a:
+	for (i = 0; i < Size.val; i += 2) {
+	    if (Rest[i].val == S.val && Rest[i].type == S.type) {
+		ASSERT(Rest[i+1].type == TAG_f);
+		op->a[0] = Rest[i+1];
+		return op;
+	    }
+	}
+	break;
     }
 
     /*
@@ -3274,36 +3705,6 @@ const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail,
     return op;
 }
 
-
-static GenOp*
-gen_func_info(LoaderState* stp, GenOpArg mod, GenOpArg func,
-	      GenOpArg arity, GenOpArg label)
-{
-    GenOp* fi;
-    GenOp* op;
-
-    NEW_GENOP(stp, fi);
-    fi->op = genop_i_func_info_4;
-    fi->arity = 4;
-    fi->a[0].type = TAG_u;	/* untagged Zero */
-    fi->a[0].val = 0;
-    fi->a[1] = mod;
-    fi->a[2] = func;
-    fi->a[3] = arity;
-
-    NEW_GENOP(stp, op);
-    op->op = genop_label_1;
-    op->arity = 1;
-    op->a[0] = label;
-    
-    fi->next = op;
-    op->next = NULL;
-
-    return fi;
-}
-
-
-
 static GenOp*
 gen_make_fun2(LoaderState* stp, GenOpArg idx)
 {
@@ -3321,52 +3722,206 @@ gen_make_fun2(LoaderState* stp, GenOpArg idx)
     op->op = genop_i_make_fun_2;
     op->arity = 2;
     op->a[0].type = TAG_u;
-    op->a[0].val = (Uint) fe;
+    op->a[0].val = (BeamInstr) fe;
     op->a[1].type = TAG_u;
     op->a[1].val = stp->lambdas[idx.val].num_free;
     op->next = NULL;
     return op;
 }
-
+/*
+ * Rewrite gc_bifs with one parameter (the common case). Utilized
+ * in ops.tab to rewrite instructions calling bif's in guards
+ * to use a garbage collecting implementation. The instructions
+ * are sometimes once again rewritten to handle literals (putting the
+ * parameter in the mostly unused r[0] before the instruction is executed).
+ */
 static GenOp*
-gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
+gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
 	      GenOpArg Src, GenOpArg Dst)
 {
     GenOp* op;
     BifFunction bf;
 
     NEW_GENOP(stp, op);
-    op->op = genop_i_gc_bif1_5;
-    op->arity = 5;
-    op->a[0] = Fail;
-    op->a[1].type = TAG_u;
+    op->next = NULL;
     bf = stp->import[Bif.val].bf;
+    /* The translations here need to have a reverse counterpart in
+       beam_emu.c:translate_gc_bif for error handling to work properly. */
     if (bf == length_1) {
-	op->a[1].val = (Uint) (void *) erts_gc_length_1;
+	op->a[1].val = (BeamInstr) (void *) erts_gc_length_1;
     } else if (bf == size_1) {
-	op->a[1].val = (Uint) (void *) erts_gc_size_1;
+	op->a[1].val = (BeamInstr) (void *) erts_gc_size_1;
     } else if (bf == bit_size_1) {
-	op->a[1].val = (Uint) (void *) erts_gc_bit_size_1;
+	op->a[1].val = (BeamInstr) (void *) erts_gc_bit_size_1;
     } else if (bf == byte_size_1) {
-	op->a[1].val = (Uint) (void *) erts_gc_byte_size_1;
+	op->a[1].val = (BeamInstr) (void *) erts_gc_byte_size_1;
     } else if (bf == abs_1) {
-	op->a[1].val = (Uint) (void *) erts_gc_abs_1;
+	op->a[1].val = (BeamInstr) (void *) erts_gc_abs_1;
     } else if (bf == float_1) {
-	op->a[1].val = (Uint) (void *) erts_gc_float_1;
+	op->a[1].val = (BeamInstr) (void *) erts_gc_float_1;
     } else if (bf == round_1) {
-	op->a[1].val = (Uint) (void *) erts_gc_round_1;
+	op->a[1].val = (BeamInstr) (void *) erts_gc_round_1;
     } else if (bf == trunc_1) {
-	op->a[1].val = (Uint) (void *) erts_gc_trunc_1;
+	op->a[1].val = (BeamInstr) (void *) erts_gc_trunc_1;
     } else {
-	abort();
+	op->op = genop_unsupported_guard_bif_3;
+	op->arity = 3;
+	op->a[0].type = TAG_a;
+	op->a[0].val = stp->import[Bif.val].module;
+	op->a[1].type = TAG_a;
+	op->a[1].val = stp->import[Bif.val].function;
+	op->a[2].type = TAG_u;
+	op->a[2].val = stp->import[Bif.val].arity;
+	return op;
     }
+    op->op = genop_i_gc_bif1_5;
+    op->arity = 5;
+    op->a[0] = Fail;
+    op->a[1].type = TAG_u;
     op->a[2] = Src;
     op->a[3] = Live;
     op->a[4] = Dst;
+    return op;
+}
+
+/*
+ * This is used by the ops.tab rule that rewrites gc_bifs with two parameters.
+ * The instruction returned is then again rewritten to an i_load instruction
+ * followed by i_gc_bif2_jIId, to handle literals properly.
+ * As opposed to the i_gc_bif1_jIsId, the instruction  i_gc_bif2_jIId is
+ * always rewritten, regardless of if there actually are any literals.
+ */
+static GenOp*
+gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
+	      GenOpArg S1, GenOpArg S2, GenOpArg Dst)
+{
+    GenOp* op;
+    BifFunction bf;
+
+    NEW_GENOP(stp, op);
+    op->next = NULL;
+    bf = stp->import[Bif.val].bf;
+    /* The translations here need to have a reverse counterpart in
+       beam_emu.c:translate_gc_bif for error handling to work properly. */
+    if (bf == binary_part_2) {
+	op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_2;
+    } else {
+	op->op = genop_unsupported_guard_bif_3;
+	op->arity = 3;
+	op->a[0].type = TAG_a;
+	op->a[0].val = stp->import[Bif.val].module;
+	op->a[1].type = TAG_a;
+	op->a[1].val = stp->import[Bif.val].function;
+	op->a[2].type = TAG_u;
+	op->a[2].val = stp->import[Bif.val].arity;
+	return op;
+    }
+    op->op = genop_ii_gc_bif2_6;
+    op->arity = 6;
+    op->a[0] = Fail;
+    op->a[1].type = TAG_u;
+    op->a[2] = S1;
+    op->a[3] = S2;
+    op->a[4] = Live;
+    op->a[5] = Dst;
+    return op;
+}
+
+/*
+ * This is used by the ops.tab rule that rewrites gc_bifs with three parameters.
+ * The instruction returned is then again rewritten to a move instruction that
+ * uses r[0] for temp storage, followed by an i_load instruction,
+ * followed by i_gc_bif3_jIsId, to handle literals properly. Rewriting
+ * always occur, as with the gc_bif2 counterpart.
+ */
+static GenOp*
+gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
+	      GenOpArg S1, GenOpArg S2, GenOpArg S3, GenOpArg Dst)
+{
+    GenOp* op;
+    BifFunction bf;
+
+    NEW_GENOP(stp, op);
+    op->next = NULL;
+    bf = stp->import[Bif.val].bf;
+    /* The translations here need to have a reverse counterpart in
+       beam_emu.c:translate_gc_bif for error handling to work properly. */
+    if (bf == binary_part_3) {
+	op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_3;
+    } else {
+	op->op = genop_unsupported_guard_bif_3;
+	op->arity = 3;
+	op->a[0].type = TAG_a;
+	op->a[0].val = stp->import[Bif.val].module;
+	op->a[1].type = TAG_a;
+	op->a[1].val = stp->import[Bif.val].function;
+	op->a[2].type = TAG_u;
+	op->a[2].val = stp->import[Bif.val].arity;
+	return op;
+    }
+    op->op = genop_ii_gc_bif3_7;
+    op->arity = 7;
+    op->a[0] = Fail;
+    op->a[1].type = TAG_u;
+    op->a[2] = S1;
+    op->a[3] = S2;
+    op->a[4] = S3;
+    op->a[5] = Live;
+    op->a[6] = Dst;
     op->next = NULL;
     return op;
 }
 
+static GenOp*
+tuple_append_put5(LoaderState* stp, GenOpArg Arity, GenOpArg Dst,
+		  GenOpArg* Puts, GenOpArg S1, GenOpArg S2, GenOpArg S3,
+		  GenOpArg S4, GenOpArg S5)
+{
+    GenOp* op;
+    int arity = Arity.val;	/* Arity of tuple, not the instruction */
+    int i;
+
+    NEW_GENOP(stp, op);
+    op->next = NULL;
+    GENOP_ARITY(op, arity+2+5);
+    op->op = genop_i_put_tuple_2;
+    op->a[0] = Dst;
+    op->a[1].type = TAG_u;
+    op->a[1].val = arity + 5;
+    for (i = 0; i < arity; i++) {
+	op->a[i+2] = Puts[i];
+    }
+    op->a[arity+2] = S1;
+    op->a[arity+3] = S2;
+    op->a[arity+4] = S3;
+    op->a[arity+5] = S4;
+    op->a[arity+6] = S5;
+    return op;
+}
+
+static GenOp*
+tuple_append_put(LoaderState* stp, GenOpArg Arity, GenOpArg Dst,
+		 GenOpArg* Puts, GenOpArg S)
+{
+    GenOp* op;
+    int arity = Arity.val;	/* Arity of tuple, not the instruction */
+    int i;
+
+    NEW_GENOP(stp, op);
+    op->next = NULL;
+    GENOP_ARITY(op, arity+2+1);
+    op->op = genop_i_put_tuple_2;
+    op->a[0] = Dst;
+    op->a[1].type = TAG_u;
+    op->a[1].val = arity + 1;
+    for (i = 0; i < arity; i++) {
+	op->a[i+2] = Puts[i];
+    }
+    op->a[arity+2] = S;
+    return op;
+}
+
+
 
 /*
  * Freeze the code in memory, move the string table into place,
@@ -3376,16 +3931,16 @@ gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif,
 static int
 freeze_code(LoaderState* stp)
 {
-    Eterm* code = stp->code;
-    Uint index;
+    BeamInstr* code = stp->code;
+    Uint *literal_end = NULL;
     int i;
     byte* str_table;
     unsigned strtab_size = stp->chunks[STR_CHUNK].size;
     unsigned attr_size = stp->chunks[ATTR_CHUNK].size;
     unsigned compile_size = stp->chunks[COMPILE_CHUNK].size;
     Uint size;
-    unsigned catches;
     Sint decoded_size;
+    Uint line_size;
 
     /*
      * Verify that there was a correct 'FunT' chunk if there were
@@ -3396,54 +3951,66 @@ freeze_code(LoaderState* stp)
 	LoadError0(stp, stp->lambda_error);
     }
 
-    
     /*
      * Calculate the final size of the code.
      */
-
-    size = (stp->ci + stp->total_literal_size) * sizeof(Eterm) +
-	strtab_size + attr_size + compile_size;
+    if (stp->line_instr == 0) {
+	line_size = 0;
+    } else {
+	line_size = (MI_LINE_FUNC_TAB + (stp->num_functions + 1) +
+		     (stp->current_li+1) + stp->num_fnames) *
+	    sizeof(Eterm) + (stp->current_li+1) * stp->loc_size;
+    }
+    size = (stp->ci * sizeof(BeamInstr)) +
+	(stp->total_literal_size * sizeof(Eterm)) +
+	strtab_size + attr_size + compile_size + line_size;
 
     /*
      * Move the code to its final location.
      */
 
-    code = (Eterm *) erts_realloc(ERTS_ALC_T_CODE, (void *) code, size);
-
+    code = (BeamInstr *) erts_realloc(ERTS_ALC_T_CODE, (void *) code, size);
+    CHKBLK(ERTS_ALC_T_CODE,code);
     /*
      * Place a pointer to the op_int_code_end instruction in the
      * function table in the beginning of the file.
      */
 
-    code[MI_FUNCTIONS+stp->num_functions] = (Eterm) (code + stp->ci - 1);
+    code[MI_FUNCTIONS+stp->num_functions] = (BeamInstr) (code + stp->ci - 1);
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
     /*
      * Store the pointer to the on_load function.
      */
 
     if (stp->on_load) {
-	code[MI_ON_LOAD_FUNCTION_PTR] = (Eterm) (code + stp->on_load);
+	code[MI_ON_LOAD_FUNCTION_PTR] = (BeamInstr) (code + stp->on_load);
     } else {
 	code[MI_ON_LOAD_FUNCTION_PTR] = 0;
     }
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
+    literal_end = (Uint *) (code+stp->ci);
     /*
      * Place the literal heap directly after the code and fix up all
-     * put_literal instructions that refer to it.
+     * instructions that refer to it.
      */
     {
-	Eterm* ptr;
-	Eterm* low;
-	Eterm* high;
+	Uint* ptr;
+	Uint* low;
+	Uint* high;
 	LiteralPatch* lp;
+	struct erl_off_heap_header* off_heap = 0;
+	struct erl_off_heap_header** off_heap_last = &off_heap;
 
-	low = code+stp->ci;
+	low = (Uint *) (code+stp->ci);
 	high = low + stp->total_literal_size;
-	code[MI_LITERALS_START] = (Eterm) low;
-	code[MI_LITERALS_END] = (Eterm) high;
+	code[MI_LITERALS_START] = (BeamInstr) low;
+	code[MI_LITERALS_END] = (BeamInstr) high;
 	ptr = low;
 	for (i = 0; i < stp->num_literals; i++) {
 	    Uint offset;
+	    struct erl_off_heap_header* t_off_heap;
 
 	    sys_memcpy(ptr, stp->literals[i].heap,
 		       stp->literals[i].heap_size*sizeof(Eterm));
@@ -3458,9 +4025,19 @@ freeze_code(LoaderState* stp)
 		    *ptr++ = offset_ptr(val, offset);
 		    break;
 		case TAG_PRIMARY_HEADER:
-		    ptr++;
-		    if (header_is_thing(val)) {
-			ptr += thing_arityval(val);
+		    if (header_is_transparent(val)) {
+			ptr++;
+		    } else {
+			if (thing_subtag(val) == REFC_BINARY_SUBTAG) {
+			    struct erl_off_heap_header* oh;
+
+			    oh = (struct erl_off_heap_header*) ptr;
+			    if (oh->next) {
+				Eterm** uptr = (Eterm **) (void *) &oh->next;
+				*uptr += offset;
+			    }
+			}
+			ptr += 1 + thing_arityval(val);
 		    }
 		    break;
 		default:
@@ -3469,10 +4046,26 @@ freeze_code(LoaderState* stp)
 		}
 	    }
 	    ASSERT(ptr == high);
+
+	    /*
+	     * Re-link the off_heap list for this term onto the
+	     * off_heap list for the entire module.
+	     */
+	    t_off_heap = stp->literals[i].off_heap.first;
+	    if (t_off_heap) {
+		t_off_heap = (struct erl_off_heap_header *)
+		    offset_ptr((UWord) t_off_heap, offset);
+		while (t_off_heap) {
+		    *off_heap_last = t_off_heap;
+		    off_heap_last = &t_off_heap->next;
+		    t_off_heap = t_off_heap->next;
+		}
+	    }
 	}
+	code[MI_LITERALS_OFF_HEAP] = (BeamInstr) off_heap;
 	lp = stp->literal_patches;
 	while (lp != 0) {
-	    Uint* op_ptr;
+	    BeamInstr* op_ptr;
 	    Uint literal;
 	    Literal* lit;
 
@@ -3485,81 +4078,124 @@ freeze_code(LoaderState* stp)
 	    op_ptr[0] = literal;
 	    lp = lp->next;
 	}
-	stp->ci += stp->total_literal_size;
+	literal_end += stp->total_literal_size;
     }
-    
+    CHKBLK(ERTS_ALC_T_CODE,code);
+
     /*
-     * Place the string table and, optionally, attributes, after the literal heap.
+     * If there is line information, place it here.
      */
+    if (stp->line_instr == 0) {
+	code[MI_LINE_TABLE] = (BeamInstr) 0;
+	str_table = (byte *) literal_end;
+    } else {
+	Eterm* line_tab = (Eterm *) literal_end;
+	Eterm* p;
+	int ftab_size = stp->num_functions;
+	int num_instrs = stp->current_li;
+	Eterm* first_line_item;
+
+	code[MI_LINE_TABLE] = (BeamInstr) line_tab;
+	p = line_tab + MI_LINE_FUNC_TAB;
+
+	first_line_item = (p + ftab_size + 1);
+	for (i = 0; i < ftab_size; i++) {
+	     *p++ = (Eterm) (BeamInstr) (first_line_item + stp->func_line[i]);
+	}
+	*p++ = (Eterm) (BeamInstr) (first_line_item + num_instrs);
+	ASSERT(p == first_line_item);
+	for (i = 0; i < num_instrs; i++) {
+		*p++ = (Eterm) (BeamInstr) (code + stp->line_instr[i].pos);
+	}
+	*p++ = (Eterm) (BeamInstr) (code + stp->ci - 1);
+
+	line_tab[MI_LINE_FNAME_PTR] = (Eterm) (BeamInstr) p;
+	memcpy(p, stp->fname, stp->num_fnames*sizeof(Eterm));
+	p += stp->num_fnames;
+
+	line_tab[MI_LINE_LOC_TAB] = (Eterm) (BeamInstr) p;
+	line_tab[MI_LINE_LOC_SIZE] = stp->loc_size;
+	if (stp->loc_size == 2) {
+	    Uint16* locp = (Uint16 *) p;
+	    for (i = 0; i < num_instrs; i++) {
+		*locp++ = (Uint16) stp->line_instr[i].loc;
+	    }
+	    *locp++ = LINE_INVALID_LOCATION;
+	    str_table = (byte *) locp;
+	} else {
+	    Uint32* locp = (Uint32 *) p;
+	    ASSERT(stp->loc_size == 4);
+	    for (i = 0; i < num_instrs; i++) {
+		*locp++ = stp->line_instr[i].loc;
+	    }
+	    *locp++ = LINE_INVALID_LOCATION;
+	    str_table = (byte *) locp;
+	}
 
-    sys_memcpy(code+stp->ci, stp->chunks[STR_CHUNK].start, strtab_size);
-    str_table = (byte *) (code+stp->ci);
+	CHKBLK(ERTS_ALC_T_CODE,code);
+    }
+
+    /*
+     * Place the string table and, optionally, attributes here.
+     */
+    sys_memcpy(str_table, stp->chunks[STR_CHUNK].start, strtab_size);
+    CHKBLK(ERTS_ALC_T_CODE,code);
     if (attr_size) {
 	byte* attr = str_table + strtab_size;
 	sys_memcpy(attr, stp->chunks[ATTR_CHUNK].start, stp->chunks[ATTR_CHUNK].size);
-	code[MI_ATTR_PTR] = (Eterm) attr;
-	code[MI_ATTR_SIZE] = (Eterm) stp->chunks[ATTR_CHUNK].size;
-	decoded_size = erts_decode_ext_size(attr, attr_size, 0);
+	code[MI_ATTR_PTR] = (BeamInstr) attr;
+	code[MI_ATTR_SIZE] = (BeamInstr) stp->chunks[ATTR_CHUNK].size;
+	decoded_size = erts_decode_ext_size(attr, attr_size);
 	if (decoded_size < 0) {
  	    LoadError0(stp, "bad external term representation of module attributes");
  	}
 	code[MI_ATTR_SIZE_ON_HEAP] = decoded_size;
     }
+    CHKBLK(ERTS_ALC_T_CODE,code);
     if (compile_size) {
 	byte* compile_info = str_table + strtab_size + attr_size;
+	CHKBLK(ERTS_ALC_T_CODE,code);
 	sys_memcpy(compile_info, stp->chunks[COMPILE_CHUNK].start,
 	       stp->chunks[COMPILE_CHUNK].size);
-	code[MI_COMPILE_PTR] = (Eterm) compile_info;
-	code[MI_COMPILE_SIZE] = (Eterm) stp->chunks[COMPILE_CHUNK].size;
-	decoded_size = erts_decode_ext_size(compile_info, compile_size, 0);
+
+	CHKBLK(ERTS_ALC_T_CODE,code);
+	code[MI_COMPILE_PTR] = (BeamInstr) compile_info;
+	CHKBLK(ERTS_ALC_T_CODE,code);
+	code[MI_COMPILE_SIZE] = (BeamInstr) stp->chunks[COMPILE_CHUNK].size;
+	CHKBLK(ERTS_ALC_T_CODE,code);
+	decoded_size = erts_decode_ext_size(compile_info, compile_size);
+	CHKBLK(ERTS_ALC_T_CODE,code);
 	if (decoded_size < 0) {
  	    LoadError0(stp, "bad external term representation of compilation information");
  	}
+	CHKBLK(ERTS_ALC_T_CODE,code);
 	code[MI_COMPILE_SIZE_ON_HEAP] = decoded_size;
     }
-
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
     /*
-     * Go through all put_strings instructions, restore the pointer to
-     * the instruction and convert string offsets to pointers (to the
-     * LAST character).
+     * Make sure that we have not overflowed the allocated code space.
      */
-
-    index = stp->put_strings;
-    while (index != 0) {
-	Uint next = code[index];
-	code[index] = BeamOpCode(op_put_string_IId);
-	code[index+2] = (Uint) (str_table + code[index+2] + code[index+1] - 1);
-	index = next;
-    }
+    ASSERT(str_table + strtab_size + attr_size + compile_size ==
+	   ((byte *) code) + size);
 
     /*
-     * Go through all i_new_bs_put_strings instructions, restore the pointer to
-     * the instruction and convert string offsets to pointers (to the
-     * FIRST character).
+     * Patch all instructions that refer to the string table.
      */
-
-    index = stp->new_bs_put_strings;
-    while (index != 0) {
-	Uint next = code[index];
-	code[index] = BeamOpCode(op_bs_put_string_II);
-	code[index+2] = (Uint) (str_table + code[index+2]);
-	index = next;
-    }
-
     {
 	StringPatch* sp = stp->string_patches;
 
 	while (sp != 0) {
-	    Uint* op_ptr;
+	    BeamInstr* op_ptr;
 	    byte* strp;
 
 	    op_ptr = code + sp->pos;
 	    strp = str_table + op_ptr[0];
-	    op_ptr[0] = (Eterm) strp;
+	    op_ptr[0] = (BeamInstr) strp;
 	    sp = sp->next;
 	}
     }
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
     /*
      * Resolve all labels.
@@ -3579,24 +4215,11 @@ freeze_code(LoaderState* stp)
 	    ASSERT(this_patch < stp->ci);
 	    next_patch = code[this_patch];
 	    ASSERT(next_patch < stp->ci);
-	    code[this_patch] = (Uint) (code + value);
+	    code[this_patch] = (BeamInstr) (code + value);
 	    this_patch = next_patch;
 	}
     }
-
-    /*
-     * Fix all catch_yf instructions.
-     */
-    index = stp->catches;
-    catches = BEAM_CATCHES_NIL;
-    while (index != 0) {
-	Uint next = code[index];
-	code[index] = BeamOpCode(op_catch_yf);
-	catches = beam_catches_cons((Uint*)code[index+2], catches);
-	code[index+2] = make_catch(catches);
-	index = next;
-    }
-    stp->catches = catches;
+    CHKBLK(ERTS_ALC_T_CODE,code);
 
     /*
      * Save the updated code pointer and code size.
@@ -3605,6 +4228,7 @@ freeze_code(LoaderState* stp)
     stp->code = code;
     stp->loaded_size = size;
 
+    CHKBLK(ERTS_ALC_T_CODE,code);
     return 1;
 
  load_error:
@@ -3622,6 +4246,26 @@ final_touch(LoaderState* stp)
 {
     int i;
     int on_load = stp->on_load;
+    unsigned catches;
+    Uint index;
+    BeamInstr* code = stp->code;
+    Module* modp;
+
+    /*
+     * Allocate catch indices and fix up all catch_yf instructions.
+     */
+
+    index = stp->catches;
+    catches = BEAM_CATCHES_NIL;
+    while (index != 0) {
+	BeamInstr next = code[index];
+	code[index] = BeamOpCode(op_catch_yf);
+	catches = beam_catches_cons((BeamInstr *)code[index+2], catches);
+	code[index+2] = make_catch(catches);
+	index = next;
+    }
+    modp = erts_put_module(stp->module);
+    modp->catches = catches;
 
     /*
      * Export functions.
@@ -3638,7 +4282,7 @@ final_touch(LoaderState* stp)
 	     * callable yet.
 	     */
 	    ep->address = ep->code+3;
-	    ep->code[4] = (Eterm) stp->export[i].address;
+	    ep->code[4] = (BeamInstr) stp->export[i].address;
 	}
     }
 
@@ -3650,14 +4294,14 @@ final_touch(LoaderState* stp)
 	Eterm mod;
 	Eterm func;
 	Uint arity;
-	Uint import;
+	BeamInstr import;
 	Uint current;
 	Uint next;
 
 	mod = stp->import[i].module;
 	func = stp->import[i].function;
 	arity = stp->import[i].arity;
-	import = (Uint) erts_export_put(mod, func, arity);
+	import = (BeamInstr) erts_export_put(mod, func, arity);
 	current = stp->import[i].patches;
 	while (current != 0) {
 	    ASSERT(current < stp->ci);
@@ -3675,7 +4319,7 @@ final_touch(LoaderState* stp)
 	for (i = 0; i < stp->num_lambdas; i++) {
 	    unsigned entry_label = stp->lambdas[i].label;
 	    ErlFunEntry* fe = stp->lambdas[i].fe;
-	    Eterm* code_ptr = (Eterm *) (stp->code + stp->labels[entry_label].value);
+	    BeamInstr* code_ptr = (BeamInstr *) (stp->code + stp->labels[entry_label].value);
 
 	    if (fe->address[0] != 0) {
 		/*
@@ -3705,6 +4349,7 @@ transform_engine(LoaderState* st)
     GenOp* instr;
     Uint* pc;
     int rval;
+    static Uint restart_fail[1] = {TOP_fail};
 
     ASSERT(gen_opc[st->genop->op].transform != -1);
     pc = op_transform + gen_opc[st->genop->op].transform;
@@ -3718,7 +4363,6 @@ transform_engine(LoaderState* st)
     ASSERT(restart != NULL);
     pc = restart;
     ASSERT(*pc < NUM_TOPS);	/* Valid instruction? */
-    ASSERT(*pc == TOP_try_me_else || *pc == TOP_fail);
     instr = st->genop;
 
 #define RETURN(r) rval = (r); goto do_return;
@@ -3731,7 +4375,9 @@ transform_engine(LoaderState* st)
 	op = *pc++;
 
 	switch (op) {
-	case TOP_is_op:
+	case TOP_next_instr:
+	    instr = instr->next;
+	    ap = 0;
 	    if (instr == NULL) {
 		/*
 		 * We'll need at least one more instruction to decide whether
@@ -3762,11 +4408,23 @@ transform_engine(LoaderState* st)
 	    if (i == 0)
 		goto restart;
 	    break;
+#if defined(TOP_is_eq)
 	case TOP_is_eq:
 	    ASSERT(ap < instr->arity);
 	    if (*pc++ != instr->a[ap].val)
 		goto restart;
 	    break;
+#endif
+	case TOP_is_type_eq:
+	    mask = *pc++;
+
+	    ASSERT(ap < instr->arity);
+	    ASSERT(instr->a[ap].type < BEAM_NUM_TAGS);
+	    if (((1 << instr->a[ap].type) & mask) == 0)
+		goto restart;
+	    if (*pc++ != instr->a[ap].val)
+		goto restart;
+	    break;
 	case TOP_is_same_var:
 	    ASSERT(ap < instr->arity);
 	    i = *pc++;
@@ -3807,7 +4465,7 @@ transform_engine(LoaderState* st)
 		if (i >= st->num_imports || st->import[i].bf == NULL)
 		    goto restart;
 		if (bif_number != -1 &&
-		    bif_export[bif_number]->code[4] != (Uint) st->import[i].bf) {
+		    bif_export[bif_number]->code[4] != (BeamInstr) st->import[i].bf) {
 		    goto restart;
 		}
 	    }
@@ -3887,14 +4545,17 @@ transform_engine(LoaderState* st)
 	case TOP_rest_args:
 	    {
 		int n = *pc++;
+		int formal_arity = gen_opc[instr->op].arity;
+		int num_vars = n + (instr->arity - formal_arity);
+		int j = formal_arity;
+
 		var = erts_alloc(ERTS_ALC_T_LOADER_TMP,
-				 instr->arity * sizeof(GenOpArg));
+				 num_vars * sizeof(GenOpArg));
 		for (i = 0; i < n; i++) {
 		    var[i] = def_vars[i];
 		}
-		while (i < instr->arity) {
-		    var[i] = instr->a[i];
-		    i++;
+		while (i < num_vars) {
+		    var[i++] = instr->a[j++];
 		}
 	    }
 	    break;
@@ -3903,10 +4564,6 @@ transform_engine(LoaderState* st)
 	case TOP_next_arg:
 	    ap++;
 	    break;
-	case TOP_next_instr:
-	    instr = instr->next;
-	    ap = 0;
-	    break;
 	case TOP_commit:
 	    instr = instr->next; /* The next_instr was optimized away. */
 
@@ -3924,8 +4581,8 @@ transform_engine(LoaderState* st)
 #endif
 	    break;
 
-#if defined(TOP_call)
-	case TOP_call:
+#if defined(TOP_call_end)
+	case TOP_call_end:
 	    {
 		GenOp** lastp;
 		GenOp* new_instr;
@@ -3962,7 +4619,7 @@ transform_engine(LoaderState* st)
 		*lastp = st->genop;
 		st->genop = new_instr;
 	    }
-	    break;
+	    RETURN(TE_OK);
 #endif
 	case TOP_new_instr:
 	    /*
@@ -3971,12 +4628,10 @@ transform_engine(LoaderState* st)
 	    NEW_GENOP(st, instr);
 	    instr->next = st->genop;
 	    st->genop = instr;
+	    instr->op = op = *pc++;
+	    instr->arity = gen_opc[op].arity;
 	    ap = 0;
 	    break;
-	case TOP_store_op:
-	    instr->op = *pc++;
-	    instr->arity = *pc++;
-	    break;
 	case TOP_store_type:
 	    i = *pc++;
 	    instr->a[ap].type = i;
@@ -3986,21 +4641,25 @@ transform_engine(LoaderState* st)
 	    i = *pc++;
 	    instr->a[ap].val = i;
 	    break;
-	case TOP_store_var:
+	case TOP_store_var_next_arg:
 	    i = *pc++;
 	    ASSERT(i < TE_MAX_VARS);
 	    instr->a[ap].type = var[i].type;
 	    instr->a[ap].val = var[i].val;
+	    ap++;
 	    break;
 	case TOP_try_me_else:
 	    restart = pc + 1;
 	    restart += *pc++;
 	    ASSERT(*pc < NUM_TOPS); /* Valid instruction? */
 	    break;
+	case TOP_try_me_else_fail:
+	    restart = restart_fail;
+	    break;
 	case TOP_end:
 	    RETURN(TE_OK);
 	case TOP_fail:
-	    RETURN(TE_FAIL)
+	    RETURN(TE_FAIL);
 	default:
 	    ASSERT(0);
 	}
@@ -4069,41 +4728,9 @@ load_printf(int line, LoaderState* context, char *fmt,...)
     erts_send_error_to_logger(context->group_leader, dsbufp);
 }
 
-
-static int
-get_int_val(LoaderState* stp, Uint len_code, Uint* result)
-{
-    Uint count;
-    Uint val;
-
-    len_code >>= 5;
-    ASSERT(len_code < 8);
-    if (len_code == 7) {
-	LoadError0(stp, "can't load integers bigger than 8 bytes yet\n");
-    }
-    count = len_code + 2;
-    if (count == 5) {
-	Uint msb;
-	GetByte(stp, msb);
-	if (msb == 0) {
-	    count--;
-	}
-	GetInt(stp, 4, *result);
-    } else if (count <= 4) {
-	GetInt(stp, count, val);
-	*result = ((val << 8*(sizeof(val)-count)) >> 8*(sizeof(val)-count));
-    } else {
-	LoadError1(stp, "too big integer; %d bytes\n", count);
-    }
-    return 1;
-
- load_error:
-    return 0;
-}
-
-
 static int
-get_erlang_integer(LoaderState* stp, Uint len_code, Uint* result)
+get_tag_and_value(LoaderState* stp, Uint len_code,
+		  unsigned tag, BeamInstr* result)
 {
     Uint count;
     Sint val;
@@ -4123,16 +4750,62 @@ get_erlang_integer(LoaderState* stp, Uint len_code, Uint* result)
     if (len_code < 7) {
 	count = len_code + 2;
     } else {
-	Uint tag;
+	unsigned sztag;
+	UWord len_word;
 
 	ASSERT(len_code == 7);
-	GetTagAndValue(stp, tag, len_code);
-	VerifyTag(stp, TAG_u, tag);
-	count = len_code + 9;
+	GetTagAndValue(stp, sztag, len_word);
+	VerifyTag(stp, sztag, TAG_u);
+	count = len_word + 9;
+    }
+
+    /*
+     * The value for tags except TAG_i must be an unsigned integer
+     * fitting in an Uint. If it does not fit, we'll indicate overflow
+     * by changing the tag to TAG_o.
+     */
+
+    if (tag != TAG_i) {
+	if (count == sizeof(Uint)+1) {
+	    Uint msb;
+
+	    /*
+	     * The encoded value has one more byte than an Uint.
+	     * It will still fit in an Uint if the most significant
+	     * byte is 0.
+	     */
+	    GetByte(stp, msb);
+	    GetInt(stp, sizeof(Uint), *result);
+	    if (msb != 0) {
+		/* Overflow: Negative or too big. */
+		return TAG_o;
+	    }
+	} else if (count == sizeof(Uint)) {
+	    /*
+	     * The value must be positive (or the encoded value would
+	     * have been one byte longer).
+	     */
+	    GetInt(stp, count, *result);
+	} else if (count < sizeof(Uint)) {
+	    GetInt(stp, count, *result);
+
+	    /*
+	     * If the sign bit is set, the value is negative
+	     * (not allowed).
+	     */
+	    if (*result & ((Uint)1 << (count*8-1))) {
+		return TAG_o;
+	    }
+	} else {
+	    GetInt(stp, count, *result);
+	    return TAG_o;
+	}
+	return tag;
     }
 
     /*
-     * Handle values up to the size of an int, meaning either a small or bignum.
+     * TAG_i: First handle values up to the size of an Uint (i.e. either
+     * a small or a bignum).
      */
 
     if (count <= sizeof(val)) {
@@ -4309,6 +4982,8 @@ new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size)
     lit->heap_size = heap_size;
     lit->heap = erts_alloc(ERTS_ALC_T_LOADER_TMP, heap_size*sizeof(Eterm));
     lit->term = make_boxed(lit->heap);
+    lit->off_heap.first = 0;
+    lit->off_heap.overhead = 0;
     *hpp = lit->heap;
     return stp->num_literals++;
 }
@@ -4376,7 +5051,7 @@ functions_in_module(Process* p, /* Process whose heap to use. */
 		     Eterm mod) /* Tagged atom for module. */
 {
     Module* modp;
-    Eterm* code;
+    BeamInstr* code;
     int i;
     Uint num_functions;
     Eterm* hp;
@@ -4394,9 +5069,9 @@ functions_in_module(Process* p, /* Process whose heap to use. */
     num_functions = code[MI_NUM_FUNCTIONS];
     hp = HAlloc(p, 5*num_functions);
     for (i = num_functions-1; i >= 0 ; i--) {
-	Eterm* func_info = (Eterm *) code[MI_FUNCTIONS+i];
-	Eterm name = func_info[3];
-	int arity = func_info[4];
+	BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i];
+	Eterm name = (Eterm) func_info[3];
+	int arity = (int) func_info[4];
 	Eterm tuple;
 
 	ASSERT(is_atom(name));
@@ -4419,7 +5094,7 @@ static Eterm
 native_addresses(Process* p, Eterm mod)
 {
     Module* modp;
-    Eterm* code;
+    BeamInstr* code;
     int i;
     Eterm* hp;
     Uint num_functions;
@@ -4442,9 +5117,9 @@ native_addresses(Process* p, Eterm mod)
     hp = HAlloc(p, need);
     hp_end = hp + need;
     for (i = num_functions-1; i >= 0 ; i--) {
-	Eterm* func_info = (Eterm *) code[MI_FUNCTIONS+i];
-	Eterm name = func_info[3];
-	int arity = func_info[4];
+	BeamInstr* func_info = (BeamInstr *) code[MI_FUNCTIONS+i];
+	Eterm name = (Eterm) func_info[3];
+	int arity = (int) func_info[4];
 	Eterm tuple;
 
 	ASSERT(is_atom(name));
@@ -4486,7 +5161,7 @@ exported_from_module(Process* p, /* Process whose heap to use. */
 	    Eterm tuple;
 	    
 	    if (ep->address == ep->code+3 &&
-		ep->code[3] == (Eterm) em_call_error_handler) {
+		ep->code[3] == (BeamInstr) em_call_error_handler) {
 		/* There is a call to the function, but it does not exist. */ 
 		continue;
 	    }
@@ -4519,7 +5194,7 @@ attributes_for_module(Process* p, /* Process whose heap to use. */
 
 {
     Module* modp;
-    Eterm* code;
+    BeamInstr* code;
     Eterm* hp;
     byte* ext;
     Eterm result = NIL;
@@ -4559,7 +5234,7 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */
 			    Eterm mod) /* Tagged atom for module. */
 {
     Module* modp;
-    Eterm* code;
+    BeamInstr* code;
     Eterm* hp;
     byte* ext;
     Eterm result = NIL;
@@ -4587,26 +5262,33 @@ compilation_info_for_module(Process* p, /* Process whose heap to use. */
     return result;
 }
 
-
 /*
- * Returns a pointer to {module, function, arity}, or NULL if not found.
+ * Find a function from the given pc and fill information in
+ * the FunctionInfo struct. If the full_info is non-zero, fill
+ * in all available information (including location in the
+ * source code). If no function is found, the 'current' field
+ * will be set to NULL.
  */
-Eterm*
-find_function_from_pc(Eterm* pc)
+
+void
+erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info)
 {
     Range* low = modules;
     Range* high = low + num_loaded_modules;
     Range* mid = mid_module;
 
+    fi->current = NULL;
+    fi->needed = 5;
+    fi->loc = LINE_INVALID_LOCATION;
     while (low < high) {
 	if (pc < mid->start) {
 	    high = mid;
 	} else if (pc > mid->end) {
 	    low = mid + 1;
 	} else {
-	    Eterm** low1 = (Eterm **) (mid->start + MI_FUNCTIONS);
-	    Eterm** high1 = low1 + mid->start[MI_NUM_FUNCTIONS];
-	    Eterm** mid1;
+	    BeamInstr** low1 = (BeamInstr **) (mid->start + MI_FUNCTIONS);
+	    BeamInstr** high1 = low1 + mid->start[MI_NUM_FUNCTIONS];
+	    BeamInstr** mid1;
 
 	    while (low1 < high1) {
 		mid1 = low1 + (high1-low1) / 2;
@@ -4614,26 +5296,160 @@ find_function_from_pc(Eterm* pc)
 		    high1 = mid1;
 		} else if (pc < mid1[1]) {
 		    mid_module = mid;
-		    return mid1[0]+2;
+		    fi->current = mid1[0]+2;
+		    if (full_info) {
+			BeamInstr** fp = (BeamInstr **) (mid->start +
+							 MI_FUNCTIONS);
+			int idx = mid1 - fp;
+			lookup_loc(fi, pc, mid->start, idx);
+		    }
+		    return;
 		} else {
 		    low1 = mid1 + 1;
 		}
 	    }
-	    return NULL;
+	    return;
 	}
 	mid = low + (high-low) / 2;
     }
-    return NULL;
+}
+
+static void
+lookup_loc(FunctionInfo* fi, BeamInstr* orig_pc, BeamInstr* modp, int idx)
+{
+    Eterm* line = (Eterm *) modp[MI_LINE_TABLE];
+    Eterm* low;
+    Eterm* high;
+    Eterm* mid;
+    Eterm pc;
+
+    if (line == 0) {
+	return;
+    }
+
+    pc = (Eterm) (BeamInstr) orig_pc;
+    fi->fname_ptr = (Eterm *) (BeamInstr) line[MI_LINE_FNAME_PTR];
+    low = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx];
+    high = (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB+idx+1];
+    while (high > low) {
+	mid = low + (high-low) / 2;
+	if (pc < mid[0]) {
+	    high = mid;
+	} else if (pc < mid[1]) {
+	    int file;
+	    int index = mid - (Eterm *) (BeamInstr) line[MI_LINE_FUNC_TAB];
+
+	    if (line[MI_LINE_LOC_SIZE] == 2) {
+		Uint16* loc_table =
+		    (Uint16 *) (BeamInstr) line[MI_LINE_LOC_TAB];
+		fi->loc = loc_table[index];
+	    } else {
+		Uint32* loc_table =
+		    (Uint32 *) (BeamInstr) line[MI_LINE_LOC_TAB];
+		ASSERT(line[MI_LINE_LOC_SIZE] == 4);
+		fi->loc = loc_table[index];
+	    }
+	    if (fi->loc == LINE_INVALID_LOCATION) {
+		return;
+	    }
+	    fi->needed += 3+2+3+2;
+	    file = LOC_FILE(fi->loc);
+	    if (file == 0) {
+		/* Special case: Module name with ".erl" appended */
+		Atom* mod_atom = atom_tab(atom_val(fi->current[0]));
+		fi->needed += 2*(mod_atom->len+4);
+	    } else {
+		Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1]));
+		fi->needed += 2*ap->len;
+	    }
+	    return;
+	} else {
+	    low = mid + 1;
+	}
+    }
+}
+
+/*
+ * Build a single {M,F,A,Loction} item to be part of
+ * a stack trace.
+ */
+Eterm*
+erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p)
+{
+    BeamInstr* current = fi->current;
+    Eterm loc = NIL;
+
+    if (fi->loc != LINE_INVALID_LOCATION) {
+	Eterm tuple;
+	int line = LOC_LINE(fi->loc);
+	int file = LOC_FILE(fi->loc);
+	Eterm file_term = NIL;
+
+	if (file == 0) {
+	    Atom* ap = atom_tab(atom_val(fi->current[0]));
+	    file_term = buf_to_intlist(&hp, ".erl", 4, NIL);
+	    file_term = buf_to_intlist(&hp, (char*)ap->name, ap->len, file_term);
+	} else {
+	    Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1]));
+	    file_term = buf_to_intlist(&hp, (char*)ap->name, ap->len, NIL);
+	}
+
+	tuple = TUPLE2(hp, am_line, make_small(line));
+	hp += 3;
+	loc = CONS(hp, tuple, loc);
+	hp += 2;
+	tuple = TUPLE2(hp, am_file, file_term);
+	hp += 3;
+	loc = CONS(hp, tuple, loc);
+	hp += 2;
+    }
+
+    if (is_list(args) || is_nil(args)) {
+	*mfa_p = TUPLE4(hp, current[0], current[1], args, loc);
+    } else {
+	Eterm arity = make_small(current[2]);
+	*mfa_p = TUPLE4(hp, current[0], current[1], arity, loc);
+    }
+    return hp + 5;
+}
+
+/*
+ * Force setting of the current function in a FunctionInfo
+ * structure. No source code location will be associated with
+ * the function.
+ */
+void
+erts_set_current_function(FunctionInfo* fi, BeamInstr* current)
+{
+    fi->current = current;
+    fi->needed = 5;
+    fi->loc = LINE_INVALID_LOCATION;
+}
+
+
+/*
+ * Returns a pointer to {module, function, arity}, or NULL if not found.
+ */
+BeamInstr*
+find_function_from_pc(BeamInstr* pc)
+{
+    FunctionInfo fi;
+
+    erts_lookup_function_info(&fi, pc, 0);
+    return fi.current;
 }
 
 /*
  * Read a specific chunk from a Beam binary.
  */
 
-Eterm
-code_get_chunk_2(Process* p, Eterm Bin, Eterm Chunk)
+BIF_RETTYPE
+code_get_chunk_2(BIF_ALIST_2)
 {
-    LoaderState state;
+    Process* p = BIF_P;
+    Eterm Bin = BIF_ARG_1;
+    Eterm Chunk = BIF_ARG_2;
+    LoaderState* stp;
     Uint chunk = 0;
     ErlSubBin* sb;
     Uint offset;
@@ -4645,15 +5461,16 @@ code_get_chunk_2(Process* p, Eterm Bin, Eterm Chunk)
     Eterm real_bin;
     byte* temp_alloc = NULL;
 
+    stp = erts_alloc_loader_state();
     if ((start = erts_get_aligned_binary_bytes(Bin, &temp_alloc)) == NULL) {
     error:
 	erts_free_aligned_binary_bytes(temp_alloc);
+	if (stp) {
+	    free_state(stp);
+	}
 	BIF_ERROR(p, BADARG);
     }
-    state.module = THE_NON_VALUE; /* Suppress diagnostiscs */
-    state.file_name = "IFF header for Beam file";
-    state.file_p = start;
-    state.file_left = binary_size(Bin);
+    stp->module = THE_NON_VALUE; /* Suppress diagnostics */
     for (i = 0; i < 4; i++) {
 	Eterm* chunkp;
 	Eterm num;
@@ -4671,25 +5488,30 @@ code_get_chunk_2(Process* p, Eterm Bin, Eterm Chunk)
     if (is_not_nil(Chunk)) {
 	goto error;
     }
-    if (!scan_iff_file(&state, &chunk, 1, 1)) {
-	erts_free_aligned_binary_bytes(temp_alloc);
-	return am_undefined;
+    if (!init_iff_file(stp, start, binary_size(Bin)) ||
+	!scan_iff_file(stp, &chunk, 1, 1) ||
+	stp->chunks[0].start == NULL) {
+	res = am_undefined;
+	goto done;
     }
     ERTS_GET_REAL_BIN(Bin, real_bin, offset, bitoffs, bitsize);
     if (bitoffs) {
-	res = new_binary(p, state.chunks[0].start, state.chunks[0].size);
+	res = new_binary(p, stp->chunks[0].start, stp->chunks[0].size);
     } else {
 	sb = (ErlSubBin *) HAlloc(p, ERL_SUB_BIN_SIZE);
 	sb->thing_word = HEADER_SUB_BIN;
 	sb->orig = real_bin;
-	sb->size = state.chunks[0].size;
+	sb->size = stp->chunks[0].size;
 	sb->bitsize = 0;
 	sb->bitoffs = 0;
-	sb->offs = offset + (state.chunks[0].start - start);
+	sb->offs = offset + (stp->chunks[0].start - start);
 	sb->is_writable = 0;
 	res = make_binary(sb);
     }
+
+ done:
     erts_free_aligned_binary_bytes(temp_alloc);
+    free_state(stp);
     return res;
 }
 
@@ -4697,32 +5519,42 @@ code_get_chunk_2(Process* p, Eterm Bin, Eterm Chunk)
  * Calculate the MD5 for a module.
  */
   
-Eterm
-code_module_md5_1(Process* p, Eterm Bin)
+BIF_RETTYPE
+code_module_md5_1(BIF_ALIST_1)
 {
-    LoaderState state;
+    Process* p = BIF_P;
+    Eterm Bin = BIF_ARG_1;
+    LoaderState* stp;
+    byte* bytes;
     byte* temp_alloc = NULL;
+    Eterm res;
 
-    if ((state.file_p = erts_get_aligned_binary_bytes(Bin, &temp_alloc)) == NULL) {
+    stp = erts_alloc_loader_state();
+    if ((bytes = erts_get_aligned_binary_bytes(Bin, &temp_alloc)) == NULL) {
+	free_state(stp);
 	BIF_ERROR(p, BADARG);
     }
-    state.module = THE_NON_VALUE; /* Suppress diagnostiscs */
-    state.file_name = "IFF header for Beam file";
-    state.file_left = binary_size(Bin);
-
-    if (!scan_iff_file(&state, chunk_types, NUM_CHUNK_TYPES, NUM_MANDATORY)) {
-	return am_undefined;
+    stp->module = THE_NON_VALUE; /* Suppress diagnostiscs */
+    if (!init_iff_file(stp, bytes, binary_size(Bin)) ||
+	!scan_iff_file(stp, chunk_types, NUM_CHUNK_TYPES, NUM_MANDATORY) ||
+	!verify_chunks(stp)) {
+	res = am_undefined;
+	goto done;
     }
+    res = new_binary(p, stp->mod_md5, sizeof(stp->mod_md5));
+
+ done:
     erts_free_aligned_binary_bytes(temp_alloc);
-    return new_binary(p, state.mod_md5, sizeof(state.mod_md5));
+    free_state(stp);
+    return res;
 }
 
 #define WORDS_PER_FUNCTION 6
 
-static Eterm*
-make_stub(Eterm* fp, Eterm mod, Eterm func, Uint arity, Uint native, Eterm OpCode)
+static BeamInstr*
+make_stub(BeamInstr* fp, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode)
 {
-    fp[0] = (Eterm) BeamOp(op_i_func_info_IaaI);
+    fp[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI);
     fp[1] = native;
     fp[2] = mod;
     fp[3] = func;
@@ -4741,15 +5573,15 @@ static byte*
 stub_copy_info(LoaderState* stp,
 	       int chunk,	/* Chunk: ATTR_CHUNK or COMPILE_CHUNK */
 	       byte* info,	/* Where to store info. */
-	       Eterm* ptr_word,	/* Where to store pointer into info. */
-	       Eterm* size_word) /* Where to store size of info. */
+	       BeamInstr* ptr_word,	/* Where to store pointer into info. */
+	       BeamInstr* size_word) /* Where to store size of info. */
 {
     Sint decoded_size;
     Uint size = stp->chunks[chunk].size;
     if (size != 0) {
 	memcpy(info, stp->chunks[chunk].start, size);
-	*ptr_word = (Eterm) info;
-	decoded_size = erts_decode_ext_size(info, size, 0);
+	*ptr_word = (BeamInstr) info;
+	decoded_size = erts_decode_ext_size(info, size);
 	if (decoded_size < 0) {
  	    return 0;
  	}
@@ -4791,7 +5623,7 @@ stub_read_export_table(LoaderState* stp)
 }
 
 static void
-stub_final_touch(LoaderState* stp, Eterm* fp)
+stub_final_touch(LoaderState* stp, BeamInstr* fp)
 {
     int i;
     int n = stp->num_exps;
@@ -4956,7 +5788,17 @@ patch_funentries(Eterm Patchlist)
 
     fe = erts_get_fun_entry(Mod, uniq, index);
     fe->native_address = (Uint *)native_address;
-    erts_refc_dec(&fe->refc, 1);
+
+    /* Deliberate MEMORY LEAK of native fun entries!!!
+     *
+     * Uncomment line below when hipe code upgrade and purging works correctly.
+     * Today we may get cases when old (leaked) native code of a purged module
+     * gets called and tries to create instances of a deleted fun entry.
+     *
+     * Reproduced on a debug emulator with stdlib_test/qlc_SUITE:join_merge
+     *
+     * erts_refc_dec(&fe->refc, 1);
+     */
 
     if (!patch(Addresses, (Uint) fe))
       return 0;
@@ -4977,29 +5819,28 @@ patch_funentries(Eterm Patchlist)
 Eterm
 erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 {
-    LoaderState state;
-    Eterm Funcs;
-    Eterm Patchlist;
+    LoaderState* stp;
+    BeamInstr Funcs;
+    BeamInstr Patchlist;
     Eterm* tp;
-    Eterm* code = NULL;
-    Eterm* ptrs;
-    Eterm* fp;
+    BeamInstr* code = NULL;
+    BeamInstr* ptrs;
+    BeamInstr* fp;
     byte* info;
     Uint ci;
     int n;
     int code_size;
     int rval;
     int i;
-    ErlDrvBinary* bin = NULL;
     byte* temp_alloc = NULL;
     byte* bytes;
     Uint size;
 
     /*
-     * Must initialize state.lambdas here because the error handling code
+     * Must initialize stp->lambdas here because the error handling code
      * at label 'error' uses it.
      */
-    init_state(&state);
+    stp = erts_alloc_loader_state();
 
     if (is_not_atom(Mod)) {
 	goto error;
@@ -5023,47 +5864,35 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
     size = binary_size(Beam);
 
     /*
-     * Uncompressed if needed.
-     */
-    if (!(size >= 4 && bytes[0] == 'F' && bytes[1] == 'O' &&
-	  bytes[2] == 'R' && bytes[3] == '1')) {
-	bin = (ErlDrvBinary *) erts_gzinflate_buffer((char*)bytes, size);
-	if (bin == NULL) {
-	    goto error;
-	}
-	bytes = (byte*)bin->orig_bytes;
-	size = bin->orig_size;
-    }
-    
-    /*
      * Scan the Beam binary and read the interesting sections.
      */
 
-    state.file_name = "IFF header for Beam file";
-    state.file_p = bytes;
-    state.file_left = size;
-    state.module = Mod;
-    state.group_leader = p->group_leader;
-    state.num_functions = n;
-    if (!scan_iff_file(&state, chunk_types, NUM_CHUNK_TYPES, NUM_MANDATORY)) {
+    stp->module = Mod;
+    stp->group_leader = p->group_leader;
+    stp->num_functions = n;
+    if (!init_iff_file(stp, bytes, size)) {
 	goto error;
     }
-    define_file(&state, "code chunk header", CODE_CHUNK);
-    if (!read_code_header(&state)) {
+    if (!scan_iff_file(stp, chunk_types, NUM_CHUNK_TYPES, NUM_MANDATORY) ||
+	!verify_chunks(stp)) {
 	goto error;
     }
-    define_file(&state, "atom table", ATOM_CHUNK);
-    if (!load_atom_table(&state)) {
+    define_file(stp, "code chunk header", CODE_CHUNK);
+    if (!read_code_header(stp)) {
 	goto error;
     }
-    define_file(&state, "export table", EXP_CHUNK);
-    if (!stub_read_export_table(&state)) {
+    define_file(stp, "atom table", ATOM_CHUNK);
+    if (!load_atom_table(stp)) {
+	goto error;
+    }
+    define_file(stp, "export table", EXP_CHUNK);
+    if (!stub_read_export_table(stp)) {
 	goto error;
     }
     
-    if (state.chunks[LAMBDA_CHUNK].size > 0) {
-	define_file(&state, "lambda (fun) table", LAMBDA_CHUNK);
-	if (!read_lambda_table(&state)) {
+    if (stp->chunks[LAMBDA_CHUNK].size > 0) {
+	define_file(stp, "lambda (fun) table", LAMBDA_CHUNK);
+	if (!read_lambda_table(stp)) {
 	    goto error;
 	}
     }
@@ -5072,9 +5901,9 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
      * Allocate memory for the stub module.
      */
 
-    code_size = ((WORDS_PER_FUNCTION+1)*n + MI_FUNCTIONS + 2) * sizeof(Eterm);
-    code_size += state.chunks[ATTR_CHUNK].size;
-    code_size += state.chunks[COMPILE_CHUNK].size;
+    code_size = ((WORDS_PER_FUNCTION+1)*n + MI_FUNCTIONS + 2) * sizeof(BeamInstr);
+    code_size += stp->chunks[ATTR_CHUNK].size;
+    code_size += stp->chunks[COMPILE_CHUNK].size;
     code = erts_alloc_fnf(ERTS_ALC_T_CODE, code_size);
     if (!code) {
 	goto error;
@@ -5086,10 +5915,15 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 
     code[MI_NUM_FUNCTIONS] = n;
     code[MI_ATTR_PTR] = 0;
+    code[MI_ATTR_SIZE] = 0;
     code[MI_ATTR_SIZE_ON_HEAP] = 0;
     code[MI_COMPILE_PTR] = 0;
+    code[MI_COMPILE_SIZE] = 0;
     code[MI_COMPILE_SIZE_ON_HEAP] = 0;
     code[MI_NUM_BREAKPOINTS] = 0;
+    code[MI_LITERALS_START] = 0;
+    code[MI_LITERALS_END] = 0;
+    code[MI_LITERALS_OFF_HEAP] = 0;
     code[MI_ON_LOAD_FUNCTION_PTR] = 0;
     ci = MI_FUNCTIONS + n + 1;
 
@@ -5141,7 +5975,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 	 * Set the pointer and make the stub. Put a return instruction
 	 * as the body until we know what kind of trap we should put there.
 	 */
-	ptrs[i] = (Eterm) fp;
+	ptrs[i] = (BeamInstr) fp;
 #ifdef HIPE
 	op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */
 #else
@@ -5154,20 +5988,20 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
      * Insert the last pointer and the int_code_end instruction.
      */
 
-    ptrs[i] = (Eterm) fp;
-    *fp++ = (Eterm) BeamOp(op_int_code_end);
+    ptrs[i] = (BeamInstr) fp;
+    *fp++ = (BeamInstr) BeamOp(op_int_code_end);
 
     /*
      * Copy attributes and compilation information.
      */
 
     info = (byte *) fp;
-    info = stub_copy_info(&state, ATTR_CHUNK, info,
+    info = stub_copy_info(stp, ATTR_CHUNK, info,
 			  code+MI_ATTR_PTR, code+MI_ATTR_SIZE_ON_HEAP);
     if (info == NULL) {
 	goto error;
     }
-    info = stub_copy_info(&state, COMPILE_CHUNK, info,
+    info = stub_copy_info(stp, COMPILE_CHUNK, info,
 			  code+MI_COMPILE_PTR, code+MI_COMPILE_SIZE_ON_HEAP);
     if (info == NULL) {
 	goto error;
@@ -5177,9 +6011,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
      * Insert the module in the module table.
      */
 
-    rval = insert_new_code(p, 0, p->group_leader, Mod, code, code_size,
-			   BEAM_CATCHES_NIL);
-    if (rval < 0) {
+    rval = insert_new_code(p, 0, p->group_leader, Mod, code, code_size);
+    if (rval != NIL) {
 	goto error;
     }
 
@@ -5189,42 +6022,27 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 
     fp = code + ci;
     for (i = 0; i < n; i++) {
-	stub_final_touch(&state, fp);
+	stub_final_touch(stp, fp);
 	fp += WORDS_PER_FUNCTION;
     }
 
     if (patch_funentries(Patchlist)) {
 	erts_free_aligned_binary_bytes(temp_alloc);
-	if (state.lambdas != state.def_lambdas) {
-	    erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.lambdas);
-	}
-	if (bin != NULL) {
-	    driver_free_binary(bin);
-	}
+	free_state(stp);
 	return Mod;
     }
 
  error:
     erts_free_aligned_binary_bytes(temp_alloc);
-    if (code != NULL) {
-	erts_free(ERTS_ALC_T_CODE, code);
-    }
-    if (state.lambdas != state.def_lambdas) {
-	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.lambdas);
-    }
-    if (bin != NULL) {
-	driver_free_binary(bin);
-    }
-
-	
+    free_state(stp);
     BIF_ERROR(p, BADARG);
 }
 
 #undef WORDS_PER_FUNCTION
 
-static int safe_mul(Uint a, Uint b, Uint* resp)
+static int safe_mul(UWord a, UWord b, UWord* resp)
 {
-    Uint res = a * b;
+    Uint res = a * b; /* XXX:Pan - used in bit syntax, the multiplication has to be stored in Uint */
     *resp = res;
 
     if (b == 0) {
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index c17844a553..997ba197db 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1999-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1999-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -23,7 +23,9 @@
 #include "beam_opcodes.h"
 #include "erl_process.h"
 
-int beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module);
+Eterm beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks,
+			    Eterm module);
+
 
 typedef struct gen_op_entry {
    char* name;
@@ -44,13 +46,13 @@ extern void** beam_ops;
 #endif
 
 
-extern Eterm beam_debug_apply[];
-extern Eterm* em_call_error_handler;
-extern Eterm* em_apply_bif;
-extern Eterm* em_call_traced_function;
+extern BeamInstr beam_debug_apply[];
+extern BeamInstr* em_call_error_handler;
+extern BeamInstr* em_apply_bif;
+extern BeamInstr* em_call_traced_function;
 typedef struct {
-    Eterm* start;		/* Pointer to start of module. */
-    Eterm* end;			/* Points one word beyond last function in module. */
+    BeamInstr* start;		/* Pointer to start of module. */
+    BeamInstr* end;			/* Points one word beyond last function in module. */
 } Range;
 
 /*
@@ -101,11 +103,18 @@ extern Uint erts_total_code_size;
  */
 #define MI_LITERALS_START	8
 #define MI_LITERALS_END		9
+#define MI_LITERALS_OFF_HEAP	10
+
 
 /*
  * Pointer to the on_load function (or NULL if none).
  */
-#define MI_ON_LOAD_FUNCTION_PTR 10
+#define MI_ON_LOAD_FUNCTION_PTR 11
+
+/*
+ * Pointer to the line table (or NULL if none).
+ */
+#define MI_LINE_TABLE 12
 
 /*
  * Start of function pointer table.  This table contains pointers to
@@ -116,5 +125,5 @@ extern Uint erts_total_code_size;
  * this table.
  */
 
-#define MI_FUNCTIONS         11
+#define MI_FUNCTIONS         13
 #endif /* _BEAM_LOAD_H */
diff --git a/erts/emulator/beam/benchmark.c b/erts/emulator/beam/benchmark.c
index 7fbf44a03c..7ac14b8e8b 100644
--- a/erts/emulator/beam/benchmark.c
+++ b/erts/emulator/beam/benchmark.c
@@ -33,17 +33,6 @@ unsigned long long messages_copied;
 unsigned long long messages_ego;
 unsigned long long minor_gc;
 unsigned long long major_gc;
-#ifdef HYBRID
-unsigned long long minor_global_gc;
-unsigned long long major_global_gc;
-unsigned long long gc_in_copy;
-#ifdef INCREMENTAL
-unsigned long long minor_gc_cycles;
-unsigned long long major_gc_cycles;
-unsigned long long minor_gc_stages;
-unsigned long long major_gc_stages;
-#endif
-#endif
 #endif /* BM_COUNTERS */
 
 #ifdef BM_TIMERS
@@ -191,17 +180,6 @@ void init_benchmarking()
     messages_ego       = 0;
     minor_gc           = 0;
     major_gc           = 0;
-#ifdef HYBRID
-    minor_global_gc    = 0;
-    major_global_gc    = 0;
-    gc_in_copy         = 0;
-#ifdef INCREMENTAL
-    minor_gc_cycles    = 0;
-    major_gc_cycles    = 0;
-    minor_gc_stages    = 0;
-    major_gc_stages    = 0;
-#endif
-#endif
 #endif /* BM_COUNTERS */
 
 #ifdef BM_HEAP_SIZES
@@ -243,16 +221,6 @@ void save_statistics()
         erts_fprintf(file,"Number of processes spawned: %lld\n",processes_spawned);
         erts_fprintf(file,"Number of local minor GCs: %lld\n",minor_gc);
         erts_fprintf(file,"Number of local major GCs: %lld\n",major_gc);
-#ifdef HYBRID
-        erts_fprintf(file,"Number of global minor GCs: %lld\n",minor_global_gc);
-        erts_fprintf(file,"Number of global major GCs: %lld\n",major_global_gc);
-#ifdef INCREMENTAL
-        erts_fprintf(file,"Number of minor GC-cycles: %lld\n",minor_gc_cycles);
-        erts_fprintf(file,"Number of major GC-cycles: %lld\n",major_gc_cycles);
-        erts_fprintf(file,"Number of minor GC-stages: %lld\n",minor_gc_stages);
-        erts_fprintf(file,"Number of major GC-stages: %lld\n",major_gc_stages);
-#endif
-#endif
         erts_fprintf(file,"Number of messages sent: %lld\n",messages_sent);
         erts_fprintf(file,"Number of messages copied: %lld\n",messages_copied);
         erts_fprintf(file,"Number of messages sent to self: %lld\n",messages_ego);
diff --git a/erts/emulator/beam/benchmark.h b/erts/emulator/beam/benchmark.h
index eedb06a1b6..003e821bce 100644
--- a/erts/emulator/beam/benchmark.h
+++ b/erts/emulator/beam/benchmark.h
@@ -99,17 +99,6 @@ extern unsigned long long messages_copied;
 extern unsigned long long messages_ego;
 extern unsigned long long minor_gc;
 extern unsigned long long major_gc;
-#ifdef HYBRID
-extern unsigned long long minor_global_gc;
-extern unsigned long long major_global_gc;
-extern unsigned long long gc_in_copy;
-#ifdef INCREMENTAL
-extern unsigned long long minor_gc_cycles;
-extern unsigned long long major_gc_cycles;
-extern unsigned long long minor_gc_stages;
-extern unsigned long long major_gc_stages;
-#endif
-#endif
 
 #define BM_COUNT(var) (var)++;
 
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 1b670585a7..fc00b42454 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -36,12 +36,16 @@
 #include "beam_bp.h"
 #include "erl_db_util.h"
 #include "register.h"
+#include "erl_thr_progress.h"
 
 static Export* flush_monitor_message_trap = NULL;
 static Export* set_cpu_topology_trap = NULL;
 static Export* await_proc_exit_trap = NULL;
 Export* erts_format_cpu_topology_trap = NULL;
 
+static Export *await_sched_wall_time_mod_trap;
+static erts_smp_atomic32_t sched_wall_time;
+
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 
 /*
@@ -368,7 +372,6 @@ static int demonitor(Process *c_p, Eterm ref)
    ErtsMonitor *mon = NULL;  /* The monitor entry to delete */
    Process  *rp;    /* Local target process */
    Eterm     to = NIL;    /* Monitor link traget */
-   Eterm     ref_p; /* Pid of this end */
    DistEntry *dep = NULL;  /* Target's distribution entry */
    int deref_de = 0;
    int res;
@@ -381,7 +384,6 @@ static int demonitor(Process *c_p, Eterm ref)
        res = ERTS_DEMONITOR_BADARG;
        goto done; /* Cannot be this monitor's ref */
    }
-   ref_p = c_p->id;
 
    mon = erts_lookup_monitor(c_p->monitors, ref);
    if (!mon) {
@@ -561,7 +563,11 @@ erts_queue_monitor_message(Process *p,
     ref_copy    = copy_struct(ref, ref_size, &hp, ohp);
 
     tup = TUPLE5(hp, am_DOWN, ref_copy, type, item_copy, reason_copy);
-    erts_queue_message(p, p_locksp, bp, tup, NIL);
+    erts_queue_message(p, p_locksp, bp, tup, NIL
+#ifdef USE_VM_PROBES
+		       , NIL
+#endif
+		       );
 }
 
 static BIF_RETTYPE
@@ -616,13 +622,15 @@ local_name_monitor(Process *p, Eterm target_name)
     rp = erts_whereis_process(p, p_locks, target_name, ERTS_PROC_LOCK_LINK,
 			      ERTS_P2P_FLG_ALLOW_OTHER_X);
     if (!rp) {
-	Eterm lhp[3];
+	DeclareTmpHeap(lhp,3,p);
 	Eterm item;
+	UseTmpHeap(3,p);
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK);
 	p_locks &= ~ERTS_PROC_LOCK_LINK;
 	item = TUPLE2(lhp, target_name, erts_this_dist_entry->sysname);
 	erts_queue_monitor_message(p, &p_locks,
 				   mon_ref, am_process, item, am_noproc);
+	UnUseTmpHeap(3,p);
     }
     else if (rp != p) {
 	erts_add_monitor(&(p->monitors), MON_ORIGIN, mon_ref, rp->id,
@@ -811,7 +819,7 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1)
     so.min_heap_size  = H_MIN_SIZE;
     so.min_vheap_size = BIN_VH_MIN_SIZE;
     so.priority       = PRIORITY_NORMAL;
-    so.max_gen_gcs    = (Uint16) erts_smp_atomic_read(&erts_max_gen_gcs);
+    so.max_gen_gcs    = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
     so.scheduler      = 0;
 
     /*
@@ -869,8 +877,6 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1)
 		}
 	    } else if (arg == am_scheduler && is_small(val)) {
 		Sint scheduler = signed_val(val);
-		if (erts_common_run_queue && erts_no_schedulers > 1)
-		    goto error;
 		if (scheduler < 0 || erts_no_schedulers < scheduler)
 		    goto error;
 		so.scheduler = (int) scheduler;
@@ -1089,17 +1095,27 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1)
 BIF_RETTYPE hibernate_3(BIF_ALIST_3)
 {
     /*
-     * hibernate/3 is implemented as an instruction; therefore
-     * this function will never be called.
+     * hibernate/3 is usually translated to an instruction; therefore
+     * this function is only called from HiPE or when the call could not
+     * be translated.
      */
-    BIF_ERROR(BIF_P, BADARG);
+    Eterm reg[3];
+
+    if (erts_hibernate(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, reg)) {
+        /*
+         * If hibernate succeeded, TRAP. The process will be suspended
+         * if status is P_WAITING or continue (if any message was in the queue).
+         */
+        BIF_TRAP_CODE_PTR_(BIF_P, BIF_P->i);
+    }
+    return THE_NON_VALUE;
 }
 
 /**********************************************************************/
 
-BIF_RETTYPE get_stacktrace_0(Process* p)
+BIF_RETTYPE get_stacktrace_0(BIF_ALIST_0)
 {
-    Eterm t = build_stacktrace(p, p->ftrace);
+    Eterm t = build_stacktrace(BIF_P, BIF_P->ftrace);
     BIF_RET(t);
 }
 
@@ -1109,10 +1125,10 @@ BIF_RETTYPE get_stacktrace_0(Process* p)
  * the process, and the final error value will be {Term,StackTrace}.
  */
 
-BIF_RETTYPE error_1(Process* p, Eterm term)
+BIF_RETTYPE error_1(BIF_ALIST_1)
 {
-    p->fvalue = term;
-    BIF_ERROR(p, EXC_ERROR);
+    BIF_P->fvalue = BIF_ARG_1;
+    BIF_ERROR(BIF_P, EXC_ERROR);
 }
 
 /**********************************************************************/
@@ -1121,12 +1137,40 @@ BIF_RETTYPE error_1(Process* p, Eterm term)
  * in the stacktrace.
  */
 
-BIF_RETTYPE error_2(Process* p, Eterm value, Eterm args)
+BIF_RETTYPE error_2(BIF_ALIST_2)
+{
+    Eterm* hp = HAlloc(BIF_P, 3);
+
+    BIF_P->fvalue = TUPLE2(hp, BIF_ARG_1, BIF_ARG_2);
+    BIF_ERROR(BIF_P, EXC_ERROR_2);
+}
+
+/**********************************************************************/
+/*
+ * This is like exactly like error/1. The only difference is
+ * that Dialyzer thinks that it it will return an arbitrary term.
+ * It is useful in stub functions for NIFs.
+ */
+
+BIF_RETTYPE nif_error_1(BIF_ALIST_1)
+{
+    BIF_P->fvalue = BIF_ARG_1;
+    BIF_ERROR(BIF_P, EXC_ERROR);
+}
+
+/**********************************************************************/
+/*
+ * This is like exactly like error/2. The only difference is
+ * that Dialyzer thinks that it it will return an arbitrary term.
+ * It is useful in stub functions for NIFs.
+ */
+
+BIF_RETTYPE nif_error_2(BIF_ALIST_2)
 {
-    Eterm* hp = HAlloc(p, 3);
+    Eterm* hp = HAlloc(BIF_P, 3);
 
-    p->fvalue = TUPLE2(hp, value, args);
-    BIF_ERROR(p, EXC_ERROR_2);
+    BIF_P->fvalue = TUPLE2(hp, BIF_ARG_1, BIF_ARG_2);
+    BIF_ERROR(BIF_P, EXC_ERROR_2);
 }
 
 /**********************************************************************/
@@ -1145,14 +1189,19 @@ BIF_RETTYPE exit_1(BIF_ALIST_1)
  * If there is an error in the argument format, 
  * return the atom 'badarg' instead.
  */
-Eterm 
-raise_3(Process *c_p, Eterm class, Eterm value, Eterm stacktrace) {
+BIF_RETTYPE raise_3(BIF_ALIST_3)
+{
+    Process *c_p = BIF_P;
+    Eterm class = BIF_ARG_1;
+    Eterm value = BIF_ARG_2;
+    Eterm stacktrace = BIF_ARG_3;
     Eterm reason;
     Eterm l, *hp, *hp_end, *tp;
     int depth, cnt;
     size_t sz;
+    int must_copy = 0;
     struct StackTrace *s;
-    
+
     if (class == am_error) {
 	c_p->fvalue = value;
 	reason = EXC_ERROR;
@@ -1168,35 +1217,74 @@ raise_3(Process *c_p, Eterm class, Eterm value, Eterm stacktrace) {
     /* Check syntax of stacktrace, and count depth.
      * Accept anything that can be returned from erlang:get_stacktrace/0,
      * as well as a 2-tuple with a fun as first element that the
-     * error_handler may need to give us.
+     * error_handler may need to give us. Also allow old-style
+     * MFA three-tuples.
      */
     for (l = stacktrace, depth = 0;  
 	 is_list(l);  
 	 l = CDR(list_val(l)), depth++) {
 	Eterm t = CAR(list_val(l));
-	int arity;
+	Eterm location = NIL;
+
 	if (is_not_tuple(t)) goto error;
 	tp = tuple_val(t);
-	arity = arityval(tp[0]);
-	if ((arity == 3) && is_atom(tp[1]) && is_atom(tp[2])) continue;
-	if ((arity == 2) && is_fun(tp[1])) continue;
-	goto error;
+	switch (arityval(tp[0])) {
+	case 2:
+	    /* {Fun,Args} */
+	    if (is_fun(tp[1])) {
+		must_copy = 1;
+	    } else {
+		goto error;
+	    }
+	    break;
+	case 3:
+	    /*
+	     * One of:
+	     * {Fun,Args,Location}
+	     * {M,F,A}
+	     */
+	    if (is_fun(tp[1])) {
+		location = tp[3];
+	    } else if (is_atom(tp[1]) && is_atom(tp[2])) {
+		must_copy = 1;
+	    } else {
+		goto error;
+	    }
+	    break;
+	case 4:
+	    if (!(is_atom(tp[1]) && is_atom(tp[2]))) {
+		goto error;
+	    }
+	    location = tp[4];
+	    break;
+	default:
+	    goto error;
+	}
+	if (is_not_list(location) && is_not_nil(location)) {
+	    goto error;
+	}
     }
     if (is_not_nil(l)) goto error;
     
     /* Create stacktrace and store */
-    if (depth <= erts_backtrace_depth) {
+    if (erts_backtrace_depth < depth) {
+	depth = erts_backtrace_depth;
+	must_copy = 1;
+    }
+    if (must_copy) {
+	cnt = depth;
+	c_p->ftrace = NIL;
+    } else {
+	/* No need to copy the stacktrace */
 	cnt = 0;
 	c_p->ftrace = stacktrace;
-    } else {
-	cnt = depth = erts_backtrace_depth;
-	c_p->ftrace = NIL;
     }
+
     tp = &c_p->ftrace;
     sz = (offsetof(struct StackTrace, trace) + sizeof(Eterm) - 1) 
 	/ sizeof(Eterm);
-    hp = HAlloc(c_p, sz + 2*(cnt + 1));
-    hp_end = hp + sz + 2*(cnt + 1);
+    hp = HAlloc(c_p, sz + (2+6)*(cnt + 1));
+    hp_end = hp + sz + (2+6)*(cnt + 1);
     s = (struct StackTrace *) hp;
     s->header = make_neg_bignum_header(sz - 1);
     s->freason = reason;
@@ -1204,13 +1292,29 @@ raise_3(Process *c_p, Eterm class, Eterm value, Eterm stacktrace) {
     s->current = NULL;
     s->depth = 0;
     hp += sz;
-    if (cnt > 0) {
+    if (must_copy) {
+	int cnt;
+
 	/* Copy list up to depth */
 	for (cnt = 0, l = stacktrace;
 	     cnt < depth;
 	     cnt++, l = CDR(list_val(l))) {
+	    Eterm t;
+	    Eterm *tpp;
+	    int arity;
+
 	    ASSERT(*tp == NIL);
-	    *tp = CONS(hp, CAR(list_val(l)), *tp);
+	    t = CAR(list_val(l));
+	    tpp = tuple_val(t);
+	    arity = arityval(tpp[0]);
+	    if (arity == 2) {
+		t = TUPLE3(hp, tpp[1], tpp[2], NIL);
+		hp += 4;
+	    } else if (arity == 3 && is_atom(tpp[1])) {
+		t = TUPLE4(hp, tpp[1], tpp[2], tpp[3], NIL);
+		hp += 5;
+	    }
+	    *tp = CONS(hp, t, *tp);
 	    tp = &CDR(list_val(*tp));
 	    hp += 2;
 	}
@@ -1218,7 +1322,7 @@ raise_3(Process *c_p, Eterm class, Eterm value, Eterm stacktrace) {
     c_p->ftrace = CONS(hp, c_p->ftrace, make_big((Eterm *) s));
     hp += 2;
     ASSERT(hp <= hp_end);
-    
+    HRelease(c_p, hp_end, hp);
     BIF_ERROR(c_p, reason);
     
  error:
@@ -1321,9 +1425,10 @@ BIF_RETTYPE exit_2(BIF_ALIST_2)
 #ifdef ERTS_SMP
 	 if (rp == BIF_P)
 	     rp_locks &= ~ERTS_PROC_LOCK_MAIN;
-	 else
+	 if (rp_locks)
+	     erts_smp_proc_unlock(rp, rp_locks);
+	 if (rp != BIF_P)
 	     erts_smp_proc_dec_refc(rp);
-	 erts_smp_proc_unlock(rp, rp_locks);
 #endif
 	 /*
 	  * We may have exited ourselves and may have to take action.
@@ -1435,8 +1540,6 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2)
        ErtsRunQueue *old;
        ErtsRunQueue *new;
        Sint sched;
-       if (erts_common_run_queue && erts_no_schedulers > 1)
-	   goto error;
        if (!is_small(BIF_ARG_2))
 	   goto error;
        sched = signed_val(BIF_ARG_2);
@@ -1635,10 +1738,10 @@ BIF_RETTYPE whereis_1(BIF_ALIST_1)
  * erlang:'!'/2
  */
 
-Eterm
-ebif_bang_2(Process* p, Eterm To, Eterm Message)
+BIF_RETTYPE
+ebif_bang_2(BIF_ALIST_2)
 {
-    return send_2(p, To, Message);
+    return erl_send(BIF_P, BIF_ARG_1, BIF_ARG_2);
 }
 
 
@@ -1845,7 +1948,11 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) {
 	if (ERTS_PROC_GET_SAVED_CALLS_BUF(p))
 	    save_calls(p, &exp_send);
 	
-	if (SEQ_TRACE_TOKEN(p) != NIL) {
+	if (SEQ_TRACE_TOKEN(p) != NIL
+#ifdef USE_VM_PROBES
+	    && SEQ_TRACE_TOKEN(p) != am_have_dt_utag
+#endif
+	    ) {
 	    seq_trace_update_send(p);
 	    seq_trace_output(SEQ_TRACE_TOKEN(p), msg, 
 			     SEQ_TRACE_SEND, portid, p);
@@ -1975,8 +2082,13 @@ do_send(Process *p, Eterm to, Eterm msg, int suspend) {
 }
 
 
-Eterm
-send_3(Process *p, Eterm to, Eterm msg, Eterm opts) {
+BIF_RETTYPE send_3(BIF_ALIST_3)
+{
+    Process *p = BIF_P;
+    Eterm to = BIF_ARG_1;
+    Eterm msg = BIF_ARG_2;
+    Eterm opts = BIF_ARG_3;
+
     int connect = !0;
     int suspend = !0;
     Eterm l = opts;
@@ -2040,8 +2152,13 @@ send_3(Process *p, Eterm to, Eterm msg, Eterm opts) {
     BIF_ERROR(p, BADARG);
 }
 
-Eterm
-send_2(Process *p, Eterm to, Eterm msg) {
+BIF_RETTYPE send_2(BIF_ALIST_2)
+{
+    return erl_send(BIF_P, BIF_ARG_1, BIF_ARG_2);
+}
+
+Eterm erl_send(Process *p, Eterm to, Eterm msg)
+{
     Sint result = do_send(p, to, msg, !0);
     
     if (result > 0) {
@@ -2129,20 +2246,146 @@ BIF_RETTYPE tl_1(BIF_ALIST_1)
 /**********************************************************************/
 /* return the size of an I/O list */
 
-BIF_RETTYPE iolist_size_1(BIF_ALIST_1)
+static Eterm
+accumulate(Eterm acc, Uint size)
 {
-    Sint size = io_list_len(BIF_ARG_1);
+    if (is_non_value(acc)) {
+	/*
+	 * There is no pre-existing accumulator. Allocate a
+	 * bignum buffer with one extra word to be used if
+	 * the bignum grows in the future.
+	 */
+	Eterm* hp = (Eterm *) erts_alloc(ERTS_ALC_T_TEMP_TERM,
+					 (BIG_UINT_HEAP_SIZE+1) *
+					 sizeof(Eterm));
+	return uint_to_big(size, hp);
+    } else {
+	Eterm* big;
+	int need_heap;
 
-    if (size == -1) {
-	BIF_ERROR(BIF_P, BADARG);
-    } else if (IS_USMALL(0, (Uint) size)) {
-	BIF_RET(make_small(size));
+	/*
+	 * Add 'size' to 'acc' in place. There is always one
+	 * extra word allocated in case the bignum grows by one word.
+	 */
+	big = big_val(acc);
+	need_heap = BIG_NEED_SIZE(BIG_SIZE(big));
+	acc = big_plus_small(acc, size, big);
+	if (BIG_NEED_SIZE(big_size(acc)) > need_heap) {
+	    /*
+	     * The extra word has been consumed. Grow the
+	     * allocation by one word.
+	     */
+	    big = (Eterm *) erts_realloc(ERTS_ALC_T_TEMP_TERM,
+					 big_val(acc),
+					 (need_heap+1) * sizeof(Eterm));
+	    acc = make_big(big);
+	}
+	return acc;
+    }
+}
+
+static Eterm
+consolidate(Process* p, Eterm acc, Uint size)
+{
+    Eterm* hp;
+
+    if (is_non_value(acc)) {
+	return erts_make_integer(size, p);
     } else {
-	Eterm* hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE);
-	BIF_RET(uint_to_big(size, hp));
+	Eterm* big;
+	Uint sz;
+	Eterm res;
+	
+	acc = accumulate(acc, size);
+	big = big_val(acc);
+	sz = BIG_NEED_SIZE(BIG_SIZE(big));
+	hp = HAlloc(p, sz);
+	res = make_big(hp);
+	while (sz--) {
+	    *hp++ = *big++;
+	}
+	erts_free(ERTS_ALC_T_TEMP_TERM, (void *) big_val(acc));
+	return res;
     }
 }
 
+BIF_RETTYPE iolist_size_1(BIF_ALIST_1)
+{
+    Eterm obj, hd;
+    Eterm* objp;
+    Uint size = 0;
+    Uint cur_size;
+    Uint new_size;
+    Eterm acc = THE_NON_VALUE;
+    DECLARE_ESTACK(s);
+
+    obj = BIF_ARG_1;
+    goto L_again;
+
+    while (!ESTACK_ISEMPTY(s)) {
+	obj = ESTACK_POP(s);
+    L_again:
+	if (is_list(obj)) {
+	L_iter_list:
+	    objp = list_val(obj);
+	    hd = CAR(objp);
+	    obj = CDR(objp);
+	    /* Head */
+	    if (is_byte(hd)) {
+		size++;
+		if (size == 0) {
+		    acc = accumulate(acc, (Uint) -1);
+		    size = 1;
+		}
+	    } else if (is_binary(hd) && binary_bitsize(hd) == 0) {
+		cur_size = binary_size(hd);
+		if ((new_size = size + cur_size) >= size) {
+		    size = new_size;
+		} else {
+		    acc = accumulate(acc, size);
+		    size = cur_size;
+		}
+	    } else if (is_list(hd)) {
+		ESTACK_PUSH(s, obj);
+		obj = hd;
+		goto L_iter_list;
+	    } else if (is_not_nil(hd)) {
+		goto L_type_error;
+	    }
+	    /* Tail */
+	    if (is_list(obj)) {
+		goto L_iter_list;
+	    } else if (is_binary(obj) && binary_bitsize(obj) == 0) {
+		cur_size = binary_size(obj);
+		if ((new_size = size + cur_size) >= size) {
+		    size = new_size;
+		} else {
+		    acc = accumulate(acc, size);
+		    size = cur_size;
+		}
+	    } else if (is_not_nil(obj)) {
+		goto L_type_error;
+	    }
+	} else if (is_binary(obj) && binary_bitsize(obj) == 0) {
+	    cur_size = binary_size(obj);
+	    if ((new_size = size + cur_size) >= size) {
+		size = new_size;
+	    } else {
+		acc = accumulate(acc, size);
+		size = cur_size;
+	    }
+	} else if (is_not_nil(obj)) {
+	    goto L_type_error;
+	}
+    }
+
+    DESTROY_ESTACK(s);
+    BIF_RET(consolidate(BIF_P, acc, size));
+
+ L_type_error:
+    DESTROY_ESTACK(s);
+    BIF_ERROR(BIF_P, BADARG);
+}
 
 /**********************************************************************/
 
@@ -3091,8 +3334,11 @@ time_to_parts(Eterm date, Sint* year, Sint* month, Sint* day,
 /* return the universal time */
 
 BIF_RETTYPE 
-localtime_to_universaltime_2(Process *p, Eterm localtime, Eterm dst)
+localtime_to_universaltime_2(BIF_ALIST_2)
 {
+    Process *p = BIF_P;
+    Eterm localtime = BIF_ARG_1;
+    Eterm dst = BIF_ARG_2;
     Sint year, month, day;
     Sint hour, minute, second;
     int isdst;
@@ -3150,6 +3396,61 @@ BIF_RETTYPE universaltime_to_localtime_1(BIF_ALIST_1)
     BIF_RET(TUPLE2(hp, res1, res2));
 }
 
+/* convert calendar:universaltime_to_seconds/1 */
+
+BIF_RETTYPE universaltime_to_posixtime_1(BIF_ALIST_1)
+{
+    Sint year, month, day;
+    Sint hour, minute, second;
+
+    Sint64 seconds = 0;
+    Eterm *hp;
+    Uint hsz = 0;
+
+    if (!time_to_parts(BIF_ARG_1, &year, &month, &day, 
+		       &hour, &minute, &second))
+	BIF_ERROR(BIF_P, BADARG);
+
+    if (!univ_to_seconds(year, month, day, hour, minute, second, &seconds)) {
+	BIF_ERROR(BIF_P, BADARG);
+    }
+
+    erts_bld_sint64(NULL, &hsz, seconds);
+    hp = HAlloc(BIF_P, hsz);
+    BIF_RET(erts_bld_sint64(&hp, NULL, seconds));
+}
+
+/* convert calendar:seconds_to_universaltime/1 */
+
+BIF_RETTYPE posixtime_to_universaltime_1(BIF_ALIST_1)
+{
+    Sint year, month, day;
+    Sint hour, minute, second;
+    Eterm res1, res2;
+    Eterm* hp;
+
+    Sint64 time = 0;
+
+    if (!term_to_Sint64(BIF_ARG_1, &time)) {
+	BIF_ERROR(BIF_P, BADARG);
+    }
+
+    if (!seconds_to_univ(time, &year, &month, &day,
+		&hour, &minute, &second)) {
+	BIF_ERROR(BIF_P, BADARG);
+    }
+
+    hp = HAlloc(BIF_P, 4+4+3);
+    res1 = TUPLE3(hp,make_small(year),make_small(month),
+		  make_small(day));
+    hp += 4;
+    res2 = TUPLE3(hp,make_small(hour),make_small(minute),
+		  make_small(second));
+    hp += 4;
+    BIF_RET(TUPLE2(hp, res1, res2));
+}
+
+
 /**********************************************************************/
 
 
@@ -3176,20 +3477,32 @@ BIF_RETTYPE garbage_collect_1(BIF_ALIST_1)
 	BIF_ERROR(BIF_P, BADARG);
     }
 
-    rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN,
+    if (BIF_P->id == BIF_ARG_1)
+	rp = BIF_P;
+    else {
+#ifdef ERTS_SMP
+	rp = erts_pid2proc_suspend(BIF_P, ERTS_PROC_LOCK_MAIN,
 				   BIF_ARG_1, ERTS_PROC_LOCK_MAIN);
-    if (!rp)
-	BIF_RET(am_false);
-    if (rp == ERTS_PROC_LOCK_BUSY)
-	ERTS_BIF_YIELD1(bif_export[BIF_garbage_collect_1], BIF_P, BIF_ARG_1);
+	if (rp == ERTS_PROC_LOCK_BUSY)
+	    ERTS_BIF_YIELD1(bif_export[BIF_garbage_collect_1], BIF_P, BIF_ARG_1);
+#else
+	rp = erts_pid2proc(BIF_P, 0, BIF_ARG_1, 0);
+#endif
+	if (!rp)
+	    BIF_RET(am_false);
+    }
 
     /* The GC cost is taken for the process executing this BIF. */
 
     FLAGS(rp) |= F_NEED_FULLSWEEP;
     reds = erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
 
-    if (BIF_P != rp)
+#ifdef ERTS_SMP
+    if (BIF_P != rp) {
+	erts_resume(rp, ERTS_PROC_LOCK_MAIN);
 	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
+    }
+#endif
 
     BIF_RET2(am_true, reds);
 }
@@ -3204,22 +3517,6 @@ BIF_RETTYPE garbage_collect_0(BIF_ALIST_0)
 }
 
 /**********************************************************************/
-/* Perform garbage collection of the message area */
-
-BIF_RETTYPE garbage_collect_message_area_0(BIF_ALIST_0)
-{
-#if defined(HYBRID) && !defined(INCREMENTAL)
-    int reds = 0;
-
-    FLAGS(BIF_P) |= F_NEED_FULLSWEEP;
-    reds = erts_global_garbage_collect(BIF_P, 0, NULL, 0);
-    BIF_RET2(am_true, reds);
-#else
-    BIF_RET(am_false);
-#endif
-}
-
-/**********************************************************************/
 /* Return a list of active ports */
 
 BIF_RETTYPE ports_0(BIF_ALIST_0)
@@ -3231,6 +3528,7 @@ BIF_RETTYPE ports_0(BIF_ALIST_0)
     Eterm* dead_ports;
     int alive, dead;
     Uint32 next_ss;
+    int i;
 
     /* To get a consistent snapshot... 
      * We add alive ports from start of the buffer
@@ -3239,27 +3537,25 @@ BIF_RETTYPE ports_0(BIF_ALIST_0)
 
     erts_smp_mtx_lock(&ports_snapshot_mtx); /* One snapshot at a time */
 
-    erts_smp_atomic_set(&erts_dead_ports_ptr, (long) (port_buf + erts_max_ports));
+    erts_smp_atomic_set_nob(&erts_dead_ports_ptr,
+			    (erts_aint_t) (port_buf + erts_max_ports));
 
-    next_ss = erts_smp_atomic_inctest(&erts_ports_snapshot);
+    next_ss = erts_smp_atomic32_inc_read_relb(&erts_ports_snapshot);
 
-    if (erts_smp_atomic_read(&erts_ports_alive) > 0) {
-	long i;
-	for (i = erts_max_ports-1; i >= 0; i--) {
-	    Port* prt = &erts_port[i];
-	    erts_smp_port_state_lock(prt);
-	    if (!(prt->status & ERTS_PORT_SFLGS_DEAD)
-		&& prt->snapshot != next_ss) {
-		ASSERT(prt->snapshot == next_ss - 1);
-		*pp++ = prt->id;		
-		prt->snapshot = next_ss; /* Consumed by this snapshot */
-	    }
-	    erts_smp_port_state_unlock(prt);
+    for (i = erts_max_ports-1; i >= 0; i--) {
+	Port* prt = &erts_port[i];
+	erts_smp_port_state_lock(prt);
+	if (!(prt->status & ERTS_PORT_SFLGS_DEAD)
+	    && prt->snapshot != next_ss) {
+	    ASSERT(prt->snapshot == next_ss - 1);
+	    *pp++ = prt->id;		
+	    prt->snapshot = next_ss; /* Consumed by this snapshot */
 	}
+	erts_smp_port_state_unlock(prt);
     }
 
-    dead_ports = (Eterm*)erts_smp_atomic_xchg(&erts_dead_ports_ptr,
-					      (long)NULL);
+    dead_ports = (Eterm*)erts_smp_atomic_xchg_nob(&erts_dead_ports_ptr,
+						  (erts_aint_t) NULL);
     erts_smp_mtx_unlock(&ports_snapshot_mtx);
 
     ASSERT(pp <= dead_ports);
@@ -3270,7 +3566,7 @@ BIF_RETTYPE ports_0(BIF_ALIST_0)
     ASSERT((alive+dead) <= erts_max_ports);
 
     if (alive+dead > 0) {
-	long i;
+	erts_aint_t i;
 	Eterm *hp = HAlloc(BIF_P, (alive+dead)*2);
 
 	for (i = 0; i < alive; i++) {
@@ -3320,7 +3616,7 @@ BIF_RETTYPE erts_debug_display_1(BIF_ALIST_1)
     erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(64);       
     pres = erts_dsprintf(dsbufp, "%.*T\n", INT_MAX, BIF_ARG_1);
     if (pres < 0)
-	erl_exit(1, "Failed to convert term to string: %d (s)\n",
+	erl_exit(1, "Failed to convert term to string: %d (%s)\n",
 		 -pres, erl_errno_id(-pres));
     hp = HAlloc(BIF_P, 2*dsbufp->str_len); /* we need length * 2 heap words */
     res = buf_to_intlist(&hp, dsbufp->str, dsbufp->str_len, NIL);
@@ -3330,9 +3626,10 @@ BIF_RETTYPE erts_debug_display_1(BIF_ALIST_1)
 }
 
 
-Eterm
-display_string_1(Process* p, Eterm string)
+BIF_RETTYPE display_string_1(BIF_ALIST_1)
 {
+    Process* p = BIF_P;
+    Eterm string = BIF_ARG_1;
     int len = is_string(string);
     char *str;
 
@@ -3348,8 +3645,7 @@ display_string_1(Process* p, Eterm string)
     BIF_RET(am_true);
 }
 
-Eterm
-display_nl_0(Process* p)
+BIF_RETTYPE display_nl_0(BIF_ALIST_0)
 {
     erts_fprintf(stderr, "\n");
     BIF_RET(am_true);
@@ -3361,43 +3657,122 @@ display_nl_0(Process* p)
 /* ARGSUSED */
 BIF_RETTYPE halt_0(BIF_ALIST_0)
 {
-    VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt/0\n"));
-    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-    erl_exit(0, "");
-    return NIL;  /* Pedantic (lint does not know about erl_exit) */
+    VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt()\n"));
+    erl_halt(0);
+    ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined);
 }
 
 /**********************************************************************/
 
-#define MSG_SIZE	200
+#define HALT_MSG_SIZE	200
+static char halt_msg[HALT_MSG_SIZE];
 
 /* stop the system with exit code */
 /* ARGSUSED */
 BIF_RETTYPE halt_1(BIF_ALIST_1)
 {
     Sint code;
-    static char msg[MSG_SIZE];
-    int i;
     
     if (is_small(BIF_ARG_1) && (code = signed_val(BIF_ARG_1)) >= 0) {
-	VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%d)\n", code));
+	VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1));
+	erl_halt((int)(- code));
+	ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined);
+    }
+    else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) {
+	VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1));
 	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erl_exit(-code, "");
-    } else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) {
-	if ((i = intlist_to_buf(BIF_ARG_1, msg, MSG_SIZE-1)) < 0) {
+	erl_exit(ERTS_ABORT_EXIT, "");
+    }
+    else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) {
+	int i;
+
+	if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE-1)) < 0) {
 	    goto error;
 	}
-	msg[i] = '\0';
-	VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%s)\n", msg));
+	halt_msg[i] = '\0';
+	VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1));
 	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erl_exit(ERTS_DUMP_EXIT, "%s\n", msg);
-    } else {
-    error:
+	erl_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg);
+    }
+    else
+	goto error;
+    return NIL;  /* Pedantic (lint does not know about erl_exit) */
+ error:
 	BIF_ERROR(BIF_P, BADARG);
+}
+
+/**********************************************************************/
+
+/* stop the system with exit code and flags */
+/* ARGSUSED */
+BIF_RETTYPE halt_2(BIF_ALIST_2)
+{
+    Sint code;
+    Eterm optlist = BIF_ARG_2;
+    int flush = 0;
+
+    for (optlist = BIF_ARG_2;
+	 is_list(optlist);
+	 optlist = CDR(list_val(optlist))) {
+	Eterm *tp, opt = CAR(list_val(optlist));
+	if (is_not_tuple(opt))
+	    goto error;
+	tp = tuple_val(opt);
+	if (tp[0] != make_arityval(2))
+	    goto error;
+	if (tp[1] == am_flush) {
+	    if (tp[2] == am_true)
+		flush = 1;
+	    else if (tp[2] == am_false)
+		flush = 0;
+	    else
+		goto error;
+	}
+	else
+	    goto error;
     }
+    if (is_not_nil(optlist))
+	goto error;
+
+    if (is_small(BIF_ARG_1) && (code = signed_val(BIF_ARG_1)) >= 0) {
+	VERBOSE(DEBUG_SYSTEM,
+		("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2));
+	if (flush) {
+	    erl_halt((int)(- code));
+	    ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined);
+	}
+	else {
+	    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	    erl_exit((int)(- code), "");
+	}
+    }
+    else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) {
+	VERBOSE(DEBUG_SYSTEM,
+		("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2));
+	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	erl_exit(ERTS_ABORT_EXIT, "");
+    }
+    else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) {
+	int i;
+
+	if ((i = intlist_to_buf(BIF_ARG_1, halt_msg, HALT_MSG_SIZE-1)) < 0) {
+	    goto error;
+	}
+	halt_msg[i] = '\0';
+	VERBOSE(DEBUG_SYSTEM,
+		("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2));
+	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	erl_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg);
+    }
+    else
+	goto error;
     return NIL;  /* Pedantic (lint does not know about erl_exit) */
+ error:
+    BIF_ERROR(BIF_P, BADARG);
 }
 
+/**********************************************************************/
+
 BIF_RETTYPE function_exported_3(BIF_ALIST_3)
 {
     if (is_not_atom(BIF_ARG_1) ||
@@ -3413,8 +3788,13 @@ BIF_RETTYPE function_exported_3(BIF_ALIST_3)
 
 /**********************************************************************/    
 
-BIF_RETTYPE is_builtin_3(Process* p, Eterm Mod, Eterm Name, Eterm Arity)
+BIF_RETTYPE is_builtin_3(BIF_ALIST_3)
 {
+    Process* p = BIF_P;
+    Eterm Mod = BIF_ARG_1;
+    Eterm Name = BIF_ARG_2;
+    Eterm Arity = BIF_ARG_3;
+
     if (is_not_atom(Mod) || is_not_atom(Name) || is_not_small(Arity)) {
 	BIF_ERROR(p, BADARG);
     }
@@ -3438,7 +3818,7 @@ term2list_dsprintf(Process *p, Eterm term)
     erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(64);       
     pres = erts_dsprintf(dsbufp, "%T", term);
     if (pres < 0)
-	erl_exit(1, "Failed to convert term to list: %d (s)\n",
+	erl_exit(1, "Failed to convert term to list: %d (%s)\n",
 		 -pres, erl_errno_id(-pres));
     hp = HAlloc(p, 2*dsbufp->str_len); /* we need length * 2 heap words */
     res = buf_to_intlist(&hp, dsbufp->str, dsbufp->str_len, NIL);
@@ -3466,15 +3846,24 @@ BIF_RETTYPE make_fun_3(BIF_ALIST_3)
     if (arity < 0) {
 	goto error;
     }
+#if HALFWORD_HEAP
+    hp = HAlloc(BIF_P, 3);
+    hp[0] = HEADER_EXPORT;
+    /* Yes, May be misaligned, but X86_64 will fix it... */
+    *((Export **) (hp+1)) = erts_export_get_or_make_stub(BIF_ARG_1, BIF_ARG_2, (Uint) arity);
+#else
     hp = HAlloc(BIF_P, 2);
     hp[0] = HEADER_EXPORT;
     hp[1] = (Eterm) erts_export_get_or_make_stub(BIF_ARG_1, BIF_ARG_2, (Uint) arity);
+#endif
     BIF_RET(make_export(hp));
 }
 
-Eterm
-fun_to_list_1(Process* p, Eterm fun)
+BIF_RETTYPE fun_to_list_1(BIF_ALIST_1)
 {
+    Process* p = BIF_P;
+    Eterm fun = BIF_ARG_1;
+
     if (is_not_any_fun(fun))
 	BIF_ERROR(p, BADARG);
     BIF_RET(term2list_dsprintf(p, fun));
@@ -3574,11 +3963,11 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1)
 
       etp = (ExternalThing *) HAlloc(BIF_P, EXTERNAL_THING_HEAD_SIZE + 1);
       etp->header = make_external_pid_header(1);
-      etp->next = MSO(BIF_P).externals;
+      etp->next = MSO(BIF_P).first;
       etp->node = enp;
       etp->data.ui[0] = make_pid_data(c, b);
 
-      MSO(BIF_P).externals = etp;
+      MSO(BIF_P).first = (struct erl_off_heap_header*) etp;
       erts_deref_dist_entry(dep);
       BIF_RET(make_external_pid(etp));
     }
@@ -3759,7 +4148,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	    goto error;
 	}
 	nval = (n > (Sint) ((Uint16) -1)) ? ((Uint16) -1) : ((Uint16) n);
-	oval = (Uint) erts_smp_atomic_xchg(&erts_max_gen_gcs, (long) nval);
+	oval = (Uint) erts_smp_atomic32_xchg_nob(&erts_max_gen_gcs,
+						 (erts_aint32_t) nval);
 	BIF_RET(make_small(oval));
     } else if (BIF_ARG_1 == am_min_heap_size) {
 	int oval = H_MIN_SIZE;
@@ -3769,11 +4159,11 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	}
 
 	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erts_smp_block_system(0);
+	erts_smp_thr_progress_block();
 
 	H_MIN_SIZE = erts_next_heap_size(n, 0);
 
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
 	BIF_RET(make_small(oval));
@@ -3785,11 +4175,11 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	}
 
 	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erts_smp_block_system(0);
+	erts_smp_thr_progress_block();
 
 	BIN_VH_MIN_SIZE = erts_next_heap_size(n, 0);
 
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
 	BIF_RET(make_small(oval));
@@ -3811,7 +4201,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	erts_backtrace_depth = n;
 	BIF_RET(make_small(oval));
     } else if (BIF_ARG_1 == am_trace_control_word) {
-	BIF_RET(db_set_trace_control_word_1(BIF_P, BIF_ARG_2));
+	BIF_RET(db_set_trace_control_word(BIF_P, BIF_ARG_2));
     } else if (BIF_ARG_1 == am_sequential_tracer) {
         Eterm old_value = erts_set_system_seq_tracer(BIF_P,
 						     ERTS_PROC_LOCK_MAIN,
@@ -3823,27 +4213,47 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	Uint i;
 	ErlMessage* mp;
 	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erts_smp_block_system(0);
+	erts_smp_thr_progress_block();
 
 	for (i = 0; i < erts_max_processes; i++) {
 	    if (process_tab[i] != (Process*) 0) {
 		Process* p = process_tab[i];
+#ifdef USE_VM_PROBES
+		p->seq_trace_token = (p->dt_utag != NIL) ? am_have_dt_utag : NIL;
+#else
 		p->seq_trace_token = NIL;
+#endif
 		p->seq_trace_clock = 0;
 		p->seq_trace_lastcnt = 0;
 		ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p);
 		mp = p->msg.first;
 		while(mp != NULL) {
+#ifdef USE_VM_PROBES
+		    ERL_MESSAGE_TOKEN(mp) = (ERL_MESSAGE_DT_UTAG(mp) != NIL) ? am_have_dt_utag : NIL;
+#else
 		    ERL_MESSAGE_TOKEN(mp) = NIL;
+#endif
 		    mp = mp->next;
 		}
 	    }
 	}
 
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
 	BIF_RET(am_true);
+    } else if (BIF_ARG_1 == am_scheduler_wall_time) {
+	if (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false) {
+	    erts_aint32_t new = BIF_ARG_2 == am_true ? 1 : 0;
+	    erts_aint32_t old = erts_smp_atomic32_xchg_nob(&sched_wall_time,
+							   new);
+	    Eterm ref = erts_sched_wall_time_request(BIF_P, 1, new);
+	    ASSERT(is_value(ref));
+	    BIF_TRAP2(await_sched_wall_time_mod_trap,
+		      BIF_P,
+		      ref,
+		      old ? am_true : am_false);
+	}
     } else if (ERTS_IS_ATOM_STR("scheduling_statistics", BIF_ARG_1)) {
 	int what;
 	if (ERTS_IS_ATOM_STR("disable", BIF_ARG_2))
@@ -3863,8 +4273,20 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	if (is_value(res))
 	    BIF_RET(res);
     } else if (ERTS_IS_ATOM_STR("cpu_topology", BIF_ARG_1)) {
+	erts_send_warning_to_logger_str(
+	    BIF_P->group_leader,
+	    "A call to erlang:system_flag(cpu_topology, _) was made.\n"
+	    "The cpu_topology argument is deprecated and scheduled\n"
+	    "for removal in erts-5.10/OTP-R16. For more information\n"
+	    "see the erlang:system_flag/2 documentation.\n");
 	BIF_TRAP1(set_cpu_topology_trap, BIF_P, BIF_ARG_2);
     } else if (ERTS_IS_ATOM_STR("scheduler_bind_type", BIF_ARG_1)) {
+	erts_send_warning_to_logger_str(
+	    BIF_P->group_leader,
+	    "A call to erlang:system_flag(scheduler_bind_type, _) was\n"
+	    "made. The scheduler_bind_type argument is deprecated and\n"
+	    "scheduled for removal in erts-5.10/OTP-R16. For more\n"
+	    "information see the erlang:system_flag/2 documentation.\n");
 	return erts_bind_schedulers(BIF_P, BIF_ARG_2);
     }
     error:
@@ -3884,7 +4306,7 @@ BIF_RETTYPE hash_2(BIF_ALIST_2)
     if ((range = signed_val(BIF_ARG_2)) <= 0) {  /* [1..MAX_SMALL] */
 	BIF_ERROR(BIF_P, BADARG);
     }
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
     if (range > ((1L << 27) - 1))
 	BIF_ERROR(BIF_P, BADARG);
 #endif
@@ -3956,7 +4378,7 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2)
     /*
      * Return either a small or a big. Use the heap for bigs if there is room.
      */
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
     BIF_RET(make_small(final_hash));
 #else
     if (IS_USMALL(0, final_hash)) {
@@ -4051,8 +4473,7 @@ void
 erts_bif_prep_await_proc_exit_data_trap(Process *c_p, Eterm pid, Eterm ret)
 {
     if (skip_current_msgq(c_p)) {
-	Eterm unused;
-	ERTS_BIF_PREP_TRAP3(unused, await_proc_exit_trap, c_p, pid, am_data, ret);
+	ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p, pid, am_data, ret);
     }
 }
 
@@ -4060,8 +4481,7 @@ void
 erts_bif_prep_await_proc_exit_reason_trap(Process *c_p, Eterm pid)
 {
     if (skip_current_msgq(c_p)) {
-	Eterm unused;
-	ERTS_BIF_PREP_TRAP3(unused, await_proc_exit_trap, c_p,
+	ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p,
 			    pid, am_reason, am_undefined);
     }
 }
@@ -4076,7 +4496,6 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p,
 {
     ASSERT(is_atom(module) && is_atom(function));
     if (skip_current_msgq(c_p)) {
-	Eterm unused;
 	Eterm term;
 	Eterm *hp;
 	int i;
@@ -4088,7 +4507,7 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p,
 	    hp += 2;
 	}
 	term = TUPLE3(hp, module, function, term);
-	ERTS_BIF_PREP_TRAP3(unused, await_proc_exit_trap, c_p, pid, am_apply, term);
+	ERTS_BIF_PREP_TRAP3_NO_RET(await_proc_exit_trap, c_p, pid, am_apply, term);
     }
 }
 
@@ -4102,7 +4521,7 @@ void erts_init_bif(void)
 
     erts_smp_spinlock_init(&make_ref_lock, "make_ref");
     erts_smp_mtx_init(&ports_snapshot_mtx, "ports_snapshot");
-    erts_smp_atomic_init(&erts_dead_ports_ptr, (long)NULL);
+    erts_smp_atomic_init_nob(&erts_dead_ports_ptr, (erts_aint_t) NULL);
 
     /*
      * bif_return_trap/1 is a hidden BIF that bifs that need to
@@ -4118,8 +4537,8 @@ void erts_init_bif(void)
 #else
     bif_return_trap_export.code[2] = 1;
 #endif
-    bif_return_trap_export.code[3] = (Eterm) em_apply_bif;
-    bif_return_trap_export.code[4] = (Eterm) &bif_return_trap;
+    bif_return_trap_export.code[3] = (BeamInstr) em_apply_bif;
+    bif_return_trap_export.code[4] = (BeamInstr) &bif_return_trap;
 
     flush_monitor_message_trap = erts_export_put(am_erlang,
 						 am_flush_monitor_message,
@@ -4132,56 +4551,11 @@ void erts_init_bif(void)
 						    am_format_cpu_topology,
 						    1);
     await_proc_exit_trap = erts_export_put(am_erlang,am_await_proc_exit,3);
+    await_sched_wall_time_mod_trap
+	= erts_export_put(am_erlang, am_await_sched_wall_time_modifications, 2);
+    erts_smp_atomic32_init_nob(&sched_wall_time, 0);
 }
 
-BIF_RETTYPE blocking_read_file_1(BIF_ALIST_1)
-{
-    Eterm bin;
-    Eterm* hp;
-    byte *buff;
-    int i, buff_size;
-    FILE *file;
-    struct stat file_info;
-    char *filename = NULL;
-    size_t size;
- 
-    i = list_length(BIF_ARG_1);
-    if (i < 0) {
-	BIF_ERROR(BIF_P, BADARG);
-    }
-    filename = erts_alloc(ERTS_ALC_T_TMP, i + 1);
-    if (intlist_to_buf(BIF_ARG_1, filename, i) != i)
-	erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
-    filename[i] = '\0';
- 
-    hp = HAlloc(BIF_P, 3);
- 
-    file = fopen(filename, "r");
-    if(file == NULL){
-	erts_free(ERTS_ALC_T_TMP, (void *) filename);
-	BIF_RET(TUPLE2(hp, am_error, am_nofile));
-    }
- 
-    stat(filename, &file_info);
-    erts_free(ERTS_ALC_T_TMP, (void *) filename);
-
-    buff_size = file_info.st_size;
-    buff = (byte *) erts_alloc_fnf(ERTS_ALC_T_TMP, buff_size);
-    if (!buff) {
-	fclose(file);
-	BIF_RET(TUPLE2(hp, am_error, am_allocator));
-    }
-    size = fread(buff, 1, buff_size, file);
-    fclose(file);
-    if (size < 0)
-	size = 0;
-    else if (size > buff_size)
-	size = (size_t) buff_size;
-    bin = new_binary(BIF_P, buff, (int) size);
-    erts_free(ERTS_ALC_T_TMP, (void *) buff);
- 
-    BIF_RET(TUPLE2(hp, am_ok, bin));
-}
 #ifdef HARDDEBUG
 /*
 You'll need this line in bif.tab to be able to use this debug bif
@@ -4256,3 +4630,193 @@ BIF_RETTYPE get_module_info_2(BIF_ALIST_2)
     }
     BIF_RET(ret);
 }
+
+BIF_RETTYPE dt_put_tag_1(BIF_ALIST_1)
+{
+#ifdef USE_VM_PROBES
+    Eterm otag;
+    if (BIF_ARG_1 == am_undefined) {
+	otag = (DT_UTAG(BIF_P) == NIL) ? am_undefined : DT_UTAG(BIF_P);
+	DT_UTAG(BIF_P) = NIL;
+	DT_UTAG_FLAGS(BIF_P) = 0;
+	if (SEQ_TRACE_TOKEN(BIF_P) == am_have_dt_utag) {
+	    SEQ_TRACE_TOKEN(BIF_P) = NIL;
+	}
+	BIF_RET(otag);
+    }
+    if (!is_binary(BIF_ARG_1)) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    otag = (DT_UTAG(BIF_P) == NIL) ? am_undefined : DT_UTAG(BIF_P);
+    DT_UTAG(BIF_P) = BIF_ARG_1;
+    DT_UTAG_FLAGS(BIF_P) |= DT_UTAG_PERMANENT;
+    if (SEQ_TRACE_TOKEN(BIF_P) == NIL) {
+	SEQ_TRACE_TOKEN(BIF_P) = am_have_dt_utag;
+    }
+    BIF_RET(otag);
+#else
+    BIF_RET(am_undefined);
+#endif
+}
+
+BIF_RETTYPE dt_get_tag_0(BIF_ALIST_0)
+{
+#ifdef USE_VM_PROBES
+    BIF_RET((DT_UTAG(BIF_P) == NIL || !(DT_UTAG_FLAGS(BIF_P) & DT_UTAG_PERMANENT)) ? am_undefined : DT_UTAG(BIF_P));
+#else
+    BIF_RET(am_undefined);
+#endif
+}
+BIF_RETTYPE dt_get_tag_data_0(BIF_ALIST_0)
+{
+#ifdef USE_VM_PROBES
+    BIF_RET((DT_UTAG(BIF_P) == NIL) ? am_undefined : DT_UTAG(BIF_P));
+#else
+    BIF_RET(am_undefined);
+#endif
+}
+BIF_RETTYPE dt_prepend_vm_tag_data_1(BIF_ALIST_1)
+{
+#ifdef USE_VM_PROBES
+    Eterm b; 
+    Eterm *hp;
+    hp = HAlloc(BIF_P,2);
+    if (is_binary((DT_UTAG(BIF_P)))) {
+	Uint sz = binary_size(DT_UTAG(BIF_P));
+	int i;
+	unsigned char *p,*q;
+	byte *temp_alloc = NULL;
+	b = new_binary(BIF_P,NULL,sz+1);
+	q = binary_bytes(b);
+	p = erts_get_aligned_binary_bytes(DT_UTAG(BIF_P),&temp_alloc);
+	for(i=0;i<sz;++i) {
+	    q[i] = p[i];
+	} 
+	erts_free_aligned_binary_bytes(temp_alloc);
+	q[sz] = '\0';
+    } else {
+	b = new_binary(BIF_P,(byte *)"\0",1);
+    }
+    BIF_RET(CONS(hp,b,BIF_ARG_1));
+#else
+    BIF_RET(BIF_ARG_1);
+#endif
+}
+BIF_RETTYPE dt_append_vm_tag_data_1(BIF_ALIST_1)
+{
+#ifdef USE_VM_PROBES
+    Eterm b; 
+    Eterm *hp;
+    hp = HAlloc(BIF_P,2);
+    if (is_binary((DT_UTAG(BIF_P)))) {
+	Uint sz = binary_size(DT_UTAG(BIF_P));
+	int i;
+	unsigned char *p,*q;
+	byte *temp_alloc = NULL;
+	b = new_binary(BIF_P,NULL,sz+1);
+	q = binary_bytes(b);
+	p = erts_get_aligned_binary_bytes(DT_UTAG(BIF_P),&temp_alloc);
+	for(i=0;i<sz;++i) {
+	    q[i] = p[i];
+	} 
+	erts_free_aligned_binary_bytes(temp_alloc);
+	q[sz] = '\0';
+    } else {
+	b = new_binary(BIF_P,(byte *)"\0",1);
+    }
+    BIF_RET(CONS(hp,BIF_ARG_1,b));
+#else
+    BIF_RET(BIF_ARG_1);
+#endif
+}
+BIF_RETTYPE dt_spread_tag_1(BIF_ALIST_1)
+{
+#ifdef USE_VM_PROBES
+    Eterm ret;
+    Eterm *hp;
+#endif
+    if (BIF_ARG_1 != am_true && BIF_ARG_1 != am_false) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+#ifdef USE_VM_PROBES
+    hp = HAlloc(BIF_P,3);
+    ret = TUPLE2(hp,make_small(DT_UTAG_FLAGS(BIF_P)),DT_UTAG(BIF_P));
+    if (DT_UTAG(BIF_P) != NIL) {
+	if (BIF_ARG_1 == am_true) {
+	    DT_UTAG_FLAGS(BIF_P) |= DT_UTAG_SPREADING;
+#ifdef DTRACE_TAG_HARDDEBUG
+	    erts_fprintf(stderr,
+			 "Dtrace -> (%T) start spreading tag %T\r\n",
+			 BIF_P->id,DT_UTAG(BIF_P));
+#endif
+	} else {
+	    DT_UTAG_FLAGS(BIF_P) &= ~DT_UTAG_SPREADING;
+#ifdef DTRACE_TAG_HARDDEBUG
+	    erts_fprintf(stderr,
+			 "Dtrace -> (%T) stop spreading tag %T\r\n",
+			 BIF_P->id,DT_UTAG(BIF_P));
+#endif
+	}
+    }
+    BIF_RET(ret);
+#else
+    BIF_RET(am_true);
+#endif
+}
+BIF_RETTYPE dt_restore_tag_1(BIF_ALIST_1)
+{
+#ifdef USE_VM_PROBES
+    Eterm *tpl;
+    Uint x;
+    if (is_not_tuple(BIF_ARG_1)) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    tpl = tuple_val(BIF_ARG_1);
+    if(arityval(*tpl) != 2 || is_not_small(tpl[1]) || (is_not_binary(tpl[2]) && tpl[2] != NIL)) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    if (tpl[2] == NIL) {
+	if (DT_UTAG(BIF_P) != NIL) {
+#ifdef DTRACE_TAG_HARDDEBUG
+	    erts_fprintf(stderr,
+			 "Dtrace -> (%T) restore Killing tag!\r\n",
+			 BIF_P->id);
+#endif
+	}
+	DT_UTAG(BIF_P) = NIL;
+	if (SEQ_TRACE_TOKEN(BIF_P) == am_have_dt_utag) {
+	    SEQ_TRACE_TOKEN(BIF_P) = NIL;
+	}
+	DT_UTAG_FLAGS(BIF_P) = 0;
+    } else {
+	x = unsigned_val(tpl[1]) & (DT_UTAG_SPREADING | DT_UTAG_PERMANENT);
+#ifdef DTRACE_TAG_HARDDEBUG
+
+	if (!(x & DT_UTAG_SPREADING) && (DT_UTAG_FLAGS(BIF_P) & 
+					 DT_UTAG_SPREADING)) {
+	    erts_fprintf(stderr,
+			 "Dtrace -> (%T) restore stop spreading "
+			 "tag %T\r\n",
+			 BIF_P->id, tpl[2]);
+	} else if ((x & DT_UTAG_SPREADING) && 
+		   !(DT_UTAG_FLAGS(BIF_P) & DT_UTAG_SPREADING)) {
+	    erts_fprintf(stderr,
+			 "Dtrace -> (%T) restore start spreading "
+			 "tag %T\r\n",BIF_P->id,tpl[2]);
+	}
+#endif
+	DT_UTAG_FLAGS(BIF_P) = x;
+	DT_UTAG(BIF_P) = tpl[2];
+	if (SEQ_TRACE_TOKEN(BIF_P) == NIL) {
+	    SEQ_TRACE_TOKEN(BIF_P) = am_have_dt_utag;
+	}
+    }
+#else    
+    if (BIF_ARG_1 != am_true) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+#endif
+    BIF_RET(am_true);
+}
+
+
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index 05e9b78c28..d20089a9fb 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -26,14 +26,14 @@ extern Export* erts_format_cpu_topology_trap;
 
 #define BIF_P A__p
 
-#define BIF_ALIST_0 Process* A__p
-#define BIF_ALIST_1 Process* A__p, Eterm A_1
-#define BIF_ALIST_2 Process* A__p, Eterm A_1, Eterm A_2
-#define BIF_ALIST_3 Process* A__p, Eterm A_1, Eterm A_2, Eterm A_3
+#define BIF_ALIST_0 Process* A__p, Eterm* BIF__ARGS
+#define BIF_ALIST_1 Process* A__p, Eterm* BIF__ARGS
+#define BIF_ALIST_2 Process* A__p, Eterm* BIF__ARGS
+#define BIF_ALIST_3 Process* A__p, Eterm* BIF__ARGS
 
-#define BIF_ARG_1  A_1
-#define BIF_ARG_2  A_2
-#define BIF_ARG_3  A_3
+#define BIF_ARG_1  (BIF__ARGS[0])
+#define BIF_ARG_2  (BIF__ARGS[1])
+#define BIF_ARG_3  (BIF__ARGS[2])
 
 #define BUMP_ALL_REDS(p) do {			\
     if (!ERTS_PROC_GET_SAVED_CALLS_BUF((p))) 	\
@@ -122,83 +122,106 @@ do {						\
 } while (0)
 
 
-#define ERTS_BIF_PREP_TRAP0(Ret, Trap, Proc)		\
-do {							\
-    (Proc)->arity = 0;					\
-    (Proc)->def_arg_reg[3] = (Eterm) (Trap->address);	\
-    (Proc)->freason = TRAP;				\
-    (Ret) = THE_NON_VALUE;				\
+#define ERTS_BIF_PREP_TRAP0(Ret, Trap, Proc)	\
+do {						\
+    (Proc)->arity = 0;				\
+    (Proc)->i = (BeamInstr*) ((Trap)->address);	\
+    (Proc)->freason = TRAP;			\
+    (Ret) = THE_NON_VALUE;			\
 } while (0)
 
-#define ERTS_BIF_PREP_TRAP1(Ret, Trap, Proc, A0)	\
-do {							\
-    (Proc)->arity = 1;					\
-    (Proc)->def_arg_reg[0] = (Eterm) (A0);		\
-    (Proc)->def_arg_reg[3] = (Eterm) ((Trap)->address);	\
-    (Proc)->freason = TRAP;				\
-    (Ret) = THE_NON_VALUE;				\
+#define ERTS_BIF_PREP_TRAP1(Ret, Trap, Proc, A0)		\
+do {								\
+    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
+    (Proc)->arity = 1;						\
+    reg[0] = (Eterm) (A0);					\
+    (Proc)->i = (BeamInstr*) ((Trap)->address);			\
+    (Proc)->freason = TRAP;					\
+    (Ret) = THE_NON_VALUE;					\
 } while (0)
 
-#define ERTS_BIF_PREP_TRAP2(Ret, Trap, Proc, A0, A1)	\
-do {							\
-    (Proc)->arity = 2;					\
-    (Proc)->def_arg_reg[0] = (Eterm) (A0);		\
-    (Proc)->def_arg_reg[1] = (Eterm) (A1);		\
-    (Proc)->def_arg_reg[3] = (Eterm) ((Trap)->address);	\
-    (Proc)->freason = TRAP;				\
-    (Ret) = THE_NON_VALUE;				\
+#define ERTS_BIF_PREP_TRAP2(Ret, Trap, Proc, A0, A1)		\
+do {								\
+    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
+    (Proc)->arity = 2;						\
+    reg[0] = (Eterm) (A0);					\
+    reg[1] = (Eterm) (A1);					\
+    (Proc)->i = (BeamInstr*) ((Trap)->address);			\
+    (Proc)->freason = TRAP;					\
+    (Ret) = THE_NON_VALUE;					\
 } while (0)
 
-#define ERTS_BIF_PREP_TRAP3(Ret, Trap, Proc, A0, A1, A2)\
+#define ERTS_BIF_PREP_TRAP3(Ret, Trap, Proc, A0, A1, A2)	\
+do {								\
+    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
+    (Proc)->arity = 3;						\
+    reg[0] = (Eterm) (A0);					\
+    reg[1] = (Eterm) (A1);					\
+    reg[2] = (Eterm) (A2);					\
+    (Proc)->i = (BeamInstr*) ((Trap)->address);			\
+    (Proc)->freason = TRAP;					\
+    (Ret) = THE_NON_VALUE;					\
+} while (0)
+
+#define ERTS_BIF_PREP_TRAP3_NO_RET(Trap, Proc, A0, A1, A2)\
 do {							\
+    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
     (Proc)->arity = 3;					\
-    (Proc)->def_arg_reg[0] = (Eterm) (A0);		\
-    (Proc)->def_arg_reg[1] = (Eterm) (A1);		\
-    (Proc)->def_arg_reg[2] = (Eterm) (A2);		\
-    (Proc)->def_arg_reg[3] = (Eterm) ((Trap)->address);	\
+    reg[0] = (Eterm) (A0);		\
+    reg[1] = (Eterm) (A1);		\
+    reg[2] = (Eterm) (A2);		\
+    (Proc)->i = (BeamInstr*) ((Trap)->address);			\
     (Proc)->freason = TRAP;				\
-    (Ret) = THE_NON_VALUE;				\
 } while (0)
 
-#define BIF_TRAP0(p, Trap_) do {			\
-      (p)->arity = 0;					\
-      (p)->def_arg_reg[3] = (Eterm) ((Trap_)->address);	\
-      (p)->freason = TRAP;				\
-      return THE_NON_VALUE;				\
+#define BIF_TRAP0(p, Trap_) do {		\
+      (p)->arity = 0;				\
+      (p)->i = (BeamInstr*) ((Trap_)->address);	\
+      (p)->freason = TRAP;			\
+      return THE_NON_VALUE;			\
+ } while(0)
+
+#define BIF_TRAP1(Trap_, p, A0) do {				\
+      Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array;	\
+      (p)->arity = 1;						\
+      reg[0] = (A0);						\
+      (p)->i = (BeamInstr*) ((Trap_)->address);			\
+      (p)->freason = TRAP;					\
+      return THE_NON_VALUE;					\
  } while(0)
 
-#define BIF_TRAP1(Trap_, p, A0) do {			\
-      (p)->arity = 1;					\
-      (p)->def_arg_reg[0] = (A0);			\
-      (p)->def_arg_reg[3] = (Eterm) ((Trap_)->address);	\
-      (p)->freason = TRAP;				\
-      return THE_NON_VALUE;				\
+#define BIF_TRAP2(Trap_, p, A0, A1) do {			\
+      Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array;	\
+      (p)->arity = 2;						\
+      reg[0] = (A0);						\
+      reg[1] = (A1);						\
+      (p)->i = (BeamInstr*) ((Trap_)->address);			\
+      (p)->freason = TRAP;					\
+      return THE_NON_VALUE;					\
  } while(0)
 
-#define BIF_TRAP2(Trap_, p, A0, A1) do {		\
-      (p)->arity = 2;					\
-      (p)->def_arg_reg[0] = (A0);			\
-      (p)->def_arg_reg[1] = (A1);			\
-      (p)->def_arg_reg[3] = (Eterm) ((Trap_)->address);	\
-      (p)->freason = TRAP;				\
-      return THE_NON_VALUE;				\
+#define BIF_TRAP3(Trap_, p, A0, A1, A2) do {			\
+      Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array;	\
+      (p)->arity = 3;						\
+      reg[0] = (A0);						\
+      reg[1] = (A1);						\
+      reg[2] = (A2);						\
+      (p)->i = (BeamInstr*) ((Trap_)->address);			\
+      (p)->freason = TRAP;					\
+      return THE_NON_VALUE;					\
  } while(0)
 
-#define BIF_TRAP3(Trap_, p, A0, A1, A2) do {		\
-      (p)->arity = 3;					\
-      (p)->def_arg_reg[0] = (A0);			\
-      (p)->def_arg_reg[1] = (A1);			\
-      (p)->def_arg_reg[2] = (A2);			\
-      (p)->def_arg_reg[3] = (Eterm) ((Trap_)->address);	\
-      (p)->freason = TRAP;				\
-      return THE_NON_VALUE;				\
+#define BIF_TRAP_CODE_PTR_0(p, Code_) do {	\
+      (p)->arity = 0;				\
+      (p)->i = (BeamInstr*) (Code_);		\
+      (p)->freason = TRAP;			\
+      return THE_NON_VALUE;			\
  } while(0)
 
-#define BIF_TRAP_CODE_PTR_0(p, Code_) do {		\
-      (p)->arity = 0;					\
-      (p)->def_arg_reg[3] = (Eterm) (Code_);		\
-      (p)->freason = TRAP;				\
-      return THE_NON_VALUE;				\
+#define BIF_TRAP_CODE_PTR_(p, Code_) do {	\
+      (p)-> i = (BeamInstr*) (Code_);		\
+      (p)->freason = TRAP;			\
+      return THE_NON_VALUE;			\
  } while(0)
 
 extern Export bif_return_trap_export;
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index b6fa06354a..797bce43ab 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1996-2010. All Rights Reserved.
+# Copyright Ericsson AB 1996-2012. All Rights Reserved.
 #
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -87,6 +87,8 @@ bif erlang:exit/2
 bif 'erl.lang.proc':signal/2		ebif_signal_2 exit_2
 bif erlang:external_size/1
 bif 'erl.lang.term':external_size/1	ebif_external_size_1
+bif erlang:external_size/2
+bif 'erl.lang.term':external_size/2	ebif_external_size_2
 ubif erlang:float/1
 ubif 'erl.lang.number':to_float/1	ebif_to_float_1 float_1
 bif erlang:float_to_list/1
@@ -97,8 +99,6 @@ bif erlang:garbage_collect/0
 bif 'erl.system':garbage_collect/0	ebif_garbage_collect_0
 bif erlang:garbage_collect/1
 bif 'erl.system':garbage_collect/1	ebif_garbage_collect_1
-bif erlang:garbage_collect_message_area/0
-bif 'erl.system':garbage_collect_message_area/0	ebif_garbage_collect_message_area_0
 bif erlang:get/0
 bif 'erl.lang.proc.pdict':get/0		ebif_get_0
 bif erlang:get/1
@@ -113,6 +113,8 @@ bif erlang:halt/0
 bif 'erl.lang.system':halt/0		ebif_halt_0
 bif erlang:halt/1
 bif 'erl.lang.system':halt/1		ebif_halt_1
+bif erlang:halt/2
+bif 'erl.lang.system':halt/2		ebif_halt_2
 bif erlang:phash/2
 bif erlang:phash2/1
 bif erlang:phash2/2
@@ -158,10 +160,6 @@ bif erlang:md5_update/2
 bif 'erl.util.crypt.md5':update/2	ebif_md5_update_2
 bif erlang:md5_final/1
 bif 'erl.util.crypt.md5':final/1	ebif_md5_final_1
-bif erlang:memory/0
-bif 'erl.lang':memory/0			ebif_memory_0
-bif erlang:memory/1
-bif 'erl.lang':memory/1			ebif_memory_1
 bif erlang:module_loaded/1
 bif 'erl.system.code':is_loaded/1	ebif_is_loaded_1 module_loaded_1
 bif erlang:function_exported/3
@@ -660,6 +658,7 @@ bif erts_debug:display/1
 bif 'erl.system.debug':display/1 ebif_erts_debug_display_1
 bif erts_debug:dist_ext_to_term/2
 bif 'erl.system.debug':dist_ext_to_term/2 ebif_erts_debug_dist_ext_to_term_2
+bif erts_debug:instructions/0
 
 #
 # Monitor testing bif's...
@@ -686,8 +685,6 @@ bif 'erl.system.code':make_stub_module/3	ebif_code_make_stub_module_3
 bif code:is_module_native/1
 bif 'erl.system.code':is_native/1	ebif_code_is_native_1 code_is_module_native_1
 
-bif erlang:blocking_read_file/1
-
 #
 # New Bifs in R9C.
 #
@@ -760,6 +757,79 @@ bif erlang:finish_after_on_load/2
 bif erlang:binary_to_term/2
 
 #
+# The binary match bifs (New in R14A - EEP9)
+#
+
+#
+# The searching/splitting/substituting thingies
+#
+ubif erlang:binary_part/2
+ubif erlang:binary_part/3
+
+bif binary:compile_pattern/1
+bif binary:match/2
+bif binary:match/3
+bif binary:matches/2
+bif binary:matches/3
+bif binary:longest_common_prefix/1
+bif binary:longest_common_suffix/1
+bif binary:first/1
+bif binary:last/1
+bif binary:at/2
+bif binary:part/2 binary_binary_part_2
+bif binary:part/3 binary_binary_part_3
+bif binary:bin_to_list/1
+bif binary:bin_to_list/2
+bif binary:bin_to_list/3
+bif binary:list_to_bin/1
+bif binary:copy/1
+bif binary:copy/2
+bif binary:referenced_byte_size/1
+bif binary:encode_unsigned/1
+bif binary:encode_unsigned/2
+bif binary:decode_unsigned/1
+bif binary:decode_unsigned/2
+
+bif erlang:nif_error/1
+bif erlang:nif_error/2
+
+#
+# Helpers for unicode filenames
+#
+bif prim_file:internal_name2native/1
+bif prim_file:internal_native2name/1
+bif prim_file:internal_normalize_utf8/1
+bif file:native_name_encoding/0
+
+#
+# New in R14B04.
+#
+bif erlang:check_old_code/1
+
+
+#
+# New in R15B
+#
+bif erlang:universaltime_to_posixtime/1
+bif erlang:posixtime_to_universaltime/1
+
+#
+# New in R15B01
+#
+
+# The dtrace BIF's are always present, but give dummy results if dynamic trace is not enabled in the build
+bif erlang:dt_put_tag/1
+bif erlang:dt_get_tag/0
+bif erlang:dt_get_tag_data/0
+bif erlang:dt_spread_tag/1
+bif erlang:dt_restore_tag/1
+
+# These are dummies even with enabled dynamic trace unless vm probes are enabled. 
+# They are also internal, for dtrace tags sent to the VM's own drivers (efile)
+bif erlang:dt_prepend_vm_tag_data/1
+bif erlang:dt_append_vm_tag_data/1
+
+#
 # Obsolete
 #
 
diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 03c88da8c6..5a5b162b9c 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -310,12 +310,12 @@
 #define DREM(a1,a0,b,r) do {				\
 	ErtsDigit __a1 = (a1);				\
 	ErtsDigit __b = (b);				\
-	ErtsDigit __q0;					\
+	ERTS_DECLARE_DUMMY(ErtsDigit __q0);		\
 	DDIVREM((__a1 % __b), (a0), __b, __q0, r);	\
     } while(0)
 
 #define DDIV(a1,a0,b,q)	do {			\
-	ErtsDigit _tmp;				\
+	ERTS_DECLARE_DUMMY(ErtsDigit _tmp);	\
 	DDIVREM(a1,a0,b,q,_tmp);		\
     } while(0)
 
@@ -413,8 +413,8 @@
     } while(0)
 
 #define DDIV2(a1,a0,b1,b0,q) do {		\
-	ErtsDigit _tmp_r1;			\
-	ErtsDigit _tmp_r0;			\
+	ERTS_DECLARE_DUMMY(ErtsDigit _tmp_r1);	\
+	ERTS_DECLARE_DUMMY(ErtsDigit _tmp_r0);	 \
 	D2DIVREM(a1,a0,b1,b0,q,_tmp_r1,_tmp_r0); \
     } while(0)
 
@@ -810,7 +810,9 @@ static dsize_t D_div(ErtsDigit* x, dsize_t xl, ErtsDigit d, ErtsDigit* q, ErtsDi
     }
 
     do {
-	ErtsDigit q0, a0, b1, b0, b;
+	ErtsDigit q0, a0, b0;
+	ERTS_DECLARE_DUMMY(ErtsDigit b);
+	ERTS_DECLARE_DUMMY(ErtsDigit b1);
 
 	if (d > a1) {
 	    a0 = *xp; 
@@ -1323,7 +1325,7 @@ static dsize_t I_lshift(ErtsDigit* x, dsize_t xl, Sint y,
 	return 1;
     }
     else {
-	long ay = (y < 0) ? -y : y;
+	SWord ay = (y < 0) ? -y : y;
 	int bw = ay / D_EXP;
 	int sw = ay % D_EXP;
 	dsize_t rl;
@@ -1448,6 +1450,20 @@ erts_make_integer(Uint x, Process *p)
 	return uint_to_big(x,hp);
     }
 }
+/*
+ * As erts_make_integer, but from a whole UWord.
+ */
+Eterm
+erts_make_integer_from_uword(UWord x, Process *p)
+{
+    Eterm* hp;
+    if (IS_USMALL(0,x))
+	return make_small(x);
+    else {
+	hp = HAlloc(p, BIG_UWORD_HEAP_SIZE(x));
+	return uword_to_big(x,hp);
+    }
+}
 
 /*
 ** convert Uint to bigint
@@ -1459,7 +1475,31 @@ Eterm uint_to_big(Uint x, Eterm *y)
     BIG_DIGIT(y, 0) = x;
     return make_big(y);
 }
+/*
+** convert UWord to bigint
+** (must only be used if x is to big to be stored as a small)
+** Allocation is tricky, the heap need has to be calculated
+** with the macro BIG_UWORD_HEAP_SIZE(x)
+*/
 
+Eterm uword_to_big(UWord x, Eterm *y)
+{
+#if HALFWORD_HEAP
+    Uint upper = x >> 32;
+    Uint lower = x & 0xFFFFFFFFUL;
+    if (upper == 0) {
+	*y = make_pos_bignum_header(1);
+    } else {
+	*y = make_pos_bignum_header(2);
+	BIG_DIGIT(y, 1) = upper;
+    }
+    BIG_DIGIT(y, 0) = lower;
+#else
+    *y = make_pos_bignum_header(1);
+    BIG_DIGIT(y, 0) = x;
+#endif
+    return make_big(y);
+}
 
 /*
 ** convert signed int to bigint
@@ -1480,19 +1520,19 @@ Eterm small_to_big(Sint x, Eterm *y)
 Eterm erts_uint64_to_big(Uint64 x, Eterm **hpp)
 {
     Eterm *hp = *hpp;
-#ifdef ARCH_32
+#if defined(ARCH_32) || HALFWORD_HEAP
     if (x >= (((Uint64) 1) << 32)) {
 	*hp = make_pos_bignum_header(2);
 	BIG_DIGIT(hp, 0) = (Uint) (x & ((Uint) 0xffffffff));
 	BIG_DIGIT(hp, 1) = (Uint) ((x >> 32) & ((Uint) 0xffffffff));
-	*hpp += 2;
+	*hpp += 3;
     }
     else
 #endif
     {
 	*hp = make_pos_bignum_header(1);
 	BIG_DIGIT(hp, 0) = (Uint) x;
-	*hpp += 1;
+	*hpp += 2;
     }
     return make_big(hp);
 }
@@ -1507,7 +1547,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp)
 	neg = 1;
 	x = -x;
     }
-#ifdef ARCH_32
+#if defined(ARCH_32) || HALFWORD_HEAP
     if (x >= (((Uint64) 1) << 32)) {
 	if (neg)
 	    *hp = make_neg_bignum_header(2);
@@ -1515,7 +1555,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp)
 	    *hp = make_pos_bignum_header(2);
 	BIG_DIGIT(hp, 0) = (Uint) (x & ((Uint) 0xffffffff));
 	BIG_DIGIT(hp, 1) = (Uint) ((x >> 32) & ((Uint) 0xffffffff));
-	*hpp += 2;
+	*hpp += 3;
     }
     else
 #endif
@@ -1525,7 +1565,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp)
 	else
 	    *hp = make_pos_bignum_header(1);
 	BIG_DIGIT(hp, 0) = (Uint) x;
-	*hpp += 1;
+	*hpp += 2;
     }
     return make_big(hp);
 }
@@ -1534,7 +1574,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp)
 ** Convert a bignum to a double float
 */
 int
-big_to_double(Eterm x, double* resp)
+big_to_double(Wterm x, double* resp)
 {
     double d = 0.0;
     Eterm* xp = big_val(x);
@@ -1560,11 +1600,67 @@ big_to_double(Eterm x, double* resp)
     return 0;
 }
 
+/*
+ * Logic has been copied from erl_bif_guard.c and slightly
+ * modified to use a static instead of dynamic heap
+ */
+Eterm
+double_to_big(double x, Eterm *heap)
+{
+    int is_negative;
+    int ds;
+    ErtsDigit* xp;
+    Eterm res;
+    int i;
+    size_t sz;
+    Eterm* hp;
+    double dbase;
+
+    if (x >= 0) {
+	is_negative = 0;
+    } else {
+	is_negative = 1;
+	x = -x;
+    }
+
+    /* Unscale & (calculate exponent) */
+    ds = 0;
+    dbase = ((double) (D_MASK) + 1);
+    while (x >= 1.0) {
+	x /= dbase; /* "shift" right */
+	ds++;
+    }
+    sz = BIG_NEED_SIZE(ds); /* number of words including arity */
+
+    hp = heap;
+    res = make_big(hp);
+    xp = (ErtsDigit*) (hp + 1);
+
+    for (i = ds - 1; i >= 0; i--) {
+	ErtsDigit d;
+
+	x *= dbase; /* "shift" left */
+	d = x; /* trunc */
+	xp[i] = d; /* store digit */
+	x -= d; /* remove integer part */
+    }
+    while ((ds & (BIG_DIGITS_PER_WORD - 1)) != 0) {
+	xp[ds++] = 0;
+    }
+
+    if (is_negative) {
+	*hp = make_neg_bignum_header(sz-1);
+    } else {
+	*hp = make_pos_bignum_header(sz-1);
+    }
+    return res;
+}
+
 
 /*
  ** Estimate the number of decimal digits (include sign)
  */
-int big_decimal_estimate(Eterm x)
+int big_decimal_estimate(Wterm x)
 {
     Eterm* xp = big_val(x);
     int lg = I_lg(BIG_V(xp), BIG_SIZE(xp));
@@ -1578,7 +1674,7 @@ int big_decimal_estimate(Eterm x)
 ** Convert a bignum into a string of decimal numbers
 */
 
-static void write_big(Eterm x, void (*write_func)(void *, char), void *arg)
+static void write_big(Wterm x, void (*write_func)(void *, char), void *arg)
 {
     Eterm* xp = big_val(x);
     ErtsDigit* dx = BIG_V(xp);
@@ -1657,7 +1753,7 @@ write_string(void *arg, char c)
     *(--(*((char **) arg))) = c;
 }
 
-char *erts_big_to_string(Eterm x, char *buf, Uint buf_sz)
+char *erts_big_to_string(Wterm x, char *buf, Uint buf_sz)
 {
     char *big_str = buf + buf_sz - 1;
     *big_str = '\0';
@@ -1701,7 +1797,7 @@ static Eterm big_norm(Eterm *x, dsize_t xl, short sign)
 /*
 ** Compare bignums
 */
-int big_comp(Eterm x, Eterm y)
+int big_comp(Wterm x, Wterm y)
 {
     Eterm* xp = big_val(x);
     Eterm* yp = big_val(y);
@@ -1748,6 +1844,7 @@ dsize_t big_bytes(Eterm x)
 /*
 ** Load a bignum from bytes
 ** xsz is the number of bytes in xp
+** *r is untouched if number fits in small
 */
 Eterm bytes_to_big(byte *xp, dsize_t xsz, int xsgn, Eterm *r)
 {
@@ -1756,7 +1853,7 @@ Eterm bytes_to_big(byte *xp, dsize_t xsz, int xsgn, Eterm *r)
     ErtsDigit d;
     int i;
 
-    while(xsz >= sizeof(ErtsDigit)) {
+    while(xsz > sizeof(ErtsDigit)) {
 	d = 0;
 	for(i = sizeof(ErtsDigit); --i >= 0;)
 	    d = (d << 8) | xp[i];
@@ -1771,11 +1868,20 @@ Eterm bytes_to_big(byte *xp, dsize_t xsz, int xsgn, Eterm *r)
 	d = 0;
 	for(i = xsz; --i >= 0;)
 	    d = (d << 8) | xp[i];
+	if (++rsz == 1 && IS_USMALL(xsgn,d)) {
+	    if (xsgn) d = -d;
+	    return make_small(d);
+	}
 	*rwp = d;
 	rwp++;
-	rsz++;
     }
-    return big_norm(r, rsz, (short) xsgn);
+    if (xsgn) {
+      *r = make_neg_bignum_header(rsz);
+    }
+    else {
+      *r = make_pos_bignum_header(rsz);
+    }
+    return make_big(r);
 }
 
 /*
@@ -1854,6 +1960,87 @@ term_to_Uint(Eterm term, Uint *up)
     }
 }
 
+int
+term_to_UWord(Eterm term, UWord *up)
+{
+#if SIZEOF_VOID_P == ERTS_SIZEOF_ETERM
+    return term_to_Uint(term,up);
+#else
+    if (is_small(term)) {
+	Sint i = signed_val(term);
+	if (i < 0) {
+	    *up = BADARG;
+	    return 0;
+	}
+	*up = (UWord) i;
+	return 1;
+    } else if (is_big(term)) {
+	ErtsDigit* xr = big_v(term);
+	dsize_t  xl = big_size(term);
+	UWord uval = 0;
+	int n = 0;
+
+	if (big_sign(term)) {
+	    *up = BADARG;
+	    return 0;
+	} else if (xl*D_EXP > sizeof(UWord)*8) {
+	    *up = SYSTEM_LIMIT;
+	    return 0;
+	}
+	while (xl-- > 0) {
+	    uval |= ((UWord)(*xr++)) << n;
+	    n += D_EXP;
+	}
+	*up = uval;
+	return 1;
+    } else {
+	*up = BADARG;
+	return 0;
+    }
+#endif
+}
+
+int
+term_to_Uint64(Eterm term, Uint64 *up)
+{
+#if SIZEOF_VOID_P == 8
+    return term_to_UWord(term,up);
+#else
+    if (is_small(term)) {
+	Sint i = signed_val(term);
+	if (i < 0) {
+	    *up = BADARG;
+	    return 0;
+	}
+	*up = (Uint64) i;
+	return 1;
+    } else if (is_big(term)) {
+	ErtsDigit* xr = big_v(term);
+	dsize_t  xl = big_size(term);
+	Uint64 uval = 0;
+	int n = 0;
+
+	if (big_sign(term)) {
+	    *up = BADARG;
+	    return 0;
+	} else if (xl*D_EXP > sizeof(Uint64)*8) {
+	    *up = SYSTEM_LIMIT;
+	    return 0;
+	}
+	while (xl-- > 0) {
+	    uval |= ((Uint64)(*xr++)) << n;
+	    n += D_EXP;
+	}
+	*up = uval;
+	return 1;
+    } else {
+	*up = BADARG;
+	return 0;
+    }
+#endif
+}
+
+
 int term_to_Sint(Eterm term, Sint *sp)
 {
     if (is_small(term)) {
@@ -1888,6 +2075,47 @@ int term_to_Sint(Eterm term, Sint *sp)
     }
 }
 
+#if HAVE_INT64
+int term_to_Sint64(Eterm term, Sint64 *sp)
+{
+#if ERTS_SIZEOF_ETERM == 8
+    return term_to_Sint(term, sp);
+#else
+    if (is_small(term)) {
+	*sp = signed_val(term);
+	return 1;
+    } else if (is_big(term)) {
+	ErtsDigit* xr = big_v(term);
+	dsize_t xl = big_size(term);
+	int sign = big_sign(term);
+	Uint64 uval = 0;
+	int n = 0;
+
+	if (xl*D_EXP > sizeof(Uint64)*8) {
+	    return 0;
+	}
+	while (xl-- > 0) {
+	    uval |= ((Uint64)(*xr++)) << n;
+	    n += D_EXP;
+	}
+	if (sign) {
+	    uval = -uval;
+	    if ((Sint64)uval > 0)
+		return 0;
+	} else {
+	    if ((Sint64)uval < 0)
+		return 0;
+	}
+	*sp = uval;
+	return 1;
+    } else {
+	return 0;
+    }
+#endif
+}
+#endif /* HAVE_INT64 */
+
+
 /*
 ** Add and subtract
 */
@@ -1914,7 +2142,7 @@ static Eterm B_plus_minus(ErtsDigit *x, dsize_t xl, short xsgn,
 /*
 ** Add bignums
 */
-Eterm big_plus(Eterm x, Eterm y, Eterm *r)
+Eterm big_plus(Wterm x, Wterm y, Eterm *r)
 {
     Eterm* xp = big_val(x);
     Eterm* yp = big_val(y);
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index b8e38d482c..7eb1e5afe2 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -34,7 +34,7 @@
 
 typedef Uint     ErtsDigit;
 
-#if (SIZEOF_VOID_P == 4) && defined(SIZEOF_LONG_LONG) && (SIZEOF_LONG_LONG == 8)
+#if ((SIZEOF_VOID_P == 4) || HALFWORD_HEAP) && defined(SIZEOF_LONG_LONG) && (SIZEOF_LONG_LONG == 8)
 /* Assume 32-bit machine with long long support */
 typedef Uint64   ErtsDoubleDigit;
 typedef Uint16   ErtsHalfDigit;
@@ -58,7 +58,7 @@ typedef Uint32   ErtsHalfDigit;
 
 typedef Uint  dsize_t;	 /* Vector size type */
 
-#define D_EXP (SIZEOF_VOID_P*8)
+#define D_EXP (ERTS_SIZEOF_ETERM*8)
 #define D_MASK     ((ErtsDigit)(-1))      /* D_BASE-1 */
 
 /* macros for bignum objects */
@@ -88,7 +88,13 @@ typedef Uint  dsize_t;	 /* Vector size type */
 
 #define BIG_UINT_HEAP_SIZE (1 + 1)	/* always, since sizeof(Uint) <= sizeof(Eterm) */
 
-#ifdef ARCH_32
+#if HALFWORD_HEAP
+#define BIG_UWORD_HEAP_SIZE(UW) (((UW) >> (sizeof(Uint) * 8)) ? 3 : 2)
+#else
+#define BIG_UWORD_HEAP_SIZE(UW) BIG_UINT_HEAP_SIZE
+#endif
+
+#if defined(ARCH_32) || HALFWORD_HEAP
 
 #define ERTS_UINT64_BIG_HEAP_SIZE__(X) \
   ((X) >= (((Uint64) 1) << 32) ? (1 + 2) : (1 + 1))
@@ -108,13 +114,13 @@ typedef Uint  dsize_t;	 /* Vector size type */
 
 #endif
 
-int big_decimal_estimate(Eterm);
+int big_decimal_estimate(Wterm);
 Eterm erts_big_to_list(Eterm, Eterm**);
-char *erts_big_to_string(Eterm x, char *buf, Uint buf_sz);
+char *erts_big_to_string(Wterm x, char *buf, Uint buf_sz);
 
 Eterm small_times(Sint, Sint, Eterm*);
 
-Eterm big_plus(Eterm, Eterm, Eterm*);
+Eterm big_plus(Wterm, Wterm, Eterm*);
 Eterm big_minus(Eterm, Eterm, Eterm*);
 Eterm big_times(Eterm, Eterm, Eterm*);
 Eterm big_div(Eterm, Eterm, Eterm*);
@@ -131,19 +137,27 @@ Eterm big_bxor(Eterm, Eterm, Eterm*);
 Eterm big_bnot(Eterm, Eterm*);
 
 Eterm big_lshift(Eterm, Sint, Eterm*);
-int big_comp (Eterm, Eterm);
+int big_comp (Wterm, Wterm);
 int big_ucomp (Eterm, Eterm);
-int big_to_double(Eterm x, double* resp);
+int big_to_double(Wterm x, double* resp);
+Eterm double_to_big(double, Eterm*);
 Eterm small_to_big(Sint, Eterm*);
 Eterm uint_to_big(Uint, Eterm*);
+Eterm uword_to_big(UWord, Eterm*);
 Eterm erts_make_integer(Uint, Process *);
+Eterm erts_make_integer_from_uword(UWord x, Process *p);
 
 dsize_t big_bytes(Eterm);
 Eterm bytes_to_big(byte*, dsize_t, int, Eterm*);
 byte* big_to_bytes(Eterm, byte*);
 
 int term_to_Uint(Eterm, Uint*);
+int term_to_UWord(Eterm, UWord*);
 int term_to_Sint(Eterm, Sint*);
+#if HAVE_INT64
+int term_to_Uint64(Eterm, Uint64*);
+int term_to_Sint64(Eterm, Sint64*);
+#endif
 
 Uint32 big_to_uint32(Eterm b);
 int term_equals_2pow32(Eterm);
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index 08c64610a2..3d2725e239 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -32,22 +32,22 @@
 #include "erl_bits.h"
 
 #ifdef DEBUG
-static int list_to_bitstr_buf(Eterm obj, char* buf, int len);
+static int list_to_bitstr_buf(Eterm obj, char* buf, Uint len);
 #else
 static int list_to_bitstr_buf(Eterm obj, char* buf);
 #endif
-static Sint bitstr_list_len(Eterm obj);
+static int bitstr_list_len(Eterm obj, Uint* num_bytes);
 
 void
 erts_init_binary(void)
 {
     /* Verify Binary alignment... */
-    if ((((Uint) &((Binary *) 0)->orig_bytes[0]) % ((Uint) 8)) != 0) {
+    if ((((UWord) &((Binary *) 0)->orig_bytes[0]) % ((UWord) 8)) != 0) {
 	/* I assume that any compiler should be able to optimize this
 	   away. If not, this test is not very expensive... */
 	erl_exit(ERTS_ABORT_EXIT,
 		 "Internal error: Address of orig_bytes[0] of a Binary"
-		 "is *not* 8-byte aligned\n");
+		 " is *not* 8-byte aligned\n");
     }
 }
 
@@ -56,7 +56,7 @@ erts_init_binary(void)
  */
 
 Eterm
-new_binary(Process *p, byte *buf, int len)
+new_binary(Process *p, byte *buf, Uint len)
 {
     ProcBin* pb;
     Binary* bptr;
@@ -88,8 +88,8 @@ new_binary(Process *p, byte *buf, int len)
     pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE);
     pb->thing_word = HEADER_PROC_BIN;
     pb->size = len;
-    pb->next = MSO(p).mso;
-    MSO(p).mso = pb;
+    pb->next = MSO(p).first;
+    MSO(p).first = (struct erl_off_heap_header*)pb;
     pb->val = bptr;
     pb->bytes = (byte*) bptr->orig_bytes;
     pb->flags = 0;
@@ -97,7 +97,7 @@ new_binary(Process *p, byte *buf, int len)
     /*
      * Miscellanous updates. Return the tagged binary.
      */
-    MSO(p).overhead += pb->size / sizeof(Eterm);
+    OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
     return make_binary(pb);
 }
 
@@ -127,8 +127,8 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, int len)
     pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE);
     pb->thing_word = HEADER_PROC_BIN;
     pb->size = len;
-    pb->next = MSO(p).mso;
-    MSO(p).mso = pb;
+    pb->next = MSO(p).first;
+    MSO(p).first = (struct erl_off_heap_header*)pb;
     pb->val = bptr;
     pb->bytes = (byte*) bptr->orig_bytes;
     pb->flags = 0;
@@ -136,7 +136,7 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, int len)
     /*
      * Miscellanous updates. Return the tagged binary.
      */
-    MSO(p).overhead += pb->size / sizeof(Eterm);
+    OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
     return make_binary(pb);
 }
 
@@ -180,7 +180,7 @@ erts_realloc_binary(Eterm bin, size_t size)
 }
 
 byte*
-erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, unsigned extra)
+erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, ErtsAlcType_t allocator, unsigned extra)
 {
     byte* bytes;
     Eterm* real_bin;
@@ -208,7 +208,7 @@ erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, unsigned extra)
 	bytes = (byte *)(&(((ErlHeapBin *) real_bin)->data)) + offs;
     }
     if (bit_offs) {
-	byte* buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, byte_size + extra);
+	byte* buf = (byte *) erts_alloc(allocator, byte_size + extra);
 	*base_ptr = buf;
 	buf += extra;
 	erts_copy_bits(bytes, bit_offs, 1, buf, 0, 1, byte_size*8);	
@@ -217,8 +217,8 @@ erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, unsigned extra)
     return bytes;
 }
 
-static Eterm
-bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs)
+Eterm
+erts_bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs)
 {
     if (bitoffs == 0) {
 	while (size) {
@@ -263,7 +263,7 @@ BIF_RETTYPE binary_to_list_1(BIF_ALIST_1)
 	Eterm* hp = HAlloc(BIF_P, 2 * size);
 	byte* bytes = binary_bytes(real_bin)+offset;
 
-	BIF_RET(bin_bytes_to_list(NIL, hp, bytes, size, bitoffs));
+	BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes, size, bitoffs));
     }
 
     error:
@@ -295,7 +295,7 @@ BIF_RETTYPE binary_to_list_3(BIF_ALIST_3)
     }
     i = stop-start+1;
     hp = HAlloc(BIF_P, 2*i);
-    BIF_RET(bin_bytes_to_list(NIL, hp, bytes+start-1, i, bitoffs));
+    BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes+start-1, i, bitoffs));
     
     error:
 	BIF_ERROR(BIF_P, BADARG);
@@ -339,36 +339,56 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1)
 	previous = CONS(hp, make_binary(last), previous);
 	hp += 2;
     }
-    BIF_RET(bin_bytes_to_list(previous, hp, bytes, size, bitoffs));
+    BIF_RET(erts_bin_bytes_to_list(previous, hp, bytes, size, bitoffs));
 }
 
 
 /* Turn a possibly deep list of ints (and binaries) into */
 /* One large binary object                               */
 
-BIF_RETTYPE list_to_binary_1(BIF_ALIST_1)
+/*
+ * This bif also exists in the binary module, under the name
+ * binary:list_to_bin/1, why it's divided into interface and
+ * implementation. Also the backend for iolist_to_binary_1.
+ */
+
+BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg)
 {
     Eterm bin;
-    int i;
-    int offset;
+    Uint size;
     byte* bytes;
-    if (is_nil(BIF_ARG_1)) {
-	BIF_RET(new_binary(BIF_P,(byte*)"",0));
+#ifdef DEBUG
+    int offset;
+#endif
+
+    if (is_nil(arg)) {
+	BIF_RET(new_binary(p,(byte*)"",0));
     }
-    if (is_not_list(BIF_ARG_1)) {
+    if (is_not_list(arg)) {
 	goto error;
     }
-    if ((i = io_list_len(BIF_ARG_1)) < 0) {
-	goto error;
+    switch (erts_iolist_size(arg, &size)) {
+    case ERTS_IOLIST_OVERFLOW: BIF_ERROR(p, SYSTEM_LIMIT);
+    case ERTS_IOLIST_TYPE: goto error;
+    default: ;
     }
-    bin = new_binary(BIF_P, (byte *)NULL, i);
+    bin = new_binary(p, (byte *)NULL, size);
     bytes = binary_bytes(bin);
-    offset = io_list_to_buf(BIF_ARG_1, (char*) bytes, i);
+#ifdef DEBUG
+    offset = 
+#endif
+	io_list_to_buf(arg, (char*) bytes, size);
+
     ASSERT(offset == 0);
     BIF_RET(bin);
     
-    error:
-	BIF_ERROR(BIF_P, BADARG);
+ error:
+    BIF_ERROR(p, BADARG);
+}
+
+BIF_RETTYPE list_to_binary_1(BIF_ALIST_1)
+{
+    return erts_list_to_binary_bif(BIF_P, BIF_ARG_1);
 }
 
 /* Turn a possibly deep list of ints (and binaries) into */
@@ -376,37 +396,17 @@ BIF_RETTYPE list_to_binary_1(BIF_ALIST_1)
 
 BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1)
 {
-    Eterm bin;
-    int i;
-    int offset;
-    byte* bytes;
-
     if (is_binary(BIF_ARG_1)) {
 	BIF_RET(BIF_ARG_1);
     }
-    if (is_nil(BIF_ARG_1)) {
-	BIF_RET(new_binary(BIF_P,(byte*)"",0));
-    }
-    if (is_not_list(BIF_ARG_1)) {
-	goto error;
-    }
-    if ((i = io_list_len(BIF_ARG_1)) < 0) {
-	goto error;
-    }
-    bin = new_binary(BIF_P, (byte *)NULL, i);
-    bytes = binary_bytes(bin);
-    offset = io_list_to_buf(BIF_ARG_1, (char*) bytes, i);
-    ASSERT(offset == 0);
-    BIF_RET(bin);
-    
-    error:
-	BIF_ERROR(BIF_P, BADARG);
+    return erts_list_to_binary_bif(BIF_P, BIF_ARG_1);
 }
 
 BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1)
 {
     Eterm bin;
-    int i,offset;
+    Uint sz;
+    int offset;
     byte* bytes;
     ErlSubBin* sb1; 
     Eterm* hp;
@@ -415,15 +415,19 @@ BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1)
 	BIF_RET(new_binary(BIF_P,(byte*)"",0));
     }
     if (is_not_list(BIF_ARG_1)) {
-	goto error;
+    error:
+	BIF_ERROR(BIF_P, BADARG);
     }
-    if ((i = bitstr_list_len(BIF_ARG_1)) < 0) {
+    switch (bitstr_list_len(BIF_ARG_1, &sz)) {
+    case ERTS_IOLIST_TYPE:
 	goto error;
+    case ERTS_IOLIST_OVERFLOW:
+	BIF_ERROR(BIF_P, SYSTEM_LIMIT);
     }
-    bin = new_binary(BIF_P, (byte *)NULL, i);
+    bin = new_binary(BIF_P, (byte *)NULL, sz);
     bytes = binary_bytes(bin);
 #ifdef DEBUG
-    offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes, i);
+    offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes, sz);
 #else
     offset = list_to_bitstr_buf(BIF_ARG_1, (char*) bytes);
 #endif
@@ -432,20 +436,16 @@ BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1)
 	hp = HAlloc(BIF_P, ERL_SUB_BIN_SIZE);
 	sb1 = (ErlSubBin *) hp;
 	sb1->thing_word = HEADER_SUB_BIN;
-	sb1->size = i-1;
+	sb1->size = sz-1;
 	sb1->offs = 0;
 	sb1->orig = bin;
 	sb1->bitoffs = 0;
 	sb1->bitsize = offset;
 	sb1->is_writable = 0;
-	hp += ERL_SUB_BIN_SIZE;
 	bin = make_binary(sb1);
     }
     
     BIF_RET(bin);
-    
-    error:
-	BIF_ERROR(BIF_P, BADARG);
 }
 
 BIF_RETTYPE split_binary_2(BIF_ALIST_2)
@@ -497,16 +497,6 @@ BIF_RETTYPE split_binary_2(BIF_ALIST_2)
 	BIF_ERROR(BIF_P, BADARG);
 }
 
-void
-erts_cleanup_mso(ProcBin* pb)
-{
-    while (pb != NULL) {
-	ProcBin* next = pb->next;
-	if (erts_refc_dectest(&pb->val->refc, 0) == 0)
-	    erts_bin_free(pb->val);
-	pb = next;
-    }
-}
 
 /*
  * Local functions.
@@ -519,7 +509,7 @@ erts_cleanup_mso(ProcBin* pb)
  */
 static int
 #ifdef DEBUG
-list_to_bitstr_buf(Eterm obj, char* buf, int len)
+list_to_bitstr_buf(Eterm obj, char* buf, Uint len)
 #else
 list_to_bitstr_buf(Eterm obj, char* buf)
 #endif
@@ -622,8 +612,8 @@ list_to_bitstr_buf(Eterm obj, char* buf)
     return offset;
 }
 
-static Sint
-bitstr_list_len(Eterm obj)
+static int
+bitstr_list_len(Eterm obj, Uint* num_bytes)
 {
     Eterm* objp;
     Uint len = 0;
@@ -631,6 +621,26 @@ bitstr_list_len(Eterm obj)
     DECLARE_ESTACK(s);
     goto L_again;
 
+#define SAFE_ADD(Var, Val)			\
+    do {					\
+        Uint valvar = (Val);			\
+	Var += valvar;				\
+	if (Var < valvar) {			\
+	    goto L_overflow_error;		\
+	}					\
+    } while (0)
+
+#define SAFE_ADD_BITSIZE(Var, Bin)					\
+    do {								\
+	if (*binary_val(Bin) == HEADER_SUB_BIN) {			\
+            Uint valvar = ((ErlSubBin *) binary_val(Bin))->bitsize;	\
+	    Var += valvar;						\
+	    if (Var < valvar) {						\
+	         goto L_overflow_error;					\
+	    }								\
+        }								\
+    } while (0)
+
     while (!ESTACK_ISEMPTY(s)) {
 	obj = ESTACK_POP(s);
     L_again:
@@ -641,9 +651,12 @@ bitstr_list_len(Eterm obj)
 	    obj = CAR(objp);
 	    if (is_byte(obj)) {
 		len++;
+		if (len == 0) {
+		    goto L_overflow_error;
+		}
 	    } else if (is_binary(obj)) {
-		len += binary_size(obj);
-		offs += binary_bitsize(obj);
+		SAFE_ADD(len, binary_size(obj));
+		SAFE_ADD_BITSIZE(offs, obj);
 	    } else if (is_list(obj)) {
 		ESTACK_PUSH(s, CDR(objp));
 		goto L_iter_list; /* on head */
@@ -655,23 +668,44 @@ bitstr_list_len(Eterm obj)
 	    if (is_list(obj))
 		goto L_iter_list; /* on tail */
 	    else if (is_binary(obj)) {
-		len += binary_size(obj);
-		offs += binary_bitsize(obj);
+		SAFE_ADD(len, binary_size(obj));
+		SAFE_ADD_BITSIZE(offs, obj);
 	    } else if (is_not_nil(obj)) {
 		goto L_type_error;
 	    }
 	} else if (is_binary(obj)) {
-	    len += binary_size(obj);
-	    offs += binary_bitsize(obj);
+	    SAFE_ADD(len, binary_size(obj));
+	    SAFE_ADD_BITSIZE(offs, obj);
 	} else if (is_not_nil(obj)) {
 	    goto L_type_error;
 	}
     }
+#undef SAFE_ADD
+#undef SAFE_ADD_BITSIZE
 
     DESTROY_ESTACK(s);
-    return (Sint) (len + (offs/8) + ((offs % 8) != 0));
+
+    /*
+     * Make sure that the number of bits in the bitstring will fit
+     * in an Uint to ensure that the binary can be matched using
+     * the binary syntax.
+     */
+    if (len << 3 < len) {
+	goto L_overflow_error;
+    }
+    len += (offs >> 3) + ((offs & 7) != 0);
+    if (len << 3 < len) {
+	goto L_overflow_error;
+    }
+    *num_bytes = len;
+    return ERTS_IOLIST_OK;
 
  L_type_error:
     DESTROY_ESTACK(s);
-    return (Sint) -1;
+    return ERTS_IOLIST_TYPE;
+
+ L_overflow_error:
+    DESTROY_ESTACK(s);
+    return ERTS_IOLIST_OVERFLOW;
 }
+
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index cc69977b79..6f5020dc14 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /* This File contains functions which are called if a user hits ^C */
@@ -37,10 +37,7 @@
 #include "beam_load.h"
 #include "erl_instrument.h"
 #include "erl_bif_timer.h"
-
-#ifdef _OSE_
-#include "time.h"
-#endif
+#include "erl_thr_progress.h"
 
 /* Forward declarations -- should really appear somewhere else */
 static void process_killer(void);
@@ -98,11 +95,11 @@ process_killer(void)
 	    erts_printf("(k)ill (n)ext (r)eturn:\n");
 	    while(1) {
 		if ((j = sys_get_key(0)) <= 0)
-		    halt_0(0);
+		    erl_exit(0, "");
 		switch(j) {
 		case 'k':
 		    if (rp->status == P_WAITING) {
-			Uint32 rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
+			ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
 			erts_smp_proc_inc_refc(rp);
 			erts_smp_proc_lock(rp, rp_locks);
 			(void) erts_send_exit_signal(NULL,
@@ -185,6 +182,7 @@ print_process_info(int to, void *to_arg, Process *p)
 {
     int garbing = 0;
     int running = 0;
+    time_t tmp_t;
     struct saved_calls *scb;
 
     /* display the PID */
@@ -247,8 +245,8 @@ print_process_info(int to, void *to_arg, Process *p)
     }
 
     erts_print(to, to_arg, "Spawned by: %T\n", p->parent);
-
-    erts_print(to, to_arg, "Started: %s", ctime((time_t*)&p->started.tv_sec));
+    tmp_t = p->started.tv_sec;
+    erts_print(to, to_arg, "Started: %s", ctime(&tmp_t));
     ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p);
     erts_print(to, to_arg, "Message queue length: %d\n", p->msg.len);
 
@@ -262,17 +260,15 @@ print_process_info(int to, void *to_arg, Process *p)
     }
 
     {
-       long s = 0;
        int frags = 0;
        ErlHeapFragment *m = p->mbuf;
        while (m != NULL) {
 	   frags++;
-	   s += m->size;
 	   m = m->next;
        }
        erts_print(to, to_arg, "Number of heap fragments: %d\n", frags);
     }
-    erts_print(to, to_arg, "Heap fragment data: %bpu\n", MBUF_SIZE(p));
+    erts_print(to, to_arg, "Heap fragment data: %beu\n", MBUF_SIZE(p));
 
     scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p);
     if (scb) {
@@ -319,15 +315,14 @@ print_process_info(int to, void *to_arg, Process *p)
     }
     
     /* print the number of reductions etc */
-    erts_print(to, to_arg, "Reductions: %bpu\n", p->reds);
+    erts_print(to, to_arg, "Reductions: %beu\n", p->reds);
 
-    erts_print(to, to_arg, "Stack+heap: %bpu\n", p->heap_sz);
+    erts_print(to, to_arg, "Stack+heap: %beu\n", p->heap_sz);
     erts_print(to, to_arg, "OldHeap: %bpu\n",
-               (OLD_HEAP(p) == NULL) ? 0 :
-               (unsigned)(OLD_HEND(p) - OLD_HEAP(p)) );
+               (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HEAP(p)) );
     erts_print(to, to_arg, "Heap unused: %bpu\n", (p->hend - p->htop));
     erts_print(to, to_arg, "OldHeap unused: %bpu\n",
-	       (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HEAP(p)) );
+	       (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HTOP(p)) );
 
     if (garbing) {
 	print_garb_info(to, to_arg, p);
@@ -381,7 +376,7 @@ loaded(int to, void *to_arg)
     int i;
     int old = 0;
     int cur = 0;
-    Eterm* code;
+    BeamInstr* code;
 
     /*
      * Calculate and print totals.
@@ -564,7 +559,7 @@ do_break(void)
 #endif
 #ifdef DEBUG
 	case 't':
-	    p_slpq();
+	    erts_p_slpq();
 	    return;
 	case 'b':
 	    bin_check();
@@ -617,29 +612,29 @@ static void
 bin_check(void)
 {
     Process  *rp;
-    ProcBin *bp;
-    int i, printed;
+    struct erl_off_heap_header* hdr;
+    int i, printed = 0;
 
     for (i=0; i < erts_max_processes; i++) {
 	if ((rp = process_tab[i]) == NULL)
 	    continue;
-	if (!(bp = rp->off_heap.mso))
-	    continue;
-	printed = 0;
-	while (bp) {
-	    if (printed == 0) {
-		erts_printf("Process %T holding binary data \n", rp->id);
-		printed = 1;
+	for (hdr = rp->off_heap.first; hdr; hdr = hdr->next) {
+	    if (hdr->thing_word == HEADER_PROC_BIN) {
+		ProcBin *bp = (ProcBin*) hdr;
+		if (!printed) {
+		    erts_printf("Process %T holding binary data \n", rp->id);
+		    printed = 1;
+		}
+		erts_printf("%p orig_size: %bpd, norefs = %bpd\n",
+			    bp->val, 
+			    bp->val->orig_size, 
+			    erts_smp_atomic_read_nob(&bp->val->refc));
 	    }
-	    erts_printf("0x%08lx orig_size: %ld, norefs = %ld\n",
-		       (unsigned long)bp->val, 
-		       (long)bp->val->orig_size, 
-		       erts_smp_atomic_read(&bp->val->refc));
-
-	    bp = bp->next;
 	}
-	if (printed == 1)
+	if (printed) {
 	    erts_printf("--------------------------------------\n");
+	    printed = 0;
+	}
     }
     /* db_bin_check() has to be rewritten for the AVL trees... */
     /*db_bin_check();*/ 
@@ -651,30 +646,32 @@ bin_check(void)
 void
 erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
 {
+#ifdef ERTS_SMP
+    ErtsThrPrgrData tpd_buf; /* in case we aren't a managed thread... */
+#endif
     int fd;
     time_t now;
     size_t dumpnamebufsize = MAXPATHLEN;
     char dumpnamebuf[MAXPATHLEN];
     char* dumpname;
 
-    if (ERTS_IS_CRASH_DUMPING)
+    if (ERTS_SOMEONE_IS_CRASH_DUMPING)
 	return;
 
-    /* Wait for all threads to block. If all threads haven't blocked
+#ifdef ERTS_SMP
+    /*
+     * Wait for all managed threads to block. If all threads haven't blocked
      * after a minute, we go anyway and hope for the best...
      *
      * We do not release system again. We expect an exit() or abort() after
      * dump has been written.
-     *
-     * NOTE: We allow gc therefore it is important not to lock *any*
-     *       process locks.
      */
-    erts_smp_emergency_block_system(60000, ERTS_BS_FLG_ALLOW_GC);
+    erts_thr_progress_fatal_error_block(60000, &tpd_buf);
     /* Either worked or not... */
 
     /* Allow us to pass certain places without locking... */
-#ifdef ERTS_SMP
-    erts_smp_atomic_inc(&erts_writing_erl_crash_dump);
+    erts_smp_atomic32_set_mb(&erts_writing_erl_crash_dump, 1);
+    erts_smp_tsd_set(erts_is_crash_dumping_key, (void *) 1);
 #else
     erts_writing_erl_crash_dump = 1;
 #endif
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 0a5050b1fe..36eda04de2 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -26,26 +26,12 @@
 #include "global.h"
 #include "erl_process.h"
 #include "erl_gc.h"
-#include "erl_nmgc.h"
 #include "big.h"
 #include "erl_binary.h"
 #include "erl_bits.h"
+#include "dtrace-wrapper.h"
 
-#ifdef HYBRID
-MA_STACK_DECLARE(src);
-MA_STACK_DECLARE(dst);
-MA_STACK_DECLARE(offset);
-#endif
-
-void
-init_copy(void)
-{
-#ifdef HYBRID
-    MA_STACK_ALLOC(src);
-    MA_STACK_ALLOC(dst);
-    MA_STACK_ALLOC(offset);
-#endif
-}
+static void move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap*);
 
 /*
  *  Copy object "obj" to process p.
@@ -57,6 +43,14 @@ copy_object(Eterm obj, Process* to)
     Eterm* hp = HAlloc(to, size);
     Eterm res;
 
+#ifdef USE_VM_PROBES
+    if (DTRACE_ENABLED(copy_object)) {
+        DTRACE_CHARBUF(proc_name, 64);
+
+        erts_snprintf(proc_name, sizeof(proc_name), "%T", to->id);
+        DTRACE2(copy_object, proc_name, size);
+    }
+#endif
     res = copy_struct(obj, size, &hp, &to->off_heap);
 #ifdef DEBUG
     if (eq(obj, res) == 0) {
@@ -70,8 +64,11 @@ copy_object(Eterm obj, Process* to)
  * Return the "flat" size of the object.
  */
 
-Uint
-size_object(Eterm obj)
+#if HALFWORD_HEAP
+Uint size_object_rel(Eterm obj, Eterm* base)
+#else
+Uint size_object(Eterm obj)
+#endif
 {
     Uint sum = 0;
     Eterm* ptr;
@@ -82,24 +79,24 @@ size_object(Eterm obj)
 	switch (primary_tag(obj)) {
 	case TAG_PRIMARY_LIST:
 	    sum += 2;
-	    ptr = list_val(obj);
+	    ptr = list_val_rel(obj,base);
 	    obj = *ptr++;
 	    if (!IS_CONST(obj)) {
 		ESTACK_PUSH(s, obj);
-	    }
+	    }	    
 	    obj = *ptr;
 	    break;
 	case TAG_PRIMARY_BOXED:
 	    {
-		Eterm hdr = *boxed_val(obj);
+		Eterm hdr = *boxed_val_rel(obj,base);
 		ASSERT(is_header(hdr));
 		switch (hdr & _TAG_HEADER_MASK) {
 		case ARITYVAL_SUBTAG:
-		    ptr = tuple_val(obj);
+		    ptr = tuple_val_rel(obj,base);
 		    arity = header_arity(hdr);
 		    sum += arity + 1;
 		    if (arity == 0) { /* Empty tuple -- unusual. */
-			goto size_common;
+			goto pop_next;
 		    }
 		    while (arity-- > 1) {
 			obj = *++ptr;
@@ -111,11 +108,10 @@ size_object(Eterm obj)
 		    break;
 		case FUN_SUBTAG:
 		    {
-			Eterm* bptr = fun_val(obj);
+			Eterm* bptr = fun_val_rel(obj,base);
 			ErlFunThing* funp = (ErlFunThing *) bptr;
 			unsigned eterms = 1 /* creator */ + funp->num_free;
 			unsigned sz = thing_arityval(hdr);
-
 			sum += 1 /* header */ + sz + eterms;
 			bptr += 1 /* header */ + sz;
 			while (eterms-- > 1) {
@@ -130,12 +126,12 @@ size_object(Eterm obj)
 		case SUB_BINARY_SUBTAG:
 		    {
 			Eterm real_bin;
-			Uint offset; /* Not used. */
+			ERTS_DECLARE_DUMMY(Uint offset); /* Not used. */
 			Uint bitsize;
 			Uint bitoffs;
 			Uint extra_bytes;
 			Eterm hdr;
-			ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize);
+			ERTS_GET_REAL_BIN_REL(obj, real_bin, offset, bitoffs, bitsize, base);
 			if ((bitsize + bitoffs) > 8) {
 			    sum += ERL_SUB_BIN_SIZE;
 			    extra_bytes = 2;
@@ -145,13 +141,13 @@ size_object(Eterm obj)
 			} else {
 			    extra_bytes = 0;
 			}
-			hdr = *binary_val(real_bin);
+			hdr = *binary_val_rel(real_bin,base);
 			if (thing_subtag(hdr) == REFC_BINARY_SUBTAG) {
 			    sum += PROC_BIN_SIZE;
 			} else {
-			    sum += heap_bin_size(binary_size(obj)+extra_bytes);
+			    sum += heap_bin_size(binary_size_rel(obj,base)+extra_bytes);
 			}
-			goto size_common;
+			goto pop_next;
 		    }
 		    break;
 		case BIN_MATCHSTATE_SUBTAG:
@@ -159,18 +155,12 @@ size_object(Eterm obj)
 			     "size_object: matchstate term not allowed");
 		default:
 		    sum += thing_arityval(hdr) + 1;
-		    /* Fall through */
-		size_common:
-		    if (ESTACK_ISEMPTY(s)) {
-			DESTROY_ESTACK(s);
-			return sum;
-		    }
-		    obj = ESTACK_POP(s);
-		    break;
+		    goto pop_next;
 		}
 	    }
 	    break;
 	case TAG_PRIMARY_IMMED1:
+	pop_next:
 	    if (ESTACK_ISEMPTY(s)) {
 		DESTROY_ESTACK(s);
 		return sum;
@@ -186,8 +176,12 @@ size_object(Eterm obj)
 /*
  *  Copy a structure to a heap.
  */
-Eterm
-copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
+#if HALFWORD_HEAP
+Eterm copy_struct_rel(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap,
+                      Eterm* src_base, Eterm* dst_base)
+#else
+Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
+#endif
 {
     char* hstart;
     Uint hsize;
@@ -211,6 +205,8 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
     if (IS_CONST(obj))
 	return obj;
 
+    DTRACE1(copy_struct, (int32_t)sz);
+
     hp = htop = *hpp;
     hbot   = htop + sz;
     hstart = (char *)htop;
@@ -219,7 +215,10 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 
     /* Copy the object onto the heap */
     switch (primary_tag(obj)) {
-    case TAG_PRIMARY_LIST: argp = &res; goto L_copy_list;
+    case TAG_PRIMARY_LIST:
+	argp = &res;
+	objp = list_val_rel(obj,src_base);
+	goto L_copy_list;
     case TAG_PRIMARY_BOXED: argp = &res; goto L_copy_boxed;
     default:
 	erl_exit(ERTS_ABORT_EXIT,
@@ -236,32 +235,46 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	    hp++;
 	    break;
 	case TAG_PRIMARY_LIST:
-	    objp = list_val(obj);
+	    objp = list_val_rel(obj,src_base);
+	#if !HALFWORD_HEAP || defined(DEBUG)
 	    if (in_area(objp,hstart,hsize)) {
+		ASSERT(!HALFWORD_HEAP);
 		hp++;
 		break;
 	    }
+	#endif
 	    argp = hp++;
 	    /* Fall through */
 
 	L_copy_list:
 	    tailp = argp;
-	    while (is_list(obj)) {
-		objp = list_val(obj);
+	    for (;;) {
 		tp = tailp;
-		elem = *objp;
+		elem = CAR(objp);
 		if (IS_CONST(elem)) {
-		    *(hbot-2) = elem;
-		    tailp = hbot-1;
 		    hbot -= 2;
+		    CAR(hbot) = elem;
+		    tailp = &CDR(hbot);
 		}
 		else {
-		    *htop = elem;
-		    tailp = htop+1;
+		    CAR(htop) = elem;
+		#if HALFWORD_HEAP
+		    CDR(htop) = CDR(objp);
+		    *tailp = make_list_rel(htop,dst_base);
 		    htop += 2;
+		    goto L_copy;
+		#else
+		    tailp = &CDR(htop);
+		    htop += 2;
+		#endif
+		}
+		ASSERT(!HALFWORD_HEAP || tp < hp || tp >= hbot);
+		*tp = make_list_rel(tailp - 1, dst_base);
+		obj = CDR(objp);
+		if (!is_list(obj)) {
+		    break;
 		}
-		*tp = make_list(tailp - 1);
-		obj = *(objp+1);
+		objp = list_val_rel(obj,src_base);
 	    }
 	    switch (primary_tag(obj)) {
 	    case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy;
@@ -273,21 +286,24 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	    }
 	    
 	case TAG_PRIMARY_BOXED:
-	    if (in_area(boxed_val(obj),hstart,hsize)) {
+	#if !HALFWORD_HEAP || defined(DEBUG)
+	    if (in_area(boxed_val_rel(obj,src_base),hstart,hsize)) {
+		ASSERT(!HALFWORD_HEAP);
 		hp++;
 		break;
 	    }
+	#endif
 	    argp = hp++;
 
 	L_copy_boxed:
-	    objp = boxed_val(obj);
+	    objp = boxed_val_rel(obj, src_base);
 	    hdr = *objp;
 	    switch (hdr & _TAG_HEADER_MASK) {
 	    case ARITYVAL_SUBTAG:
 		{
 		    int const_flag = 1; /* assume constant tuple */
 		    i = arityval(hdr);
-		    *argp = make_tuple(htop);
+		    *argp = make_tuple_rel(htop, dst_base);
 		    tp = htop;	/* tp is pointer to new arity value */
 		    *htop++ = *objp++; /* copy arity value */
 		    while (i--) {
@@ -316,13 +332,13 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    while (i--)  {
 			*tp++ = *objp++;
 		    }
-		    *argp = make_binary(hbot);
+		    *argp = make_binary_rel(hbot, dst_base);
 		    pb = (ProcBin*) hbot;
 		    erts_refc_inc(&pb->val->refc, 2);
-		    pb->next = off_heap->mso;
+		    pb->next = off_heap->first;
 		    pb->flags = 0;
-		    off_heap->mso = pb;
-		    off_heap->overhead += pb->size / sizeof(Eterm);
+		    off_heap->first = (struct erl_off_heap_header*) pb;
+		    OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
 		}
 		break;
 	    case SUB_BINARY_SUBTAG:
@@ -343,7 +359,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 			extra_bytes = 0;
 		    } 
 		    real_size = size+extra_bytes;
-		    objp = binary_val(real_bin);
+		    objp = binary_val_rel(real_bin,src_base);
 		    if (thing_subtag(*objp) == HEAP_BINARY_SUBTAG) {
 			ErlHeapBin* from = (ErlHeapBin *) objp;
 			ErlHeapBin* to;
@@ -368,12 +384,12 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 			to->val = from->val;
 			erts_refc_inc(&to->val->refc, 2);
 			to->bytes = from->bytes + offset;
-			to->next = off_heap->mso;
+			to->next = off_heap->first;
 			to->flags = 0;
-			off_heap->mso = to;
-			off_heap->overhead += to->size / sizeof(Eterm);
+			off_heap->first = (struct erl_off_heap_header*) to;
+			OH_OVERHEAD(off_heap, to->size / sizeof(Eterm));
 		    }
-		    *argp = make_binary(hbot);
+		    *argp = make_binary_rel(hbot, dst_base);
 		    if (extra_bytes != 0) {
 			ErlSubBin* res;
 			hbot -= ERL_SUB_BIN_SIZE;
@@ -385,7 +401,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 			res->offs = 0;
 			res->is_writable = 0;
 			res->orig = *argp;
-			*argp = make_binary(hbot);
+			*argp = make_binary_rel(hbot, dst_base);
 		    }
 		    break;
 		}
@@ -399,13 +415,11 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    while (i--)  {
 			*htop++ = *objp++;
 		    }
-#ifndef HYBRID /* FIND ME! */
 		    funp = (ErlFunThing *) tp;
-		    funp->next = off_heap->funs;
-		    off_heap->funs = funp;
+		    funp->next = off_heap->first;
+		    off_heap->first = (struct erl_off_heap_header*) funp;
 		    erts_refc_inc(&funp->fe->refc, 2);
-#endif
-		    *argp = make_fun(tp);
+		    *argp = make_fun_rel(tp, dst_base);
 		}
 		break;
 	    case EXTERNAL_PID_SUBTAG:
@@ -421,11 +435,11 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    *htop++ = *objp++;
 		  }
 
-		  etp->next = off_heap->externals;
-		  off_heap->externals = etp;
+		  etp->next = off_heap->first;
+		  off_heap->first = (struct erl_off_heap_header*)etp;
 		  erts_refc_inc(&etp->node->refc, 2);
 
-		  *argp = make_external(tp);
+		  *argp = make_external_rel(tp, dst_base);
 		}
 		break;
 	    case BIN_MATCHSTATE_SUBTAG:
@@ -435,7 +449,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		i = thing_arityval(hdr)+1;
 		hbot -= i;
 		tp = hbot;
-		*argp = make_boxed(hbot);
+		*argp = make_boxed_rel(hbot, dst_base);
 		while (i--) {
 		    *tp++ = *objp++;
 		}
@@ -455,7 +469,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
     if (htop != hbot)
 	erl_exit(ERTS_ABORT_EXIT,
 		 "Internal error in copy_struct() when copying %T:"
-		 " htop=%p != hbot=%p (sz=%bpu)\n",
+		 " htop=%p != hbot=%p (sz=%beu)\n",
 		 org_obj, htop, hbot, org_sz); 
 #else
     if (htop > hbot) {
@@ -467,420 +481,6 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
     return res;
 }
 
-#ifdef HYBRID
-
-#ifdef BM_MESSAGE_SIZES
-#  define BM_ADD(var,val) (var) += (val);
-#else
-#  define BM_ADD(var,val)
-#endif
-
-#ifdef DEBUG
-#  define CLEARMEM(PTR,SIZE) memset(PTR,0,SIZE*sizeof(Eterm))
-#else
-#  define CLEARMEM(PTR,SIZE)
-#endif
-
-#ifdef INCREMENTAL
-#define GlobalAlloc(p, need, hp)                                        \
-do {                                                                    \
-    Uint n = (need);                                                    \
-    BM_ADD(words_copied,n);                                             \
-    BM_SWAP_TIMER(copy,system);                                         \
-    /* If a new collection cycle is started during copy, the message *  \
-     * will end up in the old generation and all allocations         *  \
-     * thereafter must go directly into the old generation.          */ \
-    if (alloc_old) {                                                    \
-        erts_incremental_gc((p),n,&dest,1);                             \
-        (hp) = erts_inc_alloc(n);                                       \
-    } else {                                                            \
-        (hp) = IncAlloc((p),n,&dest,1);                                 \
-        if (ma_gc_flags & GC_CYCLE_START) {                             \
-            alloc_old = 1;                                              \
-            global_htop = global_heap;                                  \
-            (hp) = erts_inc_alloc(n);                                   \
-        }                                                               \
-    }                                                                   \
-    CLEARMEM((hp),(n));                                                 \
-    BM_SWAP_TIMER(system,copy);                                         \
-} while(0)
-
-#else /* no INCREMELNTAL */
-
-#define GlobalAlloc(p, need, hp)                                        \
-do {                                                                    \
-    Uint n = (need);                                                    \
-    total_need += n;                                                    \
-    if (total_need >= global_heap_sz)                                   \
-        erl_exit(ERTS_ABORT_EXIT, "Copying a message (%d words) larger than the nursery simply won't work...\n", total_need); \
-    if (global_hend - n < global_htop) {                                \
-        BM_SWAP_TIMER(copy,system);                                     \
-        erts_global_garbage_collect((p),total_need,NULL,0);             \
-        BM_SWAP_TIMER(system,copy);                                     \
-        total_need = 0;                                                 \
-        ma_src_top = 0;                                                 \
-        ma_dst_top = 0;                                                 \
-        ma_offset_top = 0;                                              \
-        goto copy_start;                                                \
-    }                                                                   \
-    (hp) = global_htop;                                                 \
-    global_htop += n;                                                   \
-    BM_ADD(words_copied,n);                                             \
-} while(0)
-#endif /* INCREMENTAL */
-
-/* Copy a message to the message area. */
-Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
-{
-    Eterm obj;
-    Eterm dest;
-#ifdef INCREMENTAL
-    int alloc_old = 0;
-#else
-    int total_need = 0;
-#endif
-
-    VERBOSE(DEBUG_MESSAGES,
-            ("COPY START; %T is sending a message @ 0x%016x\n%T\n",
-             from->id, orig, orig));
-
-#ifndef INCREMENTAL
- copy_start:
-#endif
-    MA_STACK_PUSH(src,orig);
-    MA_STACK_PUSH(dst,&dest);
-    MA_STACK_PUSH(offset,offs);
-
-    while (ma_src_top > 0) {
-        obj = MA_STACK_POP(src);
-
-        /* copy_struct_lazy should never be called with something that
-         * do not need to be copied. Within the loop, nothing that do
-         * not need copying should be placed in the src-stack.
-         */
-        ASSERT(!NO_COPY(obj));
-
-        switch (primary_tag(obj)) {
-        case TAG_PRIMARY_LIST: {
-            Eterm *hp;
-            Eterm *objp;
-
-            GlobalAlloc(from,2,hp);
-            objp = list_val(obj);
-
-            MA_STACK_UPDATE(dst,MA_STACK_POP(offset),make_list(hp));
-            MA_STACK_POP(dst);
-
-            /* TODO: Byt ordningen nedan så att CDR pushas först. */
-
-            if (NO_COPY(*objp)) {
-                hp[0] = *objp;
-#ifdef INCREMENTAL
-                if (ptr_within(ptr_val(*objp),inc_fromspc,inc_fromend))
-                    INC_STORE(gray,hp,2);
-#endif
-            } else {
-                MA_STACK_PUSH(src,*objp);
-                MA_STACK_PUSH(dst,hp);
-                MA_STACK_PUSH(offset,0);
-            }
-
-            objp++;
-
-            if (NO_COPY(*objp)) {
-                hp[1] = *objp;
-#ifdef INCREMENTAL
-                if (ptr_within(ptr_val(*objp),inc_fromspc,inc_fromend))
-                    INC_STORE(gray,hp,2);
-#endif
-            }
-            else {
-                MA_STACK_PUSH(src,*objp);
-                MA_STACK_PUSH(dst,hp);
-                MA_STACK_PUSH(offset,1);
-            }
-            continue;
-        }
-
-        case TAG_PRIMARY_BOXED: {
-            Eterm *objp = boxed_val(obj);
-
-            switch (*objp & _TAG_HEADER_MASK) {
-            case ARITYVAL_SUBTAG: {
-                Uint ari = arityval(*objp);
-                Uint i;
-                Eterm *hp;
-                GlobalAlloc(from,ari + 1,hp);
-                /* A GC above might invalidate the value of objp */
-                objp = boxed_val(obj);
-                MA_STACK_UPDATE(dst,MA_STACK_POP(offset),make_tuple(hp));
-                MA_STACK_POP(dst);
-                *hp = *objp++;
-                for (i = 1; i <= ari; i++) {
-                    switch (primary_tag(*objp)) {
-                    case TAG_PRIMARY_LIST:
-                    case TAG_PRIMARY_BOXED:
-                        if (NO_COPY(*objp)) {
-                            hp[i] = *objp;
-#ifdef INCREMENTAL
-                            if (ptr_within(ptr_val(*objp),
-                                           inc_fromspc,inc_fromend))
-                                INC_STORE(gray,hp,BOXED_NEED(hp,*hp));
-#endif
-                            objp++;
-                        } else {
-                            MA_STACK_PUSH(src,*objp++);
-                            MA_STACK_PUSH(dst,hp);
-                            MA_STACK_PUSH(offset,i);
-                        }
-                        break;
-                    default:
-                        hp[i] = *objp++;
-                    }
-                }
-                continue;
-            }
-
-            case REFC_BINARY_SUBTAG: {
-                ProcBin *pb;
-                Uint i = thing_arityval(*objp) + 1;
-                Eterm *hp;
-                GlobalAlloc(from,i,hp);
-                /* A GC above might invalidate the value of objp */
-                objp = boxed_val(obj);
-                MA_STACK_UPDATE(dst,MA_STACK_POP(offset),make_binary(hp));
-                MA_STACK_POP(dst);
-                pb = (ProcBin*) hp;
-                while (i--) {
-                    *hp++ = *objp++;
-                }
-                erts_refc_inc(&pb->val->refc, 2);
-                pb->next = erts_global_offheap.mso;
-                erts_global_offheap.mso = pb;
-                erts_global_offheap.overhead += pb->size / sizeof(Eterm);
-                continue;
-            }
-
-            case FUN_SUBTAG: {
-                ErlFunThing *funp = (ErlFunThing*) objp;
-                Uint i = thing_arityval(*objp) + 1;
-                Uint j = i + 1 + funp->num_free;
-                Uint k = i;
-                Eterm *hp, *hp_start;
-                GlobalAlloc(from,j,hp);
-                /* A GC above might invalidate the value of objp */
-                objp = boxed_val(obj);
-                hp_start = hp;
-                MA_STACK_UPDATE(dst,MA_STACK_POP(offset),make_fun(hp));
-                MA_STACK_POP(dst);
-                funp = (ErlFunThing*) hp;
-                while (i--) {
-                    *hp++ = *objp++;
-                }
-#ifndef HYBRID // FIND ME!
-                funp->next = erts_global_offheap.funs;
-                erts_global_offheap.funs = funp;
-                erts_refc_inc(&funp->fe->refc, 2);
-#endif
-                for (i = k; i < j; i++) {
-                    switch (primary_tag(*objp)) {
-                    case TAG_PRIMARY_LIST:
-                    case TAG_PRIMARY_BOXED:
-                        if (NO_COPY(*objp)) {
-#ifdef INCREMENTAL
-                            if (ptr_within(ptr_val(*objp),
-                                           inc_fromspc,inc_fromend))
-                                INC_STORE(gray,hp,BOXED_NEED(hp,*hp));
-#endif
-                            *hp++ = *objp++;
-                        } else {
-                            MA_STACK_PUSH(src,*objp++);
-                            MA_STACK_PUSH(dst,hp_start);
-                            MA_STACK_PUSH(offset,i);
-                            hp++;
-                        }
-                        break;
-                    default:
-                        *hp++ = *objp++;
-                    }
-                }
-                continue;
-            }
-
-            case EXTERNAL_PID_SUBTAG:
-            case EXTERNAL_PORT_SUBTAG:
-            case EXTERNAL_REF_SUBTAG: {
-                ExternalThing *etp;
-                Uint i =  thing_arityval(*objp) + 1;
-                Eterm *hp;
-                GlobalAlloc(from,i,hp);
-                /* A GC above might invalidate the value of objp */
-                objp = boxed_val(obj);
-                MA_STACK_UPDATE(dst,MA_STACK_POP(offset),make_external(hp));
-                MA_STACK_POP(dst);
-                etp = (ExternalThing*) hp;
-                while (i--)  {
-                    *hp++ = *objp++;
-                }
-
-                etp->next = erts_global_offheap.externals;
-                erts_global_offheap.externals = etp;
-		erts_refc_inc(&etp->node->refc, 2);
-                continue;
-            }
-
-            case SUB_BINARY_SUBTAG: {
-                ErlSubBin *sb = (ErlSubBin *) objp;
-		Eterm *hp;
-		Eterm res_binary;
-                Eterm real_bin = sb->orig;
-                Uint bit_offset = sb->bitoffs;
-		Uint bit_size = sb -> bitsize;
-		Uint sub_offset = sb->offs;
-                size_t size = sb->size;
-		Uint extra_bytes;
-		Uint real_size;
-		Uint sub_binary_heapneed;
-		if ((bit_size + bit_offset) > 8) {
-		    extra_bytes = 2;
-		    sub_binary_heapneed = ERL_SUB_BIN_SIZE;
-		} else if ((bit_size + bit_offset) > 0) {
-		    extra_bytes = 1;
-		    sub_binary_heapneed = ERL_SUB_BIN_SIZE;
-		} else {
-		    extra_bytes = 0;
-		    sub_binary_heapneed = 0;
-		}
-		
-		real_size = size+extra_bytes;
-                objp = binary_val(real_bin);
-                if (thing_subtag(*objp) == HEAP_BINARY_SUBTAG) {
-                    ErlHeapBin *from_bin;
-                    ErlHeapBin *to_bin;
-                    Uint i = heap_bin_size(real_size);
-                    GlobalAlloc(from,i+sub_binary_heapneed,hp);
-                    from_bin = (ErlHeapBin *) objp;
-                    to_bin = (ErlHeapBin *) hp;
-                    to_bin->thing_word = header_heap_bin(real_size);
-                    to_bin->size = real_size;
-                    sys_memcpy(to_bin->data, ((byte *)from_bin->data) +
-                               sub_offset, real_size);
-		    res_binary = make_binary(to_bin);
-		    hp += i;
-                } else {
-                    ProcBin *from_bin;
-                    ProcBin *to_bin;
-                    
-                    ASSERT(thing_subtag(*objp) == REFC_BINARY_SUBTAG);
-		    from_bin = (ProcBin *) objp;
-		    erts_refc_inc(&from_bin->val->refc, 2);
-                    GlobalAlloc(from,PROC_BIN_SIZE+sub_binary_heapneed,hp);
-                    to_bin = (ProcBin *) hp;
-                    to_bin->thing_word = HEADER_PROC_BIN;
-                    to_bin->size = real_size;
-                    to_bin->val = from_bin->val;
-                    to_bin->bytes = from_bin->bytes + sub_offset;
-                    to_bin->next = erts_global_offheap.mso;
-                    erts_global_offheap.mso = to_bin;
-                    erts_global_offheap.overhead += to_bin->size / sizeof(Eterm);
-		    res_binary=make_binary(to_bin);
-		    hp += PROC_BIN_SIZE;
-                }
-		if (extra_bytes != 0) {
-		    ErlSubBin* res;
-		    res = (ErlSubBin *) hp;
-		    res->thing_word = HEADER_SUB_BIN;
-		    res->size = size;
-		    res->bitsize = bit_size;
-		    res->bitoffs = bit_offset;
-		    res->offs = 0;
-		    res->is_writable = 0;
-		    res->orig = res_binary;
-		    res_binary = make_binary(hp);
-		}
-		MA_STACK_UPDATE(dst,MA_STACK_POP(offset),res_binary);
-		MA_STACK_POP(dst);
-                continue;
-            }
-
-	    case BIN_MATCHSTATE_SUBTAG:
-		erl_exit(ERTS_ABORT_EXIT,
-			 "copy_struct_lazy: matchstate term not allowed");
-
-            default: {
-                Uint size = thing_arityval(*objp) + 1;
-                Eterm *hp;
-                GlobalAlloc(from,size,hp);
-                /* A GC above might invalidate the value of objp */
-                objp = boxed_val(obj);
-                MA_STACK_UPDATE(dst,MA_STACK_POP(offset),make_boxed(hp));
-                MA_STACK_POP(dst);
-                while (size--) {
-                    *hp++ = *objp++;
-                }
-                continue;
-            }
-            }
-            continue;
-        }
-
-        case TAG_PRIMARY_HEADER:
-        ASSERT((obj & _TAG_HEADER_MASK) == ARITYVAL_SUBTAG);
-        {
-            Eterm *objp = &obj;
-            Uint ari = arityval(obj);
-            Uint i;
-            Eterm *hp;
-            GlobalAlloc(from,ari + 1,hp);
-            MA_STACK_UPDATE(dst,MA_STACK_POP(offset),make_tuple(hp));
-            MA_STACK_POP(dst);
-            *hp = *objp++;
-            for (i = 1; i <= ari; i++) {
-                switch (primary_tag(*objp)) {
-                case TAG_PRIMARY_LIST:
-                case TAG_PRIMARY_BOXED:
-                    if (NO_COPY(*objp)) {
-#ifdef INCREMENTAL
-                        if (ptr_within(ptr_val(*objp),inc_fromspc,inc_fromend))
-                            INC_STORE(gray,hp,ari + 1);
-#endif
-                        hp[i] = *objp++;
-                    } else {
-                        MA_STACK_PUSH(src,*objp++);
-                        MA_STACK_PUSH(dst,hp);
-                        MA_STACK_PUSH(offset,i);
-                    }
-                    break;
-                default:
-                    hp[i] = *objp++;
-                }
-            }
-            continue;
-        }
-
-        default:
-            erl_exit(ERTS_ABORT_EXIT,
-		     "%s, line %d: Internal error in copy_struct_lazy: 0x%08x\n",
-                     __FILE__, __LINE__,obj);
-        }
-    }
-
-    VERBOSE(DEBUG_MESSAGES,
-            ("Copy allocated @ 0x%08lx:\n%T\n",
-             (unsigned long)ptr_val(dest),dest));
-
-    ma_gc_flags &= ~GC_CYCLE_START;
-
-    ASSERT(eq(orig, dest));
-    ASSERT(ma_src_top == 0);
-    ASSERT(ma_dst_top == 0);
-    ASSERT(ma_offset_top == 0);
-    return dest;
-}
-
-#undef NO_COPY
-#endif /* HYBRID */
-
 /*
  * Copy a term that is guaranteed to be contained in a single
  * heap block. The heap block is copied word by word, and any
@@ -890,12 +490,21 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
  *
  * NOTE: Assumes that term is a tuple (ptr is an untagged tuple ptr).
  */
-Eterm
-copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
+#if HALFWORD_HEAP
+Eterm copy_shallow_rel(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap,
+		       Eterm* src_base)
+#else
+Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
+#endif
 {
     Eterm* tp = ptr;
     Eterm* hp = *hpp;
-    Sint offs = hp - tp;
+    const Eterm res = make_tuple(hp);
+#if HALFWORD_HEAP
+    const Sint offs = COMPRESS_POINTER(hp - (tp - src_base));
+#else
+    const Sint offs = (hp - tp) * sizeof(Eterm);
+#endif
 
     while (sz--) {
 	Eterm val = *tp++;
@@ -906,7 +515,7 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	    break;
 	case TAG_PRIMARY_LIST:
 	case TAG_PRIMARY_BOXED:
-	    *hp++ = offset_ptr(val, offs);
+	    *hp++ = byte_offset_ptr(val, offs);
 	    break;
 	case TAG_PRIMARY_HEADER:
 	    *hp++ = val;
@@ -915,57 +524,43 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		break;
 	    case REFC_BINARY_SUBTAG:
 		{
-		    ProcBin* pb = (ProcBin *) (hp-1);
-		    int tari = thing_arityval(val);
-
-		    sz -= tari;
-		    while (tari--) {
-			*hp++ = *tp++;
-		    }
+		    ProcBin* pb = (ProcBin *) (tp-1);
 		    erts_refc_inc(&pb->val->refc, 2);
-		    pb->next = off_heap->mso;
-		    off_heap->mso = pb;
-		    off_heap->overhead += pb->size / sizeof(Eterm);
+		    OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
 		}
-		break;
+		goto off_heap_common;
+
 	    case FUN_SUBTAG:
 		{
-#ifndef HYBRID /* FIND ME! */
-		    ErlFunThing* funp = (ErlFunThing *) (hp-1);
-#endif
-		    int tari = thing_arityval(val);
-
-		    sz -= tari;
-		    while (tari--) {
-			*hp++ = *tp++;
-		    }
-#ifndef HYBRID /* FIND ME! */
-		    funp->next = off_heap->funs;
-		    off_heap->funs = funp;
+		    ErlFunThing* funp = (ErlFunThing *) (tp-1);
 		    erts_refc_inc(&funp->fe->refc, 2);
-#endif
 		}
-		break;
+		goto off_heap_common;
+
 	    case EXTERNAL_PID_SUBTAG:
 	    case EXTERNAL_PORT_SUBTAG:
 	    case EXTERNAL_REF_SUBTAG:
 		{
-		    ExternalThing* etp = (ExternalThing *) (hp-1);
+		    ExternalThing* etp = (ExternalThing *) (tp-1);
+		    erts_refc_inc(&etp->node->refc, 2);
+		}
+	    off_heap_common:
+		{
+		    struct erl_off_heap_header* ohh = (struct erl_off_heap_header*)(hp-1);
 		    int tari = thing_arityval(val);
-
+		    
 		    sz -= tari;
 		    while (tari--) {
 			*hp++ = *tp++;
 		    }
-		    etp->next = off_heap->externals;
-		    off_heap->externals = etp;
-		    erts_refc_inc(&etp->node->refc, 2);
+		    ohh->next = off_heap->first;
+		    off_heap->first = ohh;
 		}
 		break;
 	    default:
 		{
 		    int tari = header_arity(val);
-
+    
 		    sz -= tari;
 		    while (tari--) {
 			*hp++ = *tp++;
@@ -977,5 +572,95 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 	}
     }
     *hpp = hp;
-    return make_tuple(ptr + offs); 
+
+    return res;
+}
+
+/* Move all terms in heap fragments into heap. The terms must be guaranteed to 
+ * be contained within the fragments. The source terms are destructed with
+ * move markers.
+ * Typically used to copy a multi-fragmented message (from NIF).
+ */
+void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
+		      Eterm* refs, unsigned nrefs)
+{
+    ErlHeapFragment* bp;
+    Eterm* hp_start = *hpp;
+    Eterm* hp_end;
+    Eterm* hp;
+    unsigned i;
+
+    for (bp=first; bp!=NULL; bp=bp->next) {
+	move_one_frag(hpp, bp->mem, bp->used_size, off_heap);
+	OH_OVERHEAD(off_heap, bp->off_heap.overhead);
+    }
+    hp_end = *hpp;
+    for (hp=hp_start; hp<hp_end; ++hp) {
+	Eterm* ptr;
+	Eterm val;
+	Eterm gval = *hp;
+	switch (primary_tag(gval)) {
+	case TAG_PRIMARY_BOXED:
+	    ptr = boxed_val(gval);
+	    val = *ptr;
+	    if (IS_MOVED_BOXED(val)) {
+		ASSERT(is_boxed(val));
+		*hp = val;
+	    }
+	    break;
+	case TAG_PRIMARY_LIST:
+	    ptr = list_val(gval);
+	    val = *ptr;
+	    if (IS_MOVED_CONS(val)) {
+		*hp = ptr[1];
+	    }
+	    break;
+	case TAG_PRIMARY_HEADER:
+	    if (header_is_thing(gval)) {
+		hp += thing_arityval(gval);
+	    }
+	    break;
+	}
+    }
+    for (i=0; i<nrefs; ++i) {
+	refs[i] = follow_moved(refs[i]);
+    }
+}
+
+static void
+move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap)
+{
+    Eterm* ptr = src;
+    Eterm* end = ptr + src_sz;
+    Eterm dummy_ref;
+    Eterm* hp = *hpp;
+
+    while (ptr != end) {
+	Eterm val;
+	ASSERT(ptr < end);
+	val = *ptr;
+	ASSERT(val != ERTS_HOLE_MARKER);
+	if (is_header(val)) {
+	    struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp;
+	    ASSERT(ptr + header_arity(val) < end);
+	    MOVE_BOXED(ptr, val, hp, &dummy_ref);	    
+	    switch (val & _HEADER_SUBTAG_MASK) {
+	    case REFC_BINARY_SUBTAG:
+	    case FUN_SUBTAG:
+	    case EXTERNAL_PID_SUBTAG:
+	    case EXTERNAL_PORT_SUBTAG:
+	    case EXTERNAL_REF_SUBTAG:
+		hdr->next = off_heap->first;
+		off_heap->first = hdr;
+		break;
+	    }
+	}
+	else { /* must be a cons cell */
+	    ASSERT(ptr+1 < end);
+	    MOVE_CONS(ptr, val, hp, &dummy_ref);
+	    ptr += 2;
+	}
+    }
+    *hpp = hp;
 }
+
diff --git a/erts/emulator/beam/decl.h b/erts/emulator/beam/decl.h
deleted file mode 100644
index da1be29d53..0000000000
--- a/erts/emulator/beam/decl.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-
-#ifndef __DECL_H__
-#define __DECL_H__
-
-#if defined(__STDC__) || defined(_MSC_VER)
-#define EXTERN_FUNCTION(t, f, x)  extern t f x
-#define FUNCTION(t, f, x) t f x
-#define _DOTS_ ...
-#define _VOID_      void
-#elif defined(__cplusplus)
-#define EXTERN_FUNCTION(f, x) extern "C" { f x }
-#define FUNCTION(t, f, x) t f x
-#define _DOTS_ ...
-#define _VOID_    void
-#else
-#define EXTERN_FUNCTION(t, f, x) extern t f (/*x*/)
-#define FUNCTION(t, f, x) t f (/*x*/)
-#define _DOTS_
-#define _VOID_
-#endif
-
-/*
-** Example of declarations
-**
-** EXTERN_FUNCTION(void, foo, (int, int, char));
-** FUNCTION(void, bar, (int, char));
-**
-** struct funcs {
-**    FUNCTION(int*, (*f1), (int, int));
-**    FUNCTION(void, (*f2), (int, char));
-**    FUNCTION(void, (*f3), (_VOID_));
-**    FUNCTION(int,  (*f4), (char*, _DOTS_));
-** };
-**
-*/
-
-#endif
diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c
index e3094404e2..01fd63d5d9 100644
--- a/erts/emulator/beam/dist.c
+++ b/erts/emulator/beam/dist.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -41,6 +41,8 @@
 #include "bif.h"
 #include "external.h"
 #include "erl_binary.h"
+#include "erl_thr_progress.h"
+#include "dtrace-wrapper.h"
 
 /* Turn this on to get printouts of all distribution messages
  * which go on the line
@@ -53,9 +55,9 @@
 #endif
 
 #if defined(ERTS_DIST_MSG_DBG) || defined(ERTS_RAW_DIST_MSG_DBG)
-static void bw(byte *buf, int sz)
+static void bw(byte *buf, ErlDrvSizeT sz)
 {
-    bin_write(ERTS_PRINT_STDERR,NULL,buf,sz);
+    bin_write(ERTS_PRINT_STDERR, NULL, buf, sz);
 }
 #endif
 
@@ -97,6 +99,8 @@ dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz)
 #define PASS_THROUGH 'p'        /* This code should go */
 
 int erts_is_alive; /* System must be blocked on change */
+int erts_dist_buf_busy_limit;
+
 
 /* distribution trap functions */
 Export* dsend2_trap = NULL;
@@ -126,8 +130,8 @@ delete_cache(ErtsAtomCache *cache)
 {
     if (cache) {
 	erts_free(ERTS_ALC_T_DCACHE, (void *) cache);
-	ASSERT(erts_smp_atomic_read(&no_caches) > 0);
-	erts_smp_atomic_dec(&no_caches);
+	ASSERT(erts_smp_atomic_read_nob(&no_caches) > 0);
+	erts_smp_atomic_dec_nob(&no_caches);
     }
 }
 
@@ -145,7 +149,7 @@ create_cache(DistEntry *dep)
 
     dep->cache = cp = (ErtsAtomCache*) erts_alloc(ERTS_ALC_T_DCACHE,
 						  sizeof(ErtsAtomCache));
-    erts_smp_atomic_inc(&no_caches);
+    erts_smp_atomic_inc_nob(&no_caches);
     for (i = 0; i < sizeof(cp->in_arr)/sizeof(cp->in_arr[0]); i++) {
 	cp->in_arr[i] = THE_NON_VALUE;
 	cp->out_arr[i] = THE_NON_VALUE;
@@ -154,13 +158,13 @@ create_cache(DistEntry *dep)
 
 Uint erts_dist_cache_size(void)
 {
-    return (Uint) erts_smp_atomic_read(&no_caches)*sizeof(ErtsAtomCache);
+    return (Uint) erts_smp_atomic_read_mb(&no_caches)*sizeof(ErtsAtomCache);
 }
 
 static ErtsProcList *
 get_suspended_on_de(DistEntry *dep, Uint32 unset_qflgs)
 {
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_spinlock_is_locked(&dep->qlock));
+    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&dep->qlock));
     dep->qflgs &= ~unset_qflgs;
     if (dep->qflgs & ERTS_DE_QFLG_EXIT) {
 	/* No resume when exit has been scheduled */
@@ -228,6 +232,7 @@ int is_node_name_atom(Eterm a)
 
 typedef struct {
     DistEntry *dep;
+    Eterm *lhp;
 } NetExitsContext;
 
 /* 
@@ -253,8 +258,9 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp)
 	    erts_destroy_monitor(rmon);
 	}
     } else {
-	Eterm lhp[3];
+	DeclareTmpHeapNoproc(lhp,3);
 	Eterm watched;
+	UseTmpHeapNoproc(3);
 	ASSERT(mon->type == MON_TARGET);
 	rmon = erts_remove_monitor(&(rp->monitors),mon->ref);
 	/* ASSERT(rmon != NULL); can happen during process exit */
@@ -271,6 +277,7 @@ static void doit_monitor_net_exits(ErtsMonitor *mon, void *vnecp)
 				       watched, am_noconnection);
 	    erts_destroy_monitor(rmon);
 	}
+	UnUseTmpHeapNoproc(3);
     }
     erts_smp_proc_unlock(rp, rp_locks);
  done:
@@ -375,7 +382,11 @@ static void doit_node_link_net_exits(ErtsLink *lnk, void *vnecp)
 	    Eterm tup;
 	    Eterm *hp = erts_alloc_message_heap(3,&bp,&ohp,rp,&rp_locks);
 	    tup = TUPLE2(hp, am_nodedown, name);
-	    erts_queue_message(rp, &rp_locks, bp, tup, NIL);
+	    erts_queue_message(rp, &rp_locks, bp, tup, NIL
+#ifdef USE_VM_PROBES
+			       , NIL
+#endif
+			       );
 	}
 	erts_smp_proc_unlock(rp, rp_locks);
     }
@@ -425,11 +436,11 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 	erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
 
 	nodename = erts_this_dist_entry->sysname;
-	erts_smp_block_system(ERTS_BS_FLG_ALLOW_GC);
+	erts_smp_thr_progress_block();
 	erts_set_this_node(am_Noname, 0);
 	erts_is_alive = 0;
 	send_nodes_mon_msgs(NULL, am_nodedown, nodename, am_visible, nd_reason);
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 
     }
     else { /* recursive call via erts_do_exit_port() will end up here */
@@ -439,7 +450,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 	ErtsMonitor *monitors;
 	Uint32 flags;
 
-	erts_smp_atomic_set(&dep->dist_cmd_scheduled, 1);
+	erts_smp_atomic_set_mb(&dep->dist_cmd_scheduled, 1);
 	erts_smp_de_rwlock(dep);
 
 	ERTS_SMP_LC_ASSERT(is_internal_port(dep->cid)
@@ -450,17 +461,17 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason)
 
 	if (dep->status & ERTS_DE_SFLG_EXITING) {
 #ifdef DEBUG
-	    erts_smp_spin_lock(&dep->qlock);
+	    erts_smp_mtx_lock(&dep->qlock);
 	    ASSERT(dep->qflgs & ERTS_DE_QFLG_EXIT);
-	    erts_smp_spin_unlock(&dep->qlock);
+	    erts_smp_mtx_unlock(&dep->qlock);
 #endif
 	}
 	else {
 	    dep->status |= ERTS_DE_SFLG_EXITING;
-	    erts_smp_spin_lock(&dep->qlock);
+	    erts_smp_mtx_lock(&dep->qlock);
 	    ASSERT(!(dep->qflgs & ERTS_DE_QFLG_EXIT));
 	    dep->qflgs |= ERTS_DE_QFLG_EXIT;
-	    erts_smp_spin_unlock(&dep->qlock);
+	    erts_smp_mtx_unlock(&dep->qlock);
 	}
 
 	erts_smp_de_links_lock(dep);
@@ -505,7 +516,7 @@ void init_dist(void)
 {
     init_nodes_monitors();
 
-    erts_smp_atomic_init(&no_caches, 0);
+    erts_smp_atomic_init_nob(&no_caches, 0);
 
     /* Lookup/Install all references to trap functions */
     dsend2_trap = trap_function(am_dsend,2);
@@ -530,7 +541,7 @@ alloc_dist_obuf(Uint size)
     Binary *bin = erts_bin_drv_alloc(obuf_size);
     bin->flags = BIN_FLAG_DRV;
     erts_refc_init(&bin->refc, 1);
-    bin->orig_size = (long) obuf_size;
+    bin->orig_size = (SWord) obuf_size;
     obuf = (ErtsDistOutputBuf *) &bin->orig_bytes[0];
 #ifdef DEBUG
     obuf->dbg_pattern = ERTS_DIST_OUTPUT_BUF_DBG_PATTERN;
@@ -574,7 +585,7 @@ static void clear_dist_entry(DistEntry *dep)
     erts_smp_de_links_unlock(dep);
 #endif
 
-    erts_smp_spin_lock(&dep->qlock);
+    erts_smp_mtx_lock(&dep->qlock);
 
     if (!dep->out_queue.last)
 	obuf = dep->finalized_out_queue.first;
@@ -590,8 +601,8 @@ static void clear_dist_entry(DistEntry *dep)
     dep->status = 0;
     suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL);
 
-    erts_smp_spin_unlock(&dep->qlock);
-    erts_smp_atomic_set(&dep->dist_cmd_scheduled, 0);
+    erts_smp_mtx_unlock(&dep->qlock);
+    erts_smp_atomic_set_nob(&dep->dist_cmd_scheduled, 0);
     dep->send = NULL;
     erts_smp_de_rwunlock(dep);
 
@@ -608,10 +619,10 @@ static void clear_dist_entry(DistEntry *dep)
     }
 
     if (obufsize) {
-	erts_smp_spin_lock(&dep->qlock);
+	erts_smp_mtx_lock(&dep->qlock);
 	ASSERT(dep->qsize >= obufsize);
 	dep->qsize -= obufsize;
-	erts_smp_spin_unlock(&dep->qlock);
+	erts_smp_mtx_unlock(&dep->qlock);
     }
 }
 
@@ -632,19 +643,27 @@ static void clear_dist_entry(DistEntry *dep)
 int
 erts_dsig_send_link(ErtsDSigData *dsdp, Eterm local, Eterm remote)
 {
-    Eterm ctl_heap[4];
+    DeclareTmpHeapNoproc(ctl_heap,4);
     Eterm ctl = TUPLE3(&ctl_heap[0], make_small(DOP_LINK), local, remote);
+    int res;
+    UseTmpHeapNoproc(4);
 
-    return dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    UnUseTmpHeapNoproc(4);
+    return res;
 }
 
 int
 erts_dsig_send_unlink(ErtsDSigData *dsdp, Eterm local, Eterm remote)
 {
-    Eterm ctl_heap[4];
+    DeclareTmpHeapNoproc(ctl_heap,4);
     Eterm ctl = TUPLE3(&ctl_heap[0], make_small(DOP_UNLINK), local, remote);
+    int res;
 
-    return dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    UseTmpHeapNoproc(4);
+    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    UnUseTmpHeapNoproc(4);
+    return res;
 }
 
 
@@ -656,7 +675,10 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched,
 			  Eterm ref, Eterm reason)
 {
     Eterm ctl;
-    Eterm ctl_heap[6];
+    DeclareTmpHeapNoproc(ctl_heap,6);
+    int res;
+
+    UseTmpHeapNoproc(6);
 
     ctl = TUPLE5(&ctl_heap[0], make_small(DOP_MONITOR_P_EXIT),
 		 watched, watcher, ref, reason);
@@ -667,7 +689,9 @@ erts_dsig_send_m_exit(ErtsDSigData *dsdp, Eterm watcher, Eterm watched,
     erts_smp_de_links_unlock(dsdp->dep);
 #endif
 
-    return dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+    UnUseTmpHeapNoproc(6);
+    return res;
 }
 
 /* We want to monitor a process (named or unnamed) on another node, we send:
@@ -678,13 +702,17 @@ erts_dsig_send_monitor(ErtsDSigData *dsdp, Eterm watcher, Eterm watched,
 		       Eterm ref)
 {
     Eterm ctl;
-    Eterm ctl_heap[5];
+    DeclareTmpHeapNoproc(ctl_heap,5);
+    int res;
 
+    UseTmpHeapNoproc(5);
     ctl = TUPLE4(&ctl_heap[0],
 		 make_small(DOP_MONITOR_P),
 		 watcher, watched, ref);
 
-    return dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    UnUseTmpHeapNoproc(5);
+    return res;
 }
 
 /* A local process monitoring a remote one wants to stop monitoring, either 
@@ -696,50 +724,119 @@ erts_dsig_send_demonitor(ErtsDSigData *dsdp, Eterm watcher,
 			 Eterm watched, Eterm ref, int force)
 {
     Eterm ctl;
-    Eterm ctl_heap[5];
+    DeclareTmpHeapNoproc(ctl_heap,5);
+    int res;
 
+    UseTmpHeapNoproc(5);
     ctl = TUPLE4(&ctl_heap[0],
 		 make_small(DOP_DEMONITOR_P),
 		 watcher, watched, ref);
 
-    return dsig_send(dsdp, ctl, THE_NON_VALUE, force);
+    res = dsig_send(dsdp, ctl, THE_NON_VALUE, force);
+    UnUseTmpHeapNoproc(5);
+    return res;
 }
 
 int
 erts_dsig_send_msg(ErtsDSigData *dsdp, Eterm remote, Eterm message)
 {
     Eterm ctl;
-    Eterm ctl_heap[5];
+    DeclareTmpHeapNoproc(ctl_heap,5);
     Eterm token = NIL;
     Process *sender = dsdp->proc;
+    int res;
+#ifdef USE_VM_PROBES
+    Sint tok_label = 0;
+    Sint tok_lastcnt = 0;
+    Sint tok_serial = 0;
+    Uint msize = 0;
+    DTRACE_CHARBUF(node_name, 64);
+    DTRACE_CHARBUF(sender_name, 64);
+    DTRACE_CHARBUF(receiver_name, 64);
+#endif
 
-    if (SEQ_TRACE_TOKEN(sender) != NIL) {
+    UseTmpHeapNoproc(5);
+    if (SEQ_TRACE_TOKEN(sender) != NIL 
+#ifdef USE_VM_PROBES
+	&& SEQ_TRACE_TOKEN(sender) != am_have_dt_utag 
+#endif
+	) {
 	seq_trace_update_send(sender);
 	token = SEQ_TRACE_TOKEN(sender);
 	seq_trace_output(token, message, SEQ_TRACE_SEND, remote, sender);
     }
+#ifdef USE_VM_PROBES
+    *node_name = *sender_name = *receiver_name = '\0';
+    if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) {
+        erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname);
+        erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->id);
+        erts_snprintf(receiver_name, sizeof(receiver_name), "%T", remote);
+        msize = size_object(message);
+        if (token != NIL && token != am_have_dt_utag) {
+            tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
+            tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
+            tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
+        }
+    }
+#endif
 
     if (token != NIL)
 	ctl = TUPLE4(&ctl_heap[0],
 		     make_small(DOP_SEND_TT), am_Cookie, remote, token);
     else
 	ctl = TUPLE3(&ctl_heap[0], make_small(DOP_SEND), am_Cookie, remote);
-    return dsig_send(dsdp, ctl, message, 0);
+    DTRACE6(message_send, sender_name, receiver_name,
+            msize, tok_label, tok_lastcnt, tok_serial);
+    DTRACE7(message_send_remote, sender_name, node_name, receiver_name,
+            msize, tok_label, tok_lastcnt, tok_serial);
+    res = dsig_send(dsdp, ctl, message, 0);
+    UnUseTmpHeapNoproc(5);
+    return res;
 }
 
 int
 erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message)
 {
     Eterm ctl;
-    Eterm ctl_heap[6];
+    DeclareTmpHeapNoproc(ctl_heap,6);
     Eterm token = NIL;
     Process *sender = dsdp->proc;
+    int res;
+#ifdef USE_VM_PROBES
+    Sint tok_label = 0;
+    Sint tok_lastcnt = 0;
+    Sint tok_serial = 0;
+    Uint32 msize = 0;
+    DTRACE_CHARBUF(node_name, 64);
+    DTRACE_CHARBUF(sender_name, 64);
+    DTRACE_CHARBUF(receiver_name, 128);
+#endif
 
-    if (SEQ_TRACE_TOKEN(sender) != NIL) {
+    UseTmpHeapNoproc(6);
+    if (SEQ_TRACE_TOKEN(sender) != NIL
+#ifdef USE_VM_PROBES
+	&& SEQ_TRACE_TOKEN(sender) != am_have_dt_utag 
+#endif
+	) {
 	seq_trace_update_send(sender);
 	token = SEQ_TRACE_TOKEN(sender);
 	seq_trace_output(token, message, SEQ_TRACE_SEND, remote_name, sender);
     }
+#ifdef USE_VM_PROBES
+    *node_name = *sender_name = *receiver_name = '\0';
+    if (DTRACE_ENABLED(message_send) || DTRACE_ENABLED(message_send_remote)) {
+        erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname);
+        erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->id);
+        erts_snprintf(receiver_name, sizeof(receiver_name),
+                      "{%T,%s}", remote_name, node_name);
+        msize = size_object(message);
+        if (token != NIL && token != am_have_dt_utag) {
+            tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
+            tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
+            tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
+        }
+    }
+#endif
 
     if (token != NIL)
 	ctl = TUPLE5(&ctl_heap[0], make_small(DOP_REG_SEND_TT),
@@ -747,7 +844,13 @@ erts_dsig_send_reg_msg(ErtsDSigData *dsdp, Eterm remote_name, Eterm message)
     else
 	ctl = TUPLE4(&ctl_heap[0], make_small(DOP_REG_SEND),
 		     sender->id, am_Cookie, remote_name);
-    return dsig_send(dsdp, ctl, message, 0);
+    DTRACE6(message_send, sender_name, receiver_name,
+            msize, tok_label, tok_lastcnt, tok_serial);
+    DTRACE7(message_send_remote, sender_name, node_name, receiver_name,
+            msize, tok_label, tok_lastcnt, tok_serial);
+    res = dsig_send(dsdp, ctl, message, 0);
+    UnUseTmpHeapNoproc(6);
+    return res;
 }
 
 /* local has died, deliver the exit signal to remote */
@@ -756,9 +859,25 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote,
 		       Eterm reason, Eterm token)
 {
     Eterm ctl;
-    Eterm ctl_heap[6];
+    DeclareTmpHeapNoproc(ctl_heap,6);
+    int res;
+#ifdef USE_VM_PROBES
+    Process *sender = dsdp->proc;
+    Sint tok_label = 0;
+    Sint tok_lastcnt = 0;
+    Sint tok_serial = 0;
+    DTRACE_CHARBUF(node_name, 64);
+    DTRACE_CHARBUF(sender_name, 64);
+    DTRACE_CHARBUF(remote_name, 128);
+    DTRACE_CHARBUF(reason_str, 128);
+#endif
 
-    if (token != NIL) {	
+    UseTmpHeapNoproc(6);
+    if (token != NIL 
+#ifdef USE_VM_PROBES
+	&& token != am_have_dt_utag
+#endif
+	) {	
 	seq_trace_update_send(dsdp->proc);
 	seq_trace_output_exit(token, reason, SEQ_TRACE_SEND, remote, local);
 	ctl = TUPLE5(&ctl_heap[0],
@@ -766,39 +885,76 @@ erts_dsig_send_exit_tt(ErtsDSigData *dsdp, Eterm local, Eterm remote,
     } else {
 	ctl = TUPLE4(&ctl_heap[0], make_small(DOP_EXIT), local, remote, reason);
     }
+#ifdef USE_VM_PROBES
+    *node_name = *sender_name = *remote_name = '\0';
+    if (DTRACE_ENABLED(process_exit_signal_remote)) {
+        erts_snprintf(node_name, sizeof(node_name), "%T", dsdp->dep->sysname);
+        erts_snprintf(sender_name, sizeof(sender_name), "%T", sender->id);
+        erts_snprintf(remote_name, sizeof(remote_name),
+                      "{%T,%s}", remote, node_name);
+        erts_snprintf(reason_str, sizeof(reason), "%T", reason);
+        if (token != NIL && token != am_have_dt_utag) {
+            tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
+            tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
+            tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
+        }
+    }
+#endif
+    DTRACE7(process_exit_signal_remote, sender_name, node_name,
+            remote_name, reason_str, tok_label, tok_lastcnt, tok_serial);
     /* forced, i.e ignore busy */
-    return dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+    UnUseTmpHeapNoproc(6);
+    return res;
 }
 
 int
 erts_dsig_send_exit(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason)
 {
-    Eterm ctl_heap[5];
-    Eterm ctl = TUPLE4(&ctl_heap[0],
-		       make_small(DOP_EXIT), local, remote, reason);
+    DeclareTmpHeapNoproc(ctl_heap,5);
+    int res;
+    Eterm ctl;
+
+    UseTmpHeapNoproc(5);
+    ctl = TUPLE4(&ctl_heap[0],
+		 make_small(DOP_EXIT), local, remote, reason);
     /* forced, i.e ignore busy */
-    return dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+    res =  dsig_send(dsdp, ctl, THE_NON_VALUE, 1);
+    UnUseTmpHeapNoproc(5);
+    return res;
 }
 
 int
 erts_dsig_send_exit2(ErtsDSigData *dsdp, Eterm local, Eterm remote, Eterm reason)
 {
-    Eterm ctl_heap[5];
-    Eterm ctl = TUPLE4(&ctl_heap[0],
-		       make_small(DOP_EXIT2), local, remote, reason);
+    DeclareTmpHeapNoproc(ctl_heap,5);
+    int res;
+    Eterm ctl;
+
+    UseTmpHeapNoproc(5);
+    ctl = TUPLE4(&ctl_heap[0],
+		 make_small(DOP_EXIT2), local, remote, reason);
 
-    return dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    UnUseTmpHeapNoproc(5);
+    return res;
 }
 
 
 int
 erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote)
 {
-    Eterm ctl_heap[4];
-    Eterm ctl = TUPLE3(&ctl_heap[0],
-		       make_small(DOP_GROUP_LEADER), leader, remote);
+    DeclareTmpHeapNoproc(ctl_heap,4);
+    int res;
+    Eterm ctl;
+
+    UseTmpHeapNoproc(4);
+    ctl = TUPLE3(&ctl_heap[0],
+		 make_small(DOP_GROUP_LEADER), leader, remote);
 
-    return dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    res = dsig_send(dsdp, ctl, THE_NON_VALUE, 0);
+    UnUseTmpHeapNoproc(4);
+    return res;
 }
 
 #if defined(PURIFY)
@@ -808,11 +964,15 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote)
 #include <valgrind/valgrind.h>
 #include <valgrind/memcheck.h>
 
+#ifndef HAVE_VALGRIND_PRINTF_XML
+#define VALGRIND_PRINTF_XML VALGRIND_PRINTF
+#endif
+
 #  define PURIFY_MSG(msg)                                                \
     do {								 \
 	char buf__[1]; size_t bufsz__ = sizeof(buf__);			 \
 	if (erts_sys_getenv("VALGRIND_LOG_XML", buf__, &bufsz__) >= 0) { \
-	    VALGRIND_PRINTF("<erlang_error_log>"			 \
+	    VALGRIND_PRINTF_XML("<erlang_error_log>"			 \
 			    "%s, line %d: %s</erlang_error_log>\n",	 \
 			    __FILE__, __LINE__, msg);			 \
 	} else {							 \
@@ -832,17 +992,18 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote)
 **
 **   assert  hlen == 0 !!!
 */
+
 int erts_net_message(Port *prt,
 		     DistEntry *dep,
 		     byte *hbuf,
-		     int hlen,
+		     ErlDrvSizeT hlen,
 		     byte *buf,
-		     int len)
+		     ErlDrvSizeT len)
 {
+#define DIST_CTL_DEFAULT_SIZE 64
     ErtsDistExternal ede;
     byte *t;
     Sint ctl_len;
-    int orig_ctl_len;
     Eterm arg;
     Eterm from, to;
     Eterm watcher, watched;
@@ -850,7 +1011,7 @@ int erts_net_message(Port *prt,
     Eterm *tuple;
     Eterm reason;
     Process* rp;
-    Eterm ctl_default[64];
+    DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE);
     Eterm* ctl = ctl_default;
     ErlOffHeap off_heap;
     Eterm* hp;
@@ -859,29 +1020,31 @@ int erts_net_message(Port *prt,
     Eterm token_size;
     ErtsMonitor *mon;
     ErtsLink *lnk;
+    Uint tuple_arity;
     int res;
 #ifdef ERTS_DIST_MSG_DBG
-    int orig_len = len;
+    ErlDrvSizeT orig_len = len;
 #endif
 
+    UseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
     /* Thanks to Luke Gorrie */
-    off_heap.mso = NULL;
-#ifndef HYBRID /* FIND ME! */
-    off_heap.funs = NULL;
-#endif
+    off_heap.first = NULL;
     off_heap.overhead = 0;
-    off_heap.externals = NULL;
 
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
 
-    if (!erts_is_alive)
+    if (!erts_is_alive) {
+	UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
 	return 0;
-    if (hlen > 0)
+    }
+    if (hlen != 0)
 	goto data_error;
-    if (len == 0)  /* HANDLE TICK !!! */
+    if (len == 0) {  /* HANDLE TICK !!! */
+	UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
 	return 0;
+    }
 
 #ifdef ERTS_RAW_DIST_MSG_DBG
     erts_fprintf(stderr, "<< ");
@@ -904,7 +1067,7 @@ int erts_net_message(Port *prt,
     res = erts_prepare_dist_ext(&ede, t, len, dep, dep->cache);
 
     if (res >= 0)
-	res = ctl_len = erts_decode_dist_ext_size(&ede, 0);
+	res = ctl_len = erts_decode_dist_ext_size(&ede);
     else {
 #ifdef ERTS_DIST_MSG_DBG
 	erts_fprintf(stderr, "DIST MSG DEBUG: erts_prepare_dist_ext() failed:\n");
@@ -921,8 +1084,8 @@ int erts_net_message(Port *prt,
 	PURIFY_MSG("data error");
 	goto data_error;
     }
-    orig_ctl_len = ctl_len;
-    if (ctl_len > sizeof(ctl_default)/sizeof(ctl_default[0])) {
+
+    if (ctl_len > DIST_CTL_DEFAULT_SIZE) {
 	ctl = erts_alloc(ERTS_ALC_T_DCTRL_BUF, ctl_len * sizeof(Eterm));
     }
     hp = ctl;
@@ -943,29 +1106,23 @@ int erts_net_message(Port *prt,
 #endif
 
     if (is_not_tuple(arg) || 
-	(tuple = tuple_val(arg), arityval(*tuple) < 1) ||
+	(tuple = tuple_val(arg), (tuple_arity = arityval(*tuple)) < 1) ||
 	is_not_small(tuple[1])) {
-	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	erts_dsprintf(dsbufp, "Invalid distribution message: %.200T", arg);
-	erts_send_error_to_logger_nogl(dsbufp);
-	goto data_error;
+ 	goto invalid_message;
     }
 
     token_size = 0;
 
     switch (type = unsigned_val(tuple[1])) {
     case DOP_LINK:
+	if (tuple_arity != 3) {
+	    goto invalid_message;
+	}
 	from = tuple[2];
 	to   = tuple[3];  /* local proc to link to */
 
 	if (is_not_pid(from) || is_not_pid(to)) {
-	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	    PURIFY_MSG("data error");
-	    erts_dsprintf(dsbufp,
-			  "Invalid DOP_LINK distribution message: %.200T",
-			  arg);
-	    erts_send_error_to_logger_nogl(dsbufp);
-	    goto data_error;
+	    goto invalid_message;
 	}
 
 	rp = erts_pid2proc_opt(NULL, 0,
@@ -1004,8 +1161,14 @@ int erts_net_message(Port *prt,
 
     case DOP_UNLINK: {
 	ErtsDistLinkData dld;
+	if (tuple_arity != 3) {
+	    goto invalid_message;
+	}
 	from = tuple[2];
 	to = tuple[3];
+	if (is_not_pid(from) || is_not_pid(to)) {
+	    goto invalid_message;
+	}
 	
 	rp = erts_pid2proc_opt(NULL, 0,
 			       to, ERTS_PROC_LOCK_LINK,
@@ -1032,11 +1195,19 @@ int erts_net_message(Port *prt,
 	/* A remote process wants to monitor us, we get:
 	   {DOP_MONITOR_P, Remote pid, local pid or name, ref} */
 	Eterm name;
+	
+	if (tuple_arity != 4) {
+	    goto invalid_message;
+	}
 
 	watcher = tuple[2];
 	watched = tuple[3];  /* local proc to monitor */
 	ref     = tuple[4];
 
+	if (is_not_ref(ref)) {
+	    goto invalid_message;
+	}
+
 	if (is_atom(watched)) {
 	    name = watched;
 	    rp = erts_whereis_process(NULL, 0,
@@ -1078,10 +1249,17 @@ int erts_net_message(Port *prt,
 	   We get {DOP_DEMONITOR_P, Remote pid, Local pid or name, ref},
 	   We need only the ref of course */
 
+	if (tuple_arity != 4) {
+	    goto invalid_message;
+	}
 	/* watcher = tuple[2]; */
 	/* watched = tuple[3]; May be an atom in case of monitor name */
 	ref = tuple[4];
 
+	if(is_not_ref(ref)) {
+	    goto invalid_message;
+	}
+
 	erts_smp_de_links_lock(dep);
 	mon = erts_remove_monitor(&(dep->monitors),ref);
 	erts_smp_de_links_unlock(dep);
@@ -1106,10 +1284,11 @@ int erts_net_message(Port *prt,
 	erts_destroy_monitor(mon);
 	break;
 
-    case DOP_NODE_LINK: /* XXX never sent ?? */
-	break;
-
     case DOP_REG_SEND_TT:
+	if (tuple_arity != 5) {
+	    goto invalid_message;
+	}
+
 	token_size = size_object(tuple[5]);
 	/* Fall through ... */
     case DOP_REG_SEND:
@@ -1120,12 +1299,19 @@ int erts_net_message(Port *prt,
 	 * There is intentionally no testing of the cookie (it is always '')
 	 * from R9B and onwards.
 	 */
+	if (type != DOP_REG_SEND_TT && tuple_arity != 4) {
+	    goto invalid_message;
+	}
+
 #ifdef ERTS_DIST_MSG_DBG
 	dist_msg_dbg(&ede, "MSG", buf, orig_len);
 #endif
 
 	from = tuple[2];
 	to = tuple[4];
+	if (is_not_pid(from) || is_not_atom(to)){
+	    goto invalid_message;
+	}
 	rp = erts_whereis_process(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC);
 	if (rp) {
 	    Uint xsize = (type == DOP_REG_SEND
@@ -1157,6 +1343,10 @@ int erts_net_message(Port *prt,
 	break;
 
     case DOP_SEND_TT:
+	if (tuple_arity != 4) {
+	    goto invalid_message;
+	}
+	
 	token_size = size_object(tuple[4]);
 	/* Fall through ... */
     case DOP_SEND:
@@ -1167,8 +1357,13 @@ int erts_net_message(Port *prt,
 #ifdef ERTS_DIST_MSG_DBG
 	dist_msg_dbg(&ede, "MSG", buf, orig_len);
 #endif
-
+	if (type != DOP_SEND_TT && tuple_arity != 3) {
+	    goto invalid_message;
+	}
 	to = tuple[3];
+	if (is_not_pid(to)) {
+	    goto invalid_message;
+	}
 	rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC);
 	if (rp) {
 	    Uint xsize = type == DOP_SEND ? 0 : ERTS_HEAP_FRAG_SIZE(token_size);
@@ -1202,15 +1397,23 @@ int erts_net_message(Port *prt,
 	   {DOP_MONITOR_P_EXIT, Remote pid or name, Local pid, ref, reason} */
 	   
 
-	Eterm lhp[3];
+	DeclareTmpHeapNoproc(lhp,3);
 	Eterm sysname;
 	ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_MSG_SEND|ERTS_PROC_LOCK_LINK;
 
+	if (tuple_arity != 5) {
+	    goto invalid_message;
+	}
+
 	/* watched = tuple[2]; */  /* remote proc which died */
 	/* watcher = tuple[3]; */
 	ref     = tuple[4];
 	reason  = tuple[5];
 
+	if(is_not_ref(ref)) {
+	    goto invalid_message;
+	}
+
 	erts_smp_de_links_lock(dep);
 	sysname = dep->sysname;
 	mon = erts_remove_monitor(&(dep->monitors), ref);
@@ -1237,6 +1440,7 @@ int erts_net_message(Port *prt,
 	    erts_smp_proc_unlock(rp, rp_locks);
 	    break;
 	}
+	UseTmpHeapNoproc(3);
 	
 	watched = (is_not_nil(mon->name)
 		   ? TUPLE2(&lhp[0], mon->name, sysname)
@@ -1246,6 +1450,7 @@ int erts_net_message(Port *prt,
 				   ref, am_process, watched, reason);
 	erts_smp_proc_unlock(rp, rp_locks);
 	erts_destroy_monitor(mon);
+	UnUseTmpHeapNoproc(3);
 	break;
     }
 
@@ -1255,24 +1460,25 @@ int erts_net_message(Port *prt,
 	ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCKS_XSIG_SEND;
 	/* 'from', which 'to' is linked to, died */
 	if (type == DOP_EXIT) {
-	   from = tuple[2];
-	   to = tuple[3];
-	   reason = tuple[4];
-	   token = NIL;
+	    if (tuple_arity != 4) {
+		goto invalid_message;
+	    }
+	    
+	    from = tuple[2];
+	    to = tuple[3];
+	    reason = tuple[4];
+	    token = NIL;
 	} else {
-	   from = tuple[2];
-	   to = tuple[3];
-	   token = tuple[4];
-	   reason = tuple[5];
+	    if (tuple_arity != 5) {
+		goto invalid_message;
+	    }
+	    from = tuple[2];
+	    to = tuple[3];
+	    token = tuple[4];
+	    reason = tuple[5];
 	}
-	if (is_not_internal_pid(to)) {
-	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	    PURIFY_MSG("data error");
-	    erts_dsprintf(dsbufp,
-			  "Invalid DOP_EXIT distribution message: %.200T",
-			  arg);
-	    erts_send_error_to_logger_nogl(dsbufp);
-	    goto data_error;
+	if (is_not_pid(from) || is_not_internal_pid(to)) {
+	    goto invalid_message;
 	}
 
 	rp = erts_pid2proc(NULL, 0, to, rp_locks);
@@ -1319,15 +1525,24 @@ int erts_net_message(Port *prt,
 	ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND;
 	/* 'from' is send an exit signal to 'to' */
 	if (type == DOP_EXIT2) {
-	   from = tuple[2];
-	   to = tuple[3];
-	   reason = tuple[4];
-	   token = NIL;
+	    if (tuple_arity != 4) {
+		goto invalid_message;
+	    }
+	    from = tuple[2];
+	    to = tuple[3];
+	    reason = tuple[4];
+	    token = NIL;
 	} else {
-	   from = tuple[2];
-	   to = tuple[3];
-	   token = tuple[4];
-	   reason = tuple[5];
+	    if (tuple_arity != 5) {
+		goto invalid_message;
+	    }
+	    from = tuple[2];
+	    to = tuple[3];
+	    token = tuple[4];
+	    reason = tuple[5];
+	}
+	if (is_not_pid(from) || is_not_internal_pid(to)) {
+	    goto invalid_message;
 	}
 	rp = erts_pid2proc_opt(NULL, 0, to, rp_locks,
 			       ERTS_P2P_FLG_SMP_INC_REFC);
@@ -1346,10 +1561,14 @@ int erts_net_message(Port *prt,
 	break;
     }
     case DOP_GROUP_LEADER:
+	if (tuple_arity != 3) {
+	    goto invalid_message;
+	}
 	from = tuple[2];   /* Group leader  */
 	to = tuple[3];     /* new member */
-	if (is_not_pid(from))
-	    break;
+	if (is_not_pid(from) || is_not_pid(to)) {
+	    goto invalid_message;
+	}
 
 	rp = erts_pid2proc(NULL, 0, to, ERTS_PROC_LOCK_MAIN);
 	if (!rp)
@@ -1358,57 +1577,35 @@ int erts_net_message(Port *prt,
 	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN);
 	break;
 
-    default: {
-	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	    erts_dsprintf(dsbufp,
-			  "Illegal value in distribution dispatch switch: "
-			  "%.200T",
-			  arg);
-	    erts_send_error_to_logger_nogl(dsbufp);
-	    PURIFY_MSG("data error");
-	    goto data_error;
-	}
+    default: 
+	goto invalid_message;
     }
 
-    if (off_heap.mso) {
-	erts_cleanup_mso(off_heap.mso);
-    }
-    if (off_heap.externals) {
-	erts_cleanup_externals(off_heap.externals);
-    }
-#ifndef HYBRID /* FIND ME! */
-    if (off_heap.funs) {
-	erts_cleanup_funs(off_heap.funs);
-    }
+    erts_cleanup_offheap(&off_heap);
     if (ctl != ctl_default) {
 	erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl);
     }
-#endif
+    UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     return 0;
-
- data_error:
-    if (off_heap.mso) {
-	erts_cleanup_mso(off_heap.mso);
-    }
-    if (off_heap.externals) {
-	erts_cleanup_externals(off_heap.externals);
-    }
-#ifndef HYBRID /* FIND ME! */
-    if (off_heap.funs) {
-	erts_cleanup_funs(off_heap.funs);
+ invalid_message:
+    {
+	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
+	erts_dsprintf(dsbufp, "Invalid distribution message: %.200T", arg);
+	erts_send_error_to_logger_nogl(dsbufp);
     }
+ data_error:
+    PURIFY_MSG("data error");
+    erts_cleanup_offheap(&off_heap);
     if (ctl != ctl_default) {
 	erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl);
     }
-#endif
+    UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE);
     erts_do_exit_port(prt, dep->cid, am_killed);
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     return -1;
 }
 
-#define ERTS_DE_BUSY_LIMIT (128*1024)
-
 static int
 dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy)
 {
@@ -1492,18 +1689,18 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy)
     }
     else {
 	ErtsProcList *plp = NULL;
-	erts_smp_spin_lock(&dep->qlock);
+	erts_smp_mtx_lock(&dep->qlock);
 	dep->qsize += size_obuf(obuf);
-	if (dep->qsize >= ERTS_DE_BUSY_LIMIT)
+	if (dep->qsize >= erts_dist_buf_busy_limit)
 	    dep->qflgs |= ERTS_DE_QFLG_BUSY;
 	if (!force_busy && (dep->qflgs & ERTS_DE_QFLG_BUSY)) {
-	    erts_smp_spin_unlock(&dep->qlock);
+	    erts_smp_mtx_unlock(&dep->qlock);
 
 	    plp = erts_proclist_create(c_p);
 	    plp->next = NULL;
 	    erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
 	    suspended = 1;
-	    erts_smp_spin_lock(&dep->qlock);
+	    erts_smp_mtx_lock(&dep->qlock);
 	}
 
 	/* Enqueue obuf on dist entry */
@@ -1517,6 +1714,18 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy)
 	    if (!(dep->qflgs & ERTS_DE_QFLG_BUSY)) {
 		if (suspended)
 		    resume = 1; /* was busy when we started, but isn't now */
+#ifdef USE_VM_PROBES
+                if (resume && DTRACE_ENABLED(dist_port_not_busy)) {
+                    DTRACE_CHARBUF(port_str, 64);
+                    DTRACE_CHARBUF(remote_str, 64);
+
+                    erts_snprintf(port_str, sizeof(port_str), "%T", cid);
+                    erts_snprintf(remote_str, sizeof(remote_str),
+                                  "%T", dep->sysname);
+                    DTRACE3(dist_port_not_busy, erts_this_node_sysname,
+                            port_str, remote_str);
+                }
+#endif
 	    }
 	    else {
 		/* Enqueue suspended process on dist entry */
@@ -1529,7 +1738,7 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy)
 	    }
 	}
 
-	erts_smp_spin_unlock(&dep->qlock);
+	erts_smp_mtx_unlock(&dep->qlock);
 	erts_schedule_dist_command(NULL, dep);
 	erts_smp_de_runlock(dep);
 	
@@ -1554,9 +1763,9 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy)
 	 */
 
 	data_size >>= (10-4);
-#if defined(ARCH_64)
+#if defined(ARCH_64) && !HALFWORD_HEAP
 	data_size &= 0x003fffffffffffff;
-#elif defined(ARCH_32)
+#elif defined(ARCH_32) || HALFWORD_HEAP
 	data_size &= 0x003fffff;
 #else
 #       error "Ohh come on ... !?!"
@@ -1566,6 +1775,19 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy)
     }
 
     if (suspended) {
+#ifdef USE_VM_PROBES
+        if (!resume && DTRACE_ENABLED(dist_port_busy)) {
+            DTRACE_CHARBUF(port_str, 64);
+            DTRACE_CHARBUF(remote_str, 64);
+            DTRACE_CHARBUF(pid_str, 16);
+
+            erts_snprintf(port_str, sizeof(port_str), "%T", cid);
+            erts_snprintf(remote_str, sizeof(remote_str), "%T", dep->sysname);
+            erts_snprintf(pid_str, sizeof(pid_str), "%T", c_p->id);
+            DTRACE4(dist_port_busy, erts_this_node_sysname,
+                    port_str, remote_str, pid_str);
+        }
+#endif
 	if (!resume && erts_system_monitor_flags.busy_dist_port)
 	    monitor_generic(c_p, am_busy_dist_port, cid);
 	return ERTS_DSIG_SEND_YIELD;
@@ -1586,9 +1808,21 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf)
     if (size > (Uint) INT_MAX)
 	erl_exit(ERTS_ABORT_EXIT,
 		 "Absurdly large distribution output data buffer "
-		 "(%bpu bytes) passed.\n",
+		 "(%beu bytes) passed.\n",
 		 size);
 
+#ifdef USE_VM_PROBES
+    if (DTRACE_ENABLED(dist_output)) {
+        DTRACE_CHARBUF(port_str, 64);
+        DTRACE_CHARBUF(remote_str, 64);
+
+        erts_snprintf(port_str, sizeof(port_str), "%T", prt->id);
+        erts_snprintf(remote_str, sizeof(remote_str),
+                      "%T", prt->dist_entry->sysname);
+        DTRACE4(dist_output, erts_this_node_sysname, port_str,
+                remote_str, size);
+    }
+#endif
     prt->caller = NIL;
     fpe_was_unmasked = erts_block_fpe();
     (*prt->drv_ptr->output)((ErlDrvData) prt->drv_data,
@@ -1613,7 +1847,7 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf)
     if (size > (Uint) INT_MAX)
 	erl_exit(ERTS_ABORT_EXIT,
 		 "Absurdly large distribution output data buffer "
-		 "(%bpu bytes) passed.\n",
+		 "(%beu bytes) passed.\n",
 		 size);
 
     iov[0].iov_base = NULL;
@@ -1631,6 +1865,18 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf)
 
     ASSERT(prt->drv_ptr->outputv);
 
+#ifdef USE_VM_PROBES
+    if (DTRACE_ENABLED(dist_outputv)) {
+        DTRACE_CHARBUF(port_str, 64);
+        DTRACE_CHARBUF(remote_str, 64);
+
+        erts_snprintf(port_str, sizeof(port_str), "%T", prt->id);
+        erts_snprintf(remote_str, sizeof(remote_str),
+                      "%T", prt->dist_entry->sysname);
+        DTRACE4(dist_outputv, erts_this_node_sysname, port_str,
+                remote_str, size);
+    }
+#endif
     prt->caller = NIL;
     fpe_was_unmasked = erts_block_fpe();
     (*prt->drv_ptr->outputv)((ErlDrvData) prt->drv_data, &eiov);
@@ -1640,9 +1886,9 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf)
 }
 
 
-#if defined(ARCH_64)
+#if defined(ARCH_64) && !HALFWORD_HEAP
 #define ERTS_PORT_REDS_MASK__ 0x003fffffffffffffL
-#elif defined(ARCH_32)
+#elif defined(ARCH_32) || HALFWORD_HEAP
 #define ERTS_PORT_REDS_MASK__ 0x003fffff
 #else
 #  error "Ohh come on ... !?!"
@@ -1662,10 +1908,8 @@ erts_dist_command(Port *prt, int reds_limit)
 {
     Sint reds = ERTS_PORT_REDS_DIST_CMD_START;
     int prt_busy;
-    int de_busy;
     Uint32 status;
     Uint32 flags;
-    Uint32 qflgs;
     Sint obufsize = 0;
     ErtsDistOutputQueue oq, foq;
     DistEntry *dep = prt->dist_entry;
@@ -1676,7 +1920,7 @@ erts_dist_command(Port *prt, int reds_limit)
     erts_refc_inc(&dep->refc, 1); /* Otherwise dist_entry might be
 				     removed if port command fails */
 
-    erts_smp_atomic_xchg(&dep->dist_cmd_scheduled, 0);
+    erts_smp_atomic_set_mb(&dep->dist_cmd_scheduled, 0);
 
     erts_smp_de_rlock(dep);
     flags = dep->flags;
@@ -1700,13 +1944,12 @@ erts_dist_command(Port *prt, int reds_limit)
      * a mess.
      */
 
-    erts_smp_spin_lock(&dep->qlock);
+    erts_smp_mtx_lock(&dep->qlock);
     oq.first = dep->out_queue.first;
     oq.last = dep->out_queue.last;
     dep->out_queue.first = NULL;
     dep->out_queue.last = NULL;
-    qflgs = dep->qflgs;
-    erts_smp_spin_unlock(&dep->qlock);
+    erts_smp_mtx_unlock(&dep->qlock);
 
     foq.first = dep->finalized_out_queue.first;
     foq.last = dep->finalized_out_queue.last;
@@ -1717,17 +1960,8 @@ erts_dist_command(Port *prt, int reds_limit)
 	goto preempted;
 
     prt_busy = (int) (prt->status & ERTS_PORT_SFLG_PORT_BUSY);
-    de_busy = (int) (qflgs & ERTS_DE_QFLG_BUSY);
 
-    if (prt_busy) {
-	if (!de_busy) {
-	    erts_smp_spin_lock(&dep->qlock);
-	    dep->qflgs |= ERTS_DE_QFLG_BUSY;
-	    erts_smp_spin_unlock(&dep->qlock);
-	    de_busy = 1;
-	}
-    }
-    else if (foq.first) {
+    if (!prt_busy && foq.first) {
 	int preempt = 0;
 	do {
 	    Uint size;
@@ -1745,10 +1979,7 @@ erts_dist_command(Port *prt, int reds_limit)
 	    free_dist_obuf(fob);
 	    preempt = reds > reds_limit || (prt->status & ERTS_PORT_SFLGS_DEAD);
 	    if (prt->status & ERTS_PORT_SFLG_PORT_BUSY) {
-		erts_smp_spin_lock(&dep->qlock);
-		dep->qflgs |= ERTS_DE_QFLG_BUSY;
-		erts_smp_spin_unlock(&dep->qlock);
-		de_busy = prt_busy = 1;
+		prt_busy = 1;
 		break;
 	    }
 	} while (foq.first && !preempt);
@@ -1831,10 +2062,7 @@ erts_dist_command(Port *prt, int reds_limit)
 	    free_dist_obuf(fob);
 	    preempt = reds > reds_limit || (prt->status & ERTS_PORT_SFLGS_DEAD);
 	    if (prt->status & ERTS_PORT_SFLG_PORT_BUSY) {
-		erts_smp_spin_lock(&dep->qlock);
-		dep->qflgs |= ERTS_DE_QFLG_BUSY;
-		erts_smp_spin_unlock(&dep->qlock);
-		de_busy = prt_busy = 1;
+		prt_busy = 1;
 		if (oq.first && !preempt)
 		    goto finalize_only;
 	    }
@@ -1861,22 +2089,23 @@ erts_dist_command(Port *prt, int reds_limit)
 	 * dist entry in a non-busy state and resume suspended
 	 * processes.
 	 */
-	erts_smp_spin_lock(&dep->qlock);
+	erts_smp_mtx_lock(&dep->qlock);
 	ASSERT(dep->qsize >= obufsize);
 	dep->qsize -= obufsize;
 	obufsize = 0;
-	if (de_busy && !prt_busy && dep->qsize < ERTS_DE_BUSY_LIMIT) {
+	if (!prt_busy
+	    && (dep->qflgs & ERTS_DE_QFLG_BUSY)
+	    && dep->qsize < erts_dist_buf_busy_limit) {
 	    ErtsProcList *suspendees;
 	    int resumed;
 	    suspendees = get_suspended_on_de(dep, ERTS_DE_QFLG_BUSY);
-	    erts_smp_spin_unlock(&dep->qlock);
+	    erts_smp_mtx_unlock(&dep->qlock);
 
 	    resumed = erts_resume_processes(suspendees);
 	    reds += resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED;
-	    de_busy = 0;
 	}
 	else
-	    erts_smp_spin_unlock(&dep->qlock);
+	    erts_smp_mtx_unlock(&dep->qlock);
     }
 
     ASSERT(!oq.first && !oq.last);
@@ -1885,10 +2114,10 @@ erts_dist_command(Port *prt, int reds_limit)
 
     if (obufsize != 0) {
 	ASSERT(obufsize > 0);
-	erts_smp_spin_lock(&dep->qlock);
+	erts_smp_mtx_lock(&dep->qlock);
 	ASSERT(dep->qsize >= obufsize);
 	dep->qsize -= obufsize;
-	erts_smp_spin_unlock(&dep->qlock);
+	erts_smp_mtx_unlock(&dep->qlock);
     }
 
     ASSERT(foq.first || !foq.last);
@@ -1938,9 +2167,9 @@ erts_dist_command(Port *prt, int reds_limit)
 	foq.last = NULL;
 
 #ifdef DEBUG
-	erts_smp_spin_lock(&dep->qlock);
+	erts_smp_mtx_lock(&dep->qlock);
 	ASSERT(dep->qsize == obufsize);
-	erts_smp_spin_unlock(&dep->qlock);
+	erts_smp_mtx_unlock(&dep->qlock);
 #endif
     }
     else {
@@ -1949,14 +2178,14 @@ erts_dist_command(Port *prt, int reds_limit)
 	     * Unhandle buffers need to be put back first
 	     * in out_queue.
 	     */
-	    erts_smp_spin_lock(&dep->qlock);
+	    erts_smp_mtx_lock(&dep->qlock);
 	    dep->qsize -= obufsize;
 	    obufsize = 0;
 	    oq.last->next = dep->out_queue.first;
 	    dep->out_queue.first = oq.first;
 	    if (!dep->out_queue.last)
 		dep->out_queue.last = oq.last;
-	    erts_smp_spin_unlock(&dep->qlock);
+	    erts_smp_mtx_unlock(&dep->qlock);
 	}
 
 	erts_schedule_dist_command(prt, NULL);
@@ -1967,6 +2196,18 @@ erts_dist_command(Port *prt, int reds_limit)
 void
 erts_dist_port_not_busy(Port *prt)
 {
+#ifdef USE_VM_PROBES
+    if (DTRACE_ENABLED(dist_port_not_busy)) {
+        DTRACE_CHARBUF(port_str, 64);
+        DTRACE_CHARBUF(remote_str, 64);
+
+        erts_snprintf(port_str, sizeof(port_str), "%T", prt->id);
+        erts_snprintf(remote_str, sizeof(remote_str),
+                      "%T", prt->dist_entry->sysname);
+        DTRACE3(dist_port_not_busy, erts_this_node_sysname,
+                port_str, remote_str);
+    }
+#endif
     erts_schedule_dist_command(prt, NULL);
 }
 
@@ -1980,10 +2221,10 @@ erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id)
 
 	dep->status |= ERTS_DE_SFLG_EXITING;
 
-	erts_smp_spin_lock(&dep->qlock);
+	erts_smp_mtx_lock(&dep->qlock);
 	ASSERT(!(dep->qflgs & ERTS_DE_QFLG_EXIT));
 	dep->qflgs |= ERTS_DE_QFLG_EXIT;
-	erts_smp_spin_unlock(&dep->qlock);
+	erts_smp_mtx_unlock(&dep->qlock);
 
 	erts_schedule_dist_command(NULL, dep);
     }
@@ -2246,11 +2487,11 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2)
 #endif
 
     erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-    erts_smp_block_system(ERTS_BS_FLG_ALLOW_GC);
+    erts_smp_thr_progress_block();
     erts_set_this_node(BIF_ARG_1, (Uint32) creation);
     erts_is_alive = 1;
     send_nodes_mon_msgs(NULL, am_nodeup, BIF_ARG_1, am_visible, NIL);
-    erts_smp_release_system();
+    erts_smp_thr_progress_unblock();
     erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
     BIF_RET(am_true);
@@ -2354,13 +2595,13 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
 	ErtsProcList *plp = erts_proclist_create(BIF_P);
 	plp->next = NULL;
 	erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
-	erts_smp_spin_lock(&dep->qlock);
+	erts_smp_mtx_lock(&dep->qlock);
 	if (dep->suspended.last)
 	    dep->suspended.last->next = plp;
 	else
 	    dep->suspended.first = plp;
 	dep->suspended.last = plp;
-	erts_smp_spin_unlock(&dep->qlock);
+	erts_smp_mtx_unlock(&dep->qlock);
 	goto yield;
     }
 
@@ -2388,9 +2629,9 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3)
     ASSERT(dep->send);
 
 #ifdef DEBUG
-    erts_smp_spin_lock(&dep->qlock);
+    erts_smp_mtx_lock(&dep->qlock);
     ASSERT(dep->qsize == 0);
-    erts_smp_spin_unlock(&dep->qlock);
+    erts_smp_mtx_unlock(&dep->qlock);
 #endif
 
     erts_set_dist_entry_connected(dep, BIF_ARG_2, flags);
@@ -2547,12 +2788,15 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1)
     int visible = 0;
     int hidden = 0;
     int this = 0;
-    Uint  buf[2]; /* For one cons-cell */
+    DeclareTmpHeap(buf,2,BIF_P); /* For one cons-cell */
     DistEntry *dep;
     Eterm arg_list = BIF_ARG_1;
 #ifdef DEBUG
     Eterm* endp;
 #endif
+
+    UseTmpHeap(2,BIF_P);
+
     if (is_atom(BIF_ARG_1))
       arg_list = CONS(buf, BIF_ARG_1, NIL);
 
@@ -2563,13 +2807,14 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1)
       case am_known:     visible = hidden = not_connected = this = 1; break;
       case am_this:      this = 1;                                    break;
       case am_connected: visible = hidden = 1;                        break;
-      default:           BIF_ERROR(BIF_P, BADARG);                    break;
+      default:           goto error;                                  break;
       }
       arg_list = CDR(list_val(arg_list));
     }
 
-    if (is_not_nil(arg_list))
-      BIF_ERROR(BIF_P, BADARG);
+    if (is_not_nil(arg_list)) {
+	goto error;
+    }
 
     length = 0;
 
@@ -2591,7 +2836,7 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1)
 
     if (length == 0) {
 	erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
-	BIF_RET(result);
+	goto done;
     }
 
     hp = HAlloc(BIF_P, 2*length);
@@ -2620,7 +2865,14 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1)
     }
     ASSERT(endp == hp);
     erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
+
+done:
+    UnUseTmpHeap(2,BIF_P);
     BIF_RET(result);
+
+error:
+    UnUseTmpHeap(2,BIF_P);
+    BIF_ERROR(BIF_P,BADARG);
 }
 
 /**********************************************************************/
@@ -2635,85 +2887,92 @@ BIF_RETTYPE is_alive_0(BIF_ALIST_0)
 /**********************************************************************/
 /* erlang:monitor_node(Node, Bool, Options) -> Bool */
 
-BIF_RETTYPE monitor_node_3(BIF_ALIST_3)
+static BIF_RETTYPE
+monitor_node(Process* p, Eterm Node, Eterm Bool, Eterm Options)
 {
     DistEntry *dep;
     ErtsLink *lnk;
     Eterm l;
 
-    for (l = BIF_ARG_3; l != NIL && is_list(l); l = CDR(list_val(l))) {
+    for (l = Options; l != NIL && is_list(l); l = CDR(list_val(l))) {
 	Eterm t = CAR(list_val(l));
 	/* allow_passive_connect the only available option right now */
 	if (t != am_allow_passive_connect) {
-	    BIF_ERROR(BIF_P, BADARG);
+	    BIF_ERROR(p, BADARG);
 	}
     }
     if (l != NIL) {
-	BIF_ERROR(BIF_P, BADARG);
+	BIF_ERROR(p, BADARG);
     }
 
-    if (is_not_atom(BIF_ARG_1) ||
-	((BIF_ARG_2 != am_true) && (BIF_ARG_2 != am_false)) ||
+    if (is_not_atom(Node) ||
+	((Bool != am_true) && (Bool != am_false)) ||
 	((erts_this_node->sysname == am_Noname)
-	 && (BIF_ARG_1 != erts_this_node->sysname))) {
-	BIF_ERROR(BIF_P, BADARG);
+	 && (Node != erts_this_node->sysname))) {
+	BIF_ERROR(p, BADARG);
     }
-    dep = erts_sysname_to_connected_dist_entry(BIF_ARG_1);
+    dep = erts_sysname_to_connected_dist_entry(Node);
     if (!dep) {
     do_trap:
-	BIF_TRAP3(dmonitor_node_trap, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+	BIF_TRAP3(dmonitor_node_trap, p, Node, Bool, Options);
     }
     if (dep == erts_this_dist_entry)
 	goto done;
 
-    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_LINK);
+    erts_smp_proc_lock(p, ERTS_PROC_LOCK_LINK);
     erts_smp_de_rlock(dep);
     if (ERTS_DE_IS_NOT_CONNECTED(dep)) {
-	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
+	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK);
 	erts_smp_de_runlock(dep);
 	goto do_trap;
     }
     erts_smp_de_links_lock(dep);
     erts_smp_de_runlock(dep);
 
-    if (BIF_ARG_2 == am_true) {
+    if (Bool == am_true) {
 	ASSERT(dep->cid != NIL);
 	lnk = erts_add_or_lookup_link(&(dep->node_links), LINK_NODE, 
-				      BIF_P->id);
+				      p->id);
 	++ERTS_LINK_REFC(lnk);
-	lnk = erts_add_or_lookup_link(&(BIF_P->nlinks), LINK_NODE, BIF_ARG_1);
+	lnk = erts_add_or_lookup_link(&(p->nlinks), LINK_NODE, Node);
 	++ERTS_LINK_REFC(lnk);
     }
     else  {
-	lnk = erts_lookup_link(dep->node_links, BIF_P->id);
+	lnk = erts_lookup_link(dep->node_links, p->id);
 	if (lnk != NULL) {
 	    if ((--ERTS_LINK_REFC(lnk)) == 0) {
 		erts_destroy_link(erts_remove_link(&(dep->node_links), 
-						   BIF_P->id));
+						   p->id));
 	    }
 	}
-	lnk = erts_lookup_link(BIF_P->nlinks, BIF_ARG_1);
+	lnk = erts_lookup_link(p->nlinks, Node);
 	if (lnk != NULL) {
 	    if ((--ERTS_LINK_REFC(lnk)) == 0) {
-		erts_destroy_link(erts_remove_link(&(BIF_P->nlinks), 
-						   BIF_ARG_1));
+		erts_destroy_link(erts_remove_link(&(p->nlinks),
+						   Node));
 	    }
 	}
     }
 
     erts_smp_de_links_unlock(dep);
-    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_LINK);
+    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_LINK);
 
  done:
     erts_deref_dist_entry(dep);
     BIF_RET(am_true);
 }
 
+BIF_RETTYPE monitor_node_3(BIF_ALIST_3)
+{
+    BIF_RET(monitor_node(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3));
+}
+
+
 /* monitor_node(Node, Bool) -> Bool */
 
 BIF_RETTYPE monitor_node_2(BIF_ALIST_2)
 {
-    BIF_RET(monitor_node_3(BIF_P,BIF_ARG_1,BIF_ARG_2,NIL));
+    BIF_RET(monitor_node(BIF_P, BIF_ARG_1, BIF_ARG_2, NIL));
 }
 
 BIF_RETTYPE net_kernel_dflag_unicode_io_1(BIF_ALIST_1)
@@ -2869,7 +3128,11 @@ send_nodes_mon_msg(Process *rp,
     }
 
     ASSERT(hend == hp);
-    erts_queue_message(rp, rp_locksp, bp, msg, NIL);
+    erts_queue_message(rp, rp_locksp, bp, msg, NIL
+#ifdef USE_VM_PROBES
+		       , NIL
+#endif
+		       );
 }
 
 static void
@@ -2882,6 +3145,21 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas
     ASSERT(is_immed(what));
     ASSERT(is_immed(node));
     ASSERT(is_immed(type));
+#ifdef USE_VM_PROBES
+    if (DTRACE_ENABLED(dist_monitor)) {
+        DTRACE_CHARBUF(what_str, 12);
+        DTRACE_CHARBUF(node_str, 64);
+        DTRACE_CHARBUF(type_str, 12);
+        DTRACE_CHARBUF(reason_str, 64);
+
+        erts_snprintf(what_str, sizeof(what_str), "%T", what);
+        erts_snprintf(node_str, sizeof(node_str), "%T", node);
+        erts_snprintf(type_str, sizeof(type_str), "%T", type);
+        erts_snprintf(reason_str, sizeof(reason_str), "%T", reason);
+        DTRACE5(dist_monitor, erts_this_node_sysname,
+                what_str, node_str, type_str, reason_str);
+    }
+#endif
 
     ERTS_SMP_LC_ASSERT(!c_p
 		       || (erts_proc_lc_my_proc_locks(c_p)
diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h
index ea1abcaeed..845151c895 100644
--- a/erts/emulator/beam/dist.h
+++ b/erts/emulator/beam/dist.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -38,6 +38,7 @@
 #define DFLAG_UNICODE_IO          0x1000
 #define DFLAG_DIST_HDR_ATOM_CACHE 0x2000
 #define DFLAG_SMALL_ATOM_TAGS     0x4000
+#define DFLAGS_INTERNAL_TAGS      0x8000
 
 /* All flags that should be enabled when term_to_binary/1 is used. */
 #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES	\
@@ -51,7 +52,7 @@
 #define DOP_SEND		2
 #define DOP_EXIT		3
 #define DOP_UNLINK		4
-#define DOP_NODE_LINK		5
+/* Ancient DOP_NODE_LINK (5) was here, can be reused */
 #define DOP_REG_SEND		6
 #define DOP_GROUP_LEADER	7
 #define DOP_EXIT2		8
@@ -68,7 +69,6 @@
 /* distribution trap functions */
 extern Export* dsend2_trap;
 extern Export* dsend3_trap;
-/*extern Export* dsend_nosuspend_trap;*/
 extern Export* dlink_trap;
 extern Export* dunlink_trap;
 extern Export* dmonitor_node_trap;
@@ -99,7 +99,8 @@ typedef struct {
 #define ERTS_DE_IS_CONNECTED(DEP) \
   (!ERTS_DE_IS_NOT_CONNECTED((DEP)))
 
-
+#define ERTS_DE_BUSY_LIMIT (1024*1024)
+extern int erts_dist_buf_busy_limit;
 extern int erts_is_alive;
 
 /*
@@ -153,10 +154,10 @@ erts_dsig_prepare(ErtsDSigData *dsdp,
     }
     if (no_suspend) {
 	failure = ERTS_DSIG_PREP_CONNECTED;
-	erts_smp_spin_lock(&dep->qlock);
+	erts_smp_mtx_lock(&dep->qlock);
 	if (dep->qflgs & ERTS_DE_QFLG_BUSY)
 	    failure = ERTS_DSIG_PREP_WOULD_SUSPEND;
-	erts_smp_spin_unlock(&dep->qlock);
+	erts_smp_mtx_unlock(&dep->qlock);
 	if (failure == ERTS_DSIG_PREP_WOULD_SUSPEND)
 	    goto fail;
     }
@@ -202,7 +203,7 @@ void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry)
 	id = dep->cid;
     }
 
-    if (!erts_smp_atomic_xchg(&dep->dist_cmd_scheduled, 1)) {
+    if (!erts_smp_atomic_xchg_mb(&dep->dist_cmd_scheduled, 1)) {
 	(void) erts_port_task_schedule(id,
 				       &dep->dist_cmd,
 				       ERTS_PORT_TASK_DIST_CMD,
@@ -287,4 +288,5 @@ extern void erts_kill_dist_connection(DistEntry *dep, Uint32);
 
 extern Uint erts_dist_cache_size(void);
 
+
 #endif
diff --git a/erts/emulator/beam/dtrace-wrapper.h b/erts/emulator/beam/dtrace-wrapper.h
new file mode 100644
index 0000000000..6ec0c91e21
--- /dev/null
+++ b/erts/emulator/beam/dtrace-wrapper.h
@@ -0,0 +1,111 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2012.
+ * All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifndef __DTRACE_WRAPPER_H
+#define __DTRACE_WRAPPER_H
+
+#define DTRACE_TERM_BUF_SIZE 256
+
+/*
+ * Some varieties of SystemTap macros do not like statically-sized
+ * char[N] buffers.  (For example, CentOS 6's macros.)
+ * So, we'll play a game to humor them.
+ *
+ * The code necessary to play nice with CentOS 6's SystemTap looks
+ * stupid to a C programmer's eyes, so we hide the ugliness with this
+ * macro, which expands:
+ *
+ *    DTRACE_CHARBUF(proc_name, 64);
+ *
+ * to become:
+ *
+ *    char proc_name_BUFFER[64], *proc_name = proc_name_BUFFER;
+ */
+
+#define DTRACE_CHARBUF(name, size) \
+    char name##_BUFFER[size], *name = name##_BUFFER
+
+#define DTRACE_CHARBUF_NAME(name) name##_BUFFER
+
+#if defined(USE_DYNAMIC_TRACE) && defined(USE_VM_PROBES) 
+
+#include "erlang_dtrace.h"
+
+#define DTRACE_ENABLED(name)                         \
+    erlang_##name##_enabled()
+#define DTRACE0(name)                                \
+    erlang_##name()
+#define DTRACE1(name, a0)                            \
+    erlang_##name(a0)
+#define DTRACE2(name, a0, a1)                        \
+    erlang_##name((a0), (a1))
+#define DTRACE3(name, a0, a1, a2)                    \
+    erlang_##name((a0), (a1), (a2))
+#define DTRACE4(name, a0, a1, a2, a3)                \
+    erlang_##name((a0), (a1), (a2), (a3))
+#define DTRACE5(name, a0, a1, a2, a3, a4)            \
+    erlang_##name((a0), (a1), (a2), (a3), (a4))
+#define DTRACE6(name, a0, a1, a2, a3, a4, a5)        \
+    erlang_##name((a0), (a1), (a2), (a3), (a4), (a5))
+#define DTRACE7(name, a0, a1, a2, a3, a4, a5, a6)    \
+    erlang_##name((a0), (a1), (a2), (a3), (a4), (a5), (a6))
+#define DTRACE10(name, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) \
+    erlang_##name((a0), (a1), (a2), (a3), (a4), (a5), (a6), (a7), (a8), (a9))
+#define DTRACE11(name, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) \
+    erlang_##name((a0), (a1), (a2), (a3), (a4), (a5), (a6), (a7), (a8), (a9), (a10))
+
+#if defined(_SDT_PROBE) && !defined(STAP_PROBE11)
+/* SLF: This is Ubuntu 11-style SystemTap hackery */
+/* work arround for missing STAP macro */
+#define STAP_PROBE11(provider,name,arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10,arg11) \
+  _SDT_PROBE(provider, name, 11, \
+             (arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10,arg11))
+#define _SDT_ASM_OPERANDS_11(arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10,arg11) \
+  _SDT_ASM_OPERANDS_10(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9,arg10), \
+    _SDT_ARG(11, arg11)
+#endif
+
+#ifdef STAP_PROBE_ADDR
+/* SLF: This is CentOS 5-style SystemTap hackery */
+/* SystemTap compat mode cannot support 11 args. We'll ignore the 11th */
+#define STAP_PROBE11(provider,probe,parm1,parm2,parm3,parm4,parm5,parm6,parm7,parm8,parm9,parm10,parm11) \
+    STAP_PROBE10(provider,probe,(parm1),(parm2),(parm3),(parm4),(parm5),(parm6),(parm7),(parm8),(parm9),(parm10))
+#endif /* STAP_PROBE_ADDR */
+
+#else   /* USE_DYNAMIC_TRACE && USE_VM_PROBES */
+
+/* Render all macros to do nothing */
+#define DTRACE_ENABLED(name)                         0
+#define DTRACE0(name)                                do {} while (0)
+#define DTRACE1(name, a0)                            do {} while (0)
+#define DTRACE2(name, a0, a1)                        do {} while (0)
+#define DTRACE3(name, a0, a1, a2)                    do {} while (0)
+#define DTRACE4(name, a0, a1, a2, a3)                do {} while (0)
+#define DTRACE5(name, a0, a1, a2, a3, a4)            do {} while (0)
+#define DTRACE6(name, a0, a1, a2, a3, a4, a5)        do {} while (0)
+#define DTRACE7(name, a0, a1, a2, a3, a4, a5, a6)    do {} while (0)
+#define DTRACE10(name, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) \
+                                                     do {} while (0)
+#define DTRACE11(name, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) \
+                                                     do {} while (0)
+
+#endif  /* USE_DYNAMIC_TRACE && USE_VM_PROBES */
+
+#endif  /* __DTRACE_WRAPPER_H */
diff --git a/erts/emulator/beam/elib_malloc.c b/erts/emulator/beam/elib_malloc.c
deleted file mode 100644
index b18c48d8d6..0000000000
--- a/erts/emulator/beam/elib_malloc.c
+++ /dev/null
@@ -1,2334 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-
-/*
-** Description: Faster malloc().
-*/
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-
-#include "sys.h"
-
-#ifdef ENABLE_ELIB_MALLOC
-
-#undef THREAD_SAFE_ELIB_MALLOC
-#ifdef USE_THREADS
-#define THREAD_SAFE_ELIB_MALLOC 1
-#else
-#define THREAD_SAFE_ELIB_MALLOC 0
-#endif
-
-#include "erl_driver.h"
-#include "erl_threads.h"
-#include "elib_stat.h"
-#include <stdio.h>
-#include <stdlib.h>
-
-/* To avoid clobbering of names becaure of reclaim on VxWorks,
-   we undefine all possible malloc, calloc etc. */
-#undef malloc
-#undef calloc
-#undef free
-#undef realloc
-
-#define ELIB_INLINE         /* inline all possible functions */
-
-#ifndef ELIB_ALIGN
-#define ELIB_ALIGN             sizeof(double)
-#endif
-
-#ifndef ELIB_HEAP_SIZE
-#define ELIB_HEAP_SIZE         (64*1024)  /* Default 64K */
-#endif
-
-#ifndef ELIB_HEAP_INCREAMENT
-#define ELIB_HEAP_INCREAMENT   (32*1024)  /* Default 32K */
-#endif
-
-#ifndef ELIB_FAILURE
-#define ELIB_FAILURE    abort()
-#endif
-
-#undef ASSERT
-#ifdef DEBUG
-#define ASSERT(B) \
- ((void) ((B) ? 1 : (fprintf(stderr, "%s:%d: Assertion failed: %s\n", \
-			     __FILE__, __LINE__, #B), abort(), 0)))
-#else
-#define ASSERT(B) ((void) 1)
-#endif
-
-#ifndef USE_RECURSIVE_MALLOC_MUTEX
-#define USE_RECURSIVE_MALLOC_MUTEX 0
-#endif
-
-#if USE_RECURSIVE_MALLOC_MUTEX
-static erts_mtx_t malloc_mutex = ERTS_REC_MTX_INITER;
-#else /* #if USE_RECURSIVE_MALLOC_MUTEX */
-static erts_mtx_t malloc_mutex = ERTS_MTX_INITER;
-#if THREAD_SAFE_ELIB_MALLOC
-static erts_cnd_t  malloc_cond  = ERTS_CND_INITER;
-#endif
-#endif  /* #if USE_RECURSIVE_MALLOC_MUTEX */
-
-typedef unsigned long EWord;       /* Assume 32-bit in this implementation */
-typedef unsigned short EHalfWord;  /* Assume 16-bit in this implementation */
-typedef unsigned char EByte;       /* Assume 8-bit byte */
-
-
-#define elib_printf fprintf
-#define elib_putc   fputc
-
-
-#if defined(__STDC__) || defined(__WIN32__)
-#define CONCAT(x,y) x##y
-#else
-#define CONCAT(x,y) x/**/y
-#endif
-
-
-#ifdef ELIB_DEBUG
-#define ELIB_PREFIX(fun, args) CONCAT(elib__,fun) args
-#else
-#define ELIB_PREFIX(fun, args) CONCAT(elib_,fun) args
-#endif
-
-#if defined(__STDC__)
-void *ELIB_PREFIX(malloc, (size_t));
-void *ELIB_PREFIX(calloc, (size_t, size_t));
-void ELIB_PREFIX(cfree, (EWord *));
-void ELIB_PREFIX(free, (EWord *));
-void *ELIB_PREFIX(realloc, (EWord *, size_t));
-void* ELIB_PREFIX(memresize, (EWord *, int));
-void* ELIB_PREFIX(memalign, (int, int));
-void* ELIB_PREFIX(valloc, (int));
-void* ELIB_PREFIX(pvalloc, (int));
-int ELIB_PREFIX(memsize, (EWord *));
-/* Extern interfaces used by VxWorks */
-size_t elib_sizeof(void *);
-void elib_init(EWord *, EWord);
-void elib_force_init(EWord *, EWord);
-#endif
-
-#if defined(__STDC__)
-/* define prototypes for missing */
-void* memalign(size_t a, size_t s);
-void* pvalloc(size_t nb);
-void* memresize(void *p, int nb);
-int memsize(void *p);
-#endif
-
-/* bytes to pages */
-#define PAGES(x)      (((x)+page_size-1) / page_size)
-#define PAGE_ALIGN(p) ((char*)((((EWord)(p))+page_size-1)&~(page_size-1)))
-
-/* bytes to words */
-#define WORDS(x)      (((x)+sizeof(EWord)-1) / sizeof(EWord))
-
-/* Align an address */
-#define ALIGN(p)     ((EWord*)((((EWord)(p)+ELIB_ALIGN-1)&~(ELIB_ALIGN-1))))
-
-/* Calculate the size needed to keep alignment */
-
-#define ALIGN_BSZ(nb)  ((nb+sizeof(EWord)+ELIB_ALIGN-1) & ~(ELIB_ALIGN-1))
-
-#define ALIGN_WSZ(nb)  WORDS(ALIGN_BSZ(nb))
-
-#define ALIGN_SIZE(nb) (ALIGN_WSZ(nb) - 1)
-
-
-/* PARAMETERS */
-
-#if defined(ELIB_HEAP_SBRK)
-
-#undef PAGE_SIZE
-
-/* Get the system page size (NEED MORE DEFINES HERE) */
-#ifdef _SC_PAGESIZE
-#define PAGE_SIZE   sysconf(_SC_PAGESIZE)
-#elif defined(_MSC_VER)
-#  ifdef _M_ALPHA
-#    define PAGE_SIZE      0x2000
-#  else
-#    define PAGE_SIZE      0x1000
-#  endif
-#else
-#define PAGE_SIZE   getpagesize()
-#endif
-
-#define ELIB_EXPAND(need)  expand_sbrk(need)
-static FUNCTION(int, expand_sbrk, (EWord));
-
-#elif defined(ELIB_HEAP_FIXED)
-
-#define PAGE_SIZE 1024
-#define ELIB_EXPAND(need) -1
-static EWord fix_heap[WORDS(ELIB_HEAP_SIZE)];
-
-#elif defined(ELIB_HEAP_USER)
-
-#define PAGE_SIZE 1024
-#define ELIB_EXPAND(need) -1
-
-#else
-
-#error "ELIB HEAP TYPE NOT SET"
-
-#endif
-
-
-#define STAT_ALLOCED_BLOCK(SZ)		\
-do {					\
-    tot_allocated += (SZ);		\
-    if (max_allocated < tot_allocated)	\
-	max_allocated = tot_allocated;	\
-} while (0)
-
-#define STAT_FREED_BLOCK(SZ)		\
-do {					\
-    tot_allocated -= (SZ);		\
-} while (0)
-
-static int max_allocated = 0;
-static int tot_allocated = 0;
-static EWord* eheap;        /* Align heap start */
-static EWord* eheap_top;    /* Point to end of heap */
-EWord page_size = 0;        /* Set by elib_init */
-
-#if defined(ELIB_DEBUG) || defined(DEBUG)
-#define ALIGN_CHECK(a, p)						\
- do {									\
-    if ((EWord)(p) & (a-1)) {						\
-	elib_printf(stderr,						\
-		    "RUNTIME ERROR: bad alignment (0x%lx:%d:%d)\n",	\
-		    (unsigned long) (p), (int) a, __LINE__);			\
-	ELIB_FAILURE;							\
-    }									\
-  } while(0)
-#define ELIB_ALIGN_CHECK(p) ALIGN_CHECK(ELIB_ALIGN, p)
-#else
-#define ALIGN_CHECK(a, p)
-#define ELIB_ALIGN_CHECK(p)
-#endif
-
-#define DYNAMIC 32
-
-/*
-** Free block layout
-**   1 1          30
-**  +--------------------------+
-**  |F|P|        Size          |
-**  +--------------------------+
-**
-** Where F is the free bit
-**       P is the free above bit
-**       Size is messured in words and does not include the hdr word
-**
-** If block is on the free list the size is also stored last in the block.
-** 
-*/
-typedef struct _free_block FreeBlock;
-struct _free_block {
-    EWord hdr;
-    Uint flags;
-    FreeBlock* parent;
-    FreeBlock* left;
-    FreeBlock* right;
-    EWord v[1];
-};
-
-typedef struct _allocated_block {
-    EWord hdr;
-    EWord v[5];
-} AllocatedBlock;
-
-
-/*
- * Interface to tree routines.
- */
-typedef Uint Block_t;
-
-static Block_t*	get_free_block(Uint);
-static void link_free_block(Block_t *);
-static void unlink_free_block(Block_t *del);
-
-#define FREE_BIT       0x80000000
-#define FREE_ABOVE_BIT 0x40000000
-#define SIZE_MASK      0x3fffffff     /* 2^30 words = 2^32 bytes */
-
-/* Work on both FreeBlock and AllocatedBlock */
-#define SIZEOF(p)         ((p)->hdr & SIZE_MASK)
-#define IS_FREE(p)        (((p)->hdr & FREE_BIT) != 0)
-#define IS_FREE_ABOVE(p)  (((p)->hdr & FREE_ABOVE_BIT) != 0)
-
-/* Given that we have a free block above find its size */
-#define SIZEOF_ABOVE(p)    *(((EWord*) (p)) - 1)
-
-#define MIN_BLOCK_SIZE      (sizeof(FreeBlock)/sizeof(EWord))
-#define MIN_WORD_SIZE       (MIN_BLOCK_SIZE-1)
-#define MIN_BYTE_SIZE       (sizeof(FreeBlock)-sizeof(EWord))
-
-#define MIN_ALIGN_SIZE      ALIGN_SIZE(MIN_BYTE_SIZE)
-
-
-static AllocatedBlock* heap_head = 0;
-static AllocatedBlock* heap_tail = 0;
-static EWord eheap_size = 0;
-
-static int heap_locked;
-
-static int elib_need_init = 1;
-#if THREAD_SAFE_ELIB_MALLOC
-static int elib_is_initing = 0;
-#endif
-
-typedef FreeBlock RBTree_t;
-
-static RBTree_t* root = NULL;
-
-
-static FUNCTION(void, deallocate, (AllocatedBlock*, int));
-
-/*
- * Unlink a free block
- */
-
-#define mark_allocated(p, szp) do { \
-      (p)->hdr = ((p)->hdr & FREE_ABOVE_BIT) | (szp); \
-      (p)->v[szp] &= ~FREE_ABOVE_BIT; \
-   } while(0)
-
-#define mark_free(p, szp) do { \
-      (p)->hdr = FREE_BIT | (szp); \
-      ((FreeBlock *)p)->v[szp-sizeof(FreeBlock)/sizeof(EWord)+1] = (szp); \
-   } while(0)
-
-#if 0
-/* Help macros to log2 */
-#define LOG_1(x)  (((x) > 1) ? 1 : 0)
-#define LOG_2(x)  (((x) > 3) ? 2+LOG_1((x) >> 2) : LOG_1(x))
-#define LOG_4(x)  (((x) > 15) ? 4+LOG_2((x) >> 4) : LOG_2(x))
-#define LOG_8(x)  (((x) > 255) ? 8+LOG_4((x)>>8) : LOG_4(x))
-#define LOG_16(x) (((x) > 65535) ? 16+LOG_8((x)>>16) : LOG_8(x))
-
-#define log2(x)   LOG_16(x)
-#endif
-
-/*
- * Split a block to be allocated.
- * Mark block as ALLOCATED and clear
- * FREE_ABOVE_BIT on next block
- *
- * nw is SIZE aligned and szp is SIZE aligned + 1
- */
-static void
-split_block(FreeBlock* p, EWord nw, EWord szp)
-{
-    EWord szq;
-    FreeBlock* q;
-
-    szq = szp - nw;
-    /* Preserve FREE_ABOVE bit in p->hdr !!! */
-
-    if (szq >= MIN_ALIGN_SIZE+1) {
-	szq--;
-	p->hdr = (p->hdr & FREE_ABOVE_BIT) | nw;
-
-	q = (FreeBlock*) (((EWord*) p) + nw + 1);
-	mark_free(q, szq);
-	link_free_block((Block_t *) q);
-
-	q = (FreeBlock*) (((EWord*) q) + szq + 1);
-	q->hdr |= FREE_ABOVE_BIT;
-    }
-    else {
-	mark_allocated((AllocatedBlock*)p, szp);
-    }
-}
-
-/*
- * Find a free block
- */
-static FreeBlock*
-alloc_block(EWord nw)
-{
-    for (;;) {
-	FreeBlock* p = (FreeBlock *) get_free_block(nw);
-
-	if (p != NULL) {
-	    return p;
-	} else if (ELIB_EXPAND(nw+MIN_WORD_SIZE)) {
-	    return 0;
-	}
-    }
-}
-
-
-size_t elib_sizeof(void *p)
-{
-    AllocatedBlock* pp;
-
-    if (p != 0) {
-	pp = (AllocatedBlock*) (((char *)p)-1);
-	return SIZEOF(pp);
-    }
-    return 0;
-}
-
-static void locked_elib_init(EWord*, EWord);
-static void init_elib_malloc(EWord*, EWord);
-
-/*
-** Initialize the elib
-** The addr and sz is only used when compiled with EXPAND_ADDR 
-*/
-/* Not static, this is used by VxWorks */
-void elib_init(EWord* addr, EWord sz)
-{
-    if (!elib_need_init)
-	return;
-    erts_mtx_lock(&malloc_mutex);
-    locked_elib_init(addr, sz);
-    erts_mtx_unlock(&malloc_mutex);
-}
-
-static void locked_elib_init(EWord* addr, EWord sz)
-{
-    if (!elib_need_init)
-	return;
-
-#if THREAD_SAFE_ELIB_MALLOC
-
-#if !USE_RECURSIVE_MALLOC_MUTEX
-    {
-	static erts_tid_t initer_tid;
-
-	if(elib_is_initing) {
-
-	    if(erts_equal_tids(initer_tid, erts_thr_self()))
-		return;
-
-	    /* Wait until initializing thread is done with initialization */
-
-	    while(elib_need_init)
-		erts_cnd_wait(&malloc_cond, &malloc_mutex);
-
-	    return;
-	}
-	else {
-	    initer_tid = erts_thr_self();
-	    elib_is_initing = 1;
-	}
-    }
-#else
-    if(elib_is_initing)
-	return;
-    elib_is_initing = 1;
-#endif
-
-#endif /* #if THREAD_SAFE_ELIB_MALLOC */
-
-    /* Do the actual initialization of the malloc implementation */
-    init_elib_malloc(addr, sz);
-
-#if THREAD_SAFE_ELIB_MALLOC
-
-#if !USE_RECURSIVE_MALLOC_MUTEX
-    erts_mtx_unlock(&malloc_mutex);
-#endif
-
-    /* Recursive calls to malloc are allowed here... */
-    erts_mtx_set_forksafe(&malloc_mutex);
-
-#if !USE_RECURSIVE_MALLOC_MUTEX
-    erts_mtx_lock(&malloc_mutex);
-    elib_is_initing = 0;
-#endif
-
-#endif /* #if THREAD_SAFE_ELIB_MALLOC */
-
-    elib_need_init = 0;
-
-#if THREAD_SAFE_ELIB_MALLOC && !USE_RECURSIVE_MALLOC_MUTEX
-    erts_cnd_broadcast(&malloc_cond);
-#endif
-
-}
-
-static void init_elib_malloc(EWord* addr, EWord sz)
-{
-    int i;
-    FreeBlock* freep;
-    EWord tmp_sz;
-#ifdef ELIB_HEAP_SBRK
-    char* top;
-    EWord n;
-#endif
-
-    max_allocated = 0;
-    tot_allocated = 0;
-    root = NULL;
-
-    /* Get the page size (may involve system call!!!) */
-    page_size = PAGE_SIZE;
-
-#if defined(ELIB_HEAP_SBRK)
-    sz = PAGES(ELIB_HEAP_SIZE)*page_size;
-
-    if ((top = (char*) sbrk(0)) == (char*)-1) {
-	elib_printf(stderr, "could not initialize elib, sbrk(0)");
-	ELIB_FAILURE;
-    }
-    n = PAGE_ALIGN(top) - top;
-    if ((top = (char*) sbrk(n)) == (char*)-1) {
-	elib_printf(stderr, "could not initialize elib, sbrk(n)");
-	ELIB_FAILURE;
-    }
-    if ((eheap = (EWord*) sbrk(sz)) == (EWord*)-1) {
-	elib_printf(stderr, "could not initialize elib, sbrk(SIZE)");
-	ELIB_FAILURE;
-    }
-    sz = WORDS(ELIB_HEAP_SIZE);
-#elif defined(ELIB_HEAP_FIXED)
-    eheap = fix_heap;
-    sz = WORDS(ELIB_HEAP_SIZE);
-#elif defined(ELIB_HEAP_USER)
-    eheap = addr;
-    sz = WORDS(sz);
-#else
-    return -1;
-#endif
-    eheap_size = 0;
-
-    /* Make sure that the first word of the heap_head is aligned */
-    addr = ALIGN(eheap+1);
-    sz -= ((addr - 1) - eheap);      /* Subtract unusable size */
-    eheap_top = eheap = addr - 1;    /* Set new aligned heap start */
-
-    eheap_top[sz-1] = 0;	     /* Heap stop mark */
-
-    addr = eheap;
-    heap_head = (AllocatedBlock*) addr;
-    heap_head->hdr = MIN_ALIGN_SIZE;
-    for (i = 0; i < MIN_ALIGN_SIZE; i++)
-	heap_head->v[i] = 0;
-
-    addr += (MIN_ALIGN_SIZE+1);
-    freep = (FreeBlock*) addr;
-    tmp_sz = sz - (((MIN_ALIGN_SIZE+1) + MIN_BLOCK_SIZE) + 1 + 1);
-    mark_free(freep, tmp_sz);
-    link_free_block((Block_t *) freep);
-
-    /* No need to align heap tail */
-    heap_tail = (AllocatedBlock*) &eheap_top[sz-MIN_BLOCK_SIZE-1];
-    heap_tail->hdr = FREE_ABOVE_BIT | MIN_WORD_SIZE;
-    heap_tail->v[0] = 0;
-    heap_tail->v[1] = 0;
-    heap_tail->v[2] = 0;
-
-    eheap_top += sz;
-    eheap_size += sz;
-
-    heap_locked = 0;
-}
-
-#ifdef ELIB_HEAP_USER
-void elib_force_init(EWord* addr, EWord sz)
-{
-    elib_need_init = 1;
-    elib_init(addr,sz);
-}
-#endif
-
-#ifdef ELIB_HEAP_SBRK
-
-/*
-** need in number of words (should include head and tail words)
-*/
-static int expand_sbrk(EWord sz)
-{
-    EWord* p;
-    EWord  bytes = sz * sizeof(EWord);
-    EWord  size;
-    AllocatedBlock* tail;
-
-    if (bytes < ELIB_HEAP_SIZE)
-	size = PAGES(ELIB_HEAP_INCREAMENT)*page_size;
-    else
-	size = PAGES(bytes)*page_size;
-
-    if ((p = (EWord*) sbrk(size)) == ((EWord*) -1))
-	return -1;
-
-    if (p != eheap_top) {
-	elib_printf(stderr, "panic: sbrk moved\n");
-	ELIB_FAILURE;
-    }
-
-    sz = WORDS(size);
-
-    /* Set new endof heap marker and a new heap tail */
-    eheap_top[sz-1] = 0;
-
-    tail = (AllocatedBlock*) &eheap_top[sz-MIN_BLOCK_SIZE-1];
-    tail->hdr = FREE_ABOVE_BIT | MIN_WORD_SIZE;
-    tail->v[0] = 0;
-    tail->v[1] = 0;
-    tail->v[2] = 0;
-
-    /* Patch old tail with new appended size */
-    heap_tail->hdr = (heap_tail->hdr & FREE_ABOVE_BIT) |
-	(MIN_WORD_SIZE+1+(sz-MIN_BLOCK_SIZE-1));
-    deallocate(heap_tail, 0);
-
-    heap_tail = tail;
-
-    eheap_size += sz;
-    eheap_top += sz;
-
-    return 0;
-}
-
-#endif /* ELIB_HEAP_SBRK */
-
-
-/*
-** Scan heap and check for corrupted heap
-*/
-int elib_check_heap(void)
-{
-    AllocatedBlock* p = heap_head;
-    EWord sz;
-
-    if (heap_locked) {
-	elib_printf(stderr, "heap is locked no info avaiable\n");
-	return 0;
-    }
-
-    while((sz = SIZEOF(p)) != 0) {
-	if (IS_FREE(p)) {
-	    if (p->v[sz-1] != sz) {
-		elib_printf(stderr, "panic: heap corrupted\r\n"); 
-		ELIB_FAILURE;	
-	    }
-	    p = (AllocatedBlock*) (p->v + sz);
-	    if (!IS_FREE_ABOVE(p)) {
-		elib_printf(stderr, "panic: heap corrupted\r\n"); 
-		ELIB_FAILURE;	
-	    }
-	}
-	else
-	    p = (AllocatedBlock*) (p->v + sz);
-    }
-    return 1;
-}
-
-/*
-** Load the byte vector pointed to by v of length vsz
-** with a heap image
-** The scale is defined by vsz and the current heap size
-** free = 0, full = 255
-** 
-** 
-*/
-int elib_heap_map(EByte* v, int vsz)
-{
-    AllocatedBlock* p = heap_head;
-    EWord sz;
-    int gsz = eheap_size / vsz;  /* The granuality used */
-    int fsz = 0;
-    int usz = 0;
-
-    if (gsz == 0)
-	return -1;  /* too good reolution */
-
-    while((sz = SIZEOF(p)) != 0) {
-	if (IS_FREE(p)) {
-	    fsz += sz;
-	    if ((fsz + usz) > gsz) {
-		*v++ = (255*usz)/gsz;
-		fsz -= (gsz - usz);
-		usz = 0;
-		while(fsz >= gsz) {
-		    *v++ = 0;
-		    fsz -= gsz;
-		}
-	    }
-	}
-	else {
-	    usz += sz;
-	    if ((fsz + usz) > gsz) {
-		*v++ = 255 - (255*fsz)/gsz;
-		usz -= (gsz - fsz);
-		fsz = 0;
-		while(usz >= gsz) {
-		    *v++ = 255;
-		    usz -= gsz;
-		}
-	    }
-	}
-	p = (AllocatedBlock*) (p->v + sz);
-    }
-    return 0;
-}
-
-/*
-** Generate a histogram of free/allocated blocks
-** Count granuality of 10 gives
-** (0-10],(10-100],(100-1000],(1000-10000] ...
-** (0-2], (2-4], (4-8], (8-16], ....
-*/
-static int i_logb(EWord size, int base)
-{
-    int lg = 0;
-    while(size >= base) {
-	size /= base;
-	lg++;
-    }
-    return lg;
-}
-
-int elib_histo(EWord* vf, EWord* va, int vsz, int base)
-{
-    AllocatedBlock* p = heap_head;
-    EWord sz;
-    int i;
-    int linear;
-
-    if ((vsz <= 1) || (vf == 0 && va == 0))
-	return -1;
-
-    if (base < 0) {
-	linear = 1;
-	base = -base;
-    }
-    else
-	linear = 0;
-
-    if (base <= 1)
-	return -1;
-
-    if (vf != 0) {
-	for (i = 0; i < vsz; i++)
-	    vf[i] = 0;
-    }
-    if (va != 0) {
-	for (i = 0; i < vsz; i++)
-	    va[i] = 0;
-    }
-
-    while((sz = SIZEOF(p)) != 0) {
-	if (IS_FREE(p)) {
-	    if (vf != 0) {
-		int val;
-		if (linear)
-		    val = sz / base;
-		else
-		    val = i_logb(sz, base);
-		if (val >= vsz)
-		    vf[vsz-1]++;
-		else
-		    vf[val]++;
-	    }
-	}
-	else {
-	    if (va != 0) {
-		int val;
-		if (linear)
-		    val = sz / base;
-		else
-		    val = i_logb(sz, base);
-		if (val >= vsz)
-		    va[vsz-1]++;
-		else
-		    va[val]++;
-	    }
-	}
-	p = (AllocatedBlock*) (p->v + sz);
-    }
-    return 0;
-}
-
-/*
-** Fill the info structure with actual values
-** Total
-** Allocated
-** Free
-** maxMaxFree     
-*/
-void elib_stat(struct elib_stat* info)
-{
-    EWord blks = 0;
-    EWord sz_free = 0;
-    EWord sz_alloc = 0;
-    EWord sz_max_free = 0;
-    EWord sz_min_used = 0x7fffffff;
-    EWord sz;
-    EWord num_free = 0;
-    AllocatedBlock* p = heap_head;
-
-    info->mem_total = eheap_size;
-
-    p = (AllocatedBlock*) (p->v + SIZEOF(p));
-
-    while((sz = SIZEOF(p)) != 0) {
-	blks++;
-	if (IS_FREE(p)) {
-	    if (sz > sz_max_free)
-		sz_max_free = sz;
-	    sz_free += sz;
-	    ++num_free;
-	}
-	else {
-	    if (sz < sz_min_used)
-		sz_min_used = sz;
-	    sz_alloc += sz;
-	}
-	p = (AllocatedBlock*) (p->v + sz);
-    }
-    info->mem_blocks = blks;
-    info->free_blocks = num_free;
-    info->mem_alloc = sz_alloc;
-    info->mem_free = sz_free;
-    info->min_used = sz_min_used;
-    info->max_free = sz_max_free;
-    info->mem_max_alloc = max_allocated;
-    ASSERT(sz_alloc == tot_allocated);
-}
-
-/*
-** Dump the heap
-*/
-void elib_heap_dump(char* label)
-{
-    AllocatedBlock* p = heap_head;
-    EWord sz;
-
-    elib_printf(stderr, "HEAP DUMP (%s)\n", label);
-    if (!elib_check_heap())
-	return;
-    
-    while((sz = SIZEOF(p)) != 0) {
-	if (IS_FREE(p)) {
-	    elib_printf(stderr, "%p: FREE, size = %d\n", p, (int) sz);
-	}
-	else {
-	    elib_printf(stderr, "%p: USED, size = %d %s\n", p, (int) sz,
-		       IS_FREE_ABOVE(p)?"(FREE ABOVE)":"");
-	}
-	p = (AllocatedBlock*) (p->v + sz);
-    }
-}
-
-/*
-**  Scan heaps and count:
-**  free_size, allocated_size, max_free_block
-*/
-void elib_statistics(void* to)
-{
-    struct elib_stat info;
-    EWord frag;
-
-    if (!elib_check_heap())
-	return;
-
-    elib_stat(&info);
-
-    frag = 1000 - ((1000 * info.max_free) / info.mem_free);
-
-    elib_printf(to, "Heap Statistics: total(%d), blocks(%d), frag(%d.%d%%)\n", 
-	       info.mem_total, info.mem_blocks,
-	       (int) frag/10, (int) frag % 10);
-
-    elib_printf(to, "                 allocated(%d), free(%d), "
-		"free_blocks(%d)\n",
-	       info.mem_alloc, info.mem_free,info.free_blocks);
-    elib_printf(to, "                 max_free(%d),  min_used(%d)\n",
-	       info.max_free, info.min_used);
-}
-
-/*
-** Allocate a least nb bytes with alignment a
-** Algorithm:
-**    1) Try locate a block which match exacly among the by direct index.
-**    2) Try using a fix block of greater size
-**    3) Try locate a block by searching in lists where block sizes
-**       X may vary between 2^i < X <= 2^(i+1)
-**
-** Reset memory to zero if clear is true
-*/
-static AllocatedBlock* allocate(EWord nb, EWord a, int clear)
-{
-    FreeBlock* p;
-    EWord nw;
-
-    if (a == ELIB_ALIGN) {
-	/*
-	 * Common case: Called by malloc(), realloc(), calloc().
-	 */
-	nw = nb < MIN_BYTE_SIZE ? MIN_ALIGN_SIZE : ALIGN_SIZE(nb);
-
-	if ((p = alloc_block(nw)) == 0)
-	    return NULL;
-    } else {
-	/*
-	 * Special case: Called by memalign().
-	 */
-	EWord asz, szp, szq, tmpsz;
-	FreeBlock *q;
-
-	if ((p = alloc_block((1+MIN_ALIGN_SIZE)*sizeof(EWord)+a-1+nb)) == 0)
-	    return NULL;
-
-	asz = a - ((EWord) ((AllocatedBlock *)p)->v) % a;
-
-	if (asz != a) {
-	    /* Enforce the alignment requirement by cutting of a free
-	       block at the beginning of the block. */
-
-	    if (asz < (1+MIN_ALIGN_SIZE)*sizeof(EWord) && !IS_FREE_ABOVE(p)) {
-		/* Not enough room to cut of a free block;
-		   increase align size */
-		asz += (((1+MIN_ALIGN_SIZE)*sizeof(EWord) + a - 1)/a)*a;
-	    }
-
-	    szq = ALIGN_SIZE(asz - sizeof(EWord));
-	    szp = SIZEOF(p) - szq - 1;
-
-	    q = p;
-	    p = (FreeBlock*) (((EWord*) q) + szq + 1);
-	    p->hdr = FREE_ABOVE_BIT | FREE_BIT | szp;
-
-	    if (IS_FREE_ABOVE(q)) { /* This should not be possible I think,
-				       but just in case... */
-		tmpsz = SIZEOF_ABOVE(q) + 1;
-		szq += tmpsz;
-		q = (FreeBlock*) (((EWord*) q) - tmpsz);
-		unlink_free_block((Block_t *) q);
-		q->hdr = (q->hdr & FREE_ABOVE_BIT) | FREE_BIT | szq;
-	    }
-	    mark_free(q, szq);
-	    link_free_block((Block_t *) q);
-
-	} /* else already had the correct alignment */
- 
-	nw = nb < MIN_BYTE_SIZE ? MIN_ALIGN_SIZE : ALIGN_SIZE(nb);
-    }
-
-    split_block(p, nw, SIZEOF(p));
-
-    STAT_ALLOCED_BLOCK(SIZEOF(p));
-
-    if (clear) {
-	EWord* pp = ((AllocatedBlock*)p)->v;
-
-	while(nw--)
-	    *pp++ = 0;
-    }
-
-    return (AllocatedBlock*) p;
-}
-
-
-/*
-** Deallocate memory pointed to by p
-** 1. Merge with block above if this block is free
-** 2. Merge with block below if this block is free
-** Link the block to the correct free list
-**
-** p points to the block header!
-**
-*/
-static void deallocate(AllocatedBlock* p, int stat_count)
-{
-    FreeBlock* q;
-    EWord szq;
-    EWord szp;
-
-    szp = SIZEOF(p);
-
-    if (stat_count)
-	STAT_FREED_BLOCK(SIZEOF(p));
-
-    if (IS_FREE_ABOVE(p)) {
-	szq = SIZEOF_ABOVE(p);
-	q = (FreeBlock*) ( ((EWord*) p) - szq - 1);
-	unlink_free_block((Block_t *) q);
-
-	p = (AllocatedBlock*) q;
-	szp += (szq + 1);
-    }
-    q = (FreeBlock*) (p->v + szp);
-    if (IS_FREE(q)) {
-	szq = SIZEOF(q);
-	unlink_free_block((Block_t *) q);
-	szp += (szq + 1);
-    }
-    else
-	q->hdr |= FREE_ABOVE_BIT;
-
-    /* The block above p can NEVER be free !!! */
-    p->hdr = FREE_BIT | szp;
-    p->v[szp-1] = szp;
-
-    link_free_block((Block_t *) p);
-}
-
-/*
-** Reallocate memory
-** If preserve is true then data is moved if neccesary
-*/
-static AllocatedBlock* reallocate(AllocatedBlock* p, EWord nb, int preserve)
-{
-    EWord szp;
-    EWord szq;
-    EWord sz;
-    EWord nw;
-    FreeBlock* q;
-
-    if (nb < MIN_BYTE_SIZE)
-	nw = MIN_ALIGN_SIZE;
-    else
-	nw = ALIGN_SIZE(nb);
-
-    sz = szp = SIZEOF(p);
-
-    STAT_FREED_BLOCK(szp);
-
-    /* Merge with block below */
-    q = (FreeBlock*) (p->v + szp);
-    if (IS_FREE(q)) {
-	szq = SIZEOF(q);
-	unlink_free_block((Block_t *) q);
-	szp += (szq + 1);
-    }
-
-    if (nw <= szp) {
-	split_block((FreeBlock *) p, nw, szp);
-	STAT_ALLOCED_BLOCK(SIZEOF(p));
-	return p;
-    }
-    else {
-	EWord* dp = p->v;
-	AllocatedBlock* npp;
-
-	if (IS_FREE_ABOVE(p)) {
-	    szq = SIZEOF_ABOVE(p);
-	    if (szq + szp + 1 >= nw) {
-		q = (FreeBlock*) (((EWord*) p) - szq - 1);
-		unlink_free_block((Block_t * )q);
-		szp += (szq + 1);
-		p = (AllocatedBlock*) q;
-
-		if (preserve) {
-		    EWord* pp = p->v;
-		    while(sz--)
-			*pp++ = *dp++;
-		}
-		split_block((FreeBlock *) p, nw, szp);
-		STAT_ALLOCED_BLOCK(SIZEOF(p));
-		return p;
-	    }
-	}
-
-	/*
-	 * Update p so that allocate() and deallocate() works.
-	 * (Note that allocate() may call expand_sbrk(), which in
-	 * in turn calls deallocate().)
-	 */
-
-	p->hdr = (p->hdr & FREE_ABOVE_BIT) | szp;
-	p->v[szp] &= ~FREE_ABOVE_BIT;
-
-	npp = allocate(nb, ELIB_ALIGN, 0);
-	if(npp == NULL)
-	    return NULL;
-	if (preserve) {
-	    EWord* pp = npp->v;
-	    while(sz--)
-		*pp++ = *dp++;
-	}
-	deallocate(p, 0);
-	return npp;
-    }
-}
-
-/*
-** What malloc() and friends should do (and return) when the heap is
-** exhausted.  [sverkerw]
-*/
-static void* heap_exhausted(void)
-{
-    /* Choose behaviour */
-#if 0
-    /* Crash-and-burn --- leave a usable corpse (hopefully) */
-    abort();
-#endif    
-    /* The usual ANSI-compliant behaviour */
-    return NULL;
-}
-
-/*
-** Allocate size bytes of memory
-*/
-void* ELIB_PREFIX(malloc, (size_t nb))
-{
-    void *res;
-    AllocatedBlock* p;
-
-    erts_mtx_lock(&malloc_mutex);
-    if (elib_need_init)
-	locked_elib_init(NULL,(EWord)0);
-
-    if (nb == 0)
-	res = NULL;
-    else if ((p = allocate(nb, ELIB_ALIGN, 0)) != 0) {
-	ELIB_ALIGN_CHECK(p->v);
-	res = p->v;
-    }
-    else
-	res = heap_exhausted();
-
-    erts_mtx_unlock(&malloc_mutex);
-
-    return res;
-}
-
-
-void* ELIB_PREFIX(calloc, (size_t nelem, size_t size))
-{
-    void *res;
-    int nb;
-    AllocatedBlock* p;
-    
-    erts_mtx_lock(&malloc_mutex);
-    if (elib_need_init)
-	locked_elib_init(NULL,(EWord)0);
-
-    if ((nb = nelem * size) == 0)
-	res = NULL;
-    else if ((p = allocate(nb, ELIB_ALIGN, 1)) != 0) {
-	ELIB_ALIGN_CHECK(p->v);
-	res = p->v;
-    }
-    else
-	res = heap_exhausted();
-
-    erts_mtx_unlock(&malloc_mutex);
-
-    return res;
-}
-
-/*
-** Free memory allocated by malloc
-*/
-
-void ELIB_PREFIX(free, (EWord* p))
-{
-    erts_mtx_lock(&malloc_mutex);
-    if (elib_need_init)
-	locked_elib_init(NULL,(EWord)0);
-
-    if (p != 0)
-	deallocate((AllocatedBlock*)(p-1), 1);
-
-    erts_mtx_unlock(&malloc_mutex);
-}
-
-void ELIB_PREFIX(cfree, (EWord* p))
-{
-    ELIB_PREFIX(free, (p));
-}
-
-
-/*
-** Realloc the memory allocated in p to nb number of bytes
-**
-*/
-
-void* ELIB_PREFIX(realloc, (EWord* p, size_t nb))
-{
-    void *res = NULL;
-    AllocatedBlock* pp;
-
-    erts_mtx_lock(&malloc_mutex);
-    if (elib_need_init)
-	locked_elib_init(NULL,(EWord)0);
-
-    if (p != 0) {
-	pp = (AllocatedBlock*) (p-1);
-	if (nb > 0) {
-	    if ((pp = reallocate(pp, nb, 1)) != 0) {
-		ELIB_ALIGN_CHECK(pp->v);
-		res = pp->v;
-	    }
-	}
-	else
-	    deallocate(pp, 1);
-    }
-    else if (nb > 0) {
-	if ((pp = allocate(nb, ELIB_ALIGN, 0)) != 0) {
-	    ELIB_ALIGN_CHECK(pp->v);
-	    res = pp->v;
-	}
-	else
-	    res = heap_exhausted();
-    }
-
-    erts_mtx_unlock(&malloc_mutex);
-
-    return res;
-}
-
-/*
-** Resize the memory area pointed to by p with nb number of bytes
-*/
-void* ELIB_PREFIX(memresize, (EWord* p, int nb))
-{
-    void *res = NULL;
-    AllocatedBlock* pp;
-
-    erts_mtx_lock(&malloc_mutex);
-    if (elib_need_init)
-	locked_elib_init(NULL,(EWord)0);
-
-    if (p != 0) {
-	pp = (AllocatedBlock*) (p-1);
-	if (nb > 0) {
-	    if ((pp = reallocate(pp, nb, 0)) != 0) {
-		ELIB_ALIGN_CHECK(pp->v);
-		res = pp->v;
-	    }
-	}
-	else
-	    deallocate(pp, 1);
-    }
-    else if (nb > 0) {
-	if ((pp = allocate(nb, ELIB_ALIGN, 0)) != 0) {
-	    ELIB_ALIGN_CHECK(pp->v);
-	    res = pp->v;
-	}
-	else
-	    res = heap_exhausted();
-    }
-
-    erts_mtx_unlock(&malloc_mutex);
-
-    return res;
-}
-
-
-/* Create aligned memory a must be a power of 2 !!! */
-
-void* ELIB_PREFIX(memalign, (int a, int nb))
-{
-    void *res;
-    AllocatedBlock* p;
-
-    erts_mtx_lock(&malloc_mutex);
-    if (elib_need_init)
-	locked_elib_init(NULL,(EWord)0);
-
-    if (nb == 0 || a <= 0)
-	res = NULL;
-    else if ((p = allocate(nb, a, 0)) != 0) {
-	ALIGN_CHECK(a, p->v);
-	res = p->v;
-    }
-    else
-	res = heap_exhausted();
-
-    erts_mtx_unlock(&malloc_mutex);
-
-    return res;
-}
-
-void* ELIB_PREFIX(valloc, (int nb))
-{
-    return ELIB_PREFIX(memalign, (page_size, nb));
-}
-
-
-void* ELIB_PREFIX(pvalloc, (int nb))
-{
-    return ELIB_PREFIX(memalign, (page_size, PAGES(nb)*page_size));
-}
-/* Return memory size for pointer p in bytes */
-
-int ELIB_PREFIX(memsize, (p))
-EWord* p;
-{
-    return SIZEOF((AllocatedBlock*)(p-1))*4;
-}
-
-
-/*
-** --------------------------------------------------------------------------
-**   DEBUG LIBRARY
-** --------------------------------------------------------------------------
-*/
-
-#ifdef ELIB_DEBUG
-
-#define IN_HEAP(p)  (((p) >= (char*) eheap) && (p) < (char*) eheap_top)
-/*
-** ptr_to_block: return the pointer to heap block pointed into by ptr
-** Returns 0 if not pointing into a block
-*/
-
-static EWord* ptr_to_block(char* ptr)
-{
-    AllocatedBlock* p = heap_head;
-    EWord sz;
-
-    while((sz = SIZEOF(p)) != 0) {
-	if ((ptr >= (char*) p->v) && (ptr < (char*)(p->v+sz)))
-	    return p->v;
-	p = (AllocatedBlock*) (p->v + sz);
-    }
-    return 0;
-}
-
-/*
-** Validate a pointer
-** returns:
-**      0  - if points to start of a block
-**      1  - if points outsize heap
-**     -1  - if points inside block
-**
-*/
-static int check_pointer(char* ptr)
-{
-    if (IN_HEAP(ptr)) {
-	if (ptr_to_block(ptr) == 0)
-	    return 1;
-	return 0;
-    }
-    return -1;
-}
-
-/*
-** Validate a memory area
-** returns:
-**      0 - if area is included in a block
-**     -1 - if area overlap a heap block
-**      1 - if area is outside heap
-*/
-static int check_area(char* ptr, int n)
-{
-    if (IN_HEAP(ptr)) {
-	if (IN_HEAP(ptr+n-1)) {
-	    EWord* p1 = ptr_to_block(ptr);
-	    EWord* p2 = ptr_to_block(ptr+n-1);
-
-	    if (p1 == p2)
-		return (p1 == 0) ? -1 : 0;
-	    return -1;
-	}
-    }
-    else if (IN_HEAP(ptr+n-1))
-	return -1;
-    return 1;
-}
-
-/*
-** Check if a block write will overwrite heap block
-*/
-static void check_write(char* ptr, int n, char* file, int line, char* fun)
-{
-    if (check_area(ptr, n) == -1) {
-	elib_printf(stderr, "RUNTIME ERROR: %s heap overwrite\n", fun);
-	elib_printf(stderr, "File: %s Line: %d\n", file, line);
-	ELIB_FAILURE;
-    }
-}
-
-/*
-** Check if a pointer is an allocated object
-*/
-static void check_allocated_block(char* ptr, char* file, int line, char* fun)
-{
-    EWord* q;
-
-    if (!IN_HEAP(ptr) || ((q=ptr_to_block(ptr)) == 0) || (ptr != (char*) q)) {
-	elib_printf(stderr, "RUNTIME ERROR: %s non heap pointer\n", fun);
-	elib_printf(stderr, "File: %s Line: %d\n", file, line);
-	ELIB_FAILURE;
-    }
-
-    if (IS_FREE((AllocatedBlock*)(q-1))) {
-	elib_printf(stderr, "RUNTIME ERROR: %s free pointer\n", fun);
-	elib_printf(stderr, "File: %s Line: %d\n", file, line);
-	ELIB_FAILURE;
-    }
-
-}
-
-/*
-** --------------------------------------------------------------------------
-**  DEBUG VERSIONS (COMPILED WITH THE ELIB.H)
-** --------------------------------------------------------------------------
-*/
-
-void* elib_dbg_malloc(int n, char* file, int line)
-{
-    return elib__malloc(n);
-}
-
-void* elib_dbg_calloc(int n, int s, char* file, int line)
-{
-    return elib__calloc(n, s);
-}
-
-void* elib_dbg_realloc(EWord* p, int n, char* file, int line)
-{
-    if (p == 0)
-	return elib__malloc(n);
-    check_allocated_block(p, file, line, "elib_realloc");
-    return elib__realloc(p, n);
-}
-
-void elib_dbg_free(EWord* p, char* file, int line)
-{
-    if (p == 0)
-	return;
-    check_allocated_block(p, file, line, "elib_free");
-    elib__free(p);
-}
-
-void elib_dbg_cfree(EWord* p, char* file, int line)
-{
-    if (p == 0)
-	return;
-    check_allocated_block(p, file, line, "elib_free");
-    elib__cfree(p);
-}
-
-void* elib_dbg_memalign(int a, int n, char* file, int line)
-{
-    return elib__memalign(a, n);
-}
-
-void* elib_dbg_valloc(int n, char* file, int line)
-{
-    return elib__valloc(n);
-}
-
-void* elib_dbg_pvalloc(int n, char* file, int line)
-{
-    return elib__pvalloc(n);
-}
-
-void* elib_dbg_memresize(EWord* p, int n, char* file, int line)
-{
-    if (p == 0)
-	return elib__malloc(n);
-    check_allocated_block(p, file, line, "elib_memresize");
-    return elib__memresize(p, n);
-}
-
-int elib_dbg_memsize(void* p, char* file, int line)
-{
-    check_allocated_block(p, file, line, "elib_memsize");
-    return elib__memsize(p);
-}
-
-/*
-** --------------------------------------------------------------------------
-**  LINK TIME FUNCTIONS (NOT COMPILED CALLS)
-** --------------------------------------------------------------------------
-*/
-
-void* elib_malloc(int n)
-{
-    return elib_dbg_malloc(n, "", -1);
-}
-
-void* elib_calloc(int n, int s)
-{
-    return elib_dbg_calloc(n, s, "", -1);
-}
-
-void* elib_realloc(EWord* p, int n)
-{
-    return elib_dbg_realloc(p, n, "", -1);
-}
-
-void elib_free(EWord* p)
-{
-    elib_dbg_free(p, "", -1);
-}
-
-void elib_cfree(EWord* p)
-{
-    elib_dbg_cfree(p, "", -1);
-}
-
-void* elib_memalign(int a, int n)
-{
-    return elib_dbg_memalign(a, n, "", -1);
-}
-
-void* elib_valloc(int n)
-{
-    return elib_dbg_valloc(n, "", -1);
-}
-
-void* elib_pvalloc(int n)
-{
-    return elib_dbg_pvalloc(n, "", -1);
-}
-
-void* elib_memresize(EWord* p, int n)
-{
-    return elib_dbg_memresize(p, n, "", -1);
-}
-
-
-int elib_memsize(EWord* p)
-{
-    return elib_dbg_memsize(p, "", -1);
-}
-
-#endif /* ELIB_DEBUG */
-
-/*
-** --------------------------------------------------------------------------
-** Map c library functions to elib
-** --------------------------------------------------------------------------
-*/
-
-#if defined(ELIB_ALLOC_IS_CLIB)
-void* malloc(size_t nb)
-{
-    return elib_malloc(nb);
-}
-
-void* calloc(size_t nelem, size_t size)
-{
-    return elib_calloc(nelem, size);
-}
-
-
-void free(void *p)
-{
-    elib_free(p);
-}
-
-void cfree(void *p)
-{
-    elib_cfree(p);
-}
-
-void* realloc(void* p, size_t nb)
-{
-    return elib_realloc(p, nb);
-}
-
-
-void* memalign(size_t a, size_t s)
-{
-    return elib_memalign(a, s);
-}
-
-void* valloc(size_t nb)
-{
-    return elib_valloc(nb);
-}
-
-void* pvalloc(size_t nb)
-{
-    return elib_pvalloc(nb);
-}
-
-#if 0
-void* memresize(void* p, int nb)
-{
-    return elib_memresize(p, nb);
-}
-
-int memsize(void* p)
-{
-    return elib_memsize(p);
-}
-#endif
-#endif /* ELIB_ALLOC_IS_CLIB */
-
-#endif /* ENABLE_ELIB_MALLOC */
-
-void elib_ensure_initialized(void)
-{
-#ifdef ENABLE_ELIB_MALLOC
-#ifndef ELIB_DONT_INITIALIZE
-    elib_init(NULL, 0);
-#endif
-#endif
-}
-
-#ifdef ENABLE_ELIB_MALLOC
-/**
- ** A Slightly modified version of the "address order best fit" algorithm
- ** used in erl_bestfit_alloc.c. Comments refer to that implementation.
- **/
-
-/*
- * Description:	A combined "address order best fit"/"best fit" allocator
- *              based on a Red-Black (binary search) Tree. The search,
- *              insert, and delete operations are all O(log n) operations
- *              on a Red-Black Tree. In the "address order best fit" case
- *              n equals number of free blocks, and in the "best fit" case
- *              n equals number of distinct sizes of free blocks. Red-Black
- *              Trees are described in "Introduction to Algorithms", by
- *              Thomas H. Cormen, Charles E. Leiserson, and
- *              Ronald L. Riverest.
- *
- *              This module is a callback-module for erl_alloc_util.c
- *
- * Author: 	Rickard Green
- */
-
-#ifdef DEBUG
-#if 0
-#define HARD_DEBUG
-#endif
-#else
-#undef HARD_DEBUG
-#endif
-
-#define SZ_MASK			SIZE_MASK
-#define FLG_MASK		(~(SZ_MASK))
-
-#define BLK_SZ(B)  (*((Block_t *) (B)) & SZ_MASK)
-
-#define TREE_NODE_FLG		(((Uint) 1) << 0)
-#define RED_FLG			(((Uint) 1) << 1)
-#ifdef HARD_DEBUG
-#  define LEFT_VISITED_FLG	(((Uint) 1) << 2)
-#  define RIGHT_VISITED_FLG	(((Uint) 1) << 3)
-#endif
-
-#define IS_TREE_NODE(N)		(((RBTree_t *) (N))->flags & TREE_NODE_FLG)
-#define IS_LIST_ELEM(N)		(!IS_TREE_NODE(((RBTree_t *) (N))))
-
-#define SET_TREE_NODE(N)	(((RBTree_t *) (N))->flags |= TREE_NODE_FLG)
-#define SET_LIST_ELEM(N)	(((RBTree_t *) (N))->flags &= ~TREE_NODE_FLG)
-
-#define IS_RED(N)		(((RBTree_t *) (N)) \
-				 && ((RBTree_t *) (N))->flags & RED_FLG)
-#define IS_BLACK(N)		(!IS_RED(((RBTree_t *) (N))))
-
-#define SET_RED(N)		(((RBTree_t *) (N))->flags |= RED_FLG)
-#define SET_BLACK(N)		(((RBTree_t *) (N))->flags &= ~RED_FLG)
-
-#undef ASSERT
-#define ASSERT ASSERT_EXPR
-
-#if 1
-#define RBT_ASSERT	ASSERT
-#else
-#define RBT_ASSERT(x)
-#endif
-
-
-#ifdef HARD_DEBUG
-static RBTree_t * check_tree(Uint);
-#endif
-
-#ifdef ERTS_INLINE
-#  ifndef ERTS_CAN_INLINE
-#    define ERTS_CAN_INLINE 1
-#  endif
-#else
-#  if defined(__GNUC__)
-#    define ERTS_CAN_INLINE 1
-#    define ERTS_INLINE __inline__
-#  elif defined(__WIN32__)
-#    define ERTS_CAN_INLINE 1
-#    define ERTS_INLINE __inline
-#  else
-#    define ERTS_CAN_INLINE 0
-#    define ERTS_INLINE
-#  endif
-#endif
-
-/* Types... */
-#if 0
-typedef struct RBTree_t_ RBTree_t;
-
-struct RBTree_t_ {
-    Block_t hdr;
-    Uint flags;
-    RBTree_t *parent;
-    RBTree_t *left;
-    RBTree_t *right;
-};
-#endif
-
-#if 0
-typedef struct {
-    RBTree_t t;
-    RBTree_t *next;
-} RBTreeList_t;
-
-#define LIST_NEXT(N) (((RBTreeList_t *) (N))->next)
-#define LIST_PREV(N) (((RBTreeList_t *) (N))->t.parent)
-#endif
-
-#ifdef DEBUG
-
-/* Destroy all tree fields */
-#define DESTROY_TREE_NODE(N)						\
-  sys_memset((void *) (((Block_t *) (N)) + 1),				\
-	     0xff,							\
-	     (sizeof(RBTree_t) - sizeof(Block_t)))
-
-/* Destroy all tree and list fields */
-#define DESTROY_LIST_ELEM(N)						\
-  sys_memset((void *) (((Block_t *) (N)) + 1),				\
-	     0xff,							\
-	     (sizeof(RBTreeList_t) - sizeof(Block_t)))
-
-#else
-
-#define DESTROY_TREE_NODE(N)
-#define DESTROY_LIST_ELEM(N)
-
-#endif
-
-
-/*
- * Red-Black Tree operations needed
- */
-
-static ERTS_INLINE void
-left_rotate(RBTree_t **root, RBTree_t *x)
-{
-    RBTree_t *y = x->right;
-    x->right = y->left;
-    if (y->left)
-	y->left->parent = x;
-    y->parent = x->parent;
-    if (!y->parent) {
-	RBT_ASSERT(*root == x);
-	*root = y;
-    }
-    else if (x == x->parent->left)
-	x->parent->left = y;
-    else {
-	RBT_ASSERT(x == x->parent->right);
-	x->parent->right = y;
-    }
-    y->left = x;
-    x->parent = y;
-}
-
-static ERTS_INLINE void
-right_rotate(RBTree_t **root, RBTree_t *x)
-{
-    RBTree_t *y = x->left;
-    x->left = y->right;
-    if (y->right)
-	y->right->parent = x;
-    y->parent = x->parent;
-    if (!y->parent) {
-	RBT_ASSERT(*root == x);
-	*root = y;
-    }
-    else if (x == x->parent->right)
-	x->parent->right = y;
-    else {
-	RBT_ASSERT(x == x->parent->left);
-	x->parent->left = y;
-    }
-    y->right = x;
-    x->parent = y;
-}
-
-
-/*
- * Replace node x with node y
- * NOTE: block header of y is not changed
- */
-static ERTS_INLINE void
-replace(RBTree_t **root, RBTree_t *x, RBTree_t *y)
-{
-
-    if (!x->parent) {
-	RBT_ASSERT(*root == x);
-	*root = y;
-    }
-    else if (x == x->parent->left)
-	x->parent->left = y;
-    else {
-	RBT_ASSERT(x == x->parent->right);
-	x->parent->right = y;
-    }
-    if (x->left) {
-	RBT_ASSERT(x->left->parent == x);
-	x->left->parent = y;
-    }
-    if (x->right) {
-	RBT_ASSERT(x->right->parent == x);
-	x->right->parent = y;
-    }
-
-    y->flags	= x->flags;
-    y->parent	= x->parent;
-    y->right	= x->right;
-    y->left	= x->left;
-
-    DESTROY_TREE_NODE(x);
-
-}
-
-static void
-tree_insert_fixup(RBTree_t *blk)
-{
-    RBTree_t *x = blk, *y;
-
-    /*
-     * Rearrange the tree so that it satisfies the Red-Black Tree properties
-     */
-
-    RBT_ASSERT(x != root && IS_RED(x->parent));
-    do {
-
-	/*
-	 * x and its parent are both red. Move the red pair up the tree
-	 * until we get to the root or until we can separate them.
-	 */
-
-	RBT_ASSERT(IS_RED(x));
-	RBT_ASSERT(IS_BLACK(x->parent->parent));
-	RBT_ASSERT(x->parent->parent);
-
-	if (x->parent == x->parent->parent->left) {
-	    y = x->parent->parent->right;
-	    if (IS_RED(y)) {
-		SET_BLACK(y);
-		x = x->parent;
-		SET_BLACK(x);
-		x = x->parent;
-		SET_RED(x);
-	    }
-	    else {
-
-		if (x == x->parent->right) {
-		    x = x->parent;
-		    left_rotate(&root, x);
-		}
-
-		RBT_ASSERT(x == x->parent->parent->left->left);
-		RBT_ASSERT(IS_RED(x));
-		RBT_ASSERT(IS_RED(x->parent));
-		RBT_ASSERT(IS_BLACK(x->parent->parent));
-		RBT_ASSERT(IS_BLACK(y));
-
-		SET_BLACK(x->parent);
-		SET_RED(x->parent->parent);
-		right_rotate(&root, x->parent->parent);
-
-		RBT_ASSERT(x == x->parent->left);
-		RBT_ASSERT(IS_RED(x));
-		RBT_ASSERT(IS_RED(x->parent->right));
-		RBT_ASSERT(IS_BLACK(x->parent));
-		break;
-	    }
-	}
-	else {
-	    RBT_ASSERT(x->parent == x->parent->parent->right);
-	    y = x->parent->parent->left;
-	    if (IS_RED(y)) {
-		SET_BLACK(y);
-		x = x->parent;
-		SET_BLACK(x);
-		x = x->parent;
-		SET_RED(x);
-	    }
-	    else {
-
-		if (x == x->parent->left) {
-		    x = x->parent;
-		    right_rotate(&root, x);
-		}
-
-		RBT_ASSERT(x == x->parent->parent->right->right);
-		RBT_ASSERT(IS_RED(x));
-		RBT_ASSERT(IS_RED(x->parent));
-		RBT_ASSERT(IS_BLACK(x->parent->parent));
-		RBT_ASSERT(IS_BLACK(y));
-
-		SET_BLACK(x->parent);
-		SET_RED(x->parent->parent);
-		left_rotate(&root, x->parent->parent);
-
-		RBT_ASSERT(x == x->parent->right);
-		RBT_ASSERT(IS_RED(x));
-		RBT_ASSERT(IS_RED(x->parent->left));
-		RBT_ASSERT(IS_BLACK(x->parent));
-		break;
-	    }
-	}
-    } while (x != root && IS_RED(x->parent));
-
-    SET_BLACK(root);
-}
-
-static void
-unlink_free_block(Block_t *del)
-{
-    Uint spliced_is_black;
-    RBTree_t *x, *y, *z = (RBTree_t *) del;
-    RBTree_t null_x; /* null_x is used to get the fixup started when we
-			splice out a node without children. */
-
-    null_x.parent = NULL;
-
-#ifdef HARD_DEBUG
-    check_tree(0);
-#endif
-
-    /* Remove node from tree... */
-
-    /* Find node to splice out */
-    if (!z->left || !z->right)
-	y = z;
-    else
-	/* Set y to z:s successor */
-	for(y = z->right; y->left; y = y->left);
-    /* splice out y */
-    x = y->left ? y->left : y->right;
-    spliced_is_black = IS_BLACK(y);
-    if (x) {
-	x->parent = y->parent;
-    }
-    else if (!x && spliced_is_black) {
-	x = &null_x;
-	x->flags = 0;
-	SET_BLACK(x);
-	x->right = x->left = NULL;
-	x->parent = y->parent;
-	y->left = x;
-    }
-
-    if (!y->parent) {
-	RBT_ASSERT(root == y);
-	root = x;
-    }
-    else if (y == y->parent->left)
-	y->parent->left = x;
-    else {
-	RBT_ASSERT(y == y->parent->right);
-	y->parent->right = x;
-    }
-    if (y != z) {
-	/* We spliced out the successor of z; replace z by the successor */
-	replace(&root, z, y);
-    }
-
-    if (spliced_is_black) {
-	/* We removed a black node which makes the resulting tree
-	   violate the Red-Black Tree properties. Fixup tree... */
-
-	while (IS_BLACK(x) && x->parent) {
-
-	    /*
-	     * x has an "extra black" which we move up the tree
-	     * until we reach the root or until we can get rid of it.
-	     *
-	     * y is the sibbling of x
-	     */
-
-	    if (x == x->parent->left) {
-		y = x->parent->right;
-		RBT_ASSERT(y);
-		if (IS_RED(y)) {
-		    RBT_ASSERT(y->right);
-		    RBT_ASSERT(y->left);
-		    SET_BLACK(y);
-		    RBT_ASSERT(IS_BLACK(x->parent));
-		    SET_RED(x->parent);
-		    left_rotate(&root, x->parent);
-		    y = x->parent->right;
-		}
-		RBT_ASSERT(y);
-		RBT_ASSERT(IS_BLACK(y));
-		if (IS_BLACK(y->left) && IS_BLACK(y->right)) {
-		    SET_RED(y);
-		    x = x->parent;
-		}
-		else {
-		    if (IS_BLACK(y->right)) {
-			SET_BLACK(y->left);
-			SET_RED(y);
-			right_rotate(&root, y);
-			y = x->parent->right;
-		    }
-		    RBT_ASSERT(y);
-		    if (IS_RED(x->parent)) {
-
-			SET_BLACK(x->parent);
-			SET_RED(y);
-		    }
-		    RBT_ASSERT(y->right);
-		    SET_BLACK(y->right);
-		    left_rotate(&root, x->parent);
-		    x = root;
-		    break;
-		}
-	    }
-	    else {
-		RBT_ASSERT(x == x->parent->right);
-		y = x->parent->left;
-		RBT_ASSERT(y);
-		if (IS_RED(y)) {
-		    RBT_ASSERT(y->right);
-		    RBT_ASSERT(y->left);
-		    SET_BLACK(y);
-		    RBT_ASSERT(IS_BLACK(x->parent));
-		    SET_RED(x->parent);
-		    right_rotate(&root, x->parent);
-		    y = x->parent->left;
-		}
-		RBT_ASSERT(y);
-		RBT_ASSERT(IS_BLACK(y));
-		if (IS_BLACK(y->right) && IS_BLACK(y->left)) {
-		    SET_RED(y);
-		    x = x->parent;
-		}
-		else {
-		    if (IS_BLACK(y->left)) {
-			SET_BLACK(y->right);
-			SET_RED(y);
-			left_rotate(&root, y);
-			y = x->parent->left;
-		    }
-		    RBT_ASSERT(y);
-		    if (IS_RED(x->parent)) {
-			SET_BLACK(x->parent);
-			SET_RED(y);
-		    }
-		    RBT_ASSERT(y->left);
-		    SET_BLACK(y->left);
-		    right_rotate(&root, x->parent);
-		    x = root;
-		    break;
-		}
-	    }
-	}
-	SET_BLACK(x);
-
-	if (null_x.parent) {
-	    if (null_x.parent->left == &null_x)
-		null_x.parent->left = NULL;
-	    else {
-		RBT_ASSERT(null_x.parent->right == &null_x);
-		null_x.parent->right = NULL;
-	    }
-	    RBT_ASSERT(!null_x.left);
-	    RBT_ASSERT(!null_x.right);
-	}
-	else if (root == &null_x) {
-	    root = NULL;
-	    RBT_ASSERT(!null_x.left);
-	    RBT_ASSERT(!null_x.right);
-	}
-    }
-
-
-    DESTROY_TREE_NODE(del);
-
-#ifdef HARD_DEBUG
-    check_tree(0);
-#endif
-
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
- * "Address order best fit" specific callbacks.                              *
-\*                                                                           */
-
-static void
-link_free_block(Block_t *block)
-{
-    RBTree_t *blk = (RBTree_t *) block;
-    Uint blk_sz = BLK_SZ(blk);
-
-    blk->flags	= 0;
-    blk->left	= NULL;
-    blk->right	= NULL;
-
-    if (!root) {
-	blk->parent = NULL;
-	SET_BLACK(blk);
-	root = blk;
-    } else {
-	RBTree_t *x = root;
-	while (1) {
-	    Uint size;
-
-	    size = BLK_SZ(x);
-
-	    if (blk_sz < size || (blk_sz == size && blk < x)) {
-		if (!x->left) {
-		    blk->parent = x;
-		    x->left = blk;
-		    break;
-		}
-		x = x->left;
-	    }
-	    else {
-		if (!x->right) {
-		    blk->parent = x;
-		    x->right = blk;
-		    break;
-		}
-		x = x->right;
-	    }
-
-	}
-
-	/* Insert block into size tree */
-	RBT_ASSERT(blk->parent);
-
-	SET_RED(blk);
-	if (IS_RED(blk->parent)) {
-	    tree_insert_fixup(blk);
-	}
-    }
-
-#ifdef HARD_DEBUG
-    check_tree(0);
-#endif
-}
-
-
-static Block_t *
-get_free_block(Uint size)
-{
-    RBTree_t *x = root;
-    RBTree_t *blk = NULL;
-    Uint blk_sz;
-
-    while (x) {
-	blk_sz = BLK_SZ(x);
-	if (blk_sz < size) {
-	    x = x->right;
-	}
-	else {
-	    blk = x;
-	    x = x->left;
-	}
-    }
-    
-    if (!blk)
-	return NULL;
-
-#ifdef HARD_DEBUG
-    ASSERT(blk == check_tree(size));
-#endif
-
-    unlink_free_block((Block_t *) blk);
-
-    return (Block_t *) blk;
-}
-
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
- * Debug functions                                                           *
-\*                                                                           */
-
-
-#ifdef HARD_DEBUG
-
-#define IS_LEFT_VISITED(FB)	((FB)->flags & LEFT_VISITED_FLG)
-#define IS_RIGHT_VISITED(FB)	((FB)->flags & RIGHT_VISITED_FLG)
-
-#define SET_LEFT_VISITED(FB)	((FB)->flags |= LEFT_VISITED_FLG)
-#define SET_RIGHT_VISITED(FB)	((FB)->flags |= RIGHT_VISITED_FLG)
-
-#define UNSET_LEFT_VISITED(FB)	((FB)->flags &= ~LEFT_VISITED_FLG)
-#define UNSET_RIGHT_VISITED(FB)	((FB)->flags &= ~RIGHT_VISITED_FLG)
-
-
-#if 0
-#  define PRINT_TREE
-#else
-#  undef PRINT_TREE
-#endif
-
-#ifdef PRINT_TREE
-static void print_tree(void);
-#endif
-
-/*
- * Checks that the order between parent and children are correct,
- * and that the Red-Black Tree properies are satisfied. if size > 0,
- * check_tree() returns a node that satisfies "best fit" resp.
- * "address order best fit".
- *
- * The Red-Black Tree properies are:
- *   1. Every node is either red or black.
- *   2. Every leaf (NIL) is black.
- *   3. If a node is red, then both its children are black.
- *   4. Every simple path from a node to a descendant leaf
- *      contains the same number of black nodes.
- */
-
-static RBTree_t *
-check_tree(Uint size)
-{
-    RBTree_t *res = NULL;
-    Sint blacks;
-    Sint curr_blacks;
-    RBTree_t *x;
-
-#ifdef PRINT_TREE
-    print_tree();
-#endif
-
-    if (!root)
-	return res;
-
-    x = root;
-    ASSERT(IS_BLACK(x));
-    ASSERT(!x->parent);
-    curr_blacks = 1;
-    blacks = -1;
-
-    while (x) {
-	if (!IS_LEFT_VISITED(x)) {
-	    SET_LEFT_VISITED(x);
-	    if (x->left) {
-		x = x->left;
-		if (IS_BLACK(x))
-		    curr_blacks++;
-		continue;
-	    }
-	    else {
-		if (blacks < 0)
-		    blacks = curr_blacks;
-		ASSERT(blacks == curr_blacks);
-	    }
-	}
-
-	if (!IS_RIGHT_VISITED(x)) {
-	    SET_RIGHT_VISITED(x);
-	    if (x->right) {
-		x = x->right;
-		if (IS_BLACK(x))
-		    curr_blacks++;
-		continue;
-	    }
-	    else {
-		if (blacks < 0)
-		    blacks = curr_blacks;
-		ASSERT(blacks == curr_blacks);
-	    }
-	}
-
-
-	if (IS_RED(x)) {
-	    ASSERT(IS_BLACK(x->right));
-	    ASSERT(IS_BLACK(x->left));
-	}
-
-	ASSERT(x->parent || x == root);
-
-	if (x->left) {
-	    ASSERT(x->left->parent == x);
-	    ASSERT(BLK_SZ(x->left) < BLK_SZ(x)
-		   || (BLK_SZ(x->left) == BLK_SZ(x) && x->left < x));
-	}
-
-	if (x->right) {
-	    ASSERT(x->right->parent == x);
-	    ASSERT(BLK_SZ(x->right) > BLK_SZ(x)
-		   || (BLK_SZ(x->right) == BLK_SZ(x) && x->right > x));
-	}
-
-	if (size && BLK_SZ(x) >= size) {
-	    if (!res
-		|| BLK_SZ(x) < BLK_SZ(res)
-		|| (BLK_SZ(x) == BLK_SZ(res) && x < res))
-		res = x;
-	}
-
-	UNSET_LEFT_VISITED(x);
-	UNSET_RIGHT_VISITED(x);
-	if (IS_BLACK(x))
-	    curr_blacks--;
-	x = x->parent;
-
-    }
-
-    ASSERT(curr_blacks == 0);
-
-    UNSET_LEFT_VISITED(root);
-    UNSET_RIGHT_VISITED(root);
-
-    return res;
-
-}
-
-
-#ifdef PRINT_TREE
-#define INDENT_STEP 2
-
-#include <stdio.h>
-
-static void
-print_tree_aux(RBTree_t *x, int indent)
-{
-    int i;
-
-    if (!x) {
-	for (i = 0; i < indent; i++) {
-	    putc(' ', stderr);
-	}
-	fprintf(stderr, "BLACK: nil\r\n");
-    }
-    else {
-	print_tree_aux(x->right, indent + INDENT_STEP);
-	for (i = 0; i < indent; i++) {
-	    putc(' ', stderr);
-	}
-	fprintf(stderr, "%s: sz=%lu addr=0x%lx\r\n",
-		IS_BLACK(x) ? "BLACK" : "RED",
-		BLK_SZ(x),
-		(Uint) x);
-	print_tree_aux(x->left,  indent + INDENT_STEP);
-    }
-}
-
-
-static void
-print_tree(void)
-{
-    fprintf(stderr, " --- Size-Adress tree begin ---\r\n");
-    print_tree_aux(root, 0);
-    fprintf(stderr, " --- Size-Adress tree end ---\r\n");
-}
-
-#endif
-
-#endif
-
-#endif /* ENABLE_ELIB_MALLOC */
diff --git a/erts/emulator/beam/elib_stat.h b/erts/emulator/beam/elib_stat.h
deleted file mode 100644
index d8c7f31737..0000000000
--- a/erts/emulator/beam/elib_stat.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-
-/*
-** Interface to elib statistics
-**
-*/
-#ifndef __ELIB_STAT_H__
-#define __ELIB_STAT_H__
-
-struct elib_stat {
-    int mem_total;    /* Number of heap words  */
-    int mem_blocks;   /* Number of block */
-    int mem_alloc;    /* Number of words in use */
-    int mem_free;     /* Number of words free */
-    int min_used;     /* Size of the smallest block used */
-    int max_free;     /* Size of the largest free block */
-    int free_blocks;  /* Number of fragments in free list */
-    int mem_max_alloc;/* Max number of words in use */
-};
-
-EXTERN_FUNCTION(void, elib_statistics, (void*));
-EXTERN_FUNCTION(int,  elib_check_heap, (_VOID_));
-EXTERN_FUNCTION(void, elib_heap_dump, (char*));
-EXTERN_FUNCTION(void, elib_stat, (struct elib_stat*));
-EXTERN_FUNCTION(int,  elib_heap_map, (unsigned char*, int));
-EXTERN_FUNCTION(int,  elib_histo, (unsigned long*, unsigned long*, int, int));
-
-#endif
diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c
index e8b594bb47..570cc59be2 100644
--- a/erts/emulator/beam/erl_afit_alloc.c
+++ b/erts/emulator/beam/erl_afit_alloc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -43,9 +43,9 @@
 
 /* Prototypes of callback functions */
 static Block_t *	get_free_block		(Allctr_t *, Uint,
-						 Block_t *, Uint);
-static void		link_free_block		(Allctr_t *, Block_t *);
-static void		unlink_free_block	(Allctr_t *, Block_t *);
+						 Block_t *, Uint, Uint32);
+static void		link_free_block		(Allctr_t *, Block_t *, Uint32);
+static void		unlink_free_block	(Allctr_t *, Block_t *, Uint32);
 
 
 static Eterm		info_options		(Allctr_t *, char *, int *,
@@ -65,14 +65,20 @@ erts_afalc_start(AFAllctr_t *afallctr,
 		 AFAllctrInit_t *afinit,
 		 AllctrInit_t *init)
 {
-    AFAllctr_t nulled_state = {{0}};
-    /* {{0}} is used instead of {0}, in order to avoid (an incorrect) gcc
-       warning. gcc warns if {0} is used as initializer of a struct when
-       the first member is a struct (not if, for example, the third member
-       is a struct). */
+    struct {
+	int dummy;
+	AFAllctr_t allctr;
+    } zero = {0};
+    /* The struct with a dummy element first is used in order to avoid (an
+       incorrect) gcc warning. gcc warns if {0} is used as initializer of
+       a struct when the first member is a struct (not if, for example,
+       the third member is a struct). */
+
     Allctr_t *allctr = (Allctr_t *) afallctr;
 
-    sys_memcpy((void *) afallctr, (void *) &nulled_state, sizeof(AFAllctr_t));
+    sys_memcpy((void *) afallctr, (void *) &zero.allctr, sizeof(AFAllctr_t));
+
+    init->sbmbct = 0; /* Small mbc not supported by afit */
 
     allctr->mbc_header_size		= sizeof(Carrier_t);
     allctr->min_mbc_size		= MIN_MBC_SZ;
@@ -105,7 +111,8 @@ erts_afalc_start(AFAllctr_t *afallctr,
 }
 
 static Block_t *
-get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size)
+get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size,
+	       Uint32 flags)
 {
     AFAllctr_t *afallctr = (AFAllctr_t *) allctr;
 
@@ -123,7 +130,7 @@ get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size)
 }
 
 static void
-link_free_block(Allctr_t *allctr, Block_t *block)
+link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
 {
     AFFreeBlock_t *blk = (AFFreeBlock_t *) block;
     AFAllctr_t *afallctr = (AFAllctr_t *) allctr;
@@ -144,7 +151,7 @@ link_free_block(Allctr_t *allctr, Block_t *block)
 }
 
 static void
-unlink_free_block(Allctr_t *allctr, Block_t *block)
+unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
 {
     AFFreeBlock_t *blk = (AFFreeBlock_t *) block;
     AFAllctr_t *afallctr = (AFAllctr_t *) allctr;
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index b853ec0f01..6fce032f9d 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2002-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2002-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -38,11 +38,10 @@
 #include "erl_bits.h"
 #include "erl_instrument.h"
 #include "erl_mseg.h"
-#ifdef ELIB_ALLOC_IS_CLIB
-#include "erl_version.h"
-#endif
 #include "erl_monitors.h"
 #include "erl_bif_timer.h"
+#include "erl_cpu_topology.h"
+#include "erl_thr_queue.h"
 #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE)
 #include "erl_check_io.h"
 #endif
@@ -53,8 +52,18 @@
 #include "erl_bestfit_alloc.h"
 #define GET_ERL_AF_ALLOC_IMPL
 #include "erl_afit_alloc.h"
+#define GET_ERL_AOFF_ALLOC_IMPL
+#include "erl_ao_firstfit_alloc.h"
+
+
+#if ERTS_MAX_NO_OF_SCHEDULERS > ERTS_AU_MAX_PREF_ALLOC_INSTANCES
+#  error "Too many schedulers; cannot create that many pref alloc instances"
+#endif
+
+#define ERTS_ALC_FIX_TYPE_IX(T) \
+  (ERTS_ALC_T2N((T)) - ERTS_ALC_N_MIN_A_FIXED_SIZE)
 
-#define ERTS_ALC_DEFAULT_MAX_THR_PREF 16
+#define ERTS_ALC_DEFAULT_MAX_THR_PREF ERTS_MAX_NO_OF_SCHEDULERS
 
 #if defined(SMALL_MEMORY) || defined(PURIFY) || defined(VALGRIND)
 #define AU_ALLOC_DEFAULT_ENABLE(X)	0
@@ -64,8 +73,15 @@
 
 #ifdef DEBUG
 static Uint install_debug_functions(void);
+#if 0
+#define HARD_DEBUG
+#ifdef __GNUC__
+#warning "* * * * * * * * * * * * * *"
+#warning "* HARD DEBUG IS ENABLED!  *"
+#warning "* * * * * * * * * * * * * *"
+#endif
+#endif
 #endif
-extern void elib_ensure_initialized(void);
 
 ErtsAllocatorFunctions_t erts_allctrs[ERTS_ALC_A_MAX+1];
 ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1];
@@ -81,38 +97,66 @@ typedef union {
     char align_bfa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(BFAllctr_t))];
     AFAllctr_t afa;
     char align_afa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AFAllctr_t))];
+    AOFFAllctr_t aoffa;
+    char align_aoffa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AOFFAllctr_t))];
 } ErtsAllocatorState_t;
 
-static ErtsAllocatorState_t sl_alloc_state;
+static ErtsAllocatorState_t sbmbc_alloc_state;
 static ErtsAllocatorState_t std_alloc_state;
 static ErtsAllocatorState_t ll_alloc_state;
+#if HALFWORD_HEAP
+static ErtsAllocatorState_t sbmbc_low_alloc_state;
+static ErtsAllocatorState_t std_low_alloc_state;
+static ErtsAllocatorState_t ll_low_alloc_state;
+#endif
+static ErtsAllocatorState_t sl_alloc_state;
 static ErtsAllocatorState_t temp_alloc_state;
 static ErtsAllocatorState_t eheap_alloc_state;
 static ErtsAllocatorState_t binary_alloc_state;
 static ErtsAllocatorState_t ets_alloc_state;
 static ErtsAllocatorState_t driver_alloc_state;
+static ErtsAllocatorState_t fix_alloc_state;
 
-ErtsAlcType_t erts_fix_core_allocator_ix;
-#ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE
-static void *(*fix_core_allocator)(ErtsAlcType_t, void *, Uint);
-static void *fix_core_extra;
-static void *fix_core_alloc(Uint size)
+typedef struct {
+    erts_smp_atomic32_t refc;
+    int only_sz;
+    Uint req_sched;
+    Process *proc;
+    Eterm ref;
+    Eterm ref_heap[REF_THING_SIZE];
+    int allocs[ERTS_ALC_A_MAX-ERTS_ALC_A_MIN+1+2];
+} ErtsAllocInfoReq;
+
+#define ERTS_ALC_INFO_A_ALLOC_UTIL (ERTS_ALC_A_MAX + 1)
+#define ERTS_ALC_INFO_A_MSEG_ALLOC (ERTS_ALC_A_MAX + 2)
+#define ERTS_ALC_INFO_A_MAX ERTS_ALC_INFO_A_MSEG_ALLOC
+
+#if !HALFWORD_HEAP
+ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(aireq,
+				 ErtsAllocInfoReq,
+				 5,
+				 ERTS_ALC_T_AINFO_REQ)
+#else
+static ERTS_INLINE ErtsAllocInfoReq *
+aireq_alloc(void)
 {
-    void *res;
-    res = (*fix_core_allocator)(ERTS_ALC_T_UNDEF, fix_core_extra, size);
-    if (erts_mtrace_enabled)
-	erts_mtrace_crr_alloc(res,
-			      ERTS_ALC_A_FIXED_SIZE,
-			      erts_fix_core_allocator_ix,
-			      size);
-    return res;
+    return erts_alloc(ERTS_ALC_T_AINFO_REQ, sizeof(ErtsAllocInfoReq));
+}
+
+static ERTS_INLINE void
+aireq_free(ErtsAllocInfoReq *ptr)
+{
+    erts_free(ERTS_ALC_T_AINFO_REQ, ptr);
 }
 #endif
 
+ErtsAlcType_t erts_fix_core_allocator_ix;
+
 enum allctr_type {
     GOODFIT,
     BESTFIT,
-    AFIT
+    AFIT,
+    AOFIRSTFIT
 };
 
 struct au_init {
@@ -124,6 +168,7 @@ struct au_init {
 	GFAllctrInit_t	gf;
 	BFAllctrInit_t	bf;
 	AFAllctrInit_t	af;
+	AOFFAllctrInit_t aoff;
     } init;
     struct {
 	int mmbcs;
@@ -137,7 +182,8 @@ struct au_init {
     ERTS_DEFAULT_ALLCTR_INIT,		\
     ERTS_DEFAULT_GF_ALLCTR_INIT,	\
     ERTS_DEFAULT_BF_ALLCTR_INIT,	\
-    ERTS_DEFAULT_AF_ALLCTR_INIT		\
+    ERTS_DEFAULT_AF_ALLCTR_INIT,	\
+    ERTS_DEFAULT_AOFF_ALLCTR_INIT       \
 }
 
 typedef struct {
@@ -154,6 +200,7 @@ typedef struct {
 	char *mtrace;
 	char *nodename;
     } instr;
+    struct au_init sbmbc_alloc;
     struct au_init sl_alloc;
     struct au_init std_alloc;
     struct au_init ll_alloc;
@@ -162,6 +209,12 @@ typedef struct {
     struct au_init binary_alloc;
     struct au_init ets_alloc;
     struct au_init driver_alloc;
+    struct au_init fix_alloc;
+#if HALFWORD_HEAP
+    struct au_init sbmbc_low_alloc;
+    struct au_init std_low_alloc;
+    struct au_init ll_low_alloc;
+#endif
 } erts_alc_hndl_args_init_t;
 
 #define ERTS_AU_INIT__ {0, 0, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}}
@@ -173,6 +226,34 @@ do {									\
 } while (0)
 
 static void
+set_default_sbmbc_alloc_opts(struct au_init *ip)
+{
+    SET_DEFAULT_ALLOC_OPTS(ip);
+    ip->enable			= 0;
+    ip->thr_spec		= 0;
+    ip->atype			= BESTFIT;
+    ip->init.bf.ao		= 1;
+    ip->init.util.ramv		= 0;
+    ip->init.util.mmsbc		= 0;
+    ip->init.util.mmmbc		= 500;
+    ip->init.util.sbct		= ~((UWord) 0);
+    ip->init.util.name_prefix	= "sbmbc_";
+    ip->init.util.alloc_no	= ERTS_ALC_A_SBMBC;
+#ifndef SMALL_MEMORY
+    ip->init.util.mmbcs 	= 2*1024*1024; /* Main carrier size */
+#else
+    ip->init.util.mmbcs 	= 1*1024*1024; /* Main carrier size */
+#endif
+    ip->init.util.ts 		= ERTS_ALC_MTA_SBMBC;
+    ip->init.util.asbcst	= 0;
+    ip->init.util.rsbcst	= 0;
+    ip->init.util.rsbcmt	= 0;
+    ip->init.util.rmbcmt	= 0;
+    ip->init.util.sbmbct	= 0;
+    ip->init.util.sbmbcs	= 0;
+}
+
+static void
 set_default_sl_alloc_opts(struct au_init *ip)
 {
     SET_DEFAULT_ALLOC_OPTS(ip);
@@ -189,6 +270,11 @@ set_default_sl_alloc_opts(struct au_init *ip)
 #endif
     ip->init.util.ts 		= ERTS_ALC_MTA_SHORT_LIVED;
     ip->init.util.rsbcst	= 80;
+#if HALFWORD_HEAP
+    ip->init.util.force         = 1;
+    ip->init.util.low_mem       = 1;
+#endif
+
 }
 
 static void
@@ -220,7 +306,7 @@ set_default_ll_alloc_opts(struct au_init *ip)
     ip->init.util.ramv		= 0;
     ip->init.util.mmsbc		= 0;
     ip->init.util.mmmbc		= 0;
-    ip->init.util.sbct		= ~((Uint) 0);
+    ip->init.util.sbct		= ~((UWord) 0);
     ip->init.util.name_prefix	= "ll_";
     ip->init.util.alloc_no	= ERTS_ALC_A_LONG_LIVED;
 #ifndef SMALL_MEMORY
@@ -233,6 +319,8 @@ set_default_ll_alloc_opts(struct au_init *ip)
     ip->init.util.rsbcst	= 0;
     ip->init.util.rsbcmt	= 0;
     ip->init.util.rmbcmt	= 0;
+    ip->init.util.sbmbct	= 0;
+    ip->init.util.sbmbcs	= 0;
 }
 
 static void
@@ -252,6 +340,10 @@ set_default_temp_alloc_opts(struct au_init *ip)
     ip->init.util.ts 		= ERTS_ALC_MTA_TEMPORARY;
     ip->init.util.rsbcst	= 90;
     ip->init.util.rmbcmt	= 100;
+#if HALFWORD_HEAP
+    ip->init.util.force         = 1;
+    ip->init.util.low_mem       = 1;
+#endif
 }
 
 static void
@@ -271,6 +363,10 @@ set_default_eheap_alloc_opts(struct au_init *ip)
 #endif
     ip->init.util.ts 		= ERTS_ALC_MTA_EHEAP;
     ip->init.util.rsbcst	= 50;
+#if HALFWORD_HEAP
+    ip->init.util.force         = 1;
+    ip->init.util.low_mem       = 1;
+#endif
 }
 
 static void
@@ -326,46 +422,52 @@ set_default_driver_alloc_opts(struct au_init *ip)
     ip->init.util.ts 		= ERTS_ALC_MTA_DRIVER;
 }
 
+static void
+set_default_fix_alloc_opts(struct au_init *ip,
+			   size_t *fix_type_sizes)
+{
+    SET_DEFAULT_ALLOC_OPTS(ip);
+    ip->enable			= AU_ALLOC_DEFAULT_ENABLE(1);
+    ip->thr_spec		= 1;
+    ip->atype			= BESTFIT;
+    ip->init.bf.ao = 1;
+    ip->init.util.name_prefix	= "fix_";
+    ip->init.util.fix_type_size	= fix_type_sizes;
+    ip->init.util.alloc_no	= ERTS_ALC_A_FIXED_SIZE;
+#ifndef SMALL_MEMORY
+    ip->init.util.mmbcs 	= 128*1024; /* Main carrier size */
+#else
+    ip->init.util.mmbcs 	= 128*1024; /* Main carrier size */
+#endif
+    ip->init.util.ts 		= ERTS_ALC_MTA_FIXED_SIZE;
+}
+
 #ifdef ERTS_SMP
 
 static void
 adjust_tpref(struct au_init *ip, int no_sched)
 {
     if (ip->thr_spec) {
-	Uint allocs;
-	if (ip->thr_spec < 0) {/* User specified amount */
-	    allocs = abs(ip->thr_spec);
-	    if (allocs > no_sched)
-		allocs = no_sched;
-	}
-	else if (no_sched > ERTS_ALC_DEFAULT_MAX_THR_PREF)
-	    allocs = ERTS_ALC_DEFAULT_MAX_THR_PREF;
-	else 
-	    allocs = no_sched;
-	if (allocs <= 1)
-	    ip->thr_spec = 0;
-	else {
-	    ip->thr_spec = (int) allocs;
-	    ip->thr_spec *= -1; /* thread preferred */
-
-	    /* If default ... */
-
-	    /* ... shrink main multi-block carrier size */
-	    if (ip->default_.mmbcs)
-		ip->init.util.mmbcs /= ERTS_MIN(4, allocs);
-	    /* ... shrink largest multi-block carrier size */
-	    if (ip->default_.lmbcs)
-		ip->init.util.lmbcs /= ERTS_MIN(2, allocs);
-	    /* ... shrink smallest multi-block carrier size */
-	    if (ip->default_.smbcs)
-		ip->init.util.smbcs /= ERTS_MIN(4, allocs);
-	    /* ... and more than three allocators shrink
-	       max mseg multi-block carriers */
-	    if (ip->default_.mmmbc && allocs > 2) {
-		ip->init.util.mmmbc /= ERTS_MIN(4, allocs - 1);
-		if (ip->init.util.mmmbc < 3)
-		    ip->init.util.mmmbc = 3;
-	    }
+	ip->thr_spec = no_sched;
+	ip->thr_spec *= -1; /* thread preferred */
+
+	/* If default ... */
+
+	/* ... shrink main multi-block carrier size */
+	if (ip->default_.mmbcs)
+	    ip->init.util.mmbcs /= ERTS_MIN(4, no_sched);
+	/* ... shrink largest multi-block carrier size */
+	if (ip->default_.lmbcs)
+	    ip->init.util.lmbcs /= ERTS_MIN(2, no_sched);
+	/* ... shrink smallest multi-block carrier size */
+	if (ip->default_.smbcs)
+	    ip->init.util.smbcs /= ERTS_MIN(4, no_sched);
+	/* ... and more than three allocators shrink
+	   max mseg multi-block carriers */
+	if (ip->default_.mmmbc && no_sched > 2) {
+	    ip->init.util.mmmbc /= ERTS_MIN(4, no_sched - 1);
+	    if (ip->init.util.mmmbc < 3)
+		ip->init.util.mmmbc = 3;
 	}
     }
 }
@@ -375,7 +477,7 @@ adjust_tpref(struct au_init *ip, int no_sched)
 static void handle_args(int *, char **, erts_alc_hndl_args_init_t *);
 
 static void
-set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init);
+set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu);
 
 static void
 start_au_allocator(ErtsAlcType_t alctr_n,
@@ -389,13 +491,15 @@ refuse_af_strategy(struct au_init *init)
 	init->atype = GOODFIT;
 }
 
-static void init_thr_ix(int static_ixs);
+#ifdef HARD_DEBUG
+static void hdbg_init(void);
+#endif
 
 void
 erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 {
-    Uint extra_block_size = 0;
-    int i;
+    UWord extra_block_size = 0;
+    int i, ncpu;
     erts_alc_hndl_args_init_t init = {
 	0,
 #if HAVE_ERTS_MSEG
@@ -403,13 +507,41 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 #endif
 	ERTS_DEFAULT_TRIM_THRESHOLD,
 	ERTS_DEFAULT_TOP_PAD,
-	ERTS_DEFAULT_ALCU_INIT
+	ERTS_DEFAULT_ALCU_INIT,
     };
+    size_t fix_type_sizes[ERTS_ALC_NO_FIXED_SIZES] = {0};
+
+    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_PROC)]
+	= sizeof(Process);
+#if !HALFWORD_HEAP
+    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR_SH)]
+	= ERTS_MONITOR_SH_SIZE;
+    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NLINK_SH)]
+	= ERTS_LINK_SH_SIZE;
+#endif
+    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_EV_D_STATE)]
+	= sizeof(ErtsDrvEventDataState);
+    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_SEL_D_STATE)]
+	= sizeof(ErtsDrvSelectDataState);
+    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MSG_REF)]
+	= sizeof(ErlMessage);
+#ifdef ERTS_SMP
+    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_THR_Q_EL_SL)]
+	= sizeof(ErtsThrQElement_t);
+#endif
+#ifdef HARD_DEBUG
+    hdbg_init();
+#endif
+
+    erts_have_sbmbc_alloc = 0;
+    ncpu = eaiop->ncpu;
+    if (ncpu < 1)
+	ncpu = 1;
 
     erts_sys_alloc_init();
-    init_thr_ix(erts_no_schedulers);
     erts_init_utils_mem();
 
+    set_default_sbmbc_alloc_opts(&init.sbmbc_alloc);
     set_default_sl_alloc_opts(&init.sl_alloc);
     set_default_std_alloc_opts(&init.std_alloc);
     set_default_ll_alloc_opts(&init.ll_alloc);
@@ -418,22 +550,27 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     set_default_binary_alloc_opts(&init.binary_alloc);
     set_default_ets_alloc_opts(&init.ets_alloc);
     set_default_driver_alloc_opts(&init.driver_alloc);
+    set_default_fix_alloc_opts(&init.fix_alloc,
+			       fix_type_sizes);
 
     if (argc && argv)
 	handle_args(argc, argv, &init);
 
-    if (erts_no_schedulers <= 1) {
-	init.sl_alloc.thr_spec = 0;
-	init.std_alloc.thr_spec = 0;
-	init.ll_alloc.thr_spec = 0;
-	init.eheap_alloc.thr_spec = 0;
-	init.binary_alloc.thr_spec = 0;
-	init.ets_alloc.thr_spec = 0;
-	init.driver_alloc.thr_spec = 0;
-    }
+#ifndef ERTS_SMP
+    init.sbmbc_alloc.thr_spec = 0;
+    init.sl_alloc.thr_spec = 0;
+    init.std_alloc.thr_spec = 0;
+    init.ll_alloc.thr_spec = 0;
+    init.eheap_alloc.thr_spec = 0;
+    init.binary_alloc.thr_spec = 0;
+    init.ets_alloc.thr_spec = 0;
+    init.driver_alloc.thr_spec = 0;
+    init.fix_alloc.thr_spec = 0;
+#endif
 
     if (init.erts_alloc_config) {
 	/* Adjust flags that erts_alloc_config won't like */
+	init.sbmbc_alloc.thr_spec = 0;
 	init.temp_alloc.thr_spec = 0;
 	init.sl_alloc.thr_spec = 0;
 	init.std_alloc.thr_spec = 0;
@@ -442,6 +579,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	init.binary_alloc.thr_spec = 0;
 	init.ets_alloc.thr_spec = 0;
 	init.driver_alloc.thr_spec = 0;
+	init.fix_alloc.thr_spec = 0;	
     }
 
 #ifdef ERTS_SMP
@@ -450,6 +588,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	init.temp_alloc.thr_spec = erts_no_schedulers;
 
     /* Others must use thread preferred interface */
+    adjust_tpref(&init.sbmbc_alloc, erts_no_schedulers);
     adjust_tpref(&init.sl_alloc, erts_no_schedulers);
     adjust_tpref(&init.std_alloc, erts_no_schedulers);
     adjust_tpref(&init.ll_alloc, erts_no_schedulers);
@@ -457,6 +596,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     adjust_tpref(&init.binary_alloc, erts_no_schedulers);
     adjust_tpref(&init.ets_alloc, erts_no_schedulers);
     adjust_tpref(&init.driver_alloc, erts_no_schedulers);
+    adjust_tpref(&init.fix_alloc, erts_no_schedulers);
 
 #else
     /* No thread specific if not smp */
@@ -467,6 +607,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
      * The following allocators cannot be run with afit strategy.
      * Make sure they don't...
      */
+    refuse_af_strategy(&init.sbmbc_alloc);
     refuse_af_strategy(&init.sl_alloc);
     refuse_af_strategy(&init.std_alloc);
     refuse_af_strategy(&init.ll_alloc);
@@ -474,6 +615,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     refuse_af_strategy(&init.binary_alloc);
     refuse_af_strategy(&init.ets_alloc);
     refuse_af_strategy(&init.driver_alloc);
+    refuse_af_strategy(&init.fix_alloc);
 
 #ifdef ERTS_SMP 
     if (!init.temp_alloc.thr_spec)
@@ -482,12 +624,14 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 
     erts_mtrace_pre_init();
 #if HAVE_ERTS_MSEG
+    init.mseg.nos = erts_no_schedulers;
     erts_mseg_init(&init.mseg);
 #endif
     erts_alcu_init(&init.alloc_util);
     erts_afalc_init();
     erts_bfalc_init();
     erts_gfalc_init();
+    erts_aoffalc_init();
 
     for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
 	erts_allctrs[i].alloc		= NULL;
@@ -500,33 +644,45 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	erts_allctrs_info[i].extra	= NULL;
     }
 
-#ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE
-#if !defined(PURIFY) && !defined(VALGRIND)
-    erts_allctrs[ERTS_ALC_A_FIXED_SIZE].alloc		= erts_fix_alloc;
-    erts_allctrs[ERTS_ALC_A_FIXED_SIZE].realloc		= erts_fix_realloc;
-    erts_allctrs[ERTS_ALC_A_FIXED_SIZE].free		= erts_fix_free;
-    erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].enabled	= 1;
-#else
-    erts_allctrs[ERTS_ALC_A_FIXED_SIZE].alloc		= erts_sys_alloc;
-    erts_allctrs[ERTS_ALC_A_FIXED_SIZE].realloc		= erts_sys_realloc;
-    erts_allctrs[ERTS_ALC_A_FIXED_SIZE].free		= erts_sys_free;
-    erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].enabled	= 0;
-#endif
-#endif
-
     erts_allctrs[ERTS_ALC_A_SYSTEM].alloc		= erts_sys_alloc;
     erts_allctrs[ERTS_ALC_A_SYSTEM].realloc		= erts_sys_realloc;
     erts_allctrs[ERTS_ALC_A_SYSTEM].free		= erts_sys_free;
     erts_allctrs_info[ERTS_ALC_A_SYSTEM].enabled	= 1;
 
-    set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc);
-    set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc);
-    set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc);
-    set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc);
-    set_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc);
-    set_au_allocator(ERTS_ALC_A_BINARY, &init.binary_alloc);
-    set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc);
-    set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc);
+#if HALFWORD_HEAP
+    /* Init low memory variants by cloning */
+    init.sbmbc_low_alloc = init.sbmbc_alloc;
+    init.sbmbc_low_alloc.init.util.name_prefix = "sbmbc_low_";
+    init.sbmbc_low_alloc.init.util.alloc_no = ERTS_ALC_A_SBMBC_LOW;
+    init.sbmbc_low_alloc.init.util.low_mem = 1;
+
+    init.std_low_alloc = init.std_alloc;
+    init.std_low_alloc.init.util.name_prefix	= "std_low_";
+    init.std_low_alloc.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW;
+    init.std_low_alloc.init.util.force = 1;
+    init.std_low_alloc.init.util.low_mem = 1;
+
+    init.ll_low_alloc = init.ll_alloc;
+    init.ll_low_alloc.init.util.name_prefix	= "ll_low_";
+    init.ll_low_alloc.init.util.alloc_no = ERTS_ALC_A_LONG_LIVED_LOW;
+    init.ll_low_alloc.init.util.force = 1;
+    init.ll_low_alloc.init.util.low_mem = 1;
+
+    set_au_allocator(ERTS_ALC_A_SBMBC_LOW, &init.sbmbc_low_alloc, ncpu);
+    set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc, ncpu);
+    set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc, ncpu);
+#endif /* HALFWORD */
+
+    set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, ncpu);
+    set_au_allocator(ERTS_ALC_A_SBMBC, &init.sbmbc_alloc, ncpu);
+    set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc, ncpu);
+    set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc, ncpu);
+    set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc, ncpu);
+    set_au_allocator(ERTS_ALC_A_EHEAP, &init.eheap_alloc, ncpu);
+    set_au_allocator(ERTS_ALC_A_BINARY, &init.binary_alloc, ncpu);
+    set_au_allocator(ERTS_ALC_A_ETS, &init.ets_alloc, ncpu);
+    set_au_allocator(ERTS_ALC_A_DRIVER, &init.driver_alloc, ncpu);
+    set_au_allocator(ERTS_ALC_A_FIXED_SIZE, &init.fix_alloc, ncpu);
 
     for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
 	if (!erts_allctrs[i].alloc)
@@ -543,13 +699,22 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     sys_alloc_opt(SYS_ALLOC_OPT_TRIM_THRESHOLD, init.trim_threshold);
     sys_alloc_opt(SYS_ALLOC_OPT_TOP_PAD, init.top_pad);
 
-    if (erts_allctrs_info[ERTS_FIX_CORE_ALLOCATOR].enabled)
-	erts_fix_core_allocator_ix = ERTS_FIX_CORE_ALLOCATOR;
-    else
-	erts_fix_core_allocator_ix = ERTS_ALC_A_SYSTEM;
-
     erts_mtrace_init(init.instr.mtrace, init.instr.nodename);
 
+    /* sbmbc_alloc() needs to be started first */
+    start_au_allocator(ERTS_ALC_A_SBMBC,
+		       &init.sbmbc_alloc,
+		       &sbmbc_alloc_state);
+#if HALFWORD_HEAP
+    start_au_allocator(ERTS_ALC_A_SBMBC_LOW,
+		       &init.sbmbc_low_alloc,
+		       &sbmbc_low_alloc_state);
+    erts_have_sbmbc_alloc = (init.sbmbc_alloc.enable
+			     && init.sbmbc_low_alloc.enable);
+#else
+    erts_have_sbmbc_alloc = init.sbmbc_alloc.enable;
+#endif
+
     start_au_allocator(ERTS_ALC_A_TEMPORARY,
 		       &init.temp_alloc,
 		       &temp_alloc_state);
@@ -565,7 +730,14 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     start_au_allocator(ERTS_ALC_A_LONG_LIVED,
 		       &init.ll_alloc,
 		       &ll_alloc_state);
-
+#if HALFWORD_HEAP
+    start_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW,
+		       &init.ll_low_alloc,
+		       &ll_low_alloc_state);
+    start_au_allocator(ERTS_ALC_A_STANDARD_LOW,
+		       &init.std_low_alloc,
+		       &std_low_alloc_state);
+#endif
     start_au_allocator(ERTS_ALC_A_EHEAP,
 		       &init.eheap_alloc,
 		       &eheap_alloc_state);
@@ -582,51 +754,57 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 		       &init.driver_alloc,
 		       &driver_alloc_state);
 
-    fix_core_allocator	= erts_allctrs[erts_fix_core_allocator_ix].alloc;
-    fix_core_extra	= erts_allctrs[erts_fix_core_allocator_ix].extra;
+    start_au_allocator(ERTS_ALC_A_FIXED_SIZE,
+		       &init.fix_alloc,
+		       &fix_alloc_state);
 
     erts_mtrace_install_wrapper_functions();
     extra_block_size += erts_instr_init(init.instr.stat, init.instr.map);
 
+#if !HALFWORD_HEAP
+    init_aireq_alloc();
+#endif
+
 #ifdef DEBUG
     extra_block_size += install_debug_functions();
 #endif
 
-#ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE
-
-    erts_init_fix_alloc(extra_block_size, fix_core_alloc);
+}
 
+void
+erts_alloc_late_init(void)
+{
 
-#if !defined(PURIFY) && !defined(VALGRIND)
-    erts_set_fix_size(ERTS_ALC_T_PROC,		sizeof(Process));
-    erts_set_fix_size(ERTS_ALC_T_DB_TABLE,	sizeof(DbTable));
-    erts_set_fix_size(ERTS_ALC_T_ATOM,		sizeof(Atom));
-    erts_set_fix_size(ERTS_ALC_T_EXPORT,	sizeof(Export));
-    erts_set_fix_size(ERTS_ALC_T_MODULE,	sizeof(Module));
-    erts_set_fix_size(ERTS_ALC_T_REG_PROC,	sizeof(RegProc));
-    erts_set_fix_size(ERTS_ALC_T_MONITOR_SH,	ERTS_MONITOR_SH_SIZE*sizeof(Uint));
-    erts_set_fix_size(ERTS_ALC_T_NLINK_SH,	ERTS_LINK_SH_SIZE*sizeof(Uint));
-    erts_set_fix_size(ERTS_ALC_T_FUN_ENTRY,	sizeof(ErlFunEntry));
-#ifdef ERTS_ALC_T_DRV_EV_D_STATE
-    erts_set_fix_size(ERTS_ALC_T_DRV_EV_D_STATE,
-		      sizeof(ErtsDrvEventDataState));
-#endif
-#ifdef ERTS_ALC_T_DRV_SEL_D_STATE
-    erts_set_fix_size(ERTS_ALC_T_DRV_SEL_D_STATE,
-		      sizeof(ErtsDrvSelectDataState));
-#endif
-#endif
-#endif
+}
 
+static void *
+erts_realloc_fixed_size(ErtsAlcType_t type, void *extra, void *p, Uint size)
+{
+    erl_exit(ERTS_ABORT_EXIT,
+	     "Attempt to reallocate a block of the fixed size type %s\n",
+	     ERTS_ALC_T2TD(type));
 }
 
+
 static void
-set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init)
+set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu)
 {
     ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n];
     ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n];
     ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n];
 
+    /*
+     * Some allocators are forced on if halfword heap is used.
+     */
+    if (init->init.util.force)
+	init->enable = 1;
+
+    tspec->enabled = 0;
+    tspec->dd = 0;
+    tspec->aix = alctr_n;
+    tspec->size	= 0;
+    ai->thr_spec = 0;
+
     if (!init->enable) {
 	af->alloc = erts_sys_alloc;
 	af->realloc = erts_sys_realloc;
@@ -638,14 +816,14 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init)
 	return;
     }
 
-    tspec->enabled = 0;
-    tspec->all_thr_safe = 0;
-    ai->thr_spec = 0;
 #ifdef USE_THREADS
+#ifdef ERTS_SMP
     if (init->thr_spec) {
 	if (init->thr_spec > 0) {
 	    af->alloc = erts_alcu_alloc_thr_spec;
-	    if (init->init.util.ramv)
+	    if (init->init.util.fix_type_size)
+		af->realloc = erts_realloc_fixed_size;
+	    else if (init->init.util.ramv)
 		af->realloc = erts_alcu_realloc_mv_thr_spec;
 	    else
 		af->realloc = erts_alcu_realloc_thr_spec;
@@ -653,12 +831,14 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init)
 	}
 	else {
 	    af->alloc = erts_alcu_alloc_thr_pref;
-	    if (init->init.util.ramv)
+	    if (init->init.util.fix_type_size)
+		af->realloc = erts_realloc_fixed_size;
+	    else if (init->init.util.ramv)
 		af->realloc = erts_alcu_realloc_mv_thr_pref;
 	    else
 		af->realloc = erts_alcu_realloc_thr_pref;
 	    af->free = erts_alcu_free_thr_pref;
-	    tspec->all_thr_safe = 1;
+	    tspec->dd = 1;
 	}
 
 	tspec->enabled	= 1;
@@ -666,9 +846,13 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init)
 
 	ai->thr_spec	= tspec->size;
     }
-    else if (init->init.util.ts) {
+    else
+#endif
+	if (init->init.util.ts) {
 	af->alloc = erts_alcu_alloc_ts;
-	if (init->init.util.ramv)
+	if (init->init.util.fix_type_size)
+	    af->realloc = erts_realloc_fixed_size;
+	else if (init->init.util.ramv)
 	    af->realloc = erts_alcu_realloc_mv_ts;
 	else
 	    af->realloc = erts_alcu_realloc_ts;
@@ -678,7 +862,9 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init)
 #endif
     {
 	af->alloc = erts_alcu_alloc;
-	if (init->init.util.ramv)
+	if (init->init.util.fix_type_size)
+	    af->realloc = erts_realloc_fixed_size;
+	else if (init->init.util.ramv)
 	    af->realloc = erts_alcu_realloc_mv;
 	else
 	    af->realloc = erts_alcu_realloc;
@@ -701,12 +887,14 @@ start_au_allocator(ErtsAlcType_t alctr_n,
     ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n];
     ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n];
     ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n];
+    ErtsAlcFixList_t *fix_lists = NULL;
+    size_t fix_list_size = 0;
 
     if (!init->enable)
 	return;
 
     if (init->thr_spec) {
-	void *states = erts_sys_alloc(0,
+	char *states = erts_sys_alloc(0,
 				      NULL,
 				      ((sizeof(Allctr_t *)
 					* (tspec->size + 1))
@@ -718,18 +906,40 @@ start_au_allocator(ErtsAlcType_t alctr_n,
 		     "Failed to allocate allocator states for %salloc\n",
 		     init->init.util.name_prefix);
 	tspec->allctr = (Allctr_t **) states;
-	states = ((char *) states) + sizeof(Allctr_t *) * (tspec->size + 1);
-	states = ((((Uint) states) & ERTS_CACHE_LINE_MASK)
-		  ? (void *) ((((Uint) states) & ~ERTS_CACHE_LINE_MASK)
+	states += sizeof(Allctr_t *) * (tspec->size + 1);
+	states = ((((UWord) states) & ERTS_CACHE_LINE_MASK)
+		  ? (char *) ((((UWord) states) & ~ERTS_CACHE_LINE_MASK)
 			      + ERTS_CACHE_LINE_SIZE)
-		  : (void *) states);
-	tspec->allctr[0] = init->thr_spec > 0 ? (Allctr_t *) state : (Allctr_t *) NULL;
+		  : (char *) states);
+	tspec->allctr[0] = (Allctr_t *) state;
 	size = tspec->size;
 	for (i = 1; i < size; i++)
 	    tspec->allctr[i] = (Allctr_t *)
 		&((ErtsAllocatorState_t *) states)[i-1];
     }
 
+    if (init->init.util.fix_type_size) {
+	size_t tot_fix_list_size;
+	fix_list_size = sizeof(ErtsAlcFixList_t)*ERTS_ALC_NO_FIXED_SIZES;
+	fix_list_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(fix_list_size);
+	tot_fix_list_size = fix_list_size;
+	if (init->thr_spec)
+	    tot_fix_list_size *= tspec->size;
+	fix_lists = erts_sys_alloc(0,
+				   NULL,
+				   (tot_fix_list_size
+				    + ERTS_CACHE_LINE_SIZE - 1));
+	if (!fix_lists)
+	    erl_exit(ERTS_ABORT_EXIT,
+		     "Failed to allocate fix lists for %salloc\n",
+		     init->init.util.name_prefix);
+
+	if (((UWord) fix_lists) & ERTS_CACHE_LINE_MASK)
+		fix_lists = ((ErtsAlcFixList_t *)
+		       ((((UWord) fix_lists) & ~ERTS_CACHE_LINE_MASK)
+			+ ERTS_CACHE_LINE_SIZE));
+    }
+
     for (i = 0; i < size; i++) {
 	void *as;
 	atype = init->atype;
@@ -740,25 +950,32 @@ start_au_allocator(ErtsAlcType_t alctr_n,
 	    as0 = (void *) tspec->allctr[i];
 	    if (!as0)
 		continue;
-	    if (i == 0) {
-		if (atype == AFIT)
-		    atype = GOODFIT;
-		init->init.util.ts = 1;
+	    if (init->thr_spec < 0) {
+		init->init.util.ts = i == 0;
+		init->init.util.tspec = 0;
+		init->init.util.tpref = -1*init->thr_spec + 1;
 	    }
 	    else {
-		if (init->thr_spec < 0) {
+		if (i != 0)
+		    init->init.util.ts = 0;
+		else {
+		    if (atype == AFIT)
+			atype = GOODFIT;
 		    init->init.util.ts = 1;
-		    init->init.util.tspec = 0;
-		    init->init.util.tpref = -1*init->thr_spec;
 		}
-		else {
-		    init->init.util.ts = 0;
-		    init->init.util.tspec = init->thr_spec + 1;
-		    init->init.util.tpref = 0;
-		}   
-	    }
+		init->init.util.tspec = init->thr_spec + 1;
+		init->init.util.tpref = 0;
+	    }   
 	}
 
+	if (fix_lists) {
+	    init->init.util.fix = fix_lists;
+	    fix_lists = ((ErtsAlcFixList_t *)
+			 (((char *) fix_lists) + fix_list_size));
+	}
+
+	init->init.util.ix = i;
+
 	switch (atype) {
 	case GOODFIT:
 	    as = (void *) erts_gfalc_start((GFAllctr_t *) as0,
@@ -775,6 +992,12 @@ start_au_allocator(ErtsAlcType_t alctr_n,
 					   &init->init.af,
 					   &init->init.util);
 	    break;
+    	case AOFIRSTFIT:
+	    as = (void *) erts_aoffalc_start((AOFFAllctr_t *) as0,
+					     &init->init.aoff,
+					     &init->init.util);
+	    break;
+
 	default:
 	    as = NULL;
 	    ASSERT(0);
@@ -788,11 +1011,8 @@ start_au_allocator(ErtsAlcType_t alctr_n,
 	af->extra = as;
     }
 
-    if (init->thr_spec) {
+    if (init->thr_spec)
 	af->extra = tspec;
-	init->init.util.ts = 1;
-    }
-
     ai->extra = af->extra;
 }
 
@@ -875,7 +1095,7 @@ get_kb_value(char *param_end, char** argv, int* ip)
     char *param = argv[*ip]+1;
     char *value = get_value(param_end, argv, ip);
     errno = 0;
-    tmp = (Sint) strtol(value, &rest, 10);
+    tmp = (Sint) ErtsStrToSint(value, &rest, 10);
     if (errno != 0 || rest == value || tmp < 0 || max < ((Uint) tmp))
 	bad_value(param, param_end, value);
     if (max == (Uint) tmp)
@@ -885,45 +1105,31 @@ get_kb_value(char *param_end, char** argv, int* ip)
 }
 
 static Uint
-get_amount_value(char *param_end, char** argv, int* ip)
+get_byte_value(char *param_end, char** argv, int* ip)
 {
     Sint tmp;
     char *rest;
     char *param = argv[*ip]+1;
     char *value = get_value(param_end, argv, ip);
     errno = 0;
-    tmp = (Sint) strtol(value, &rest, 10);
+    tmp = (Sint) ErtsStrToSint(value, &rest, 10);
     if (errno != 0 || rest == value || tmp < 0)
 	bad_value(param, param_end, value);
     return (Uint) tmp;
 }
 
-static int
-get_bool_or_possitive_amount_value(int *bool, Uint *amount,
-				   char *param_end, char** argv, int* ip)
+static Uint
+get_amount_value(char *param_end, char** argv, int* ip)
 {
+    Sint tmp;
+    char *rest;
     char *param = argv[*ip]+1;
     char *value = get_value(param_end, argv, ip);
-    if (strcmp(value, "true") == 0) {
-	*bool = 1; 
-	return 1;
-    }
-    else if (strcmp(value, "false") == 0) {
-	*bool = 0; 
-	return 1;
-    }
-    else {
-	Sint tmp;
-	char *rest;
-	errno = 0;
-	tmp = (Sint) strtol(value, &rest, 10);
-	if (errno != 0 || rest == value || tmp <= 0) {
-	    bad_value(param, param_end, value);
-	    return -1;
-	}
-	*amount = (Uint) tmp;
-	return 0;
-    }
+    errno = 0;
+    tmp = (Sint) ErtsStrToSint(value, &rest, 10);
+    if (errno != 0 || rest == value || tmp < 0)
+	bad_value(param, param_end, value);
+    return (Uint) tmp;
 }
 
 static void
@@ -955,6 +1161,9 @@ handle_au_arg(struct au_init *auip,
 	    else if (strcmp("af", alg) == 0) {
 		auip->atype = AFIT;
 	    }
+	    else if (strcmp("aoff", alg) == 0) {
+		auip->atype = AOFIRSTFIT;
+	    }
 	    else {
 		bad_value(param, sub_param + 1, alg);
 	    }
@@ -1023,6 +1232,12 @@ handle_au_arg(struct au_init *auip,
 	if(has_prefix("sbct", sub_param)) {
 	    auip->init.util.sbct = get_kb_value(sub_param + 4, argv, ip);
 	}
+	else if (has_prefix("sbmbcs", sub_param)) {
+	    auip->init.util.sbmbcs = get_byte_value(sub_param + 6, argv, ip);
+	}
+	else if (has_prefix("sbmbct", sub_param)) {
+	    auip->init.util.sbmbct = get_byte_value(sub_param + 6, argv, ip);
+	}
 	else if (has_prefix("smbcs", sub_param)) {
 	    auip->default_.smbcs = 0;
 	    auip->init.util.smbcs = get_kb_value(sub_param + 5, argv, ip);
@@ -1031,25 +1246,16 @@ handle_au_arg(struct au_init *auip,
 	    goto bad_switch;
 	break;
     case 't': {
-	Uint no;
-	int enable;
-	int res = get_bool_or_possitive_amount_value(&enable,
-						     &no,
-						     sub_param+1,
-						     argv,
-						     ip);
-	if (res > 0)
-	    auip->thr_spec = enable ? 1 : 0;
+	int res = get_bool_value(sub_param+1, argv, ip);
+	if (res > 0) {
+	    auip->thr_spec = 1;
+	    break;
+	}
 	else if (res == 0) {
-	    int allocs = (int) no;
-	    if (allocs < 0)
-		allocs = INT_MIN;
-	    else {
-		allocs *= -1;
-	    }
-	    auip->thr_spec = allocs;
+	    auip->thr_spec = 0;
+	    break;
 	}
-	break;
+	goto bad_switch;
     }
     default:
     bad_switch:
@@ -1061,12 +1267,14 @@ static void
 handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 {
     struct au_init *aui[] = {
+	&init->sbmbc_alloc,
 	&init->binary_alloc,
 	&init->std_alloc,
 	&init->ets_alloc,
 	&init->eheap_alloc,
 	&init->ll_alloc,
 	&init->driver_alloc,
+	&init->fix_alloc,
 	&init->sl_alloc,
 	&init->temp_alloc
     };
@@ -1088,20 +1296,17 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 		case 'B':
 		    handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i);
 		    break;
+		case 'C':
+		    handle_au_arg(&init->sbmbc_alloc, &argv[i][3], argv, &i);
+		    break;
 		case 'D':
 		    handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i);
 		    break;
 		case 'E':
 		    handle_au_arg(&init->ets_alloc, &argv[i][3], argv, &i);
 		    break;
-		case 'F': /* fix_alloc */
-		    if (has_prefix("e", param+2)) {
-			arg = get_value(param+3, argv, &i);
-			if (strcmp("true", arg) != 0)
-			    bad_value(param, param+3, arg);
-		    }
-		    else
-			bad_param(param, param+2);
+		case 'F':
+		    handle_au_arg(&init->fix_alloc, &argv[i][3], argv, &i);
 		    break;
 		case 'H':
 		    handle_au_arg(&init->eheap_alloc, &argv[i][3], argv, &i);
@@ -1128,12 +1333,6 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 #endif
 			    get_amount_value(argv[i]+6, argv, &i);
 		    }
-		    else if (has_prefix("cci", argv[i]+3)) {
-#if HAVE_ERTS_MSEG
-			init->mseg.cci =
-#endif
-			    get_amount_value(argv[i]+6, argv, &i);
-		    }
 		    else {
 			bad_param(param, param+2);
 		    }
@@ -1219,6 +1418,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 			    set_default_binary_alloc_opts(&init->binary_alloc);
 			    set_default_ets_alloc_opts(&init->ets_alloc);
 			    set_default_driver_alloc_opts(&init->driver_alloc);
+			    set_default_driver_alloc_opts(&init->fix_alloc);
 
 			    init->driver_alloc.enable = 0;
 			    if (strcmp("r9c", arg) == 0) {
@@ -1337,7 +1537,6 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
 	    argv[j++] = argv[i];
     }
     *argc = j;
-    
 }
 
 static char *type_no_str(ErtsAlcType_t n)
@@ -1354,43 +1553,103 @@ static char *type_no_str(ErtsAlcType_t n)
 
 #define type_str(T) type_no_str(ERTS_ALC_T2N((T)))
 
-erts_tsd_key_t thr_ix_key;
-erts_spinlock_t alloc_thr_ix_lock;
-int last_thr_ix;
-int first_dyn_thr_ix;
-
-static void
-init_thr_ix(int static_ixs)
+void
+erts_alloc_register_scheduler(void *vesdp)
 {
-    erts_tsd_key_create(&thr_ix_key);
-    erts_spinlock_init(&alloc_thr_ix_lock, "alloc_thr_ix_lock");
-    last_thr_ix = -4711;
-    first_dyn_thr_ix = static_ixs+1;
+    ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
+    int ix = (int) esdp->no;
+    int aix;
+
+    for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) {
+	ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix];
+	esdp->alloc_data.deallctr[aix] = NULL;
+	esdp->alloc_data.pref_ix[aix] = -1;
+	if (tspec->enabled) {
+	    if (!tspec->dd)
+		esdp->alloc_data.pref_ix[aix] = ix;
+	    else {
+		Allctr_t *allctr = tspec->allctr[ix];
+		ASSERT(allctr);
+		esdp->alloc_data.deallctr[aix] = allctr;
+		esdp->alloc_data.pref_ix[aix] = ix;
+	    }
+	}
+    }
 }
 
-int
-erts_alc_get_thr_ix(void)
+#ifdef ERTS_SMP
+void
+erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
+					    int *need_thr_progress,
+					    ErtsThrPrgrVal *thr_prgr_p,
+					    int *more_work)
 {
-    int ix = (int)(long) erts_tsd_get(thr_ix_key);
-    if (ix == 0) {
-	erts_spin_lock(&alloc_thr_ix_lock);
-	last_thr_ix++;
-	if (last_thr_ix < 0) 
-	    last_thr_ix = first_dyn_thr_ix;
-	ix = last_thr_ix;
-	erts_spin_unlock(&alloc_thr_ix_lock);
-	erts_tsd_set(thr_ix_key, (void *)(long) ix);
+    ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
+    int aix;
+    for (aix = ERTS_ALC_A_MIN; aix <= ERTS_ALC_A_MAX; aix++) {
+	Allctr_t *allctr;
+	if (esdp)
+	    allctr = esdp->alloc_data.deallctr[aix];
+	else {
+	    ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[aix];
+	    if (tspec->enabled && tspec->dd)
+		allctr = tspec->allctr[0];
+	    else
+		allctr = NULL;
+	}
+	if (allctr) {
+	    erts_alcu_check_delayed_dealloc(allctr,
+					    1,
+					    need_thr_progress,
+					    thr_prgr_p,
+					    more_work);
+	}
     }
-    ASSERT(ix > 0);
-    return ix;
+}
+#endif
+
+erts_aint32_t
+erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs)
+{
+#ifdef ERTS_SMP
+    ErtsAllocatorThrSpec_t *tspec;
+    tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE];
+    if (erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].thr_spec && tspec->enabled)
+	return erts_alcu_fix_alloc_shrink(tspec->allctr[ix], flgs);
+    if (ix == 0 && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra)
+	return erts_alcu_fix_alloc_shrink(
+	    erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra, flgs);
+#else
+    if (ix == 1 && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra)
+	return erts_alcu_fix_alloc_shrink(
+	    erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra, flgs);
+#endif
+    return 0;
 }
 
-void erts_alloc_reg_scheduler_id(Uint id)
+static void
+no_verify(Allctr_t *allctr)
 {
-    int ix = (int) id;
-    ASSERT(0 < ix && ix <= first_dyn_thr_ix);
-    ASSERT(0 == (int) (long) erts_tsd_get(thr_ix_key));
-    erts_tsd_set(thr_ix_key, (void *)(long) ix);
+
+}
+
+erts_alloc_verify_func_t
+erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr)
+{
+    if (erts_allctrs_info[ERTS_ALC_A_TEMPORARY].alloc_util
+	&& erts_allctrs_info[ERTS_ALC_A_TEMPORARY].thr_spec) {
+	ErtsAllocatorThrSpec_t *tspec;
+	int ix = ERTS_ALC_GET_THR_IX();
+	tspec = &erts_allctr_thr_spec[ERTS_ALC_A_TEMPORARY];
+
+	if (ix < tspec->size) {
+	    *allctr = tspec->allctr[ix];
+	    return erts_alcu_verify_unused;
+	}
+    }
+
+    *allctr = NULL;
+    return no_verify;
 }
 
 __decl_noreturn void
@@ -1483,10 +1742,10 @@ erts_realloc_n_enomem(ErtsAlcType_t n, void *ptr, Uint size)
     erts_alc_fatal_error(ERTS_ALC_E_NOMEM, ERTS_ALC_O_REALLOC, n, size);
 }
 
-static ERTS_INLINE Uint
-alcu_size(ErtsAlcType_t ai)
+static ERTS_INLINE UWord
+alcu_size(ErtsAlcType_t ai, ErtsAlcUFixInfo_t *fi, int fisz)
 {
-    Uint res = 0;
+    UWord res = 0;
 
     ASSERT(erts_allctrs_info[ai].enabled);
     ASSERT(erts_allctrs_info[ai].alloc_util);
@@ -1494,22 +1753,20 @@ alcu_size(ErtsAlcType_t ai)
     if (!erts_allctrs_info[ai].thr_spec) {
 	Allctr_t *allctr = erts_allctrs_info[ai].extra;
 	AllctrSize_t asize;
-	erts_alcu_current_size(allctr, &asize);
+	erts_alcu_current_size(allctr, &asize, fi, fisz);
 	res += asize.blocks;
     }
     else {
 	ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ai];
 	int i;
 
-	ASSERT(tspec->all_thr_safe);
-
 	ASSERT(tspec->enabled);
 
 	for (i = tspec->size - 1; i >= 0; i--) {
 	    Allctr_t *allctr = tspec->allctr[i];
 	    AllctrSize_t asize;
 	    if (allctr) {
-		erts_alcu_current_size(allctr, &asize);
+		erts_alcu_current_size(allctr, &asize, fi, fisz);
 		res += asize.blocks;
 	    }
 	}
@@ -1518,11 +1775,66 @@ alcu_size(ErtsAlcType_t ai)
     return res;
 }
 
+#if HALFWORD_HEAP
+static ERTS_INLINE int
+alcu_is_low(ErtsAlcType_t ai)
+{
+    int is_low = 0;
+    ASSERT(erts_allctrs_info[ai].enabled);
+    ASSERT(erts_allctrs_info[ai].alloc_util);
+
+    if (!erts_allctrs_info[ai].thr_spec) {
+	Allctr_t *allctr = erts_allctrs_info[ai].extra;
+	is_low = allctr->mseg_opt.low_mem;
+    }
+    else {
+	ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ai];
+	int i;
+# ifdef DEBUG
+	int found_one = 0;
+# endif
+
+	ASSERT(tspec->enabled);
+
+	for (i = tspec->size - 1; i >= 0; i--) {
+	    Allctr_t *allctr = tspec->allctr[i];
+	    if (allctr) {
+# ifdef DEBUG
+		if (!found_one) {
+		    is_low = allctr->mseg_opt.low_mem;
+		    found_one = 1;
+		}
+		else ASSERT(is_low == allctr->mseg_opt.low_mem);
+# else
+		is_low = allctr->mseg_opt.low_mem;
+		break;
+# endif
+	    }
+	}
+	ASSERT(found_one);
+    }
+    return is_low;
+}
+#endif /* HALFWORD */
+
+static ERTS_INLINE void
+add_fix_values(UWord *ap, UWord *up, ErtsAlcUFixInfo_t *fi, ErtsAlcType_t type)
+{
+    int ix = ERTS_ALC_T2N(type) - ERTS_ALC_N_MIN_A_FIXED_SIZE;
+    ASSERT(0 <= ix && ix < ERTS_ALC_NO_FIXED_SIZES);
+
+    *ap += (UWord) fi[ix].allocated;
+    *up += (UWord) fi[ix].used;
+}
+
 Eterm
 erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 {
+/*
+ * NOTE! When updating this function, make sure to also update
+ *       erlang:memory/[0,1] in $ERL_TOP/erts/preloaded/src/erlang.erl
+ */
 #define ERTS_MEM_NEED_ALL_ALCU (!erts_instr_stat && want_tot_or_sys)
-    ErtsFixInfo efi;
     struct {
 	int total;
 	int processes;
@@ -1534,28 +1846,36 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	int code;
 	int ets;
 	int maximum;
+#if HALFWORD_HEAP
+	int low;
+#endif
     } want = {0};
     struct {
-	Uint total;
-	Uint processes;
-	Uint processes_used;
-	Uint system;
-	Uint atom;
-	Uint atom_used;
-	Uint binary;
-	Uint code;
-	Uint ets;
-	Uint maximum;
+	UWord total;
+	UWord processes;
+	UWord processes_used;
+	UWord system;
+	UWord atom;
+	UWord atom_used;
+	UWord binary;
+	UWord code;
+	UWord ets;
+	UWord maximum;
+#if HALFWORD_HEAP
+	UWord low;
+#endif
     } size = {0};
-    Eterm atoms[sizeof(size)/sizeof(Uint)];
-    Uint *uintps[sizeof(size)/sizeof(Uint)];
-    Eterm euints[sizeof(size)/sizeof(Uint)];
-    int need_atom;
+    Eterm atoms[sizeof(size)/sizeof(UWord)];
+    UWord *uintps[sizeof(size)/sizeof(UWord)];
+    Eterm euints[sizeof(size)/sizeof(UWord)];
     int want_tot_or_sys;
     int length;
     Eterm res = THE_NON_VALUE;
     ErtsAlcType_t ai;
     int only_one_value = 0;
+    ErtsAlcUFixInfo_t fi[ERTS_ALC_NO_FIXED_SIZES] = {{0,0}};
+
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
 
     /* Figure out whats wanted... */
 
@@ -1602,15 +1922,20 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	    atoms[length] = am_maximum;
 	    uintps[length++] = &size.maximum;
 	}
-
+#if HALFWORD_HEAP
+	want.low = 1;
+	atoms[length] = am_low;
+	uintps[length++] = &size.low;
+#endif
     }
     else {
-	Eterm tmp_heap[2];
+	DeclareTmpHeapNoproc(tmp_heap,2);
 	Eterm wanted_list;
 
 	if (is_nil(earg))
 	    return NIL;
 
+	UseTmpHeapNoproc(2);
 	if (is_not_atom(earg))
 	    wanted_list = earg;
 	else {
@@ -1690,25 +2015,40 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 			atoms[length] = am_maximum;
 			uintps[length++] = &size.maximum;
 		    }
-		}
-		else
+		} else {
+		    UnUseTmpHeapNoproc(2);
 		    return am_badarg;
+		}
 		break;
+#if HALFWORD_HEAP
+	    case am_low:
+		if (!want.low) {
+		    want.low = 1;
+		    atoms[length] = am_low;
+		    uintps[length++] = &size.low;
+		}
+		break;
+#endif
 	    default:
+		UnUseTmpHeapNoproc(2);
 		return am_badarg;
 	    }
 	    wanted_list = CDR(list_val(wanted_list));
 	}
+	UnUseTmpHeapNoproc(2);
 	if (is_not_nil(wanted_list))
 	    return am_badarg;
     }
 
-    /* All alloc_util allocators *have* to be enabled */
+    /* All alloc_util allocators except sbmbc_alloc *have* to be enabled */
     
     for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) {
 	switch (ai) {
 	case ERTS_ALC_A_SYSTEM:
-	case ERTS_ALC_A_FIXED_SIZE:
+	case ERTS_ALC_A_SBMBC:
+#if HALFWORD_HEAP
+	case ERTS_ALC_A_SBMBC_LOW:
+#endif
 	    break;
 	default:
 	    if (!erts_allctrs_info[ai].enabled
@@ -1721,7 +2061,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 
     ASSERT(length <= sizeof(atoms)/sizeof(Eterm));
     ASSERT(length <= sizeof(euints)/sizeof(Eterm));
-    ASSERT(length <= sizeof(uintps)/sizeof(Uint));
+    ASSERT(length <= sizeof(uintps)/sizeof(UWord));
 
 
     if (proc) {
@@ -1734,21 +2074,26 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
     /* Calculate values needed... */
 
     want_tot_or_sys = want.total || want.system;
-    need_atom = ERTS_MEM_NEED_ALL_ALCU || want.atom;
 
     if (ERTS_MEM_NEED_ALL_ALCU) {
 	size.total = 0;
 
 	for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) {
 	    if (erts_allctrs_info[ai].alloc_util) {
-		Uint *save;
-		Uint asz;
+		UWord *save;
+		UWord asz;
 		switch (ai) {
 		case ERTS_ALC_A_TEMPORARY:
 		     /*
 		      * Often not thread safe and usually never
 		      * contain any allocated memory.
 		      */
+		case ERTS_ALC_A_SBMBC:
+		    /* Included in other allocators */
+#if HALFWORD_HEAP
+		case ERTS_ALC_A_SBMBC_LOW:
+		    /* Included in other allocators */
+#endif
 		    continue;
 		case ERTS_ALC_A_EHEAP:
 		    save = &size.processes;
@@ -1759,14 +2104,23 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 		case ERTS_ALC_A_BINARY:
 		    save = &size.binary;
 		    break;
+		case ERTS_ALC_A_FIXED_SIZE:
+		    asz = alcu_size(ai, fi, ERTS_ALC_NO_FIXED_SIZES);
+		    size.total += asz;
+		    continue;
 		default:
 		    save = NULL;
 		    break;
 		}
-		asz = alcu_size(ai);
+		asz = alcu_size(ai, NULL, 0);
 		if (save)
 		    *save = asz;
 		size.total += asz;
+#if HALFWORD_HEAP
+		if (alcu_is_low(ai)) {
+		    size.low += asz;
+		}
+#endif
 	    }
 	}
     }
@@ -1774,76 +2128,69 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 
 
     if (want_tot_or_sys || want.processes || want.processes_used) {
-	Uint tmp;
+	UWord tmp;
 
 	if (ERTS_MEM_NEED_ALL_ALCU)
 	    tmp = size.processes;
-	else
-	    tmp = alcu_size(ERTS_ALC_A_EHEAP);
-	tmp += erts_max_processes*sizeof(Process*);
-#ifdef HYBRID
+	else {
+	    alcu_size(ERTS_ALC_A_FIXED_SIZE,
+		      fi, ERTS_ALC_NO_FIXED_SIZES);
+	    tmp = alcu_size(ERTS_ALC_A_EHEAP, NULL, 0);
+	}
 	tmp += erts_max_processes*sizeof(Process*);
-#endif
 	tmp += erts_bif_timer_memory_size();
 	tmp += erts_tot_link_lh_size();
 
 	size.processes = size.processes_used = tmp;
 
-	erts_fix_info(ERTS_ALC_T_NLINK_SH, &efi);
-	size.processes += efi.total;
-	size.processes_used += efi.used;
-
-	erts_fix_info(ERTS_ALC_T_MONITOR_SH, &efi);
-	size.processes += efi.total;
-	size.processes_used += efi.used;
-
-	erts_fix_info(ERTS_ALC_T_PROC, &efi);
-	size.processes += efi.total;
-	size.processes_used += efi.used;
-
-	erts_fix_info(ERTS_ALC_T_REG_PROC, &efi);
-	size.processes += efi.total;
-	size.processes_used += efi.used;
-
+	add_fix_values(&size.processes,
+		       &size.processes_used,
+		       fi,
+		       ERTS_ALC_T_PROC);
+#if !HALFWORD_HEAP
+	add_fix_values(&size.processes,
+		       &size.processes_used,
+		       fi,
+		       ERTS_ALC_T_MONITOR_SH);
+
+	add_fix_values(&size.processes,
+		       &size.processes_used,
+		       fi,
+		       ERTS_ALC_T_NLINK_SH);
+#endif
+	add_fix_values(&size.processes,
+		       &size.processes_used,
+		       fi,
+		       ERTS_ALC_T_MSG_REF);
     }
 
     if (want.atom || want.atom_used) {
 	Uint reserved_atom_space, atom_space;
 	erts_atom_get_text_space_sizes(&reserved_atom_space, &atom_space);
 	size.atom = size.atom_used = atom_table_sz();
-	erts_fix_info(ERTS_ALC_T_ATOM, &efi);
 
-	if (want.atom) {
+	if (want.atom)
 	    size.atom += reserved_atom_space;
-	    size.atom += efi.total;
-	}
 
-	if (want.atom_used) {
+	if (want.atom_used)
 	    size.atom_used += atom_space;
-	    size.atom_used += efi.used;
-	}
     }
 
     if (!ERTS_MEM_NEED_ALL_ALCU && want.binary)
-	size.binary = alcu_size(ERTS_ALC_A_BINARY);
+	size.binary = alcu_size(ERTS_ALC_A_BINARY, NULL, 0);
 
     if (want.code) {
 	size.code = module_table_sz();
-	erts_fix_info(ERTS_ALC_T_MODULE, &efi);
-	size.code += efi.used;
 	size.code += export_table_sz();
-	erts_fix_info(ERTS_ALC_T_EXPORT, &efi);
-	size.code += efi.used;
+	size.code += export_list_size() * sizeof(Export);
 	size.code += erts_fun_table_sz();
-	erts_fix_info(ERTS_ALC_T_FUN_ENTRY, &efi);
-	size.code += efi.used;
 	size.code += allocated_modules*sizeof(Range);
 	size.code += erts_total_code_size;
     }
 
     if (want.ets) {
 	if (!ERTS_MEM_NEED_ALL_ALCU)
-	    size.ets = alcu_size(ERTS_ALC_A_ETS);
+	    size.ets = alcu_size(ERTS_ALC_A_ETS, NULL, 0);
 	size.ets += erts_get_ets_misc_mem_size();
     }
 
@@ -1879,9 +2226,9 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	if (only_one_value) {
 	    ASSERT(length == 1);
 	    hsz = 0;
-	    erts_bld_uint(NULL, &hsz, *uintps[0]);
+	    erts_bld_uword(NULL, &hsz, *uintps[0]);
 	    hp = hsz ? HAlloc((Process *) proc, hsz) : NULL;
-	    res = erts_bld_uint(&hp, NULL, *uintps[0]);
+	    res = erts_bld_uword(&hp, NULL, *uintps[0]);
 	}
 	else {
 	    Uint **hpp = NULL;
@@ -1891,7 +2238,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 	    while (1) {
 		int i;
 		for (i = 0; i < length; i++)
-		    euints[i] = erts_bld_uint(hpp, hszp, *uintps[i]);
+		    euints[i] = erts_bld_uword(hpp, hszp, *uintps[i]);
 		res = erts_bld_2tup_list(hpp, hszp, length, atoms, euints);
 		if (hpp)
 		    break;
@@ -1916,13 +2263,10 @@ struct aa_values {
 Eterm
 erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc)
 {
-#define MAX_AA_VALUES \
-  (20 + (ERTS_ALC_N_MAX_A_FIXED_SIZE - ERTS_ALC_N_MIN_A_FIXED_SIZE + 1))
-
+#define MAX_AA_VALUES (23)
     struct aa_values values[MAX_AA_VALUES];
     Eterm res = THE_NON_VALUE;
     int i, length;
-    ErtsFixInfo efi;
     Uint reserved_atom_space, atom_space;
 
     if (proc) {
@@ -1987,6 +2331,11 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc)
     i++;
 
     values[i].arity = 2;
+    values[i].name = "export_list";
+    values[i].ui[0] = export_list_size() * sizeof(Export);
+    i++;
+
+    values[i].arity = 2;
     values[i].name = "register_table";
     values[i].ui[0] = process_reg_sz();
     i++;
@@ -2031,22 +2380,15 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc)
     values[i].ui[0] = erts_tot_link_lh_size();
     i++;
 
-    {
-	Uint n;
-
-	for (n = ERTS_ALC_N_MIN_A_FIXED_SIZE;
-	     n <= ERTS_ALC_N_MAX_A_FIXED_SIZE;
-	     n++) {
-	    erts_fix_info(ERTS_ALC_N2T(n), &efi);
-
-	    values[i].arity = 3;
-	    values[i].name = ERTS_ALC_N2TD(n);
-	    values[i].ui[0] = efi.total;
-	    values[i].ui[1] = efi.used;
-	    i++;
-	}    
+    values[i].arity = 2;
+    values[i].name = "process_table";
+    values[i].ui[0] = erts_max_processes*sizeof(Process*);
+    i++;
 
-    }
+    values[i].arity = 2;
+    values[i].name = "ets_misc";
+    values[i].ui[0] = erts_get_ets_misc_mem_size();
+    i++;
 
     length = i;
     ASSERT(length <= MAX_AA_VALUES);
@@ -2060,11 +2402,11 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc)
 	for (i = 0; i < length; i++) {
 	    switch (values[i].arity) {
 	    case 2:
-		erts_print(to, arg, "%s: %bpu\n",
+		erts_print(to, arg, "%s: %beu\n",
 			   values[i].name, values[i].ui[0]);
 		break;
 	    case 3:
-		erts_print(to, arg, "%s: %bpu %bpu\n",
+		erts_print(to, arg, "%s: %beu %beu\n",
 			   values[i].name, values[i].ui[0], values[i].ui[1]);
 		break;
 	    default:
@@ -2140,17 +2482,16 @@ erts_alloc_util_allocators(void *proc)
     Uint sz;
     int i;
     /*
-     * Currently all allocators except sys_alloc and fix_alloc are
+     * Currently all allocators except sys_alloc are
      * alloc_util allocators.
      */
-    sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 2)*2;
+    sz = ((ERTS_ALC_A_MAX + 1 - ERTS_ALC_A_MIN) - 1)*2;
     ASSERT(sz > 0);
     hp = HAlloc((Process *) proc, sz);
     res = NIL;
     for (i = ERTS_ALC_A_MAX; i >= ERTS_ALC_A_MIN; i--) {
 	switch (i) {
 	case ERTS_ALC_A_SYSTEM:
-	case ERTS_ALC_A_FIXED_SIZE:
 	    break;
 	default: {
 	    char *alc_str = (char *) ERTS_ALC_A2AD(i);
@@ -2164,272 +2505,12 @@ erts_alloc_util_allocators(void *proc)
     return res;
 }
 
-Eterm
-erts_allocator_info_term(void *proc, Eterm which_alloc, int only_sz)
-{
-#define ERTS_AIT_RET(R) \
-  do { res = (R); goto done; } while (0)
-#define ERTS_AIT_HALLOC(P, S) \
-  do { hp = HAlloc((P), (S)); hp_end = hp + (S); } while (0)
-
-    ErtsAlcType_t i;
-    Uint sz = 0;
-    Uint *hp = NULL;
-    Uint *hp_end = NULL;
-    Eterm res = am_undefined;
-
-    if (is_not_atom(which_alloc))
-	goto done;
-
-    for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
-	if (erts_is_atom_str((char *) ERTS_ALC_A2AD(i), which_alloc)) {
-	    if (!erts_allctrs_info[i].enabled)
-		ERTS_AIT_RET(am_false);
-	    else {
-		if (erts_allctrs_info[i].alloc_util) {
-		    Eterm ires, tmp;
-		    Eterm **hpp;
-		    Uint *szp;
-		    Eterm (*info_func)(Allctr_t *,
-				       int,
-				       int *,
-				       void *,
-				       Uint **,
-				       Uint *);
-
-		    info_func = (only_sz
-				 ? erts_alcu_sz_info
-				 : erts_alcu_info);
-
-		    if (erts_allctrs_info[i].thr_spec) {
-			ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[i];
-			int j;
-			int block_system = !tspec->all_thr_safe;
-
-			if (block_system) {
-			    erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
-			    erts_smp_block_system(0);
-			}
-			ASSERT(tspec->enabled);
-
-			szp = &sz;
-			hpp = NULL;
-
-			while (1) {
-			    ires = NIL;
-			    for (j = tspec->size - 1; j >= 0; j--) {
-				Allctr_t *allctr = tspec->allctr[j];
-				if (allctr) {
-				    tmp = erts_bld_tuple(hpp,
-							 szp,
-							 3,
-							 erts_bld_atom(hpp,
-								       szp,
-								       "instance"),
-							 make_small((Uint) j),
-							 (*info_func)(allctr,
-								      hpp != NULL,
-								      NULL,
-								      NULL,
-								      hpp,
-								      szp));
-				    ires = erts_bld_cons(hpp, szp, tmp, ires);
-				}
-			    }
-			    if (hpp)
-				break;
-			    ERTS_AIT_HALLOC((Process *) proc, sz);
-			    hpp = &hp;
-			    szp = NULL;
-			}
-
-			if (block_system) {
-			    erts_smp_release_system();
-			    erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
-			}
-		    }
-		    else {
-			Allctr_t *allctr = erts_allctrs_info[i].extra;
-			szp = &sz;
-			hpp = NULL;
-			while (1) {
-			    ires = NIL;
-			    tmp = erts_bld_tuple(hpp,
-						 szp,
-						 3,
-						 erts_bld_atom(hpp,
-							       szp,
-							       "instance"),
-						 make_small((Uint) 0),
-						 (*info_func)(allctr,
-							      hpp != NULL,
-							      NULL,
-							      NULL,
-							      hpp,
-							      szp));
-			    ires = erts_bld_cons(hpp, szp, tmp, ires);
-			    if (hpp)
-				break;
-			    ERTS_AIT_HALLOC((Process *) proc, sz);
-			    hpp = &hp;
-			    szp = NULL;
-			}
-		    }
-		    ERTS_AIT_RET(ires);
-		}
-		else {
-		    Eterm *szp, **hpp;
-		    
-		    switch (i) {
-		    case ERTS_ALC_A_SYSTEM: {
-			SysAllocStat sas;
-			Eterm opts_am;
-			Eterm opts;
-			Eterm as[4];
-			Eterm ts[4];
-			int l;
-
-			if (only_sz)
-			    ERTS_AIT_RET(NIL);
-
-			sys_alloc_stat(&sas);
-			opts_am = am_atom_put("options", 7);
-
-			szp = &sz;
-			hpp = NULL;
-
-		    restart_sys_alloc:
-			l = 0;
-			as[l] = am_atom_put("e", 1);
-			ts[l++] = am_true;
-#ifdef ELIB_ALLOC_IS_CLIB
-			as[l] = am_atom_put("m", 1);
-			ts[l++] = am_atom_put("elib", 4);
-#else
-			as[l] = am_atom_put("m", 1);
-			ts[l++] = am_atom_put("libc", 4);
-#endif
-			if(sas.trim_threshold >= 0) {
-			    as[l] = am_atom_put("tt", 2);
-			    ts[l++] = erts_bld_uint(hpp, szp,
-						    (Uint) sas.trim_threshold);
-			}
-			if(sas.top_pad >= 0) {
-			    as[l] = am_atom_put("tp", 2);
-			    ts[l++] = erts_bld_uint(hpp, szp, (Uint) sas.top_pad);
-			}
-
-			opts = erts_bld_2tup_list(hpp, szp, l, as, ts);
-			res = erts_bld_2tup_list(hpp, szp, 1, &opts_am, &opts);
-			
-			if (szp) {
-			    ERTS_AIT_HALLOC((Process *) proc, sz);
-			    szp = NULL;
-			    hpp = &hp;
-			    goto restart_sys_alloc;
-			}
-			ERTS_AIT_RET(res);
-		    }
-		    case ERTS_ALC_A_FIXED_SIZE: {
-			ErtsAlcType_t n;
-			Eterm as[2], vs[2];
-
-			if (only_sz)
-			    ERTS_AIT_RET(NIL);
-
-			as[0] = am_atom_put("options", 7);
-			as[1] = am_atom_put("pools", 5);
-
-			szp = &sz;
-			hpp = NULL;
-
-		    restart_fix_alloc:
-
-			vs[0] = erts_bld_cons(hpp, szp,
-					      erts_bld_tuple(hpp, szp, 2, 
-							     am_atom_put("e",
-									 1),
-							     am_true),
-					      NIL);
-
-			vs[1] = NIL;
-			for (n = ERTS_ALC_N_MIN_A_FIXED_SIZE;
-			     n <= ERTS_ALC_N_MAX_A_FIXED_SIZE;
-			     n++) {
-			    ErtsFixInfo efi;
-			    erts_fix_info(ERTS_ALC_N2T(n), &efi);
-
-			    vs[1] = erts_bld_cons(
-				hpp, szp,
-				erts_bld_tuple(
-				    hpp, szp, 3,
-				    am_atom_put((char *) ERTS_ALC_N2TD(n),
-						strlen(ERTS_ALC_N2TD(n))),
-				    erts_bld_uint(hpp, szp, efi.total),
-				    erts_bld_uint(hpp, szp, efi.used)),
-				vs[1]);
-
-			}
-			
-			res = erts_bld_2tup_list(hpp, szp, 2, as, vs);
-			if (szp) {
-			    ERTS_AIT_HALLOC((Process *) proc, sz);
-			    szp = NULL;
-			    hpp = &hp;
-			    goto restart_fix_alloc;
-			}
-			ERTS_AIT_RET(res);
-		    }
-		    default:
-			ASSERT(0);
-			goto done;
-		    }
-		}
-	    }
-	}
-    }
-
-    if (ERTS_IS_ATOM_STR("mseg_alloc", which_alloc)) {
-#if HAVE_ERTS_MSEG
-	if (only_sz)
-	    ERTS_AIT_RET(NIL);
-	erts_mseg_info(NULL, NULL, 0, NULL, &sz);
-	if (sz)
-	    ERTS_AIT_HALLOC((Process *) proc, sz);
-	ERTS_AIT_RET(erts_mseg_info(NULL, NULL, 1, &hp, NULL));
-#else
-	ERTS_AIT_RET(am_false);
-#endif
-
-    }
-    else if (ERTS_IS_ATOM_STR("alloc_util", which_alloc)) {
-	if (only_sz)
-	    ERTS_AIT_RET(NIL);
-	erts_alcu_au_info_options(NULL, NULL, NULL, &sz);
-	if (sz)
-	    ERTS_AIT_HALLOC((Process *) proc, sz);
-	ERTS_AIT_RET(erts_alcu_au_info_options(NULL, NULL, &hp, NULL));
-    }
-
- done:
-    if (hp) {
-	ASSERT(hp_end >= hp);
-	HRelease((Process *) proc, hp_end, hp);
-    }
-    return res;
-
-#undef ERTS_AIT_RET
-#undef ERTS_AIT_HALLOC
-}
-
 void
 erts_allocator_info(int to, void *arg)
 {
     ErtsAlcType_t a;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)
-		       || (ERTS_IS_CRASH_DUMPING
-			   && erts_smp_is_system_blocked(ERTS_BS_FLG_ALLOW_GC)));
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
 
     for (a = ERTS_ALC_A_MIN; a <= ERTS_ALC_A_MAX; a++) {
 	int ai;
@@ -2462,11 +2543,7 @@ erts_allocator_info(int to, void *arg)
 		    case ERTS_ALC_A_SYSTEM: {
 			SysAllocStat sas;
 			erts_print(to, arg, "option e: true\n");
-#ifdef ELIB_ALLOC_IS_CLIB
-			erts_print(to, arg, "option m: elib\n");
-#else
 			erts_print(to, arg, "option m: libc\n");
-#endif
 			sys_alloc_stat(&sas);
 			if(sas.trim_threshold >= 0)
 			    erts_print(to, arg, "option tt: %d\n", sas.trim_threshold);
@@ -2474,22 +2551,6 @@ erts_allocator_info(int to, void *arg)
 			    erts_print(to, arg, "option tp: %d\n", sas.top_pad);
 			break;
 		    }
-		    case ERTS_ALC_A_FIXED_SIZE: {
-			ErtsAlcType_t n;
-			erts_print(to, arg, "option e: true\n");
-
-			for (n = ERTS_ALC_N_MIN_A_FIXED_SIZE;
-			     n <= ERTS_ALC_N_MAX_A_FIXED_SIZE;
-			     n++) {
-			    ErtsFixInfo efi;
-			    erts_fix_info(ERTS_ALC_N2T(n), &efi);
-			    erts_print(to, arg, "%s: %lu %lu\n",
-				       ERTS_ALC_N2TD(n),
-				       efi.total,
-				       efi.used);
-			}
-			break;
-		    }
 		    default:
 			ASSERT(0);
 			break;
@@ -2500,8 +2561,18 @@ erts_allocator_info(int to, void *arg)
     }
 
 #if HAVE_ERTS_MSEG
-    erts_print(to, arg, "=allocator:mseg_alloc\n");
-    erts_mseg_info(&to, arg, 0, NULL, NULL);
+    {
+#ifdef ERTS_SMP
+	int max = (int) erts_no_schedulers;
+#else
+	int max = 0;
+#endif
+	int i;
+	for (i = 0; i <= max; i++) {
+	    erts_print(to, arg, "=allocator:mseg_alloc[%d]\n", i);
+	    erts_mseg_info(i, &to, arg, 0, NULL, NULL);
+	}
+    }
 #endif
 
     erts_print(to, arg, "=allocator:alloc_util\n");
@@ -2555,7 +2626,7 @@ erts_allocator_options(void *proc)
 		use_mseg++;
 #endif
 		if (erts_allctr_thr_spec[a].enabled)
-		    allctr = erts_allctr_thr_spec[a].allctr[1];
+		    allctr = erts_allctr_thr_spec[a].allctr[0];
 		else
 		    allctr = erts_allctrs_info[a].extra;
 		tmp = erts_alcu_info_options(allctr, NULL, NULL, hpp, szp);
@@ -2570,13 +2641,8 @@ erts_allocator_options(void *proc)
 
 		switch (a) {
 		case ERTS_ALC_A_SYSTEM:
-#ifdef ELIB_ALLOC_IS_CLIB
-		    as[l] = am_atom_put("m", 1);
-		    ts[l++] = am_atom_put("elib", 4);
-#else
 		    as[l] = am_atom_put("m", 1);
 		    ts[l++] = am_atom_put("libc", 4);
-#endif
 		    if(sas.trim_threshold >= 0) {
 			as[l] = am_atom_put("tt", 2);
 			ts[l++] = erts_bld_uint(hpp, szp,
@@ -2609,7 +2675,7 @@ erts_allocator_options(void *proc)
 #if HAVE_ERTS_MSEG
     if (use_mseg) {
 	atoms[length] = am_atom_put("mseg_alloc", 10);
-	terms[length++] = erts_mseg_info_options(NULL, NULL, hpp, szp);
+	terms[length++] = erts_mseg_info_options(0, NULL, NULL, hpp, szp);
     }
 #endif
 
@@ -2647,23 +2713,7 @@ erts_allocator_options(void *proc)
 
     features = length ? erts_bld_list(hpp, szp, length, terms) : NIL;
 
-#if defined(ELIB_ALLOC_IS_CLIB)
-    {
-	Eterm version;
-	int i;
-	int ver[5];
-	i = sscanf(ERLANG_VERSION,
-		   "%d.%d.%d.%d.%d",
-		   &ver[0], &ver[1], &ver[2], &ver[3], &ver[4]);
-
-	version = NIL;
-	for(i--; i >= 0; i--)
-	  version = erts_bld_cons(hpp, szp, make_small(ver[i]), version);
-
-	res = erts_bld_tuple(hpp, szp, 4,
-			     am_elib_malloc, version, features, settings);
-    }
-#elif defined(__GLIBC__)
+#if defined(__GLIBC__)
     {
 	Eterm AM_glibc = am_atom_put("glibc", 5);
 	Eterm version;
@@ -2707,6 +2757,343 @@ erts_allocator_options(void *proc)
     return res;
 }
 
+void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size)
+{
+    UWord v = (UWord) erts_alloc(type, size + (ERTS_CACHE_LINE_SIZE-1)
+#ifdef VALGRIND
+				  + sizeof(UWord)
+#endif
+				 );
+
+#ifdef VALGRIND
+    {   /* Link them to avoid Leak_PossiblyLost */
+	static UWord* first_in_list = NULL;
+        *(UWord**)v = first_in_list;
+	first_in_list = (UWord*) v;
+	v += sizeof(UWord);
+    }
+#endif
+
+    if (v & ERTS_CACHE_LINE_MASK) {
+	v = (v & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE;
+    }
+    ASSERT((v & ERTS_CACHE_LINE_MASK) == 0);
+    return (void*)v;
+}
+
+static void
+reply_alloc_info(void *vair)
+{
+    ErtsAllocInfoReq *air = (ErtsAllocInfoReq *) vair;
+    Uint sched_id = erts_get_scheduler_id();
+    int global_instances = air->req_sched == sched_id;
+    ErtsProcLocks rp_locks;
+    Process *rp = air->proc;
+    Eterm ref_copy = NIL, ai_list, msg;
+    Eterm *hp = NULL, *hp_end = NULL, *hp_start = NULL;
+    Eterm **hpp;
+    Uint sz, *szp;
+    ErlOffHeap *ohp = NULL;
+    ErlHeapFragment *bp = NULL;
+    int i;
+    Eterm (*info_func)(Allctr_t *,
+		       int,
+		       int *,
+		       void *,
+		       Uint **,
+		       Uint *) = (air->only_sz
+				  ? erts_alcu_sz_info
+				  : erts_alcu_info);
+
+    rp_locks = air->req_sched == sched_id ? ERTS_PROC_LOCK_MAIN : 0;
+
+    sz = 0;
+    hpp = NULL;
+    szp = &sz;
+
+    while (1) {
+
+	if (hpp)
+	    ref_copy = STORE_NC(hpp, ohp, air->ref);
+	else
+	    *szp += REF_THING_SIZE;
+
+	ai_list = NIL;
+	for (i = 0; air->allocs[i] != ERTS_ALC_A_INVALID; i++);
+	for (i--; i >= 0; i--) {
+	    int ai = air->allocs[i];
+	    Allctr_t *allctr;
+	    Eterm ainfo;
+	    Eterm alloc_atom;
+	    if (global_instances) {
+		switch (ai) {
+		case ERTS_ALC_A_SYSTEM: {
+		    alloc_atom = erts_bld_atom(hpp, szp, "sys_alloc");
+		    ainfo = NIL;
+		    if (!air->only_sz) {
+			SysAllocStat sas;
+			if (hpp)
+			    sys_alloc_stat(&sas);
+			if (szp) {
+			    /* ensure ehough heap */
+			    sas.top_pad = INT_MAX;
+			    sas.trim_threshold = INT_MAX;
+			}
+			if (sas.top_pad >= 0) {
+			    ainfo = erts_bld_cons(
+				hpp, szp,
+				erts_bld_tuple(
+				    hpp, szp, 2,
+				    erts_bld_atom(hpp, szp, "tp"),
+				    erts_bld_uint(
+					hpp, szp,
+					(Uint) sas.top_pad)),
+				ainfo);
+			}
+			if (sas.trim_threshold >= 0) {
+			    ainfo = erts_bld_cons(
+				hpp, szp,
+				erts_bld_tuple(
+				    hpp, szp, 2,
+				    erts_bld_atom(hpp, szp, "tt"),
+				    erts_bld_uint(
+					hpp, szp,
+					(Uint) sas.trim_threshold)),
+				ainfo);
+			}
+			ainfo = erts_bld_cons(hpp, szp,
+					      erts_bld_tuple(
+						  hpp, szp, 2,
+						  erts_bld_atom(hpp, szp,
+								"m"),
+						  erts_bld_atom(hpp, szp,
+								"libc")),
+					      ainfo);
+			ainfo = erts_bld_cons(hpp, szp,
+					      erts_bld_tuple(
+						  hpp, szp, 2,
+						  erts_bld_atom(hpp, szp,
+								"e"),
+						  am_true),
+					      ainfo);
+			ainfo = erts_bld_tuple(hpp, szp, 2,
+					       erts_bld_atom(hpp, szp,
+							     "options"),
+					       ainfo);
+			ainfo = erts_bld_cons(hpp, szp,ainfo,NIL);
+		    }
+		    ainfo = erts_bld_tuple(hpp, szp, 3,
+					   alloc_atom,
+					   make_small(0),
+					   ainfo);
+		    break;
+		}
+		case ERTS_ALC_INFO_A_ALLOC_UTIL:
+		    alloc_atom = erts_bld_atom(hpp, szp, "alloc_util");
+		    ainfo = (air->only_sz
+			     ? NIL
+			     : erts_alcu_au_info_options(NULL, NULL,
+							 hpp, szp));
+		    ainfo = erts_bld_tuple(hpp, szp, 3,
+					   alloc_atom,
+					   make_small(0),
+					   ainfo);
+		    break;
+		case ERTS_ALC_INFO_A_MSEG_ALLOC:
+		    alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc");
+#if HAVE_ERTS_MSEG
+		    ainfo = (air->only_sz
+			     ? NIL
+			     : erts_mseg_info(0, NULL, NULL, hpp != NULL,
+					      hpp, szp));
+		    ainfo = erts_bld_tuple(hpp, szp, 3,
+					   alloc_atom,
+					   make_small(0),
+					   ainfo);
+#else
+		    ainfo = erts_bld_tuple(hpp, szp, 2, alloc_atom,
+					   am_false);
+#endif
+			break;
+		default:
+		    alloc_atom = erts_bld_atom(hpp, szp,
+					       (char *) ERTS_ALC_A2AD(ai));
+		    if (!erts_allctrs_info[ai].enabled) 
+			ainfo = erts_bld_tuple(hpp, szp, 2, alloc_atom,
+					       am_false);
+		    else if (erts_allctrs_info[ai].alloc_util) {
+			if (erts_allctrs_info[ai].thr_spec)
+			    allctr = erts_allctr_thr_spec[ai].allctr[0];
+			else
+			    allctr = erts_allctrs_info[ai].extra;
+			ainfo = info_func(allctr, hpp != NULL, NULL,
+					  NULL, hpp, szp);
+			ainfo = erts_bld_tuple(hpp, szp, 3, alloc_atom,
+					       make_small(0), ainfo);
+		    }
+		    else {
+			erl_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n",
+				 __FILE__, __LINE__);
+		    }
+		}
+		ai_list = erts_bld_cons(hpp, szp,
+					ainfo, ai_list);
+	    }
+	    switch (ai) {
+	    case ERTS_ALC_A_SYSTEM:
+	    case ERTS_ALC_INFO_A_ALLOC_UTIL:
+		break;
+	    case ERTS_ALC_INFO_A_MSEG_ALLOC:
+#if HAVE_ERTS_MSEG && defined(ERTS_SMP)
+		alloc_atom = erts_bld_atom(hpp, szp, "mseg_alloc");
+		ainfo = (air->only_sz
+			 ? NIL
+			 : erts_mseg_info(sched_id, NULL, NULL,
+					  hpp != NULL, hpp, szp));
+		ainfo = erts_bld_tuple(hpp, szp, 3,
+				       alloc_atom,
+				       make_small(sched_id),
+				       ainfo);
+		ai_list = erts_bld_cons(hpp, szp, ainfo, ai_list);
+#endif
+		break;
+	    default:
+		if (erts_allctrs_info[ai].thr_spec) {
+		    alloc_atom = erts_bld_atom(hpp, szp,
+					       (char *) ERTS_ALC_A2AD(ai));
+		    allctr = erts_allctr_thr_spec[ai].allctr[sched_id];
+		    ainfo = info_func(allctr, hpp != NULL, NULL,
+				      NULL, hpp, szp);
+		    ai_list = erts_bld_cons(hpp, szp,
+					    erts_bld_tuple(
+						hpp, szp,
+						3,
+						alloc_atom,
+						make_small(sched_id),
+						ainfo),
+					    ai_list);
+		}
+		break;
+	    }
+	    msg = erts_bld_tuple(hpp, szp,
+				 3,
+				 ref_copy,
+				 make_small(sched_id),
+				 ai_list);
+
+	}
+	if (hpp)
+	    break;
+
+	hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks);
+	hp_start = hp;
+	hp_end = hp + sz;
+	szp = NULL;
+	hpp = &hp;
+    }
+    if (bp)
+	bp = erts_resize_message_buffer(bp, hp - hp_start, &msg, 1);
+    else {
+	ASSERT(hp);
+	HRelease(rp, hp_end, hp);	    
+    }
+
+    erts_queue_message(rp, &rp_locks, bp, msg, NIL
+#ifdef USE_VM_PROBES
+		       , NIL
+#endif
+		       );
+
+    if (air->req_sched == sched_id)
+	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
+ 
+    erts_smp_proc_unlock(rp, rp_locks);
+    erts_smp_proc_dec_refc(rp);
+
+    if (erts_smp_atomic32_dec_read_nob(&air->refc) == 0)
+	aireq_free(air);
+}
+
+int
+erts_request_alloc_info(struct process *c_p,
+			Eterm ref,
+			Eterm allocs,
+			int only_sz)
+{
+    ErtsAllocInfoReq *air = aireq_alloc();
+    Eterm req_ai[ERTS_ALC_A_MAX+1+2] = {0};
+    Eterm alist;
+    Eterm *hp;
+    int airix = 0, ai;
+
+    air->req_sched = erts_get_scheduler_id();
+
+    air->only_sz = only_sz;
+
+    air->proc = c_p;
+
+    if (is_not_internal_ref(ref))
+	return 0;
+
+    hp = &air->ref_heap[0];
+    air->ref = STORE_NC(&hp, NULL, ref);
+
+    if (is_not_list(allocs))
+	return 0;
+
+    alist = allocs;
+
+    while (is_list(alist)) {
+	int saved = 0;
+	Eterm* consp = list_val(alist);
+	Eterm alloc = CAR(consp);
+
+	for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++)
+	    if (erts_is_atom_str((char *) erts_alc_a2ad[ai], alloc))
+		goto save_alloc;
+	if (erts_is_atom_str("mseg_alloc", alloc)) {
+	    ai = ERTS_ALC_INFO_A_MSEG_ALLOC;
+	    goto save_alloc;
+	}
+	if (erts_is_atom_str("alloc_util", alloc)) {
+	    ai = ERTS_ALC_INFO_A_ALLOC_UTIL;
+	save_alloc:
+	    if (req_ai[ai])
+		return 0;
+	    air->allocs[airix++] = ai;
+	    req_ai[ai] = 1;
+	    saved = 1;
+	}
+
+	if (!saved)
+	    return 0;
+
+	alist = CDR(consp);
+    }
+
+    if (is_not_nil(alist))
+	return 0;
+
+    air->allocs[airix] = ERTS_ALC_A_INVALID;
+
+    erts_smp_atomic32_init_nob(&air->refc,
+			       (erts_aint32_t) erts_no_schedulers);
+
+    erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers);
+
+#ifdef ERTS_SMP
+    if (erts_no_schedulers > 1)
+	erts_schedule_multi_misc_aux_work(1,
+					  erts_no_schedulers,
+					  reply_alloc_info,
+					  (void *) air);
+#endif
+
+    reply_alloc_info((void *) air);
+
+    return 1;
+}
+
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
  * Deprecated functions                                                    *
  *                                                                         *
@@ -2736,10 +3123,10 @@ void *safe_realloc(void *ptr, Uint sz)
 \*                                                                           */
 #define ERTS_ALC_TEST_ABORT erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n")
 
-unsigned long erts_alc_test(unsigned long op,
-			    unsigned long a1,
-			    unsigned long a2,
-			    unsigned long a3)
+UWord erts_alc_test(UWord op,
+			    UWord a1,
+			    UWord a2,
+			    UWord a3)
 {
     switch (op >> 8) {
     case 0x0:	return erts_alcu_test(op,  a1, a2);
@@ -2747,29 +3134,30 @@ unsigned long erts_alc_test(unsigned long op,
     case 0x2:	return erts_bfalc_test(op, a1, a2);
     case 0x3:	return erts_afalc_test(op, a1, a2);
     case 0x4:	return erts_mseg_test(op,  a1, a2, a3);
+    case 0x5:	return erts_aoffalc_test(op, a1, a2);
     case 0xf:
 	switch (op) {
 	case 0xf00:
 #ifdef USE_THREADS
 	    if (((Allctr_t *) a1)->thread_safe)
-		return (unsigned long) erts_alcu_alloc_ts(ERTS_ALC_T_UNDEF,
+		return (UWord) erts_alcu_alloc_ts(ERTS_ALC_T_UNDEF,
 							  (void *) a1,
 							  (Uint) a2);
 	    else
 #endif
-		return (unsigned long) erts_alcu_alloc(ERTS_ALC_T_UNDEF,
+		return (UWord) erts_alcu_alloc(ERTS_ALC_T_UNDEF,
 						       (void *) a1,
 						       (Uint) a2);
 	case 0xf01:
 #ifdef USE_THREADS
 	    if (((Allctr_t *) a1)->thread_safe)
-		return (unsigned long) erts_alcu_realloc_ts(ERTS_ALC_T_UNDEF,
+		return (UWord) erts_alcu_realloc_ts(ERTS_ALC_T_UNDEF,
 							    (void *) a1,
 							    (void *) a2,
 							    (Uint) a3);
 	    else
 #endif
-		return (unsigned long) erts_alcu_realloc(ERTS_ALC_T_UNDEF,
+		return (UWord) erts_alcu_realloc(ERTS_ALC_T_UNDEF,
 							 (void *) a1,
 							 (void *) a2,
 							 (Uint) a3);
@@ -2790,6 +3178,7 @@ unsigned long erts_alc_test(unsigned long op,
 	    init.atype = GOODFIT;
 	    init.init.util.name_prefix = (char *) a1;
 	    init.init.util.ts = a2 ? 1 : 0;
+	    init.init.util.sbmbct = 0;
 
 	    if ((char **) a3) {
 		char **argv = (char **) a3;
@@ -2798,7 +3187,7 @@ unsigned long erts_alc_test(unsigned long op,
 		    if (argv[i][0] == '-' && argv[i][1] == 't')
 			handle_au_arg(&init, &argv[i][2], argv, &i);
 		    else
-			return (unsigned long) NULL;
+			return (UWord) NULL;
 		    i++;
 		}
 	    }
@@ -2825,31 +3214,39 @@ unsigned long erts_alc_test(unsigned long op,
 					  &init.init.af,
 					  &init.init.util);
 		break;
+	    case AOFIRSTFIT:
+		allctr = erts_aoffalc_start((AOFFAllctr_t *)
+					  erts_alloc(ERTS_ALC_T_UNDEF,
+						     sizeof(AOFFAllctr_t)),
+					  &init.init.aoff,
+					  &init.init.util);
+		break;
+
 	    default:
 		ASSERT(0);
 		allctr = NULL;
 		break;
 	    }
 
-	    return (unsigned long) allctr;
+	    return (UWord) allctr;
 	}
 	case 0xf04:
 	    erts_alcu_stop((Allctr_t *) a1);
 	    erts_free(ERTS_ALC_T_UNDEF, (void *) a1);
 	    break;
 #ifdef USE_THREADS
-	case 0xf05: return (unsigned long) 1;
-	case 0xf06: return (unsigned long) ((Allctr_t *) a1)->thread_safe;
+	case 0xf05: return (UWord) 1;
+	case 0xf06: return (UWord) ((Allctr_t *) a1)->thread_safe;
 #ifdef ETHR_NO_FORKSAFETY
-	case 0xf07: return (unsigned long) 0;
+	case 0xf07: return (UWord) 0;
 #else
-	case 0xf07: return (unsigned long) ((Allctr_t *) a1)->thread_safe;
+	case 0xf07: return (UWord) ((Allctr_t *) a1)->thread_safe;
 #endif
 	case 0xf08: {
 	    ethr_mutex *mtx = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_mutex));
 	    if (ethr_mutex_init(mtx) != 0)
 		ERTS_ALC_TEST_ABORT;
-	    return (unsigned long) mtx;
+	    return (UWord) mtx;
 	}
 	case 0xf09: {
 	    ethr_mutex *mtx = (ethr_mutex *) a1;
@@ -2859,18 +3256,16 @@ unsigned long erts_alc_test(unsigned long op,
 	    break;
 	}
 	case 0xf0a:
-	    if (ethr_mutex_lock((ethr_mutex *) a1) != 0)
-		ERTS_ALC_TEST_ABORT;
+	    ethr_mutex_lock((ethr_mutex *) a1);
 	    break;
 	case 0xf0b:
-	    if (ethr_mutex_unlock((ethr_mutex *) a1) != 0)
-		ERTS_ALC_TEST_ABORT;
+	    ethr_mutex_unlock((ethr_mutex *) a1);
 	    break;
 	case 0xf0c: {
 	    ethr_cond *cnd = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_cond));
 	    if (ethr_cond_init(cnd) != 0)
 		ERTS_ALC_TEST_ABORT;
-	    return (unsigned long) cnd;
+	    return (UWord) cnd;
 	}
 	case 0xf0d: {
 	    ethr_cond *cnd = (ethr_cond *) a1;
@@ -2880,33 +3275,23 @@ unsigned long erts_alc_test(unsigned long op,
 	    break;
 	}
 	case 0xf0e:
-	    if (ethr_cond_broadcast((ethr_cond *) a1) != 0)
-		ERTS_ALC_TEST_ABORT;
+	    ethr_cond_broadcast((ethr_cond *) a1);
 	    break;
 	case 0xf0f: {
 	    int res;
 	    do {
 		res = ethr_cond_wait((ethr_cond *) a1, (ethr_mutex *) a2);
 	    } while (res == EINTR);
-	    if (res != 0)
-		ERTS_ALC_TEST_ABORT;
 	    break;
 	}
 	case 0xf10: {
 	    ethr_tid *tid = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_tid));
-#ifdef ERTS_ENABLE_LOCK_COUNT
-	    if (erts_lcnt_thr_create(tid,
-				(void * (*)(void *)) a1,
-				(void *) a2,
-				NULL) != 0)
-#else
 	    if (ethr_thr_create(tid,
 				(void * (*)(void *)) a1,
 				(void *) a2,
 				NULL) != 0)
-#endif
 		ERTS_ALC_TEST_ABORT;
-	    return (unsigned long) tid;
+	    return (UWord) tid;
 	}
 	case 0xf11: {
 	    ethr_tid *tid = (ethr_tid *) a1;
@@ -2923,13 +3308,13 @@ unsigned long erts_alc_test(unsigned long op,
 	default:
 	    break;
 	}
-	return (unsigned long) 0;
+	return (UWord) 0;
     default:
 	break;
     }
 
     ASSERT(0);
-    return ~((unsigned long) 0);
+    return ~((UWord) 0);
 }
 
 #ifdef DEBUG
@@ -2943,10 +3328,13 @@ unsigned long erts_alc_test(unsigned long op,
 #undef PRINT_OPS
 #endif
 
+#ifdef HARD_DEBUG
+#define FENCE_SZ		(4*sizeof(UWord))
+#else
+#define FENCE_SZ		(3*sizeof(UWord))
+#endif
 
-#define FENCE_SZ		(3*sizeof(Uint))
-
-#ifdef ARCH_64
+#if defined(ARCH_64)
 #define FENCE_PATTERN 0xABCDEF97ABCDEF97
 #else
 #define FENCE_PATTERN 0xABCDEF97
@@ -2956,7 +3344,7 @@ unsigned long erts_alc_test(unsigned long op,
 #define TYPE_PATTERN_SHIFT 16
 
 #define FIXED_FENCE_PATTERN_MASK \
-  (~((Uint) (TYPE_PATTERN_MASK << TYPE_PATTERN_SHIFT)))
+  (~((UWord) (TYPE_PATTERN_MASK << TYPE_PATTERN_SHIFT)))
 #define FIXED_FENCE_PATTERN \
   (FENCE_PATTERN & FIXED_FENCE_PATTERN_MASK)
 
@@ -2966,22 +3354,170 @@ unsigned long erts_alc_test(unsigned long op,
 #define GET_TYPE_OF_PATTERN(P) \
   (((P) >> TYPE_PATTERN_SHIFT) & TYPE_PATTERN_MASK)
 
+#ifdef HARD_DEBUG
+
+#define ERL_ALC_HDBG_MAX_MBLK 100000
+#define ERTS_ALC_O_CHECK -1
+
+typedef struct hdbg_mblk_ hdbg_mblk;
+struct hdbg_mblk_ {
+    hdbg_mblk *next;
+    hdbg_mblk *prev;
+    void *p;
+    Uint s;
+    ErtsAlcType_t n;
+};
+
+static hdbg_mblk hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK];
+
+static hdbg_mblk *free_hdbg_mblks;
+static hdbg_mblk *used_hdbg_mblks;
+static erts_mtx_t hdbg_mblk_mtx;
+
+static void
+hdbg_init(void)
+{
+    int i;
+    for (i = 0; i < ERL_ALC_HDBG_MAX_MBLK-1; i++)
+	hdbg_mblks[i].next = &hdbg_mblks[i+1];
+    hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK-1].next = NULL;
+    free_hdbg_mblks = &hdbg_mblks[0];
+    used_hdbg_mblks = NULL;
+    erts_mtx_init(&hdbg_mblk_mtx, "erts_alloc_hard_debug");
+}
+
+static void *check_memory_fence(void *ptr,
+				Uint *size,
+				ErtsAlcType_t n,
+				int func);
+void erts_hdbg_chk_blks(void);
+
+void
+erts_hdbg_chk_blks(void)
+{
+    hdbg_mblk *mblk;
+
+    erts_mtx_lock(&hdbg_mblk_mtx);
+    for (mblk = used_hdbg_mblks; mblk; mblk = mblk->next) {
+	Uint sz;
+	check_memory_fence(mblk->p, &sz, mblk->n, ERTS_ALC_O_CHECK);
+	ASSERT(sz == mblk->s);
+    }
+    erts_mtx_unlock(&hdbg_mblk_mtx);
+}
+
+static hdbg_mblk *
+hdbg_alloc(void *p, Uint s, ErtsAlcType_t n)
+{
+    hdbg_mblk *mblk;
+
+    erts_mtx_lock(&hdbg_mblk_mtx);
+    mblk = free_hdbg_mblks;
+    if (!mblk) {
+	erts_fprintf(stderr,
+		     "Ran out of debug blocks; please increase "
+		     "ERL_ALC_HDBG_MAX_MBLK=%d and recompile!\n",
+		     ERL_ALC_HDBG_MAX_MBLK);
+	abort();
+    }
+    free_hdbg_mblks = mblk->next;
+
+    mblk->p = p;
+    mblk->s = s;
+    mblk->n = n;
+
+    mblk->next = used_hdbg_mblks;
+    mblk->prev = NULL;
+    if (used_hdbg_mblks)
+	used_hdbg_mblks->prev = mblk;
+    used_hdbg_mblks = mblk;
+    erts_mtx_unlock(&hdbg_mblk_mtx);
+    return (void *) mblk;
+}
+
+static void
+hdbg_free(hdbg_mblk *mblk)
+{
+    erts_mtx_lock(&hdbg_mblk_mtx);
+    if (mblk->next)
+	mblk->next->prev = mblk->prev;
+    if (mblk->prev)
+	mblk->prev->next = mblk->next;
+    else
+	used_hdbg_mblks = mblk->next;
+
+    mblk->next = free_hdbg_mblks;
+    free_hdbg_mblks = mblk;
+    erts_mtx_unlock(&hdbg_mblk_mtx);
+}
+
+#endif
+
+#ifdef  ERTS_ALLOC_UTIL_HARD_DEBUG
+static void *check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func);
+
+void check_allocated_block( Uint type, void *blk)
+{
+    Uint dummy;
+    check_memory_fence(blk, &dummy, ERTS_ALC_T2N(type), ERTS_ALC_O_FREE);
+}
+
+void check_allocators(void)
+{
+    int i;
+    if (!erts_initialized)
+	return;
+    for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; ++i) {
+	if (erts_allctrs_info[i].alloc_util) {
+	    ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) erts_allctrs[i].extra;
+	    Allctr_t *allctr = real_af->extra;
+	    Carrier_t *ct;
+#ifdef USE_THREADS
+	if (allctr->thread_safe)
+	    erts_mtx_lock(&allctr->mutex);
+#endif
+
+	    if (allctr->check_mbc) {
+		for (ct = allctr->mbc_list.first; ct; ct = ct->next) {
+		    fprintf(stderr,"Checking allocator %d\r\n",i);
+		    allctr->check_mbc(allctr,ct);
+		}
+	    }
+#ifdef USE_THREADS
+	if (allctr->thread_safe)
+	    erts_mtx_unlock(&allctr->mutex);
+#endif
+	}
+    }
+}
+#endif
 
 static void *
 set_memory_fence(void *ptr, Uint sz, ErtsAlcType_t n)
 {
-    Uint *ui_ptr;
-    Uint pattern;
+    UWord *ui_ptr;
+    UWord pattern;
+#ifdef HARD_DEBUG
+    hdbg_mblk **mblkpp;
+#endif
 
     if (!ptr)
 	return NULL;
 
-    ui_ptr = (Uint *) ptr;
+    ui_ptr = (UWord *) ptr;
     pattern = MK_PATTERN(n);
-    
+
+#ifdef HARD_DEBUG
+    mblkpp = (hdbg_mblk **) ui_ptr++;
+#endif
+
     *(ui_ptr++) = sz;
     *(ui_ptr++) = pattern;
-    memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(Uint));
+    memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(UWord));
+
+#ifdef HARD_DEBUG
+    *mblkpp = hdbg_alloc((void *) ui_ptr, sz, n);
+#endif
 
     return (void *) ui_ptr;
 }
@@ -2991,16 +3527,22 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
 {
     Uint sz;
     Uint found_type;
-    Uint pre_pattern;
-    Uint post_pattern;
-    Uint *ui_ptr;
+    UWord pre_pattern;
+    UWord post_pattern;
+    UWord *ui_ptr;
+#ifdef HARD_DEBUG
+    hdbg_mblk *mblk;
+#endif
 
     if (!ptr)
 	return NULL;
 
-    ui_ptr = (Uint *) ptr;
+    ui_ptr = (UWord *) ptr;
     pre_pattern = *(--ui_ptr);
     *size = sz = *(--ui_ptr);
+#ifdef HARD_DEBUG
+    mblk = (hdbg_mblk *) *(--ui_ptr);
+#endif
 
     found_type = GET_TYPE_OF_PATTERN(pre_pattern);
     if (pre_pattern != MK_PATTERN(n)) {
@@ -3008,10 +3550,10 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
 	    erl_exit(ERTS_ABORT_EXIT,
 		     "ERROR: Fence at beginning of memory block (p=0x%u) "
 		     "clobbered.\n",
-		     (unsigned long) ptr);
+		     (UWord) ptr);
     }
 
-    memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(Uint));
+    memcpy((void *) &post_pattern, (void *) (((char *)ptr)+sz), sizeof(UWord));
 
     if (post_pattern != MK_PATTERN(n)
 	|| pre_pattern != post_pattern) {
@@ -3025,12 +3567,12 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
 	    erl_exit(ERTS_ABORT_EXIT,
 		     "ERROR: Fence at end of memory block (p=0x%u, sz=%u) "
 		     "clobbered.\n",
-		     (unsigned long) ptr, (unsigned long) sz);
+		     (UWord) ptr, (UWord) sz);
 	if (found_type != GET_TYPE_OF_PATTERN(post_pattern))
 	    erl_exit(ERTS_ABORT_EXIT,
 		     "ERROR: Fence around memory block (p=0x%u, sz=%u) "
 		     "clobbered.\n",
-		     (unsigned long) ptr, (unsigned long) sz);
+		     (UWord) ptr, (UWord) sz);
 
 	ftype = type_no_str(found_type);
 	if (!ftype) {
@@ -3053,9 +3595,20 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func)
 	erl_exit(ERTS_ABORT_EXIT,
 		 "ERROR: Memory block (p=0x%u, sz=%u) allocated as type \"%s\","
 		 " but %s as type \"%s\".\n",
-		 (unsigned long) ptr, (unsigned long) sz, ftype, op_str, otype);
+		 (UWord) ptr, (UWord) sz, ftype, op_str, otype);
     }
 
+#ifdef HARD_DEBUG
+    switch (func) {
+    case ERTS_ALC_O_REALLOC:
+    case ERTS_ALC_O_FREE:
+	hdbg_free(mblk);
+	break;
+    default:
+	break;
+    }
+#endif
+
     return (void *) ui_ptr;
 }
 
@@ -3068,6 +3621,10 @@ debug_alloc(ErtsAlcType_t n, void *extra, Uint size)
     Uint dsize;
     void *res;
 
+#ifdef HARD_DEBUG
+    erts_hdbg_chk_blks();
+#endif
+
     ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX);
     dsize = size + FENCE_SZ;
     res = (*real_af->alloc)(n, real_af->extra, dsize);
@@ -3097,13 +3654,17 @@ debug_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size)
     dsize = size + FENCE_SZ;
     dptr = check_memory_fence(ptr, &old_size, n, ERTS_ALC_O_REALLOC);
 
+#ifdef HARD_DEBUG
+    erts_hdbg_chk_blks();
+#endif
+
     if (old_size > size)
 	sys_memset((void *) (((char *) ptr) + size),
 		   0xf,
 		   sizeof(Uint) + old_size - size);
 
     res = (*real_af->realloc)(n, real_af->extra, dptr, dsize);
-    
+
     res = set_memory_fence(res, size, n);
 
 #ifdef PRINT_OPS
@@ -3133,6 +3694,10 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr)
     fprintf(stderr, "free(%s, 0x%lx)\r\n", ERTS_ALC_N2TD(n), (Uint) ptr);
 #endif
 
+#ifdef HARD_DEBUG
+    erts_hdbg_chk_blks();
+#endif
+
 }
 
 static Uint
@@ -3152,6 +3717,4 @@ install_debug_functions(void)
     return FENCE_SZ;
 }
 
-
-
 #endif /* #ifdef DEBUG */
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index e7a203002f..e475f9d8a2 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2002-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2002-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -21,6 +21,10 @@
 #define ERL_ALLOC_H__
 
 #include "erl_alloc_types.h"
+#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
+#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
+#include "erl_thr_progress.h"
+#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
 #include "erl_alloc_util.h"
 #ifdef USE_THREADS
 #include "erl_threads.h"
@@ -43,50 +47,47 @@
 #  define ERTS_ALC_INLINE
 #endif
 
-#define ERTS_FIX_CORE_ALLOCATOR ERTS_ALC_A_LONG_LIVED
-extern ErtsAlcType_t erts_fix_core_allocator_ix;
-
-typedef struct {
-    Uint total;
-    Uint used;
-} ErtsFixInfo;
+#define ERTS_ALC_NO_FIXED_SIZES \
+  (ERTS_ALC_N_MAX_A_FIXED_SIZE - ERTS_ALC_N_MIN_A_FIXED_SIZE + 1)
 
 void erts_sys_alloc_init(void);
 void *erts_sys_alloc(ErtsAlcType_t, void *, Uint);
 void *erts_sys_realloc(ErtsAlcType_t, void *, void *, Uint);
 void erts_sys_free(ErtsAlcType_t, void *, void *);
 
-
-void erts_init_fix_alloc(Uint, void *(*)(Uint));
-Uint erts_get_fix_size(ErtsAlcType_t);
-void erts_set_fix_size(ErtsAlcType_t, Uint);
-void erts_fix_info(ErtsAlcType_t, ErtsFixInfo *);
-void *erts_fix_alloc(ErtsAlcType_t, void *, Uint);
-void *erts_fix_realloc(ErtsAlcType_t, void *, void*, Uint);
-void erts_fix_free(ErtsAlcType_t, void *, void*);
-
-
 Eterm erts_memory(int *, void *, void *, Eterm);
 Eterm erts_allocated_areas(int *, void *, void *);
 
 Eterm erts_alloc_util_allocators(void *proc);
 void erts_allocator_info(int, void *);
-Eterm erts_allocator_info_term(void *proc, Eterm which_alloc, int only_sz);
 Eterm erts_allocator_options(void *proc);
 
+struct process;
+
+int erts_request_alloc_info(struct process *c_p, Eterm ref, Eterm allocs,
+			    int only_sz);
+
 #define ERTS_ALLOC_INIT_DEF_OPTS_INITER {0}
 typedef struct {
-    int dummy;
+    int ncpu;
 } ErtsAllocInitOpts;
 
+typedef struct {
+    Allctr_t *deallctr[ERTS_ALC_A_MAX+1];
+    int pref_ix[ERTS_ALC_A_MAX+1];
+    int flist_ix[ERTS_ALC_A_MAX+1];
+    int pre_alc_ix;
+} ErtsSchedAllocData;
+
 void erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop);
+void erts_alloc_late_init(void);
 
 #if defined(GET_ERTS_ALC_TEST) || defined(ERTS_ALC_INTERNAL__)
 /* Only for testing */
-unsigned long erts_alc_test(unsigned long,
-			    unsigned long,
-			    unsigned long,
-			    unsigned long);
+UWord erts_alc_test(UWord,
+		    UWord,
+		    UWord,
+		    UWord);
 #endif
 
 #define ERTS_ALC_O_ALLOC		0
@@ -99,6 +100,14 @@ unsigned long erts_alc_test(unsigned long,
 
 #define ERTS_ALC_MIN_LONG_LIVED_TIME	(10*60*1000)
 
+#if HALFWORD_HEAP
+#define ERTS_IS_SBMBC_ALLOCATOR_NO__(NO) \
+  ((NO) == ERTS_ALC_A_SBMBC || (NO) == ERTS_ALC_A_SBMBC_LOW)
+#else
+#define ERTS_IS_SBMBC_ALLOCATOR_NO__(NO) \
+  ((NO) == ERTS_ALC_A_SBMBC)
+#endif
+
 typedef struct {
     int alloc_util;
     int enabled;
@@ -118,15 +127,22 @@ extern ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1];
 
 typedef struct {
     int enabled;
-    int all_thr_safe;
+    int dd;
+    int aix;
     int size;
     Allctr_t **allctr;
 } ErtsAllocatorThrSpec_t;
 
 extern ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1];
 
-int erts_alc_get_thr_ix(void);
-void erts_alloc_reg_scheduler_id(Uint id);
+void erts_alloc_register_scheduler(void *vesdp);
+#ifdef ERTS_SMP
+void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
+						 int *need_thr_progress,
+						 ErtsThrPrgrVal *thr_prgr_p,
+						 int *more_work);
+#endif
+erts_aint32_t erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs);
 
 __decl_noreturn void erts_alloc_enomem(ErtsAlcType_t,Uint)		
      __noreturn;
@@ -175,6 +191,14 @@ void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size);
 
 #endif /* #if !ERTS_ALC_DO_INLINE */
 
+void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size);
+
+#ifndef ERTS_CACHE_LINE_SIZE
+/* Assume a cache line size of 64 bytes */
+#  define ERTS_CACHE_LINE_SIZE ((UWord) 64)
+#  define ERTS_CACHE_LINE_MASK (ERTS_CACHE_LINE_SIZE - 1)
+#endif
+
 #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__)
 
 ERTS_ALC_INLINE
@@ -236,11 +260,13 @@ void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size)
 
 #endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */
 
-#ifndef ERTS_CACHE_LINE_SIZE
-/* Assume a cache line size of 64 bytes */
-#  define ERTS_CACHE_LINE_SIZE ((Uint) 64)
-#  define ERTS_CACHE_LINE_MASK (ERTS_CACHE_LINE_SIZE - 1)
-#endif
+#define ERTS_ALC_GET_THR_IX() ((int) erts_get_scheduler_id())
+
+typedef void (*erts_alloc_verify_func_t)(Allctr_t *);
+
+erts_alloc_verify_func_t
+erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr);
+
 
 #define ERTS_ALC_CACHE_LINE_ALIGN_SIZE(SZ) \
   (((((SZ) - 1) / ERTS_CACHE_LINE_SIZE) + 1) * ERTS_CACHE_LINE_SIZE)
@@ -420,140 +446,45 @@ NAME##_free(TYPE *p)							\
     }									\
 }
 
-typedef struct {
-    void *start;
-    void *end;
-    int chunks_mem_size;
-} erts_sched_pref_quick_alloc_data_t;
-
-#ifdef DEBUG
-#define ERTS_SPPA_DBG_CHK_IN_CHNK(A, C, P)				\
-do {									\
-    ASSERT((void *) (C) < (void *) (P));				\
-    ASSERT((void *) (P)							\
-	   < (void *) (((char *) (C)) + (A)->chunks_mem_size));		\
-} while (0)
-#else
-#define ERTS_SPPA_DBG_CHK_IN_CHNK(A, C, P)
-#endif
+#include "erl_sched_spec_pre_alloc.h"
 
 #define ERTS_SCHED_PREF_PRE_ALLOC_IMPL(NAME, TYPE, PASZ)		\
-union erts_qa_##NAME##__ {						\
+union erts_sspa_##NAME##__ {						\
+    erts_sspa_blk_t next;						\
     TYPE type;								\
-    union erts_qa_##NAME##__ *next;					\
 };									\
-typedef struct {							\
-    erts_smp_spinlock_t lock;						\
-    union erts_qa_##NAME##__ *freelist;					\
-    union erts_qa_##NAME##__ pre_alloced[1];				\
-} erts_qa_##NAME##_chunk__;						\
-static erts_sched_pref_quick_alloc_data_t *qa_data_##NAME##__;		\
-static ERTS_INLINE erts_qa_##NAME##_chunk__ *				\
-get_##NAME##_chunk_ix(int cix)						\
-{									\
-    char *ptr = (char *) qa_data_##NAME##__->start;			\
-    ptr += cix*qa_data_##NAME##__->chunks_mem_size;			\
-    return (erts_qa_##NAME##_chunk__ *) ptr;				\
-}									\
-static ERTS_INLINE erts_qa_##NAME##_chunk__ *				\
-get_##NAME##_chunk_ptr(void *ptr)					\
-{									\
-    int cix;								\
-    size_t diff;							\
-    if (ptr < qa_data_##NAME##__->start || qa_data_##NAME##__->end <= ptr)\
-	return NULL;							\
-    diff = ((char *) ptr) -  ((char *) qa_data_##NAME##__->start);	\
-    cix = diff / qa_data_##NAME##__->chunks_mem_size;			\
-    return get_##NAME##_chunk_ix(cix);					\
-}									\
+									\
+static erts_sspa_data_t *sspa_data_##NAME##__;				\
+									\
 static void								\
 init_##NAME##_alloc(void)						\
 {									\
-    size_t tot_size;							\
-    size_t chunk_mem_size;						\
-    char *chunk_start;							\
-    int cix;								\
-    int no_blocks = ERTS_PRE_ALLOC_SIZE((PASZ));			\
-    int no_blocks_per_chunk = 2*((no_blocks-1)/erts_no_schedulers + 1);	\
-    no_blocks = no_blocks_per_chunk * erts_no_schedulers;		\
-    chunk_mem_size = sizeof(erts_qa_##NAME##_chunk__);			\
-    chunk_mem_size += (sizeof(union erts_qa_##NAME##__)			\
-		       * (no_blocks_per_chunk - 1));			\
-    chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(chunk_mem_size);	\
-    tot_size = sizeof(erts_sched_pref_quick_alloc_data_t);		\
-    tot_size += ERTS_CACHE_LINE_SIZE - 1;				\
-    tot_size += chunk_mem_size*erts_no_schedulers;			\
-    qa_data_##NAME##__ = erts_alloc(ERTS_ALC_T_PRE_ALLOC_DATA,tot_size);\
-    chunk_start = (((char *) qa_data_##NAME##__)			\
-		   + sizeof(erts_sched_pref_quick_alloc_data_t));	\
-    if ((((Uint) chunk_start) & ERTS_CACHE_LINE_MASK) != ((Uint) 0))	\
-	chunk_start = ((char *)						\
-		       ((((Uint) chunk_start) & ~ERTS_CACHE_LINE_MASK)	\
-		       + ERTS_CACHE_LINE_SIZE));			\
-    qa_data_##NAME##__->chunks_mem_size = chunk_mem_size;		\
-    qa_data_##NAME##__->start = (void *) chunk_start;			\
-    qa_data_##NAME##__->end = (chunk_start				\
-			       + chunk_mem_size*erts_no_schedulers);	\
-    for (cix = 0; cix < erts_no_schedulers; cix++) {			\
-	int i;								\
-	erts_qa_##NAME##_chunk__ *chunk = get_##NAME##_chunk_ix(cix);	\
-	erts_smp_spinlock_init(&chunk->lock, #NAME "_alloc_lock");	\
-	chunk->freelist = &chunk->pre_alloced[0];			\
-	for (i = 1; i < no_blocks_per_chunk; i++) {			\
-	    ERTS_PRE_ALLOC_CLOBBER(&chunk->pre_alloced[i-1],		\
-				   union erts_qa_##NAME##__);		\
-	    chunk->pre_alloced[i-1].next = &chunk->pre_alloced[i];	\
-	}								\
-	ERTS_PRE_ALLOC_CLOBBER(&chunk->pre_alloced[no_blocks_per_chunk-1],\
-			       union erts_qa_##NAME##__);		\
-	chunk->pre_alloced[no_blocks_per_chunk-1].next = NULL;		\
-    }									\
+    sspa_data_##NAME##__ =						\
+	erts_sspa_create(sizeof(union erts_sspa_##NAME##__),		\
+			 ERTS_PRE_ALLOC_SIZE((PASZ)));			\
 }									\
-static ERTS_INLINE TYPE *						\
+									\
+static TYPE *								\
 NAME##_alloc(void)							\
 {									\
-    int cix = ((int) erts_get_scheduler_id()) - 1;			\
-    TYPE *res;								\
-    if (cix < 0)							\
-	res = NULL;							\
-    else {								\
-	erts_qa_##NAME##_chunk__ *chunk = get_##NAME##_chunk_ix(cix);	\
-	erts_smp_spin_lock(&chunk->lock);				\
-	if (!chunk->freelist)						\
-	    res = NULL;							\
-	else {								\
-	    res = &chunk->freelist->type;				\
-	    chunk->freelist = chunk->freelist->next;			\
-	    ERTS_SPPA_DBG_CHK_IN_CHNK(qa_data_##NAME##__, chunk, res);	\
-	}								\
-	erts_smp_spin_unlock(&chunk->lock);				\
-    }									\
-    return res;								\
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();		\
+    if (!esdp)								\
+	return NULL;							\
+    return (TYPE *) erts_sspa_alloc(sspa_data_##NAME##__,		\
+				    (int) esdp->no - 1);		\
 }									\
-static ERTS_INLINE int							\
+									\
+static int								\
 NAME##_free(TYPE *p)							\
 {									\
-    erts_qa_##NAME##_chunk__ *chunk;					\
-    chunk = get_##NAME##_chunk_ptr((void *) p);				\
-    if (!chunk)								\
-	return 0;							\
-    else {								\
-	union erts_qa_##NAME##__ *up;					\
-	ERTS_SPPA_DBG_CHK_IN_CHNK(qa_data_##NAME##__, chunk, p);	\
-	up = ((union erts_qa_##NAME##__ *)				\
-	      (((char *) p)						\
-	       - ((char *) &((union erts_qa_##NAME##__ *) 0)->type)));	\
-	erts_smp_spin_lock(&chunk->lock);				\
-	ERTS_PRE_ALLOC_CLOBBER(up, union erts_qa_##NAME##__);		\
-	up->next = chunk->freelist;					\
-	chunk->freelist = up;						\
-	erts_smp_spin_unlock(&chunk->lock);				\
-	return 1;							\
-    }									\
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();		\
+    return erts_sspa_free(sspa_data_##NAME##__,				\
+			  esdp ? (int) esdp->no - 1 : -1,		\
+			  (char *) p);					\
 }
 
 #ifdef DEBUG
-#define ERTS_ALC_DBG_BLK_SZ(PTR) (*(((Uint *) (PTR)) - 2))
+#define ERTS_ALC_DBG_BLK_SZ(PTR) (*(((UWord *) (PTR)) - 2))
 #endif /* #ifdef DEBUG */
 
 #undef ERTS_ALC_INLINE
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index f701f71c7d..4aa8fa82fb 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -1,19 +1,19 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2003-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2003-2012. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
 
@@ -50,6 +50,15 @@
 #   command line argument to make_alloc_types. The variable X is false
 #   after a "+disable X" statement or if it has never been mentioned.
 
++if smp
++disable threads_no_smp
++else
++if threads
++enable threads_no_smp
++else
++disable threads_no_smp
++endif
++endif
 
 # --- Allocator declarations -------------------------------------------------
 #
@@ -65,6 +74,11 @@
 
 allocator	SYSTEM		true		sys_alloc
 
+allocator	SBMBC		true		sbmbc_alloc
++if halfword
+allocator	SBMBC_LOW	true		sbmbc_low_alloc
++endif
+
 +if smp
 
 allocator	TEMPORARY	true		temp_alloc
@@ -75,6 +89,11 @@ allocator	EHEAP		true		eheap_alloc
 allocator	ETS		true		ets_alloc
 allocator	FIXED_SIZE	true		fix_alloc
 
++if halfword
+allocator	LONG_LIVED_LOW	true		ll_low_alloc
+allocator	STANDARD_LOW	true		std_low_alloc
++endif
+
 +else # Non smp build
 
 allocator	TEMPORARY	false		temp_alloc
@@ -85,12 +104,18 @@ allocator	EHEAP		false		eheap_alloc
 allocator	ETS		false		ets_alloc
 allocator	FIXED_SIZE	false		fix_alloc
 
++if halfword
+allocator	LONG_LIVED_LOW	false		ll_low_alloc
+allocator	STANDARD_LOW	false		std_low_alloc
++endif
+
 +endif
 
 allocator	BINARY		true		binary_alloc
 allocator	DRIVER		true		driver_alloc
 
 
+
 # --- Class declarations -----------------------------------------------------
 #
 # Syntax: class <CLASS> <DESCRIPTION>
@@ -117,32 +142,25 @@ class	SYSTEM		system_data
 #       should be deallocated before the emulator starts executing Erlang
 #       code again.
 #
-# NOTE: When adding or removing a type which uses the FIXED_SIZE allocator,
-#       also add or remove initialization of the type in erts_alloc_init()
-#       (erl_alloc.c).
-#
 #	<TYPE>		<ALLOCATOR>	<CLASS>		<DESCRIPTION>
 
+type	SBMBC		SBMBC		SYSTEM		small_block_mbc
 type	PROC		FIXED_SIZE	PROCESSES	proc
-type	ATOM		FIXED_SIZE	ATOM		atom_entry
-type	EXPORT		FIXED_SIZE	CODE		export_entry
-type	MODULE		FIXED_SIZE	CODE		module_entry
-type	REG_PROC	FIXED_SIZE	PROCESSES	reg_proc
+type	ATOM		LONG_LIVED	ATOM		atom_entry
+type	MODULE		LONG_LIVED	CODE		module_entry
+type	REG_PROC	STANDARD	PROCESSES	reg_proc
 type	LINK_LH		STANDARD	PROCESSES	link_lh
-type	MONITOR_SH	FIXED_SIZE	PROCESSES	monitor_sh
-type	MONITOR_LH	STANDARD	PROCESSES	monitor_lh
-type	NLINK_SH	FIXED_SIZE	PROCESSES	nlink_sh
-type	NLINK_LH	STANDARD	PROCESSES	nlink_lh
 type	SUSPEND_MON	STANDARD	PROCESSES	suspend_monitor
 type	PEND_SUSPEND	SHORT_LIVED	PROCESSES	pending_suspend
 type	PROC_LIST	SHORT_LIVED	PROCESSES	proc_list
-type	FUN_ENTRY	FIXED_SIZE	CODE		fun_entry
+type	FUN_ENTRY	LONG_LIVED	CODE		fun_entry
 type	ATOM_TXT	LONG_LIVED	ATOM		atom_text
+type 	BEAM_REGISTER	EHEAP		PROCESSES	beam_register
 type	HEAP		EHEAP		PROCESSES	heap
 type	OLD_HEAP	EHEAP		PROCESSES	old_heap
 type	HEAP_FRAG	EHEAP		PROCESSES	heap_frag
 type	TMP_HEAP	TEMPORARY	PROCESSES	tmp_heap
-type	MSG_REF		SHORT_LIVED	PROCESSES	msg_ref
+type	MSG_REF		FIXED_SIZE	PROCESSES	msg_ref
 type	MSG_ROOTS	TEMPORARY	PROCESSES	msg_roots
 type	ROOTSET		TEMPORARY	PROCESSES	root_set
 type	LOADER_TMP	TEMPORARY	CODE		loader_tmp
@@ -174,7 +192,6 @@ type	DRIVER		STANDARD	SYSTEM		driver
 type	NIF		DRIVER		SYSTEM		nif_internal
 type	BINARY		BINARY		BINARIES	binary
 type	NBIF_TABLE	SYSTEM		SYSTEM		nbif_tab
-type	CODE		LONG_LIVED	CODE		code
 type	ARG_REG		STANDARD	PROCESSES	arg_reg
 type	PROC_DICT	STANDARD	PROCESSES	proc_dict
 type	CALLS_BUF	STANDARD	PROCESSES	calls_buf
@@ -184,17 +201,16 @@ type	LINEBUF		STANDARD	SYSTEM		line_buf
 type	IOQ		STANDARD	SYSTEM		io_queue
 type	BITS_BUF	STANDARD	SYSTEM		bits_buf
 type	TMP_DIST_BUF	TEMPORARY	SYSTEM		tmp_dist_buf
-type	ASYNC_Q		LONG_LIVED	SYSTEM		async_queue
+type	ASYNC_DATA	LONG_LIVED	SYSTEM		internal_async_data
 type	ESTACK		TEMPORARY	SYSTEM		estack
 type	PORT_CALL_BUF	TEMPORARY	SYSTEM		port_call_buf
-type	DB_TABLE	FIXED_SIZE	ETS		db_tab
+type	DB_TABLE	ETS		ETS		db_tab
 type	DB_FIXATION	SHORT_LIVED	ETS		db_fixation
 type	DB_FIX_DEL	SHORT_LIVED	ETS		fixed_del
 type	DB_TABLES	LONG_LIVED	ETS		db_tabs
 type    DB_NTAB_ENT	STANDARD	ETS		db_named_table_entry
 type	DB_TMP		TEMPORARY	ETS		db_tmp
 type	DB_MC_STK	TEMPORARY	ETS		db_mc_stack
-type	DB_MS_PSDO_PROC	LONG_LIVED	ETS		db_match_pseudo_proc
 type	DB_MS_RUN_HEAP	SHORT_LIVED	ETS		db_match_spec_run_heap
 type	DB_MS_CMPL_HEAP	TEMPORARY	ETS		db_match_spec_cmpl_heap
 type	DB_SEG		ETS		ETS		db_segment
@@ -211,9 +227,8 @@ type	LOGGER_DSBUF	TEMPORARY	SYSTEM		logger_dsbuf
 type	TMP_DSBUF	TEMPORARY	SYSTEM		tmp_dsbuf
 type	INFO_DSBUF	SYSTEM		SYSTEM		info_dsbuf
 # INFO_DSBUF have to use the SYSTEM allocator; otherwise, a deadlock might occur
-type	SCHDLR_DATA	LONG_LIVED	PROCESSES	scheduler_data
+type	SCHDLR_SLP_INFO	LONG_LIVED	SYSTEM		scheduler_sleep_info
 type	RUNQS		LONG_LIVED	SYSTEM		run_queues
-type	DDLL_PROCESS	STANDARD	SYSTEM		ddll_processes
 type	DDLL_HANDLE	STANDARD	SYSTEM		ddll_handle
 type	DDLL_ERRCODES	LONG_LIVED	SYSTEM		ddll_errcodes
 type	DDLL_TMP_BUF	TEMPORARY	SYSTEM		ddll_tmp_buf
@@ -231,6 +246,7 @@ type    RE_SUBJECT      SHORT_LIVED     SYSTEM          re_subject
 type  	RE_HEAP 	STANDARD	SYSTEM		re_heap
 type	RE_STACK 	SHORT_LIVED	SYSTEM		re_stack
 type	UNICODE_BUFFER 	SHORT_LIVED	SYSTEM		unicode_buffer
+type	BINARY_BUFFER 	SHORT_LIVED	SYSTEM		binary_buffer
 type	PRE_ALLOC_DATA	LONG_LIVED	SYSTEM		pre_alloc_data
 type	DRV_THR_OPTS	DRIVER		SYSTEM		driver_thread_opts
 type	DRV_TID		DRIVER		SYSTEM		driver_tid
@@ -244,6 +260,24 @@ type	CPUDATA		LONG_LIVED	SYSTEM		cpu_data
 type	TMP_CPU_IDS	SHORT_LIVED	SYSTEM		tmp_cpu_ids
 type	EXT_TERM_DATA	SHORT_LIVED	PROCESSES	external_term_data
 type	ZLIB		STANDARD	SYSTEM		zlib
+type	CPU_GRPS_MAP	LONG_LIVED	SYSTEM		cpu_groups_map
+type	AUX_WORK_TMO	LONG_LIVED	SYSTEM		aux_work_timeouts
+type	MISC_AUX_WORK_Q	LONG_LIVED	SYSTEM		misc_aux_work_q
+
++if threads_no_smp
+# Need thread safe allocs, but std_alloc and fix_alloc are not;
+# use driver_alloc which is...
+type	THR_Q_EL	DRIVER 	   	SYSTEM		thr_q_element
+type	THR_Q_EL_SL	DRIVER		SYSTEM		sl_thr_q_element
+type	MISC_AUX_WORK	DRIVER		SYSTEM		misc_aux_work
++else
+type	THR_Q_EL	STANDARD   	SYSTEM		thr_q_element
+type	THR_Q_EL_SL	FIXED_SIZE	SYSTEM		sl_thr_q_element
+type	MISC_AUX_WORK	SHORT_LIVED	SYSTEM		misc_aux_work
++endif
+type	THR_Q		STANDARD	SYSTEM		thr_queue
+type	THR_Q_SL	SHORT_LIVED	SYSTEM		short_lived_thr_queue
+type	THR_Q_LL	LONG_LIVED	SYSTEM		long_lived_thr_queue
 
 +if smp
 type	ASYNC		SHORT_LIVED	SYSTEM		async
@@ -257,8 +291,10 @@ type	PORT_LOCK	STANDARD	SYSTEM		port_lock
 type	DRIVER_LOCK	STANDARD	SYSTEM		driver_lock
 type	XPORTS_LIST	SHORT_LIVED	SYSTEM		extra_port_list
 type	PROC_LCK_WTR	LONG_LIVED	SYSTEM		proc_lock_waiter
-type	PROC_LCK_QS	LONG_LIVED	SYSTEM		proc_lock_queues
 type	RUNQ_BLNS	LONG_LIVED	SYSTEM		run_queue_balancing
+type	THR_PRGR_IDATA	LONG_LIVED	SYSTEM		thr_prgr_internal_data
+type	THR_PRGR_DATA	LONG_LIVED	SYSTEM		thr_prgr_data
+type	T_THR_PRGR_DATA	SHORT_LIVED	SYSTEM		temp_thr_prgr_data
 +endif
 
 #
@@ -267,13 +303,9 @@ type	RUNQ_BLNS	LONG_LIVED	SYSTEM		run_queue_balancing
 
 +if threads
 
-type	ETHR_INTERNAL	SYSTEM		SYSTEM		ethread_internal
-
-+ifnot smp
-
-type	ARCALLBACK	LONG_LIVED	SYSTEM		async_ready_callback
-
-+endif
+type	ETHR_STD	STANDARD	SYSTEM		ethread_standard
+type	ETHR_SL		SHORT_LIVED	SYSTEM		ethread_short_lived
+type	ETHR_LL		LONG_LIVED	SYSTEM		ethread_long_lived
 
 +endif
 
@@ -284,19 +316,6 @@ type	ACTIVE_PROCS	STANDARD	PROCESSES	active_procs
 
 +endif
 
-+if hybrid
-
-type	ACTIVE_PROCS	STANDARD	PROCESSES	active_procs
-
-# Used for all memory involved in incremental gc of the message area
-# that is, young (x2) and old generation, forwarding pointers and blackmap
-type	MESSAGE_AREA	LONG_LIVED	PROCESSES	message_area
-
-# Used in MA_STACK (global.h) and INC_STORAGE (erl_nmgc.h)
-type	OBJECT_STACK	STANDARD	PROCESSES	object_stack
-
-+endif
-
 +if smp
 type	SL_PTIMER	SHORT_LIVED	SYSTEM		ptimer_sl
 type	LL_PTIMER	STANDARD	SYSTEM		ptimer_ll
@@ -318,10 +337,49 @@ type	SSB		SHORT_LIVED	PROCESSES	ssb
 +endif
 
 
++if halfword
+
+type	SBMBC_LOW	SBMBC_LOW	SYSTEM		small_block_mbc_low
+type	DDLL_PROCESS	STANDARD_LOW	SYSTEM		ddll_processes
+type	MONITOR_LH	STANDARD_LOW	PROCESSES	monitor_lh
+type	NLINK_LH	STANDARD_LOW	PROCESSES	nlink_lh
+type	CODE		LONG_LIVED_LOW	CODE		code
+type	DB_HEIR_DATA	STANDARD_LOW	ETS		db_heir_data
+type	DB_MS_PSDO_PROC	LONG_LIVED_LOW	ETS		db_match_pseudo_proc
+type	SCHDLR_DATA	LONG_LIVED_LOW	SYSTEM		scheduler_data
+type  	LL_TEMP_TERM 	LONG_LIVED_LOW	SYSTEM		ll_temp_term
+
+type	EXPORT		LONG_LIVED_LOW	CODE		export_entry
+type	MONITOR_SH	STANDARD_LOW	PROCESSES	monitor_sh
+type	NLINK_SH	STANDARD_LOW	PROCESSES	nlink_sh
+type	AINFO_REQ	STANDARD_LOW	SYSTEM		alloc_info_request
+type	SCHED_WTIME_REQ	STANDARD_LOW	SYSTEM		sched_wall_time_request
+
++else # "fullword"
+
+type	DDLL_PROCESS	STANDARD	SYSTEM		ddll_processes
+type	MONITOR_LH	STANDARD	PROCESSES	monitor_lh
+type	NLINK_LH	STANDARD	PROCESSES	nlink_lh
+type	CODE		LONG_LIVED	CODE		code
+type	DB_HEIR_DATA	STANDARD	ETS		db_heir_data
+type	DB_MS_PSDO_PROC	LONG_LIVED	ETS		db_match_pseudo_proc
+type	SCHDLR_DATA	LONG_LIVED	SYSTEM		scheduler_data
+type  	LL_TEMP_TERM 	LONG_LIVED	SYSTEM		ll_temp_term
+
+type	EXPORT		LONG_LIVED	CODE		export_entry
+type	MONITOR_SH	FIXED_SIZE	PROCESSES	monitor_sh
+type	NLINK_SH	FIXED_SIZE	PROCESSES	nlink_sh
+type	AINFO_REQ	SHORT_LIVED	SYSTEM		alloc_info_request
+type	SCHED_WTIME_REQ	SHORT_LIVED	SYSTEM		sched_wall_time_request
+
++endif
+
+
 #
 # Types used by system specific code
 #
 
+type  	TEMP_TERM 	TEMPORARY	SYSTEM		temp_term
 type	DRV_TAB		LONG_LIVED	SYSTEM		drv_tab
 type	DRV_EV_STATE	LONG_LIVED	SYSTEM		driver_event_state
 type	DRV_EV_D_STATE	FIXED_SIZE	SYSTEM		driver_event_data_state
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 9b7bc24c1c..97ba306a79 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2002-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2002-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -46,6 +46,7 @@
 #include "erl_alloc_util.h"
 #include "erl_mseg.h"
 #include "erl_threads.h"
+#include "erl_thr_progress.h"
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
 #include "erl_lock_count.h"
@@ -61,22 +62,30 @@
 #warning "* * * * * * * * * *"
 #endif
 
+#define ERTS_ALCU_DD_OPS_LIM_HIGH 20
+#define ERTS_ALCU_DD_OPS_LIM_LOW 2
+
+/* Fix alloc limit */
+#define ERTS_ALCU_FIX_MAX_LIST_SZ 1000
+#define ERTS_ALC_FIX_MAX_SHRINK_OPS 30
+
 #define ALLOC_ZERO_EQ_NULL 0
 
 static int atoms_initialized = 0;
 static int initialized = 0;
 
+int erts_have_sbmbc_alloc;
 
 #if HAVE_ERTS_MSEG
 
-#define INV_MSEG_UNIT_MASK	((Uint) (mseg_unit_size - 1))
+#define INV_MSEG_UNIT_MASK	((UWord) (mseg_unit_size - 1))
 #define MSEG_UNIT_MASK		(~INV_MSEG_UNIT_MASK)
 #define MSEG_UNIT_FLOOR(X)	((X) & MSEG_UNIT_MASK)
 #define MSEG_UNIT_CEILING(X)	MSEG_UNIT_FLOOR((X) + INV_MSEG_UNIT_MASK)
 
 #endif
 
-#define INV_SYS_ALLOC_CARRIER_MASK	((Uint) (sys_alloc_carrier_size - 1))
+#define INV_SYS_ALLOC_CARRIER_MASK	((UWord) (sys_alloc_carrier_size - 1))
 #define SYS_ALLOC_CARRIER_MASK		(~INV_SYS_ALLOC_CARRIER_MASK)
 #define SYS_ALLOC_CARRIER_FLOOR(X)	((X) & SYS_ALLOC_CARRIER_MASK)
 #define SYS_ALLOC_CARRIER_CEILING(X) \
@@ -85,8 +94,6 @@ static int initialized = 0;
 #undef ASSERT
 #define ASSERT ASSERT_EXPR
 
-#define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE ((Uint) 1)
-
 #if 0
 /* Can be useful for debugging */
 #define MBC_REALLOC_ALWAYS_MOVES
@@ -114,12 +121,12 @@ static Uint mseg_unit_size;
 
 /* Blocks ... */
 
-#define SBC_BLK_FTR_FLG		(((Uint) 1) << 0)
-#define UNUSED1_BLK_FTR_FLG	(((Uint) 1) << 1)
-#define UNUSED2_BLK_FTR_FLG	(((Uint) 1) << 2)
+#define SBC_BLK_FTR_FLG		(((UWord) 1) << 0)
+#define UNUSED1_BLK_FTR_FLG	(((UWord) 1) << 1)
+#define UNUSED2_BLK_FTR_FLG	(((UWord) 1) << 2)
 
 #define ABLK_HDR_SZ (sizeof(Block_t))
-#define FBLK_FTR_SZ (sizeof(Uint))
+#define FBLK_FTR_SZ (sizeof(UWord))
 
 #define UMEMSZ2BLKSZ(AP, SZ)						\
   (ABLK_HDR_SZ + (SZ) <= (AP)->min_block_size				\
@@ -130,14 +137,14 @@ static Uint mseg_unit_size;
 #define BLK2UMEM(P) ((void *)    (((char *) (P)) + ABLK_HDR_SZ))
 
 #define PREV_BLK_SZ(B) \
-  ((Uint) (*(((Uint *) (B)) - 1) & SZ_MASK))
+  ((UWord) (*(((UWord *) (B)) - 1) & SZ_MASK))
 
 #define SET_BLK_SZ_FTR(B, SZ) \
-  (*((Uint *) (((char *) (B)) + (SZ) - sizeof(Uint))) = (SZ))
+  (*((UWord *) (((char *) (B)) + (SZ) - sizeof(UWord))) = (SZ))
 
-#define THIS_FREE_BLK_HDR_FLG 	(((Uint) 1) << 0)
-#define PREV_FREE_BLK_HDR_FLG 	(((Uint) 1) << 1)
-#define LAST_BLK_HDR_FLG 	(((Uint) 1) << 2)
+#define THIS_FREE_BLK_HDR_FLG 	(((UWord) 1) << 0)
+#define PREV_FREE_BLK_HDR_FLG 	(((UWord) 1) << 1)
+#define LAST_BLK_HDR_FLG 	(((UWord) 1) << 2)
 
 #define SET_BLK_SZ(B, SZ) \
   (ASSERT(((SZ) & FLG_MASK) == 0), \
@@ -156,11 +163,11 @@ static Uint mseg_unit_size;
   (*((Block_t *) (B)) &= ~LAST_BLK_HDR_FLG)
 
 #define SBH_THIS_FREE		THIS_FREE_BLK_HDR_FLG
-#define SBH_THIS_ALLOCED	((Uint) 0)
+#define SBH_THIS_ALLOCED	((UWord) 0)
 #define SBH_PREV_FREE		PREV_FREE_BLK_HDR_FLG
-#define SBH_PREV_ALLOCED	((Uint) 0)
+#define SBH_PREV_ALLOCED	((UWord) 0)
 #define SBH_LAST_BLK		LAST_BLK_HDR_FLG
-#define SBH_NOT_LAST_BLK	((Uint) 0)
+#define SBH_NOT_LAST_BLK	((UWord) 0)
 
 #define SET_BLK_HDR(B, Sz, F) \
   (ASSERT(((Sz) & FLG_MASK) == 0), *((Block_t *) (B)) = ((Sz) | (F)))
@@ -200,7 +207,7 @@ static Uint mseg_unit_size;
   ((FTR) = 0)
 
 #define IS_SBC_BLK(B) \
-  (IS_PREV_BLK_FREE((B)) && (((Uint *) (B))[-1] & SBC_BLK_FTR_FLG))
+  (IS_PREV_BLK_FREE((B)) && (((UWord *) (B))[-1] & SBC_BLK_FTR_FLG))
 #define IS_MBC_BLK(B) \
   (!IS_SBC_BLK((B)))
 
@@ -211,8 +218,8 @@ static Uint mseg_unit_size;
 
 /* Carriers ... */
 
-#define MSEG_CARRIER_HDR_FLAG		(((Uint) 1) << 0)
-#define SBC_CARRIER_HDR_FLAG		(((Uint) 1) << 1)
+#define MSEG_CARRIER_HDR_FLAG		(((UWord) 1) << 0)
+#define SBC_CARRIER_HDR_FLAG		(((UWord) 1) << 1)
 
 #define SCH_SYS_ALLOC			0
 #define SCH_MSEG			MSEG_CARRIER_HDR_FLAG
@@ -270,19 +277,30 @@ static void check_blk_carrier(Allctr_t *, Block_t *);
 #define HARD_CHECK_BLK_CARRIER(A, B)
 #endif
 
-
 /* Statistics updating ... */
 
 #ifdef DEBUG
 #define DEBUG_CHECK_CARRIER_NO_SZ(AP)					\
-    ASSERT(((AP)->sbcs.curr_mseg.no && (AP)->sbcs.curr_mseg.size)	\
-	   || (!(AP)->sbcs.curr_mseg.no && !(AP)->sbcs.curr_mseg.size));\
-    ASSERT(((AP)->sbcs.curr_sys_alloc.no && (AP)->sbcs.curr_sys_alloc.size)\
-	   || (!(AP)->sbcs.curr_sys_alloc.no && !(AP)->sbcs.curr_sys_alloc.size));\
-    ASSERT(((AP)->mbcs.curr_mseg.no && (AP)->mbcs.curr_mseg.size)	\
-	   || (!(AP)->mbcs.curr_mseg.no && !(AP)->mbcs.curr_mseg.size));\
-    ASSERT(((AP)->mbcs.curr_sys_alloc.no && (AP)->mbcs.curr_sys_alloc.size)\
-	   || (!(AP)->mbcs.curr_sys_alloc.no && !(AP)->mbcs.curr_sys_alloc.size))
+    ASSERT(((AP)->sbcs.curr.norm.mseg.no				\
+	    && (AP)->sbcs.curr.norm.mseg.size)				\
+	   || (!(AP)->sbcs.curr.norm.mseg.no				\
+	       && !(AP)->sbcs.curr.norm.mseg.size));			\
+    ASSERT(((AP)->sbcs.curr.norm.sys_alloc.no				\
+	    && (AP)->sbcs.curr.norm.sys_alloc.size)			\
+	   || (!(AP)->sbcs.curr.norm.sys_alloc.no			\
+	       && !(AP)->sbcs.curr.norm.sys_alloc.size));		\
+    ASSERT(((AP)->mbcs.curr.norm.mseg.no				\
+	    && (AP)->mbcs.curr.norm.mseg.size)				\
+	   || (!(AP)->mbcs.curr.norm.mseg.no				\
+	       && !(AP)->mbcs.curr.norm.mseg.size));			\
+    ASSERT(((AP)->mbcs.curr.norm.sys_alloc.no				\
+	    && (AP)->mbcs.curr.norm.sys_alloc.size)			\
+	   || (!(AP)->mbcs.curr.norm.sys_alloc.no			\
+	       && !(AP)->mbcs.curr.norm.sys_alloc.size));		\
+    ASSERT(((AP)->sbmbcs.curr.small_block.no				\
+	    && (AP)->sbmbcs.curr.small_block.size)			\
+	   || (!(AP)->sbmbcs.curr.small_block.no			\
+	       && !(AP)->sbmbcs.curr.small_block.size))
 
 #else
 #define DEBUG_CHECK_CARRIER_NO_SZ(AP)
@@ -292,27 +310,27 @@ static void check_blk_carrier(Allctr_t *, Block_t *);
     (AP)->sbcs.blocks.curr.size += (BSZ);				\
     if ((AP)->sbcs.blocks.max.size < (AP)->sbcs.blocks.curr.size)	\
 	(AP)->sbcs.blocks.max.size = (AP)->sbcs.blocks.curr.size;	\
-    if ((AP)->sbcs.max.no < ((AP)->sbcs.curr_mseg.no			\
-			     + (AP)->sbcs.curr_sys_alloc.no))		\
-	(AP)->sbcs.max.no = ((AP)->sbcs.curr_mseg.no			\
-			     + (AP)->sbcs.curr_sys_alloc.no);		\
-    if ((AP)->sbcs.max.size < ((AP)->sbcs.curr_mseg.size		\
-			       + (AP)->sbcs.curr_sys_alloc.size))	\
-	(AP)->sbcs.max.size = ((AP)->sbcs.curr_mseg.size		\
-			       + (AP)->sbcs.curr_sys_alloc.size)
+    if ((AP)->sbcs.max.no < ((AP)->sbcs.curr.norm.mseg.no		\
+			     + (AP)->sbcs.curr.norm.sys_alloc.no))	\
+	(AP)->sbcs.max.no = ((AP)->sbcs.curr.norm.mseg.no		\
+			     + (AP)->sbcs.curr.norm.sys_alloc.no);	\
+    if ((AP)->sbcs.max.size < ((AP)->sbcs.curr.norm.mseg.size		\
+			       + (AP)->sbcs.curr.norm.sys_alloc.size))	\
+	(AP)->sbcs.max.size = ((AP)->sbcs.curr.norm.mseg.size		\
+			       + (AP)->sbcs.curr.norm.sys_alloc.size)
 
 #define STAT_MSEG_SBC_ALLOC(AP, CSZ, BSZ)				\
 do {									\
-    (AP)->sbcs.curr_mseg.no++;						\
-    (AP)->sbcs.curr_mseg.size += (CSZ);					\
+    (AP)->sbcs.curr.norm.mseg.no++;					\
+    (AP)->sbcs.curr.norm.mseg.size += (CSZ);				\
     STAT_SBC_ALLOC((AP), (BSZ));					\
     DEBUG_CHECK_CARRIER_NO_SZ((AP));					\
 } while (0)
 
 #define STAT_SYS_ALLOC_SBC_ALLOC(AP, CSZ, BSZ)				\
 do {									\
-    (AP)->sbcs.curr_sys_alloc.no++;					\
-    (AP)->sbcs.curr_sys_alloc.size += (CSZ);				\
+    (AP)->sbcs.curr.norm.sys_alloc.no++;				\
+    (AP)->sbcs.curr.norm.sys_alloc.size += (CSZ);			\
     STAT_SBC_ALLOC((AP), (BSZ));					\
     DEBUG_CHECK_CARRIER_NO_SZ((AP));					\
 } while (0)
@@ -324,101 +342,127 @@ do {									\
 
 #define STAT_MSEG_SBC_FREE(AP, CSZ, BSZ)				\
 do {									\
-    ASSERT((AP)->sbcs.curr_mseg.no > 0);				\
-    (AP)->sbcs.curr_mseg.no--;						\
-    ASSERT((AP)->sbcs.curr_mseg.size >= (CSZ));				\
-    (AP)->sbcs.curr_mseg.size -= (CSZ);					\
+    ASSERT((AP)->sbcs.curr.norm.mseg.no > 0);				\
+    (AP)->sbcs.curr.norm.mseg.no--;					\
+    ASSERT((AP)->sbcs.curr.norm.mseg.size >= (CSZ));			\
+    (AP)->sbcs.curr.norm.mseg.size -= (CSZ);				\
     STAT_SBC_FREE((AP), (BSZ));						\
     DEBUG_CHECK_CARRIER_NO_SZ((AP));					\
 } while (0)
 
 #define STAT_SYS_ALLOC_SBC_FREE(AP, CSZ, BSZ)				\
 do {									\
-    ASSERT((AP)->sbcs.curr_sys_alloc.no > 0);				\
-    (AP)->sbcs.curr_sys_alloc.no--;					\
-    ASSERT((AP)->sbcs.curr_sys_alloc.size >= (CSZ));			\
-    (AP)->sbcs.curr_sys_alloc.size -= (CSZ);				\
+    ASSERT((AP)->sbcs.curr.norm.sys_alloc.no > 0);			\
+    (AP)->sbcs.curr.norm.sys_alloc.no--;				\
+    ASSERT((AP)->sbcs.curr.norm.sys_alloc.size >= (CSZ));		\
+    (AP)->sbcs.curr.norm.sys_alloc.size -= (CSZ);			\
     STAT_SBC_FREE((AP), (BSZ));						\
     DEBUG_CHECK_CARRIER_NO_SZ((AP));					\
 } while (0)
 
 #define STAT_MBC_ALLOC(AP)						\
-    if ((AP)->mbcs.max.no < ((AP)->mbcs.curr_mseg.no			\
-			     + (AP)->mbcs.curr_sys_alloc.no))		\
-	(AP)->mbcs.max.no = ((AP)->mbcs.curr_mseg.no			\
-			     + (AP)->mbcs.curr_sys_alloc.no);		\
-    if ((AP)->mbcs.max.size < ((AP)->mbcs.curr_mseg.size		\
-			       + (AP)->mbcs.curr_sys_alloc.size))	\
-	(AP)->mbcs.max.size = ((AP)->mbcs.curr_mseg.size		\
-			       + (AP)->mbcs.curr_sys_alloc.size)
+    if ((AP)->mbcs.max.no < ((AP)->mbcs.curr.norm.mseg.no		\
+			     + (AP)->mbcs.curr.norm.sys_alloc.no))	\
+	(AP)->mbcs.max.no = ((AP)->mbcs.curr.norm.mseg.no		\
+			     + (AP)->mbcs.curr.norm.sys_alloc.no);	\
+    if ((AP)->mbcs.max.size < ((AP)->mbcs.curr.norm.mseg.size		\
+			       + (AP)->mbcs.curr.norm.sys_alloc.size))	\
+	(AP)->mbcs.max.size = ((AP)->mbcs.curr.norm.mseg.size		\
+			       + (AP)->mbcs.curr.norm.sys_alloc.size)
 
 
+#define STAT_SBMBC_ALLOC(AP, CSZ)					\
+do {									\
+    (AP)->sbmbcs.curr.small_block.no++;					\
+    (AP)->sbmbcs.curr.small_block.size += (CSZ);			\
+    if ((AP)->sbmbcs.max.no < (AP)->sbmbcs.curr.small_block.no)		\
+	(AP)->sbmbcs.max.no = (AP)->sbmbcs.curr.small_block.no;		\
+    if ((AP)->sbmbcs.max.size < (AP)->sbmbcs.curr.small_block.size)	\
+	(AP)->sbmbcs.max.size = (AP)->sbmbcs.curr.small_block.size;	\
+    DEBUG_CHECK_CARRIER_NO_SZ((AP));					\
+} while (0)
+
 #define STAT_MSEG_MBC_ALLOC(AP, CSZ)					\
 do {									\
-    (AP)->mbcs.curr_mseg.no++;						\
-    (AP)->mbcs.curr_mseg.size += (CSZ);					\
+    (AP)->mbcs.curr.norm.mseg.no++;					\
+    (AP)->mbcs.curr.norm.mseg.size += (CSZ);				\
     STAT_MBC_ALLOC((AP));						\
     DEBUG_CHECK_CARRIER_NO_SZ((AP));					\
 } while (0)
 
 #define STAT_SYS_ALLOC_MBC_ALLOC(AP, CSZ)				\
 do {									\
-    (AP)->mbcs.curr_sys_alloc.no++;					\
-    (AP)->mbcs.curr_sys_alloc.size += (CSZ);				\
+    (AP)->mbcs.curr.norm.sys_alloc.no++;				\
+    (AP)->mbcs.curr.norm.sys_alloc.size += (CSZ);			\
     STAT_MBC_ALLOC((AP));						\
     DEBUG_CHECK_CARRIER_NO_SZ((AP));					\
 } while (0)
 
+#define STAT_SBMBC_FREE(AP, CSZ)					\
+do {									\
+    ASSERT((AP)->sbmbcs.curr.small_block.no > 0);			\
+    (AP)->sbmbcs.curr.small_block.no--;					\
+    ASSERT((AP)->sbmbcs.curr.small_block.size >= (CSZ));		\
+    (AP)->sbmbcs.curr.small_block.size -= (CSZ);			\
+    DEBUG_CHECK_CARRIER_NO_SZ((AP));					\
+} while (0)
+
 #define STAT_MSEG_MBC_FREE(AP, CSZ)					\
 do {									\
-    ASSERT((AP)->mbcs.curr_mseg.no > 0);				\
-    (AP)->mbcs.curr_mseg.no--;						\
-    ASSERT((AP)->mbcs.curr_mseg.size >= (CSZ));				\
-    (AP)->mbcs.curr_mseg.size -= (CSZ);					\
+    ASSERT((AP)->mbcs.curr.norm.mseg.no > 0);				\
+    (AP)->mbcs.curr.norm.mseg.no--;					\
+    ASSERT((AP)->mbcs.curr.norm.mseg.size >= (CSZ));			\
+    (AP)->mbcs.curr.norm.mseg.size -= (CSZ);				\
     DEBUG_CHECK_CARRIER_NO_SZ((AP));					\
 } while (0)
 
 #define STAT_SYS_ALLOC_MBC_FREE(AP, CSZ)				\
 do {									\
-    ASSERT((AP)->mbcs.curr_sys_alloc.no > 0);				\
-    (AP)->mbcs.curr_sys_alloc.no--;					\
-    ASSERT((AP)->mbcs.curr_sys_alloc.size >= (CSZ));			\
-    (AP)->mbcs.curr_sys_alloc.size -= (CSZ);				\
+    ASSERT((AP)->mbcs.curr.norm.sys_alloc.no > 0);			\
+    (AP)->mbcs.curr.norm.sys_alloc.no--;				\
+    ASSERT((AP)->mbcs.curr.norm.sys_alloc.size >= (CSZ));		\
+    (AP)->mbcs.curr.norm.sys_alloc.size -= (CSZ);			\
     DEBUG_CHECK_CARRIER_NO_SZ((AP));					\
 } while (0)
 
-#define STAT_MBC_BLK_ALLOC(AP, BSZ)					\
+#define STAT_MBC_BLK_ALLOC(AP, BSZ, FLGS)	       			\
 do {									\
-    (AP)->mbcs.blocks.curr.no++;					\
-    if ((AP)->mbcs.blocks.max.no < (AP)->mbcs.blocks.curr.no)		\
-	(AP)->mbcs.blocks.max.no = (AP)->mbcs.blocks.curr.no;		\
-    (AP)->mbcs.blocks.curr.size += (BSZ);				\
-    if ((AP)->mbcs.blocks.max.size < (AP)->mbcs.blocks.curr.size)	\
-	(AP)->mbcs.blocks.max.size = (AP)->mbcs.blocks.curr.size;	\
+    CarriersStats_t *cstats__ = (((FLGS) & ERTS_ALCU_FLG_SBMBC)		\
+				 ? &(AP)->sbmbcs			\
+				 : &(AP)->mbcs);			\
+    cstats__->blocks.curr.no++;						\
+    if (cstats__->blocks.max.no < cstats__->blocks.curr.no)		\
+	cstats__->blocks.max.no = cstats__->blocks.curr.no;		\
+    cstats__->blocks.curr.size += (BSZ);				\
+    if (cstats__->blocks.max.size < cstats__->blocks.curr.size)		\
+	cstats__->blocks.max.size = cstats__->blocks.curr.size;		\
 } while (0)
 
-#define STAT_MBC_BLK_FREE(AP, BSZ)					\
+#define STAT_MBC_BLK_FREE(AP, BSZ, FLGS)				\
 do {									\
-    ASSERT((AP)->mbcs.blocks.curr.no > 0);				\
-    (AP)->mbcs.blocks.curr.no--;					\
-    ASSERT((AP)->mbcs.blocks.curr.size >= (BSZ));			\
-    (AP)->mbcs.blocks.curr.size -= (BSZ);				\
+    CarriersStats_t *cstats__ = (((FLGS) & ERTS_ALCU_FLG_SBMBC)		\
+				 ? &(AP)->sbmbcs			\
+				 : &(AP)->mbcs);			\
+    ASSERT(cstats__->blocks.curr.no > 0);				\
+    cstats__->blocks.curr.no--;						\
+    ASSERT(cstats__->blocks.curr.size >= (BSZ));			\
+    cstats__->blocks.curr.size -= (BSZ);				\
 } while (0)
 
 /* Debug stuff... */
 #ifdef DEBUG
-static Uint carrier_alignment;
+static UWord carrier_alignment;
 #define DEBUG_SAVE_ALIGNMENT(C)						\
 do {									\
-    Uint algnmnt__ = sizeof(Unit_t) - (((Uint) (C)) % sizeof(Unit_t));	\
+    UWord algnmnt__ = sizeof(Unit_t) - (((UWord) (C)) % sizeof(Unit_t));\
     carrier_alignment = MIN(carrier_alignment, algnmnt__);		\
-    ASSERT(((Uint) (C)) % sizeof(Uint) == 0);				\
+    ASSERT(((UWord) (C)) % sizeof(UWord) == 0);				\
 } while (0)
 #define DEBUG_CHECK_ALIGNMENT(P)					\
 do {									\
-    ASSERT(sizeof(Unit_t) - (((Uint) (P)) % sizeof(Unit_t))		\
+    ASSERT(sizeof(Unit_t) - (((UWord) (P)) % sizeof(Unit_t))		\
 	   >= carrier_alignment);					\
-    ASSERT(((Uint) (P)) % sizeof(Uint) == 0);				\
+    ASSERT(((UWord) (P)) % sizeof(UWord) == 0);				\
 } while (0)
 
 #else
@@ -428,26 +472,34 @@ do {									\
 
 #ifdef DEBUG
 #ifdef USE_THREADS
-#define ERTS_ALCU_DBG_CHK_THR_SPEC(A)					\
+#define ERTS_ALCU_DBG_CHK_THR_ACCESS(A)					\
 do {									\
     if (!(A)->thread_safe) {						\
-	if (!(A)->debug.saved_tid)					\
+	if (!(A)->debug.saved_tid) {					\
 	    (A)->debug.tid = erts_thr_self();				\
+	    (A)->debug.saved_tid = 1;					\
+	}								\
 	else {								\
-	    ASSERT(ethr_equal_tids((A)->debug.tid, erts_thr_self()));	\
+	    ERTS_SMP_LC_ASSERT(						\
+		ethr_equal_tids((A)->debug.tid, erts_thr_self())	\
+		|| erts_thr_progress_is_blocking());			\
 	}								\
     }									\
 } while (0)
 #else
-#define ERTS_ALCU_DBG_CHK_THR_SPEC(A)
+#define ERTS_ALCU_DBG_CHK_THR_ACCESS(A)
 #endif
 #else
-#define ERTS_ALCU_DBG_CHK_THR_SPEC(A)
+#define ERTS_ALCU_DBG_CHK_THR_ACCESS(A)
 #endif
 
 
 static void make_name_atoms(Allctr_t *allctr);
 
+static Block_t *create_carrier(Allctr_t *, Uint, UWord);
+static void destroy_carrier(Allctr_t *, Block_t *);
+static void mbc_free(Allctr_t *allctr, void *p);
+
 
 /* mseg ... */
 
@@ -524,8 +576,8 @@ static Uint
 get_next_mbc_size(Allctr_t *allctr)
 {
     Uint size;
-    int cs = (allctr->mbcs.curr_mseg.no
-	      + allctr->mbcs.curr_sys_alloc.no
+    int cs = (allctr->mbcs.curr.norm.mseg.no
+	      + allctr->mbcs.curr.norm.sys_alloc.no
 	      - (allctr->main_carrier ? 1 : 0));
 
     ASSERT(cs >= 0);
@@ -609,43 +661,576 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr)
     }
 }
 
-
-static Block_t *create_carrier(Allctr_t *, Uint, Uint);
+static Block_t *create_sbmbc(Allctr_t *allctr, Uint umem_sz);
+static void destroy_sbmbc(Allctr_t *allctr, Block_t *blk);
+static Block_t *create_carrier(Allctr_t *, Uint, UWord);
 static void destroy_carrier(Allctr_t *, Block_t *);
 
+#if 0
+#define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B)			\
+    do { if ((FIX)) chk_fix_list((A), (FIX), (IX), (B)); } while (0)
+static void
+chk_fix_list(Allctr_t *allctr, ErtsAlcFixList_t *fix, int ix, int before)
+{
+    void *p;
+    int n;
+    for (n = 0, p = fix[ix].list; p; p = *((void **) p))
+	n++;
+    if (n != fix[ix].list_size) {
+	erts_fprintf(stderr, "FOUND IT ts=%d, sched=%d, ix=%d, n=%d, ls=%d %s!\n",
+		     allctr->thread_safe, allctr->ix, ix, n, fix[ix].list_size, before ? "before" : "after");
+	abort();
+    }
+}
+#else
+#define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B)
+#endif
+
+erts_aint32_t
+erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs)
+{
+    int all_empty = 1;
+    erts_aint32_t res = 0;
+    int ix, o;
+    ErtsAlcFixList_t *fix = allctr->fix;
+    int flush = flgs == 0;
+
+#ifdef USE_THREADS
+    if (allctr->thread_safe)
+	erts_mtx_lock(&allctr->mutex);
+#endif
+
+    for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) {
+	ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
+	if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) {
+	    fix[ix].limit = fix[ix].max_used;
+	    if (fix[ix].limit < fix[ix].used)
+		fix[ix].limit = fix[ix].used;
+	    fix[ix].max_used = fix[ix].used;
+	    ASSERT(fix[ix].limit >= 0);
+
+	}
+	if (flush) {
+	    fix[ix].limit = 0;
+	    fix[ix].max_used = fix[ix].used;
+	    ASSERT(fix[ix].limit >= 0);
+	}
+	for (o = 0; o < ERTS_ALC_FIX_MAX_SHRINK_OPS || flush; o++) {
+	    Block_t *blk;
+	    void *ptr;
+
+	    if (!flush && fix[ix].limit >= fix[ix].allocated)
+		break;
+	    if (fix[ix].list_size == 0)
+		break;
+	    ptr = fix[ix].list;
+	    fix[ix].list = *((void **) ptr);
+	    fix[ix].list_size--;	    
+
+	    blk = UMEM2BLK(ptr);
+
+	    if (IS_SBC_BLK(blk))
+		destroy_carrier(allctr, blk);
+	    else
+		mbc_free(allctr, ptr);
+
+	    fix[ix].allocated--;
+	}
+	if (fix[ix].list_size != 0) {
+	    if (fix[ix].limit < fix[ix].allocated)
+		res |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC;
+	    all_empty = 0;
+	}
+	ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
+    }
+
+    if (all_empty && allctr->fix_shrink_scheduled) {
+	allctr->fix_shrink_scheduled = 0;
+	erts_set_aux_work_timeout(allctr->ix,
+				  (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
+				   | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
+				  0);
+    }
+
+#ifdef USE_THREADS
+    if (allctr->thread_safe)
+	erts_mtx_unlock(&allctr->mutex);
+#endif
+
+    return res;
+}
+
+#ifdef ERTS_SMP
+
+#define ERTS_ALCU_DD_FIX_TYPE_OFFS \
+  ((sizeof(ErtsAllctrDDBlock_t)-1)/sizeof(UWord) + 1)
+
+#define ERTS_AU_PREF_ALLOC_IX_MASK \
+   ((((UWord) 1) << ERTS_AU_PREF_ALLOC_BITS) - 1)
+#define ERTS_AU_PREF_ALLOC_SIZE_MASK \
+   ((((UWord) 1) << (sizeof(UWord)*8 - ERTS_AU_PREF_ALLOC_BITS)) - 1)
+
+static ERTS_INLINE int
+get_pref_allctr(void *extra, Allctr_t **allctr)
+{
+    ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra;
+    int pref_ix;
+
+    pref_ix = ERTS_ALC_GET_THR_IX();
+
+    ASSERT(sizeof(UWord) == sizeof(Allctr_t *));
+    ASSERT(0 <= pref_ix && pref_ix < tspec->size);
+
+    *allctr = tspec->allctr[pref_ix];
+    return pref_ix;
+}
+
+static ERTS_INLINE void *
+get_used_allctr(void *extra, void *p, Allctr_t **allctr, UWord *sizep)
+{
+    ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra;
+    void *ptr = (void *) (((char *) p) - sizeof(UWord));
+    UWord ainfo = *((UWord *) ptr);
+    int aix = (int) (ainfo & ERTS_AU_PREF_ALLOC_IX_MASK);
+    *allctr = tspec->allctr[aix];
+    if (sizep)
+	*sizep = ((ainfo >> ERTS_AU_PREF_ALLOC_BITS)
+		  & ERTS_AU_PREF_ALLOC_SIZE_MASK);
+    return ptr;
+}
+
+static ERTS_INLINE void *
+put_used_allctr(void *p, int ix, UWord size)
+{
+    UWord ainfo = (size >= ERTS_AU_PREF_ALLOC_SIZE_MASK
+		   ? ERTS_AU_PREF_ALLOC_SIZE_MASK
+		   : size);
+    ainfo <<= ERTS_AU_PREF_ALLOC_BITS;
+    ainfo |= (UWord) ix;
+    *((UWord *) p) = ainfo;
+    return (void *) (((char *) p) + sizeof(UWord));
+}
+
+static void
+init_dd_queue(ErtsAllctrDDQueue_t *ddq)
+{
+    erts_atomic_init_nob(&ddq->tail.data.marker.atmc_next, ERTS_AINT_NULL);
+    erts_atomic_init_nob(&ddq->tail.data.last,
+			 (erts_aint_t) &ddq->tail.data.marker);
+    erts_atomic_init_nob(&ddq->tail.data.um_refc[0], 0);
+    erts_atomic_init_nob(&ddq->tail.data.um_refc[1], 0);
+    erts_atomic32_init_nob(&ddq->tail.data.um_refc_ix, 0);
+    ddq->head.first = &ddq->tail.data.marker;
+    ddq->head.unref_end = &ddq->tail.data.marker;
+    ddq->head.next.thr_progress = erts_thr_progress_current();
+    ddq->head.next.thr_progress_reached = 1;
+    ddq->head.next.um_refc_ix = 1;
+    ddq->head.next.unref_end = &ddq->tail.data.marker;
+    ddq->head.used_marker = 1;
+}
+
+static ERTS_INLINE int
+ddq_managed_thread_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit)
+{
+    erts_aint_t itmp;
+    ErtsAllctrDDBlock_t *enq, *this = ptr;
+
+    erts_atomic_init_nob(&this->atmc_next, ERTS_AINT_NULL);
+    /* Enqueue at end of list... */
+
+    enq = (ErtsAllctrDDBlock_t *) erts_atomic_read_nob(&ddq->tail.data.last);
+    itmp = erts_atomic_cmpxchg_relb(&enq->atmc_next,
+				    (erts_aint_t) this,
+				    ERTS_AINT_NULL);
+    if (itmp == ERTS_AINT_NULL) {
+	/* We are required to move last pointer */
+#ifdef DEBUG
+	ASSERT(ERTS_AINT_NULL == erts_atomic_read_nob(&this->atmc_next));
+	ASSERT(((erts_aint_t) enq)
+	       == erts_atomic_xchg_relb(&ddq->tail.data.last,
+					(erts_aint_t) this));
+#else
+	erts_atomic_set_relb(&ddq->tail.data.last, (erts_aint_t) this);
+#endif
+	return 1;
+    }
+    else {
+	/*
+	 * We *need* to insert element somewhere in between the
+	 * last element we read earlier and the actual last element.
+	 */
+	int i = cinit;
+
+	while (1) {
+	    erts_aint_t itmp2;
+	    erts_atomic_set_nob(&this->atmc_next, itmp);
+	    itmp2 = erts_atomic_cmpxchg_relb(&enq->atmc_next,
+					     (erts_aint_t) this,
+					     itmp);
+	    if (itmp == itmp2)
+		return 0; /* inserted this */
+	    if ((i & 1) == 0)
+		itmp = itmp2;
+	    else {
+		enq = (ErtsAllctrDDBlock_t *) itmp2;
+		itmp = erts_atomic_read_acqb(&enq->atmc_next);
+		ASSERT(itmp != ERTS_AINT_NULL);
+	    }
+	    i++;
+	}
+    }
+}
+
+static ERTS_INLINE erts_aint_t
+check_insert_marker(ErtsAllctrDDQueue_t *ddq, erts_aint_t ilast)
+{
+    if (!ddq->head.used_marker
+	&& ddq->head.unref_end == (ErtsAllctrDDBlock_t *) ilast) {
+	erts_aint_t itmp;
+	ErtsAllctrDDBlock_t *last = (ErtsAllctrDDBlock_t *) ilast;
+
+	erts_atomic_init_nob(&ddq->tail.data.marker.atmc_next, ERTS_AINT_NULL);
+	itmp = erts_atomic_cmpxchg_relb(&last->atmc_next,
+					(erts_aint_t) &ddq->tail.data.marker,
+					ERTS_AINT_NULL);
+	if (itmp == ERTS_AINT_NULL) {
+	    ilast = (erts_aint_t) &ddq->tail.data.marker;
+	    ddq->head.used_marker = !0;
+	    erts_atomic_set_relb(&ddq->tail.data.last, ilast);
+	}
+    }
+    return ilast;
+}
+
+static ERTS_INLINE int
+ddq_enqueue(ErtsAlcType_t type, ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit)
+{
+    int last_elem;
+    int um_refc_ix = 0;
+    int managed_thread = erts_thr_progress_is_managed_thread();
+    if (!managed_thread) {
+	um_refc_ix = erts_atomic32_read_acqb(&ddq->tail.data.um_refc_ix);
+	while (1) {
+	    int tmp_um_refc_ix;
+	    erts_atomic_inc_acqb(&ddq->tail.data.um_refc[um_refc_ix]);
+	    tmp_um_refc_ix = erts_atomic32_read_acqb(&ddq->tail.data.um_refc_ix);
+	    if (tmp_um_refc_ix == um_refc_ix)
+		break;
+	    erts_atomic_dec_relb(&ddq->tail.data.um_refc[um_refc_ix]);
+	    um_refc_ix = tmp_um_refc_ix;
+	}
+    }
+
+    last_elem = ddq_managed_thread_enqueue(ddq, ptr, cinit);
+
+    if (!managed_thread)
+	erts_atomic_dec_relb(&ddq->tail.data.um_refc[um_refc_ix]);
+    return last_elem;
+}
+
+static ERTS_INLINE void *
+ddq_dequeue(ErtsAllctrDDQueue_t *ddq)
+{
+    ErtsAllctrDDBlock_t *blk;
+
+    if (ddq->head.first == ddq->head.unref_end)
+	return NULL;
+
+    blk = ddq->head.first;
+    if (blk == &ddq->tail.data.marker) {
+	ASSERT(ddq->head.used_marker);
+	ddq->head.used_marker = 0;
+	blk = ((ErtsAllctrDDBlock_t *)
+	       erts_atomic_read_nob(&blk->atmc_next));
+	if (blk == ddq->head.unref_end) {
+	    ddq->head.first = blk;
+	    return NULL;
+	}
+    }
+
+    ddq->head.first = ((ErtsAllctrDDBlock_t *)
+		       erts_atomic_read_nob(&blk->atmc_next));
+
+    ASSERT(ddq->head.first);
+
+    return (void *) blk;
+}
+
+static int
+ddq_check_incoming(ErtsAllctrDDQueue_t *ddq)
+{
+    erts_aint_t ilast = erts_atomic_read_nob(&ddq->tail.data.last);
+    if (((ErtsAllctrDDBlock_t *) ilast) == &ddq->tail.data.marker
+	&& ddq->head.first == &ddq->tail.data.marker) {
+	/* Nothing more to do... */
+	return 0;
+    }
+
+    if (ddq->head.next.thr_progress_reached
+	|| erts_thr_progress_has_reached(ddq->head.next.thr_progress)) {
+	int um_refc_ix;
+	ddq->head.next.thr_progress_reached = 1;
+	um_refc_ix = ddq->head.next.um_refc_ix;
+	if (erts_atomic_read_nob(&ddq->tail.data.um_refc[um_refc_ix]) == 0) {
+	    /* Move unreferenced end pointer forward... */
+
+	    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+
+	    ddq->head.unref_end = ddq->head.next.unref_end;
+
+	    ilast = check_insert_marker(ddq, ilast);
+
+	    if (ddq->head.unref_end != (ErtsAllctrDDBlock_t *) ilast) {
+		ddq->head.next.unref_end = (ErtsAllctrDDBlock_t *) ilast;
+		ddq->head.next.thr_progress = erts_thr_progress_later(NULL);
+		erts_atomic32_set_relb(&ddq->tail.data.um_refc_ix,
+				       um_refc_ix);
+		ddq->head.next.um_refc_ix = um_refc_ix == 0 ? 1 : 0;
+		ddq->head.next.thr_progress_reached = 0;
+	    }
+	}
+    }
+    return 1;
+}
+
+static ERTS_INLINE void
+store_earliest_thr_prgr(ErtsThrPrgrVal *prev_val, ErtsAllctrDDQueue_t *ddq)
+{
+    if (!ddq->head.next.thr_progress_reached
+	&& (*prev_val == ERTS_THR_PRGR_INVALID
+	    || erts_thr_progress_cmp(ddq->head.next.thr_progress,
+				     *prev_val) < 0)) {
+	*prev_val = ddq->head.next.thr_progress;
+    }
+}
+
+static ERTS_INLINE int
+handle_delayed_dealloc(Allctr_t *allctr,
+		       int allctr_locked,
+		       int use_limit,
+		       int ops_limit,
+		       int *need_thr_progress,
+		       ErtsThrPrgrVal *thr_prgr_p,
+		       int *need_more_work)
+{
+    int need_thr_prgr = 0;
+    int need_mr_wrk = 0;
+    int have_checked_incoming = 0;
+    int ops = 0;
+    ErtsAlcFixList_t *fix;
+    int res;
+    ErtsAllctrDDQueue_t *ddq;
+
+    if (allctr->thread_safe && !allctr_locked)
+	erts_mtx_lock(&allctr->mutex);
+
+    ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr);
+
+    fix = allctr->fix;
+
+    ddq = &allctr->dd.q;
+
+    res = 0;
+
+    while (1) {
+	Block_t *blk;
+	void *ptr;
+	int ix;
+
+	if (use_limit && ++ops > ops_limit) {
+	    if (ddq->head.first != ddq->head.unref_end) {
+		need_mr_wrk = 1;
+		if (need_more_work)
+		    *need_more_work |= 1;
+	    }
+	    break;
+	}
+
+    dequeue:
+	ptr = ddq_dequeue(ddq);
+	if (!ptr) {
+	    if (have_checked_incoming)
+		break;
+	    need_thr_prgr = ddq_check_incoming(ddq);
+	    if (need_thr_progress) {
+		*need_thr_progress |= need_thr_prgr;
+		if (need_thr_prgr)
+		    store_earliest_thr_prgr(thr_prgr_p, ddq);
+
+	    }
+	    have_checked_incoming = 1;
+	    goto dequeue;
+	}
+
+	res = 1;
+
+	INC_CC(allctr->calls.this_free);
+
+	if (fix) {
+	    ErtsAlcType_t type;
+
+	    type = (ErtsAlcType_t) ((UWord *) ptr)[ERTS_ALCU_DD_FIX_TYPE_OFFS];
+	    ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE;
+	    ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
+	    fix[ix].used--;
+	    if (fix[ix].allocated < fix[ix].limit
+		&& fix[ix].list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) {
+		*((void **) ptr) = fix[ix].list;
+		fix[ix].list = ptr;
+		fix[ix].list_size++;
+		if (!allctr->fix_shrink_scheduled) {
+		    allctr->fix_shrink_scheduled = 1;
+		    erts_set_aux_work_timeout(
+			allctr->ix,
+			(ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
+			 | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
+			1);
+		}
+		ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
+		continue;
+	    }
+	    fix[ix].allocated--;
+	    if (fix[ix].list && fix[ix].allocated > fix[ix].limit) {
+		blk = UMEM2BLK(ptr);
+		if (IS_SBC_BLK(blk))
+		    destroy_carrier(allctr, blk);
+		else
+		    mbc_free(allctr, ptr);
+		ptr = fix[ix].list;
+		fix[ix].list = *((void **) ptr);
+		fix[ix].list_size--;
+		fix[ix].allocated--;
+	    }
+	}
+
+	blk = UMEM2BLK(ptr);
+
+	if (IS_SBC_BLK(blk))
+	    destroy_carrier(allctr, blk);
+	else
+	    mbc_free(allctr, ptr);
+	ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
+    }
+
+    if (need_thr_progress && !(need_thr_prgr | need_mr_wrk)) {
+	need_thr_prgr = ddq_check_incoming(ddq);
+	*need_thr_progress |= need_thr_prgr;
+	if (need_thr_prgr)
+	    store_earliest_thr_prgr(thr_prgr_p, ddq);
+    }
+
+    if (allctr->thread_safe && !allctr_locked)
+	erts_mtx_unlock(&allctr->mutex);
+   return res;
+}
+
+static ERTS_INLINE void
+enqueue_dealloc_other_instance(ErtsAlcType_t type,
+			       Allctr_t *allctr,
+			       void *ptr,
+			       int cinit)
+{
+    if (allctr->fix)
+	((UWord *) ptr)[ERTS_ALCU_DD_FIX_TYPE_OFFS] = (UWord) type;
+
+    if (ddq_enqueue(type, &allctr->dd.q, ptr, cinit))
+	erts_alloc_notify_delayed_dealloc(allctr->ix);
+}
+
+#endif
+
+#ifdef ERTS_SMP
+void
+erts_alcu_check_delayed_dealloc(Allctr_t *allctr,
+				int limit,
+				int *need_thr_progress,
+				ErtsThrPrgrVal *thr_prgr_p,
+				int *more_work)
+{
+    handle_delayed_dealloc(allctr,
+			   0,
+			   limit,
+			   ERTS_ALCU_DD_OPS_LIM_HIGH,
+			   need_thr_progress,
+			   thr_prgr_p,
+			   more_work);
+}
+#endif
+
+#define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \
+    handle_delayed_dealloc((Allctr), (Locked), 1, \
+			   ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL, NULL)
+
 /* Multi block carrier alloc/realloc/free ... */
 
 /* NOTE! mbc_alloc() may in case of memory shortage place the requested
  * block in a sbc.
  */
 static ERTS_INLINE void *
-mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp)
+mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp, Uint32 *alcu_flgsp)
 {
     Block_t *blk;
+    Uint get_blk_sz;
+    Uint sbmbct;
 
     ASSERT(size);
     ASSERT(size < allctr->sbc_threshold);
 
-    *blk_szp = UMEMSZ2BLKSZ(allctr, size);
+    *blk_szp = get_blk_sz = UMEMSZ2BLKSZ(allctr, size);
+
+    sbmbct = allctr->sbmbc_threshold;
+    if (sbmbct) {
+	if (get_blk_sz < sbmbct) {
+	    *alcu_flgsp |= ERTS_ALCU_FLG_SBMBC;
+	    if (get_blk_sz + allctr->min_block_size > sbmbct) {
+		/* Since we use block size to determine if blocks are
+		   located in sbmbc or not... */
+		get_blk_sz += allctr->min_block_size;
+	    }
+	}
+    }
+
+#ifdef ERTS_SMP
+    if (allctr->dd.use)
+	ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1);
+#endif
+
+    blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0, *alcu_flgsp);
 
-    blk = (*allctr->get_free_block)(allctr, *blk_szp, NULL, 0);
+#ifdef ERTS_SMP
+    if (!blk && allctr->dd.use) {
+	if (ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1))
+	    blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0,
+					    *alcu_flgsp);
+    }
+#endif
 
     if (!blk) {
-	blk = create_carrier(allctr, *blk_szp, CFLG_MBC);
-	if (!blk) {
-	    /* Emergency! We couldn't create the carrier as we wanted.
-	       Try to place it in a sys_alloced sbc. */
-	    blk = create_carrier(allctr,
-				 size,
-				 CFLG_SBC|CFLG_FORCE_SIZE|CFLG_FORCE_SYS_ALLOC);
+	if ((*alcu_flgsp) & ERTS_ALCU_FLG_SBMBC) 
+	    blk = create_sbmbc(allctr, get_blk_sz);
+	else {
+#if HALFWORD_HEAP
+	    blk = create_carrier(allctr, get_blk_sz, CFLG_MBC|CFLG_FORCE_MSEG);
+#else
+	    blk = create_carrier(allctr, get_blk_sz, CFLG_MBC);
+	    if (!blk) {
+		/* Emergency! We couldn't create the carrier as we wanted.
+		   Try to place it in a sys_alloced sbc. */
+		blk = create_carrier(allctr,
+				     size,
+				     (CFLG_SBC
+				      | CFLG_FORCE_SIZE
+				      | CFLG_FORCE_SYS_ALLOC));
+	    }
+#endif
 	}
     }
 
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
     if (IS_MBC_BLK(blk)) {
-	(*allctr->link_free_block)(allctr, blk);
+	(*allctr->link_free_block)(allctr, blk, *alcu_flgsp);
 	HARD_CHECK_BLK_CARRIER(allctr, blk);
-	(*allctr->unlink_free_block)(allctr, blk);
+	(*allctr->unlink_free_block)(allctr, blk, *alcu_flgsp);
     }
 #endif
 
@@ -656,14 +1241,15 @@ static ERTS_INLINE void
 mbc_alloc_finalize(Allctr_t *allctr,
 		   Block_t *blk,
 		   Uint org_blk_sz,
-		   Uint flags,
+		   UWord flags,
 		   Uint want_blk_sz,
-		   int valid_blk_info)
+		   int valid_blk_info,
+		   Uint32 alcu_flgs)
 {
     Uint blk_sz;
     Uint nxt_blk_sz;
     Block_t *nxt_blk;
-    Uint prev_free_flg = flags & PREV_FREE_BLK_HDR_FLG;
+    UWord prev_free_flg = flags & PREV_FREE_BLK_HDR_FLG;
 
     ASSERT(org_blk_sz >= want_blk_sz);
     ASSERT(blk);
@@ -694,7 +1280,7 @@ mbc_alloc_finalize(Allctr_t *allctr,
 		SET_PREV_BLK_FREE(nxt_nxt_blk);
 	    }
 	}
-	(*allctr->link_free_block)(allctr, nxt_blk);
+	(*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs);
 
 	ASSERT(IS_NOT_LAST_BLK(blk));
 	ASSERT(IS_FREE_BLK(nxt_blk));
@@ -735,7 +1321,7 @@ mbc_alloc_finalize(Allctr_t *allctr,
 	       : IS_NOT_LAST_BLK(blk));
     }
 
-    STAT_MBC_BLK_ALLOC(allctr, blk_sz);
+    STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs);
 
     ASSERT(IS_ALLOCED_BLK(blk));
     ASSERT(blk_sz == BLK_SZ(blk));
@@ -755,7 +1341,8 @@ mbc_alloc(Allctr_t *allctr, Uint size)
 {
     Block_t *blk;
     Uint blk_sz;
-    blk = mbc_alloc_block(allctr, size, &blk_sz);
+    Uint32 alcu_flgs = 0;
+    blk = mbc_alloc_block(allctr, size, &blk_sz, &alcu_flgs);
     if (!blk)
 	return NULL;
     if (IS_MBC_BLK(blk))
@@ -764,7 +1351,8 @@ mbc_alloc(Allctr_t *allctr, Uint size)
 			   BLK_SZ(blk),
 			   GET_BLK_HDR_FLGS(blk),
 			   blk_sz,
-			   1);
+			   1,
+			   alcu_flgs);
     return BLK2UMEM(blk);
 }
 
@@ -773,6 +1361,7 @@ mbc_free(Allctr_t *allctr, void *p)
 {
     Uint is_first_blk;
     Uint is_last_blk;
+    Uint32 alcu_flgs = 0;
     Uint blk_sz;
     Block_t *blk;
     Block_t *nxt_blk;
@@ -782,13 +1371,15 @@ mbc_free(Allctr_t *allctr, void *p)
 
     blk = UMEM2BLK(p);
     blk_sz = BLK_SZ(blk);
+    if (blk_sz < allctr->sbmbc_threshold)
+	alcu_flgs |= ERTS_ALCU_FLG_SBMBC;
 
     ASSERT(IS_MBC_BLK(blk));
     ASSERT(blk_sz >= allctr->min_block_size);
 
     HARD_CHECK_BLK_CARRIER(allctr, blk);
 
-    STAT_MBC_BLK_FREE(allctr, blk_sz);
+    STAT_MBC_BLK_FREE(allctr, blk_sz, alcu_flgs);
 
     is_first_blk = IS_FIRST_BLK(blk);
     is_last_blk = IS_LAST_BLK(blk);
@@ -796,7 +1387,7 @@ mbc_free(Allctr_t *allctr, void *p)
     if (!is_first_blk && IS_PREV_BLK_FREE(blk)) {
 	/* Coalesce with previous block... */
 	blk = PREV_BLK(blk);
-	(*allctr->unlink_free_block)(allctr, blk);
+	(*allctr->unlink_free_block)(allctr, blk, alcu_flgs);
 
 	blk_sz += BLK_SZ(blk);
 	is_first_blk = IS_FIRST_BLK(blk);
@@ -812,7 +1403,7 @@ mbc_free(Allctr_t *allctr, void *p)
 	nxt_blk = NXT_BLK(blk);
 	if (IS_FREE_BLK(nxt_blk)) {
 	    /* Coalesce with next block... */
-	    (*allctr->unlink_free_block)(allctr, nxt_blk);
+	    (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs);
 	    blk_sz += BLK_SZ(nxt_blk);
 	    SET_BLK_SZ(blk, blk_sz);
 
@@ -844,16 +1435,20 @@ mbc_free(Allctr_t *allctr, void *p)
 
     if (is_first_blk
 	&& is_last_blk
-	&& allctr->main_carrier != FBLK2MBC(allctr, blk))
-	destroy_carrier(allctr, blk);
+	&& allctr->main_carrier != FBLK2MBC(allctr, blk)) {
+	if (alcu_flgs & ERTS_ALCU_FLG_SBMBC)
+	    destroy_sbmbc(allctr, blk);
+	else
+	    destroy_carrier(allctr, blk);
+    }
     else {
-	(*allctr->link_free_block)(allctr, blk);
+	(*allctr->link_free_block)(allctr, blk, alcu_flgs);
 	HARD_CHECK_BLK_CARRIER(allctr, blk);
     }
 }
 
 static void *
-mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
+mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs)
 {
     void *new_p;
     Uint old_blk_sz;
@@ -861,12 +1456,17 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
 #ifndef MBC_REALLOC_ALWAYS_MOVES
     Block_t *new_blk, *cand_blk;
     Uint cand_blk_sz;
-    Uint blk_sz;
+    Uint blk_sz, get_blk_sz;
     Block_t *nxt_blk;
     Uint nxt_blk_sz;
     Uint is_last_blk;
 #endif /* #ifndef MBC_REALLOC_ALWAYS_MOVES */
 
+#ifdef ERTS_SMP
+    if (allctr->dd.use)
+	ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1);
+#endif
+
     ASSERT(p);
     ASSERT(size);
     ASSERT(size < allctr->sbc_threshold);
@@ -877,10 +1477,16 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
     ASSERT(old_blk_sz >= allctr->min_block_size);
 
 #ifdef MBC_REALLOC_ALWAYS_MOVES
-    if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
+    if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
 	return NULL;
 #else /* !MBC_REALLOC_ALWAYS_MOVES */
-    blk_sz = UMEMSZ2BLKSZ(allctr, size);
+    get_blk_sz = blk_sz = UMEMSZ2BLKSZ(allctr, size);
+    if ((alcu_flgs & ERTS_ALCU_FLG_SBMBC)
+	&& (blk_sz + allctr->min_block_size > allctr->sbmbc_threshold)) {
+	/* Since we use block size to determine if blocks are
+	   located in sbmbc or not... */
+	get_blk_sz = blk_sz + allctr->min_block_size;
+    }
 
     ASSERT(IS_ALLOCED_BLK(blk));
     ASSERT(IS_MBC_BLK(blk));
@@ -895,6 +1501,9 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
 	Uint diff_sz_val = old_blk_sz - blk_sz;
 	Uint old_blk_sz_val = old_blk_sz;
 
+	if (get_blk_sz >= old_blk_sz)
+	    return p;
+
 	if (diff_sz_val >= (~((Uint) 0) / 100)) {
 	    /* div both by 128 */
 	    old_blk_sz_val >>= 7;
@@ -903,7 +1512,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
 
 	/* Avoid fragmentation by moving the block if it is shrunk much */
 	if (100*diff_sz_val > allctr->mbc_move_threshold*old_blk_sz_val) {
-	    if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
+	    if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
 		return NULL;
 
 	    cand_blk_sz = old_blk_sz;
@@ -920,10 +1529,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
 	    }
 
 	    new_blk = (*allctr->get_free_block)(allctr,
-						blk_sz,
+						get_blk_sz,
 						cand_blk,
-						cand_blk_sz);
-
+						cand_blk_sz,
+						alcu_flgs);
 	    if (new_blk || cand_blk != blk)
 		goto move_into_new_blk;
 	}
@@ -946,8 +1555,8 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
 		    nxt_blk_sz,
 		    SBH_THIS_FREE|SBH_PREV_ALLOCED|SBH_NOT_LAST_BLK);
 
-	STAT_MBC_BLK_FREE(allctr, old_blk_sz);
-	STAT_MBC_BLK_ALLOC(allctr, blk_sz);
+	STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs);
+	STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs);
 
 	ASSERT(BLK_SZ(blk) >= allctr->min_block_size);
 
@@ -958,7 +1567,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
 	    if (IS_FREE_BLK(nxt_nxt_blk)) {
 		/* Coalesce with next free block... */
 		nxt_blk_sz += BLK_SZ(nxt_nxt_blk);
-		(*allctr->unlink_free_block)(allctr, nxt_nxt_blk);
+		(*allctr->unlink_free_block)(allctr, nxt_nxt_blk, alcu_flgs);
 		SET_BLK_SZ(nxt_blk, nxt_blk_sz);
 
 		is_last_blk = IS_LAST_BLK(nxt_nxt_blk);
@@ -973,7 +1582,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
 	    }
 	}
 
-	(*allctr->link_free_block)(allctr, nxt_blk);
+	(*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs);
 
 
 	ASSERT(IS_ALLOCED_BLK(blk));
@@ -1003,12 +1612,12 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
     if (!is_last_blk) {
 	nxt_blk = NXT_BLK(blk);
 	nxt_blk_sz = BLK_SZ(nxt_blk);
-	if (IS_FREE_BLK(nxt_blk) && blk_sz <= old_blk_sz + nxt_blk_sz) {
+	if (IS_FREE_BLK(nxt_blk) && get_blk_sz <= old_blk_sz + nxt_blk_sz) {
 	    /* Grow into next block... */
 
 	    HARD_CHECK_BLK_CARRIER(allctr, blk);
 
-	    (*allctr->unlink_free_block)(allctr, nxt_blk);
+	    (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs);
 	    nxt_blk_sz -= blk_sz - old_blk_sz;
 
 	    is_last_blk = IS_LAST_BLK(nxt_blk);
@@ -1045,13 +1654,13 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
 		else
 		    SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz);
 
-		(*allctr->link_free_block)(allctr, nxt_blk);
+		(*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs);
 
 		ASSERT(IS_FREE_BLK(nxt_blk));
 	    }
 
-	    STAT_MBC_BLK_FREE(allctr, old_blk_sz);
-	    STAT_MBC_BLK_ALLOC(allctr, blk_sz);
+	    STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs);
+	    STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs);
 
 
 	    ASSERT(IS_ALLOCED_BLK(blk));
@@ -1082,7 +1691,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
 	}
     }
 
-    if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
+    if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
 	return NULL;
 
     /* Need to grow in another block */
@@ -1102,7 +1711,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
 	}
     }
 
-    if (cand_blk_sz < blk_sz) {
+    if (cand_blk_sz < get_blk_sz) {
 	/* We wont fit in cand_blk get a new one */
 #endif /* !MBC_REALLOC_ALWAYS_MOVES */
 
@@ -1121,9 +1730,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
 	/* We will at least fit in cand_blk */
 
 	new_blk = (*allctr->get_free_block)(allctr,
-					    blk_sz,
+					    get_blk_sz,
 					    cand_blk,
-					    cand_blk_sz);
+					    cand_blk_sz,
+					    alcu_flgs);
     move_into_new_blk:
 	/*
 	 * new_blk, and cand_blk have to be correctly set
@@ -1136,7 +1746,8 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
 			       BLK_SZ(new_blk),
 			       GET_BLK_HDR_FLGS(new_blk),
 			       blk_sz,
-			       1);
+			       1,
+			       alcu_flgs);
 	    new_p = BLK2UMEM(new_blk);
 	    sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ));
 	    mbc_free(allctr, p);
@@ -1144,7 +1755,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
 	}
 	else {
 	    Uint new_blk_sz;
-	    Uint new_blk_flgs;
+	    UWord new_blk_flgs;
 	    Uint prev_blk_sz;
 	    Uint blk_cpy_sz;
 
@@ -1158,7 +1769,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
 
 	    HARD_CHECK_BLK_CARRIER(allctr, blk);
 
-	    (*allctr->unlink_free_block)(allctr, new_blk); /* prev */
+	    (*allctr->unlink_free_block)(allctr, new_blk, alcu_flgs); /* prev */
 
 	    if (is_last_blk) 
 		new_blk_flgs |= LAST_BLK_HDR_FLG;
@@ -1167,7 +1778,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
 		if (IS_FREE_BLK(nxt_blk)) {
 		    new_blk_flgs |= GET_LAST_BLK_HDR_FLG(nxt_blk);
 		    new_blk_sz += BLK_SZ(nxt_blk);
-		    (*allctr->unlink_free_block)(allctr, nxt_blk);
+		    (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs);
 		}
 	    }
 
@@ -1190,9 +1801,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint flgs)
 			       new_blk_sz,
 			       new_blk_flgs,
 			       blk_sz,
-			       0);
+			       0,
+			       alcu_flgs);
 
-	    STAT_MBC_BLK_FREE(allctr, old_blk_sz);
+	    STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs);
 
 	    return new_p;
 	}
@@ -1237,9 +1849,103 @@ do {									\
 #define CHECK_1BLK_CARRIER(A, SBC, MSEGED, C, CSZ, B, BSZ)
 #endif
 
+static Block_t *
+create_sbmbc(Allctr_t *allctr, Uint umem_sz)
+{
+    Block_t *blk;
+    Uint blk_sz;
+    Uint crr_sz = allctr->sbmbc_size;
+    Carrier_t *crr;
+
+#if HALFWORD_HEAP
+    if (allctr->mseg_opt.low_mem)
+	crr = erts_alloc(ERTS_ALC_T_SBMBC_LOW, crr_sz);
+    else
+#endif
+	crr = erts_alloc(ERTS_ALC_T_SBMBC, crr_sz);
+
+    INC_CC(allctr->calls.sbmbc_alloc);
+    SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_MBC);
+
+    blk = MBC2FBLK(allctr, crr);
+
+#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
+    if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
+	crr_sz -= sizeof(UWord);
+#endif
+
+    blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size);
+
+    SET_MBC_BLK_FTR(((UWord *) blk)[-1]);
+    SET_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK);
+
+#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
+    *((Carrier_t **) NXT_BLK(blk)) = crr;
+#endif
+
+    link_carrier(&allctr->sbmbc_list, crr);
+
+#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
+    if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
+	crr_sz += sizeof(UWord);
+#endif
+
+    STAT_SBMBC_ALLOC(allctr, crr_sz);
+    CHECK_1BLK_CARRIER(allctr, 0, 0, crr, crr_sz, blk, blk_sz);
+#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
+    if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
+	crr_sz -= sizeof(UWord);
+#endif
+    if (allctr->creating_mbc)
+	(*allctr->creating_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC);
+
+    DEBUG_SAVE_ALIGNMENT(crr);
+    return blk;
+}
+
+static void
+destroy_sbmbc(Allctr_t *allctr, Block_t *blk)
+{
+    Uint crr_sz;
+    Carrier_t *crr;
+
+    ASSERT(IS_FIRST_BLK(blk));
+
+    ASSERT(IS_MBC_BLK(blk));
+
+    crr = FBLK2MBC(allctr, blk);
+    crr_sz = CARRIER_SZ(crr);
+
+#ifdef DEBUG
+    if (!allctr->stopped) {
+	ASSERT(IS_LAST_BLK(blk));
+
+#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
+	(*allctr->link_free_block)(allctr, blk, ERTS_ALCU_FLG_SBMBC);
+	HARD_CHECK_BLK_CARRIER(allctr, blk);
+	(*allctr->unlink_free_block)(allctr, blk, ERTS_ALCU_FLG_SBMBC);
+#endif
+    }
+#endif
+
+    STAT_SBMBC_FREE(allctr, crr_sz);
+
+    unlink_carrier(&allctr->sbmbc_list, crr);
+    if (allctr->destroying_mbc)
+	(*allctr->destroying_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC);
+
+    INC_CC(allctr->calls.sbmbc_free);
+
+#if HALFWORD_HEAP
+    if (allctr->mseg_opt.low_mem)
+	erts_free(ERTS_ALC_T_SBMBC_LOW, crr);
+    else
+#endif
+	erts_free(ERTS_ALC_T_SBMBC, crr);
+}
 
 static Block_t *
-create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags)
+create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 {
     Block_t *blk;
     Carrier_t *crr;
@@ -1262,14 +1968,14 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags)
 	goto try_sys_alloc;
     if (flags & CFLG_FORCE_MSEG)
 	goto try_mseg;
-    if (erts_mseg_no() >= max_mseg_carriers)
+    if (erts_mseg_no(&allctr->mseg_opt) >= max_mseg_carriers)
 	goto try_sys_alloc;
     if (flags & CFLG_SBC) {
-	if (allctr->sbcs.curr_mseg.no >= allctr->max_mseg_sbcs)
+	if (allctr->sbcs.curr.norm.mseg.no >= allctr->max_mseg_sbcs)
 	    goto try_sys_alloc;
     }
     else {
-	if (allctr->mbcs.curr_mseg.no >= allctr->max_mseg_mbcs)
+	if (allctr->mbcs.curr.norm.mseg.no >= allctr->max_mseg_mbcs)
 	    goto try_sys_alloc;
     }
 
@@ -1283,8 +1989,8 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags)
 	if (crr_sz < allctr->mbc_header_size + blk_sz)
 	    crr_sz = allctr->mbc_header_size + blk_sz;
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
-	if (sizeof(Unit_t) == sizeof(Uint))
-	    crr_sz += sizeof(Uint);
+	if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
+	    crr_sz += sizeof(UWord);
 #endif
     }
     crr_sz = MSEG_UNIT_CEILING(crr_sz);
@@ -1324,8 +2030,8 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags)
 	    && bcrr_sz < allctr->smallest_mbc_size)
 	    bcrr_sz = allctr->smallest_mbc_size;
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
-	if (sizeof(Unit_t) == sizeof(Uint))
-	    bcrr_sz += sizeof(Uint);
+	if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
+	    bcrr_sz += sizeof(UWord);
 #endif
 
     }
@@ -1360,7 +2066,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags)
 
 	blk = SBC2BLK(allctr, crr);
 
-	SET_SBC_BLK_FTR(((Uint *) blk)[-1]);
+	SET_SBC_BLK_FTR(((UWord *) blk)[-1]);
 	SET_BLK_HDR(blk, blk_sz, SBH_THIS_ALLOCED|SBH_PREV_FREE|SBH_LAST_BLK);
 
 	link_carrier(&allctr->sbc_list, crr);
@@ -1379,13 +2085,13 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags)
 	blk = MBC2FBLK(allctr, crr);
 
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
-	if (sizeof(Unit_t) == sizeof(Uint))
-	    crr_sz -= sizeof(Uint);
+	if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
+	    crr_sz -= sizeof(UWord);
 #endif
 
 	blk_sz = UNIT_FLOOR(crr_sz - allctr->mbc_header_size);
 
-	SET_MBC_BLK_FTR(((Uint *) blk)[-1]);
+	SET_MBC_BLK_FTR(((UWord *) blk)[-1]);
 	SET_BLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_PREV_FREE|SBH_LAST_BLK);
 
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
@@ -1400,16 +2106,16 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags)
 	link_carrier(&allctr->mbc_list, crr);
 
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
-	if (sizeof(Unit_t) == sizeof(Uint))
-	    crr_sz += sizeof(Uint);
+	if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
+	    crr_sz += sizeof(UWord);
 #endif
 	CHECK_1BLK_CARRIER(allctr, 0, is_mseg, crr, crr_sz, blk, blk_sz);
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
-	if (sizeof(Unit_t) == sizeof(Uint))
-	    crr_sz -= sizeof(Uint);
+	if (allctr->mbc_header_size % sizeof(Unit_t) == 0)
+	    crr_sz -= sizeof(UWord);
 #endif
 	if (allctr->creating_mbc)
-	    (*allctr->creating_mbc)(allctr, crr);
+	    (*allctr->creating_mbc)(allctr, crr, 0);
 
     }
 
@@ -1418,11 +2124,12 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, Uint flags)
 }
 
 static Block_t *
-resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, Uint flags)
+resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags)
 {
     Block_t *new_blk;
     Carrier_t *new_crr, *old_crr;
-    Uint create_flags, old_crr_sz, old_blk_sz, new_blk_sz, new_crr_sz;
+    UWord create_flags;
+    Uint old_crr_sz, old_blk_sz, new_blk_sz, new_crr_sz;
     Uint new_bcrr_sz;
 
     if (flags & CFLG_MBC) {
@@ -1588,9 +2295,9 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
 	    ASSERT(IS_LAST_BLK(blk));
 
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
-	    (*allctr->link_free_block)(allctr, blk);
+	    (*allctr->link_free_block)(allctr, blk, 0);
 	    HARD_CHECK_BLK_CARRIER(allctr, blk);
-	    (*allctr->unlink_free_block)(allctr, blk);
+	    (*allctr->unlink_free_block)(allctr, blk, 0);
 #endif
 	}
 #endif
@@ -1607,7 +2314,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk)
 
 	unlink_carrier(&allctr->mbc_list, crr);
 	if (allctr->destroying_mbc)
-	    (*allctr->destroying_mbc)(allctr, crr);
+	    (*allctr->destroying_mbc)(allctr, crr, 0);
     }
 
 
@@ -1632,6 +2339,9 @@ static struct {
     Eterm e;
     Eterm t;
     Eterm ramv;
+#if HALFWORD_HEAP
+    Eterm low;
+#endif
     Eterm sbct;
 #if HAVE_ERTS_MSEG
     Eterm asbcst;
@@ -1648,14 +2358,21 @@ static struct {
     Eterm lmbcs;
     Eterm smbcs;
     Eterm mbcgs;
+    Eterm sbmbcs;
+    Eterm sbmbct;
 
 #if HAVE_ERTS_MSEG
     Eterm mmc;
 #endif
     Eterm ycs;
 
+    /* Eterm sbmbcs; */
+
+    Eterm fix_types;
+
     Eterm mbcs;
     Eterm sbcs;
+
     Eterm sys_alloc_carriers_size;
 #if HAVE_ERTS_MSEG
     Eterm mseg_alloc_carriers_size;
@@ -1678,11 +2395,15 @@ static struct {
     Eterm mseg_dealloc;
     Eterm mseg_realloc;
 #endif
+    Eterm sbmbc_alloc;
+    Eterm sbmbc_free;
 #ifdef DEBUG
     Eterm end_of_atoms;
 #endif
 } am;
 
+static Eterm fix_type_atoms[ERTS_ALC_NO_FIXED_SIZES];
+
 static ERTS_INLINE void atom_init(Eterm *atom, char *name)
 {
     *atom = am_atom_put(name, strlen(name));
@@ -1703,6 +2424,7 @@ init_atoms(Allctr_t *allctr)
     erts_mtx_lock(&init_atoms_mtx);
 
     if (!atoms_initialized) {
+	int ix;
 #ifdef DEBUG
 	Eterm *atom;
 
@@ -1717,6 +2439,9 @@ init_atoms(Allctr_t *allctr)
 	AM_INIT(e);
 	AM_INIT(t);
 	AM_INIT(ramv);
+#if HALFWORD_HEAP
+	AM_INIT(low);
+#endif
 	AM_INIT(sbct);
 #if HAVE_ERTS_MSEG
 	AM_INIT(asbcst);
@@ -1733,14 +2458,21 @@ init_atoms(Allctr_t *allctr)
 	AM_INIT(lmbcs);
 	AM_INIT(smbcs);
 	AM_INIT(mbcgs);
+	AM_INIT(sbmbcs);
+	AM_INIT(sbmbct);
 
 #if HAVE_ERTS_MSEG
 	AM_INIT(mmc);
 #endif
 	AM_INIT(ycs);
 
+	/*AM_INIT(sbmbcs);*/
+
+	AM_INIT(fix_types);
+
 	AM_INIT(mbcs);
 	AM_INIT(sbcs);
+
 	AM_INIT(sys_alloc_carriers_size);
 #if HAVE_ERTS_MSEG
 	AM_INIT(mseg_alloc_carriers_size);
@@ -1763,12 +2495,21 @@ init_atoms(Allctr_t *allctr)
 	AM_INIT(mseg_dealloc);
 	AM_INIT(mseg_realloc);
 #endif
+	AM_INIT(sbmbc_free);
+	AM_INIT(sbmbc_alloc);
 
 #ifdef DEBUG
 	for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) {
 	    ASSERT(*atom != THE_NON_VALUE);
 	}
 #endif
+
+	for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) {
+	    ErtsAlcType_t n = ERTS_ALC_N_MIN_A_FIXED_SIZE + ix;
+	    char *name = (char *) ERTS_ALC_N2TD(n);
+	    size_t len = strlen(name);
+	    fix_type_atoms[ix] = am_atom_put(name, len);
+	}
     }
 
     
@@ -1847,6 +2588,48 @@ add_4tup(Uint **hpp, Uint *szp, Eterm *lp,
 }
 
 static Eterm
+sz_info_fix(Allctr_t *allctr,
+	    int *print_to_p,
+	    void *print_to_arg,
+	    Uint **hpp,
+	    Uint *szp)
+{
+    Eterm res;
+    int ix;
+    ErtsAlcFixList_t *fix = allctr->fix;
+
+    ASSERT(fix);
+
+    res = NIL;
+
+    for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) {
+	ErtsAlcType_t n = ix + ERTS_ALC_N_MIN_A_FIXED_SIZE;
+	Uint alloced = (fix[ix].type_size * fix[ix].allocated);
+	Uint used = fix[ix].type_size*fix[ix].used;
+
+	if (print_to_p) {
+	    int to = *print_to_p;
+	    void *arg = print_to_arg;
+	    erts_print(to,
+		       arg,
+		       "fix type: %s %bpu %bpu\n",
+		       (char *) ERTS_ALC_N2TD(n),
+		       alloced,
+		       used);
+	}
+
+	if (hpp || szp) {
+	    add_3tup(hpp, szp, &res,
+		     fix_type_atoms[ix],
+		     bld_unstable_uint(hpp, szp, alloced),
+		     bld_unstable_uint(hpp, szp, used));
+	}
+    }
+
+    return res;
+}
+
+static Eterm
 sz_info_carriers(Allctr_t *allctr,
 		 CarriersStats_t *cs,
 		 char *prefix,
@@ -1856,7 +2639,9 @@ sz_info_carriers(Allctr_t *allctr,
 		 Uint *szp)
 {
     Eterm res = THE_NON_VALUE;
-    Uint curr_size = cs->curr_mseg.size + cs->curr_sys_alloc.size;
+    Uint curr_size = (cs == &allctr->sbmbcs
+		      ? cs->curr.small_block.size
+		      : cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size);
 
     if (print_to_p) {
 	int to = *print_to_p;
@@ -1870,7 +2655,7 @@ sz_info_carriers(Allctr_t *allctr,
 		   cs->blocks.max_ever.size);
 	erts_print(to,
 		   arg,
-		   "%scarriers size: %bpu %bpu %bpu\n",
+		   "%scarriers size: %beu %bpu %bpu\n",
 		   prefix,
 		   curr_size,
 		   cs->max.size,
@@ -1904,8 +2689,17 @@ info_carriers(Allctr_t *allctr,
 	      Uint *szp)
 {
     Eterm res = THE_NON_VALUE;
-    Uint curr_no   = cs->curr_mseg.no   + cs->curr_sys_alloc.no;
-    Uint curr_size = cs->curr_mseg.size + cs->curr_sys_alloc.size;
+    Uint curr_no, curr_size;
+    int small_block = cs == &allctr->sbmbcs;
+
+    if (small_block) {
+	curr_no = cs->curr.small_block.no;
+	curr_size = cs->curr.small_block.size;
+    }
+    else {
+	curr_no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no;
+	curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size;
+    }
 
     if (print_to_p) {
 	int to = *print_to_p;
@@ -1926,67 +2720,75 @@ info_carriers(Allctr_t *allctr,
 		   cs->blocks.max_ever.size);
 	erts_print(to,
 		   arg,
-		   "%scarriers: %bpu %bpu %bpu\n",
+		   "%scarriers: %beu %bpu %bpu\n",
 		   prefix,
 		   curr_no,
 		   cs->max.no,
 		   cs->max_ever.no);
+	if (!small_block) {
 #if HAVE_ERTS_MSEG
-	erts_print(to,
-		   arg,
-		   "%smseg carriers: %bpu\n",
-		   prefix,
-		   cs->curr_mseg.no);
+	    erts_print(to,
+		       arg,
+		       "%smseg carriers: %bpu\n",
+		       prefix,
+		       cs->curr.norm.mseg.no);
 #endif
+	    erts_print(to,
+		       arg,
+		       "%ssys_alloc carriers: %bpu\n",
+		       prefix,
+		       cs->curr.norm.sys_alloc.no);
+	}
 	erts_print(to,
 		   arg,
-		   "%ssys_alloc carriers: %bpu\n",
-		   prefix,
-		   cs->curr_sys_alloc.no);
-	erts_print(to,
-		   arg,
-		   "%scarriers size: %bpu %bpu %bpu\n",
+		   "%scarriers size: %beu %bpu %bpu\n",
 		   prefix,
 		   curr_size,
 		   cs->max.size,
 		   cs->max_ever.size);
+	if (!small_block) {
 #if HAVE_ERTS_MSEG
-	erts_print(to,
-		   arg,
-		   "%smseg carriers size: %bpu\n",
-		   prefix,
-		   cs->curr_mseg.size);
+	    erts_print(to,
+		       arg,
+		       "%smseg carriers size: %bpu\n",
+		       prefix,
+		       cs->curr.norm.mseg.size);
 #endif
-	erts_print(to,
-		   arg,
-		   "%ssys_alloc carriers size: %bpu\n",
-		   prefix,
-		   cs->curr_sys_alloc.size);
+	    erts_print(to,
+		       arg,
+		       "%ssys_alloc carriers size: %bpu\n",
+		       prefix,
+		       cs->curr.norm.sys_alloc.size);
+	}
     }
 
     if (hpp || szp) {
 	res = NIL;
-	add_2tup(hpp, szp, &res,
-		 am.sys_alloc_carriers_size,
-		 bld_unstable_uint(hpp, szp, cs->curr_sys_alloc.size));
+	if (!small_block) {
+	    add_2tup(hpp, szp, &res,
+		     am.sys_alloc_carriers_size,
+		     bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.size));
 #if HAVE_ERTS_MSEG
-	add_2tup(hpp, szp, &res,
-		 am.mseg_alloc_carriers_size,
-		 bld_unstable_uint(hpp, szp, cs->curr_mseg.size));
+	    add_2tup(hpp, szp, &res,
+		     am.mseg_alloc_carriers_size,
+		     bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.size));
 #endif
+	}
 	add_4tup(hpp, szp, &res,
 		 am.carriers_size,
 		 bld_unstable_uint(hpp, szp, curr_size),
 		 bld_unstable_uint(hpp, szp, cs->max.size),
 		 bld_unstable_uint(hpp, szp, cs->max_ever.size));
-	add_2tup(hpp, szp, &res,
-		 am.sys_alloc_carriers,
-		 bld_unstable_uint(hpp, szp, cs->curr_sys_alloc.no));
+	if (!small_block) {
+	    add_2tup(hpp, szp, &res,
+		     am.sys_alloc_carriers,
+		     bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.no));
 #if HAVE_ERTS_MSEG
-	add_2tup(hpp, szp, &res,
-		 am.mseg_alloc_carriers,
-		 bld_unstable_uint(hpp, szp, cs->curr_mseg.no));
+	    add_2tup(hpp, szp, &res,
+		     am.mseg_alloc_carriers,
+		     bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.no));
 #endif
+	}
 	add_4tup(hpp, szp, &res,
 		 am.carriers,
 		 bld_unstable_uint(hpp, szp, curr_no),
@@ -2046,15 +2848,15 @@ info_calls(Allctr_t *allctr,
 
 #define PRINT_CC_4(TO, TOA, NAME, CC)					\
     if ((CC).giga_no == 0)						\
-	erts_print(TO, TOA, "%s calls: %bpu\n", NAME, CC.no);		\
+	erts_print(TO, TOA, "%s calls: %b32u\n", NAME, CC.no);		\
     else								\
-	erts_print(TO, TOA, "%s calls: %bpu%09lu\n", NAME, CC.giga_no, CC.no)
+	erts_print(TO, TOA, "%s calls: %b32u%09lu\n", NAME, CC.giga_no, CC.no)
 
 #define PRINT_CC_5(TO, TOA, PRFX, NAME, CC)				\
     if ((CC).giga_no == 0)						\
-	erts_print(TO, TOA, "%s%s calls: %bpu\n",PRFX,NAME,CC.no);	\
+	erts_print(TO, TOA, "%s%s calls: %b32u\n",PRFX,NAME,CC.no);	\
     else								\
-	erts_print(TO, TOA, "%s%s calls: %bpu%09lu\n",PRFX,NAME,CC.giga_no,CC.no)
+	erts_print(TO, TOA, "%s%s calls: %b32u%09lu\n",PRFX,NAME,CC.giga_no,CC.no)
 
 	char *prefix = allctr->name_prefix;
 	int to = *print_to_p;
@@ -2064,6 +2866,9 @@ info_calls(Allctr_t *allctr,
 	PRINT_CC_5(to, arg, prefix, "free",         allctr->calls.this_free);
 	PRINT_CC_5(to, arg, prefix, "realloc",      allctr->calls.this_realloc);
 
+	PRINT_CC_4(to, arg,         "sbmbc_alloc",  allctr->calls.sbmbc_alloc);
+	PRINT_CC_4(to, arg,         "sbmbc_free",   allctr->calls.sbmbc_free);
+
 #if HAVE_ERTS_MSEG
 	PRINT_CC_4(to, arg,         "mseg_alloc",   allctr->calls.mseg_alloc);
 	PRINT_CC_4(to, arg,         "mseg_dealloc", allctr->calls.mseg_dealloc);
@@ -2115,6 +2920,14 @@ info_calls(Allctr_t *allctr,
 		 bld_unstable_uint(hpp, szp, allctr->calls.mseg_alloc.no));
 #endif
 	add_3tup(hpp, szp, &res,
+		 am.sbmbc_free,
+		 bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_free.giga_no),
+		 bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_free.no));
+	add_3tup(hpp, szp, &res,
+		 am.sbmbc_alloc,
+		 bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_alloc.giga_no),
+		 bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_alloc.no));
+	add_3tup(hpp, szp, &res,
 		 allctr->name.realloc,
 		 bld_unstable_uint(hpp, szp, allctr->calls.this_realloc.giga_no),
 		 bld_unstable_uint(hpp, szp, allctr->calls.this_realloc.no));
@@ -2161,23 +2974,31 @@ info_options(Allctr_t *allctr,
 		   "option e: true\n"
 		   "option t: %s\n"
 		   "option ramv: %s\n"
-		   "option sbct: %bpu\n"
+#if HALFWORD_HEAP
+		   "option low: %s\n"
+#endif
+		   "option sbct: %beu\n"
 #if HAVE_ERTS_MSEG
 		   "option asbcst: %bpu\n"
 		   "option rsbcst: %bpu\n"
 #endif
-		   "option rsbcmt: %bpu\n"
-		   "option rmbcmt: %bpu\n"
-		   "option mmbcs: %bpu\n"
+		   "option rsbcmt: %beu\n"
+		   "option rmbcmt: %beu\n"
+		   "option mmbcs: %beu\n"
 #if HAVE_ERTS_MSEG
-		   "option mmsbc: %bpu\n"
-		   "option mmmbc: %bpu\n"
+		   "option mmsbc: %beu\n"
+		   "option mmmbc: %beu\n"
 #endif
-		   "option lmbcs: %bpu\n"
-		   "option smbcs: %bpu\n"
-		   "option mbcgs: %bpu\n",
+		   "option lmbcs: %beu\n"
+		   "option smbcs: %beu\n"
+		   "option mbcgs: %beu\n"
+		   "option sbmbcs: %beu\n"
+		   "option sbmbct: %beu\n",
 		   topt,
 		   allctr->ramv ? "true" : "false",
+#if HALFWORD_HEAP
+		   allctr->mseg_opt.low_mem ? "true" : "false",
+#endif
 		   allctr->sbc_threshold,
 #if HAVE_ERTS_MSEG
 		   allctr->mseg_opt.abs_shrink_th,
@@ -2192,7 +3013,9 @@ info_options(Allctr_t *allctr,
 #endif
 		   allctr->largest_mbc_size,
 		   allctr->smallest_mbc_size,
-		   allctr->mbc_growth_stages);
+		   allctr->mbc_growth_stages,
+		   allctr->sbmbc_size,
+		   allctr->sbmbc_threshold);
     }
 
     res = (*allctr->info_options)(allctr, "option ", print_to_p, print_to_arg,
@@ -2200,6 +3023,12 @@ info_options(Allctr_t *allctr,
 
     if (hpp || szp) {
 	add_2tup(hpp, szp, &res,
+		 am.sbmbct,
+		 bld_uint(hpp, szp, allctr->sbmbc_threshold));
+	add_2tup(hpp, szp, &res,
+		 am.sbmbcs,
+		 bld_uint(hpp, szp, allctr->sbmbc_size));
+	add_2tup(hpp, szp, &res,
 		 am.mbcgs,
 		 bld_uint(hpp, szp, allctr->mbc_growth_stages));
 	add_2tup(hpp, szp, &res,
@@ -2236,10 +3065,11 @@ info_options(Allctr_t *allctr,
 	add_2tup(hpp, szp, &res,
 		 am.sbct,
 		 bld_uint(hpp, szp, allctr->sbc_threshold));
+#if HALFWORD_HEAP
+	add_2tup(hpp, szp, &res, am.low, allctr->mseg_opt.low_mem ? am_true : am_false);
+#endif
 	add_2tup(hpp, szp, &res, am.ramv, allctr->ramv ? am_true : am_false);
-	add_2tup(hpp, szp, &res, am.t, (allctr->t
-					? bld_uint(hpp, szp, (Uint) allctr->t)
-					: am_false));
+	add_2tup(hpp, szp, &res, am.t, (allctr->t ? am_true : am_false));
 	add_2tup(hpp, szp, &res, am.e, am_true);
     }
 
@@ -2263,10 +3093,10 @@ update_max_ever_values(CarriersStats_t *cs)
 static ERTS_INLINE void
 reset_max_values(CarriersStats_t *cs)
 {
-    cs->max.no			= cs->curr_mseg.no   + cs->curr_sys_alloc.no;
-    cs->max.size		= cs->curr_mseg.size + cs->curr_sys_alloc.size;
-    cs->blocks.max.no		= cs->blocks.curr.no;
-    cs->blocks.max.size		= cs->blocks.curr.size;
+    cs->max.no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no;
+    cs->max.size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size;
+    cs->blocks.max.no = cs->blocks.curr.no;
+    cs->blocks.max.size = cs->blocks.curr.size;
 }
 
 
@@ -2285,9 +3115,9 @@ erts_alcu_au_info_options(int *print_to_p, void *print_to_arg,
 	erts_print(*print_to_p,
 		   print_to_arg,
 #if HAVE_ERTS_MSEG
-		   "option mmc: %bpu\n"
+		   "option mmc: %beu\n"
 #endif
-		   "option ycs: %bpu\n",
+		   "option ycs: %beu\n",
 #if HAVE_ERTS_MSEG
 		   max_mseg_carriers,
 #endif
@@ -2345,7 +3175,7 @@ erts_alcu_sz_info(Allctr_t *allctr,
 		  Uint **hpp,
 		  Uint *szp)
 {
-    Eterm res, mbcs, sbcs;
+    Eterm res, sbmbcs, mbcs, sbcs, fix = THE_NON_VALUE;
 
     res  = THE_NON_VALUE;
 
@@ -2362,29 +3192,40 @@ erts_alcu_sz_info(Allctr_t *allctr,
 	erts_mtx_lock(&allctr->mutex);
 #endif
 
+    ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr);
+
     if (hpp || szp)
 	ensure_atoms_initialized(allctr);
 
     /* Update sbc values not continously updated */
     allctr->sbcs.blocks.curr.no
-	= allctr->sbcs.curr_mseg.no + allctr->sbcs.curr_sys_alloc.no;
+	= allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no;
     allctr->sbcs.blocks.max.no = allctr->sbcs.max.no;
 
+    update_max_ever_values(&allctr->sbmbcs);
     update_max_ever_values(&allctr->mbcs);
     update_max_ever_values(&allctr->sbcs);
 
-    mbcs  = sz_info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p,
-			     print_to_arg, hpp, szp);
-    sbcs  = sz_info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p,
-			     print_to_arg, hpp, szp);
+    if (allctr->fix)
+	fix = sz_info_fix(allctr, print_to_p, print_to_arg, hpp, szp);
+    sbmbcs = sz_info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p,
+			      print_to_arg, hpp, szp);
+    mbcs = sz_info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p,
+			    print_to_arg, hpp, szp);
+    sbcs = sz_info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p,
+			    print_to_arg, hpp, szp);
 
     if (hpp || szp) {
 	res = NIL;
 	add_2tup(hpp, szp, &res, am.sbcs, sbcs);
 	add_2tup(hpp, szp, &res, am.mbcs, mbcs);
+	add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs);
+	if (allctr->fix)
+	    add_2tup(hpp, szp, &res, am.fix_types, fix);
     }
 
     if (begin_max_period) {
+	reset_max_values(&allctr->sbmbcs);
 	reset_max_values(&allctr->mbcs);
 	reset_max_values(&allctr->sbcs);
     }
@@ -2406,7 +3247,7 @@ erts_alcu_info(Allctr_t *allctr,
 	       Uint **hpp,
 	       Uint *szp)
 {
-    Eterm res, sett, mbcs, sbcs, calls;
+    Eterm res, sett, sbmbcs, mbcs, sbcs, calls, fix = THE_NON_VALUE;
 
     res  = THE_NON_VALUE;
 
@@ -2423,14 +3264,17 @@ erts_alcu_info(Allctr_t *allctr,
 	erts_mtx_lock(&allctr->mutex);
 #endif
 
+    ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr);
+
     if (hpp || szp)
 	ensure_atoms_initialized(allctr);
 
     /* Update sbc values not continously updated */
     allctr->sbcs.blocks.curr.no
-	= allctr->sbcs.curr_mseg.no + allctr->sbcs.curr_sys_alloc.no;
+	= allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no;
     allctr->sbcs.blocks.max.no = allctr->sbcs.max.no;
 
+    update_max_ever_values(&allctr->sbmbcs);
     update_max_ever_values(&allctr->mbcs);
     update_max_ever_values(&allctr->sbcs);
 
@@ -2442,11 +3286,15 @@ erts_alcu_info(Allctr_t *allctr,
 		   ERTS_ALCU_VSN_STR);
     }
 
-    sett  = info_options(allctr, print_to_p, print_to_arg, hpp, szp);
-    mbcs  = info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p,
-			  print_to_arg, hpp, szp);
-    sbcs  = info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p,
-			  print_to_arg, hpp, szp);
+    sett = info_options(allctr, print_to_p, print_to_arg, hpp, szp);
+    if (allctr->fix)
+	fix = sz_info_fix(allctr, print_to_p, print_to_arg, hpp, szp);
+    sbmbcs = info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p,
+			   print_to_arg, hpp, szp);
+    mbcs = info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p,
+			 print_to_arg, hpp, szp);
+    sbcs = info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p,
+			 print_to_arg, hpp, szp);
     calls = info_calls(allctr, print_to_p, print_to_arg, hpp, szp);
 
     if (hpp || szp) {
@@ -2455,6 +3303,9 @@ erts_alcu_info(Allctr_t *allctr,
 	add_2tup(hpp, szp, &res, am.calls, calls);
 	add_2tup(hpp, szp, &res, am.sbcs, sbcs);
 	add_2tup(hpp, szp, &res, am.mbcs, mbcs);
+	add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs);
+	if (allctr->fix)
+	    add_2tup(hpp, szp, &res, am.fix_types, fix);
 	add_2tup(hpp, szp, &res, am.options, sett);
 	add_3tup(hpp, szp, &res,
 		 am.versions,
@@ -2463,6 +3314,7 @@ erts_alcu_info(Allctr_t *allctr,
     }
 
     if (begin_max_period) {
+	reset_max_values(&allctr->sbmbcs);
 	reset_max_values(&allctr->mbcs);
 	reset_max_values(&allctr->sbcs);
     }
@@ -2478,7 +3330,7 @@ erts_alcu_info(Allctr_t *allctr,
 
 
 void
-erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size)
+erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t *fi, int fisz)
 {
 
 #ifdef USE_THREADS
@@ -2486,14 +3338,28 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size)
 	erts_mtx_lock(&allctr->mutex);
 #endif
 
-    size->carriers = allctr->mbcs.curr_mseg.size;
-    size->carriers += allctr->mbcs.curr_sys_alloc.size;
-    size->carriers += allctr->sbcs.curr_mseg.size;
-    size->carriers += allctr->sbcs.curr_sys_alloc.size;
+    size->carriers = allctr->mbcs.curr.norm.mseg.size;
+    size->carriers += allctr->mbcs.curr.norm.sys_alloc.size;
+    size->carriers += allctr->sbmbcs.curr.small_block.size;
+    size->carriers += allctr->sbcs.curr.norm.mseg.size;
+    size->carriers += allctr->sbcs.curr.norm.sys_alloc.size;
 
     size->blocks = allctr->mbcs.blocks.curr.size;
+    size->blocks += allctr->sbmbcs.blocks.curr.size;
     size->blocks += allctr->sbcs.blocks.curr.size;
 
+    if (fi) {
+	int ix;
+	for (ix = 0; ix < fisz; ix++) {
+	    if (allctr->fix) {
+		fi[ix].allocated += (allctr->fix[ix].type_size
+				     * allctr->fix[ix].allocated);
+		fi[ix].used += (allctr->fix[ix].type_size
+				* allctr->fix[ix].used);
+	    }
+	}
+    }
+
 #ifdef USE_THREADS
     if (allctr->thread_safe)
 	erts_mtx_unlock(&allctr->mutex);
@@ -2507,12 +3373,16 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
 {
     Allctr_t *allctr = (Allctr_t *) extra; 
     void *res;
+    ErtsAlcFixList_t *fix;
 
     ASSERT(initialized);
 
     ASSERT(allctr);
 
-    ERTS_ALCU_DBG_CHK_THR_SPEC(allctr);
+    ERTS_SMP_LC_ASSERT(!allctr->thread_safe
+		       || erts_lc_mtx_is_locked(&allctr->mutex));
+
+    ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr);
 
 #if ALLOC_ZERO_EQ_NULL
     if (!size)
@@ -2521,13 +3391,61 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
 
     INC_CC(allctr->calls.this_alloc);
 
+    fix = allctr->fix;
+    if (fix) {
+	int ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE;
+	ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
+	fix[ix].used++;
+	res = fix[ix].list;
+	if (res) {
+	    fix[ix].list_size--;
+	    fix[ix].list = *((void **) res);
+	    if (fix[ix].list && fix[ix].allocated > fix[ix].limit) {
+		void *p = fix[ix].list;
+		Block_t *blk;
+		fix[ix].list = *((void **) p);
+		fix[ix].list_size--;
+		blk = UMEM2BLK(p);
+		if (IS_SBC_BLK(blk))
+		    destroy_carrier(allctr, blk);
+		else
+		    mbc_free(allctr, p);
+		fix[ix].allocated--;
+	    }
+	    ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
+	    return res;
+	}
+	if (size < 2*sizeof(UWord))
+	    size += sizeof(UWord);
+	if (fix[ix].limit < fix[ix].used)
+	    fix[ix].limit = fix[ix].used;
+	if (fix[ix].max_used < fix[ix].used)
+	    fix[ix].max_used = fix[ix].used;
+	fix[ix].allocated++;
+    }
+
     if (size >= allctr->sbc_threshold) {
-	Block_t *blk = create_carrier(allctr, size, CFLG_SBC);
+	Block_t *blk;
+#ifdef ERTS_SMP
+	if (allctr->dd.use)
+	    ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1);
+#endif
+#if HALFWORD_HEAP
+	blk = create_carrier(allctr, size,
+			     CFLG_SBC | CFLG_FORCE_MSEG);
+#else
+	blk = create_carrier(allctr, size, CFLG_SBC);
+#endif
 	res = blk ? BLK2UMEM(blk) : NULL;
     }
     else
 	res = mbc_alloc(allctr, size);
 
+    if (!res && fix) {
+	int ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE;
+	fix[ix].allocated--;
+	fix[ix].used--;
+    }
     return res;
 }
 
@@ -2556,29 +3474,28 @@ erts_alcu_alloc_ts(ErtsAlcType_t type, void *extra, Uint size)
     return res;
 }
 
+#ifdef ERTS_SMP
+
 void *
 erts_alcu_alloc_thr_spec(ErtsAlcType_t type, void *extra, Uint size)
 {
     ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra;
-    int ix = erts_alc_get_thr_ix();
+    int ix;
     Allctr_t *allctr;
-    int unlock;
     void *res;
 
-    ASSERT(ix > 0);
-    if (ix < tspec->size) {
-	allctr = tspec->allctr[ix];
-	unlock = 0;
-    }
-    else {
-	allctr = tspec->allctr[0];
-	unlock = 1;
+    ix = ERTS_ALC_GET_THR_IX();
+
+    ASSERT(0 <= ix && ix < tspec->size);
+
+    allctr = tspec->allctr[ix];
+
+    if (allctr->thread_safe)
 	erts_mtx_lock(&allctr->mutex);
-    }
 
     res = do_erts_alcu_alloc(type, allctr, size);
 
-    if (unlock)
+    if (allctr->thread_safe)
 	erts_mtx_unlock(&allctr->mutex);
 
     DEBUG_CHECK_ALIGNMENT(res);
@@ -2589,51 +3506,96 @@ erts_alcu_alloc_thr_spec(ErtsAlcType_t type, void *extra, Uint size)
 void *
 erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size)
 {
-    ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra;
-    int ix = erts_alc_get_thr_ix();
-    Allctr_t *allctr;
+    int pref_ix;
+    Allctr_t *pref_allctr;
     void *res;
 
-    ASSERT(sizeof(Uint) == sizeof(Allctr_t *));
-    ASSERT(ix > 0);
-    if (ix >= tspec->size)
-	ix = (ix % (tspec->size - 1)) + 1;
-    allctr = tspec->allctr[ix];
-    erts_mtx_lock(&allctr->mutex);
-    res = do_erts_alcu_alloc(type, allctr, size + sizeof(Uint));
-    if (res) {
-	*((Allctr_t **) res) = allctr;
-	res = (void *) (((char *) res) + sizeof(Uint));
-    }
-    erts_mtx_unlock(&allctr->mutex);
+    pref_ix = get_pref_allctr(extra, &pref_allctr);
+
+    if (pref_allctr->thread_safe)
+	erts_mtx_lock(&pref_allctr->mutex);
+
+    ERTS_ALCU_DBG_CHK_THR_ACCESS(pref_allctr);
+
+    res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(UWord));
+    if (pref_allctr->thread_safe)
+	erts_mtx_unlock(&pref_allctr->mutex);
+
+    if (res)
+	res = put_used_allctr(res, pref_ix, size);
+
     DEBUG_CHECK_ALIGNMENT(res);
+
+
     return res;
 }
 
 #endif
 
+#endif
+
 /* ------------------------------------------------------------------------- */
 
 static ERTS_INLINE void
 do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p)
 {
+    int ix;
     Allctr_t *allctr = (Allctr_t *) extra; 
     ASSERT(initialized);
 
     ASSERT(allctr);
 
-    ERTS_ALCU_DBG_CHK_THR_SPEC(allctr);
+    ERTS_SMP_LC_ASSERT(!allctr->thread_safe
+		       || erts_lc_mtx_is_locked(&allctr->mutex));
+
+    ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr);
 
     if (p) {
+	ErtsAlcFixList_t *fix = allctr->fix;
 	Block_t *blk;
 
 	INC_CC(allctr->calls.this_free);
 
+	if (fix) {
+	    ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE;
+	    ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1);
+	    fix[ix].used--;
+	    if (fix[ix].allocated < fix[ix].limit
+		&& fix[ix].list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) {
+		*((void **) p) = fix[ix].list;
+		fix[ix].list = p;
+		fix[ix].list_size++;
+		if (!allctr->fix_shrink_scheduled) {
+		    allctr->fix_shrink_scheduled = 1;
+		    erts_set_aux_work_timeout(
+			allctr->ix,
+			(ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
+			 | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
+			1);
+		}
+		ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
+		return;
+	    }
+	    fix[ix].allocated--;
+	    if (fix[ix].list && fix[ix].allocated > fix[ix].limit) {
+		blk = UMEM2BLK(p);
+		if (IS_SBC_BLK(blk))
+		    destroy_carrier(allctr, blk);
+		else
+		    mbc_free(allctr, p);
+		p = fix[ix].list;
+		fix[ix].list = *((void **) p);
+		fix[ix].list_size--;
+		fix[ix].allocated--;
+	    }
+	}
+
 	blk = UMEM2BLK(p);
 	if (IS_SBC_BLK(blk))
 	    destroy_carrier(allctr, blk);
 	else
 	    mbc_free(allctr, p);
+	ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0);
     }
 }
 
@@ -2653,44 +3615,60 @@ erts_alcu_free_ts(ErtsAlcType_t type, void *extra, void *p)
     erts_mtx_unlock(&allctr->mutex);
 }
 
+#ifdef ERTS_SMP
+
 void
 erts_alcu_free_thr_spec(ErtsAlcType_t type, void *extra, void *p)
 {
     ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra;
-    int ix = erts_alc_get_thr_ix();
-    int unlock;
+    int ix;
     Allctr_t *allctr;
 
-    ASSERT(ix > 0);
-    if (ix < tspec->size) {
-	allctr = tspec->allctr[ix];
-	unlock = 0;
-    }
-    else {
-	allctr = tspec->allctr[0];
-	unlock = 1;
+    ix = ERTS_ALC_GET_THR_IX();
+
+    ASSERT(0 <= ix && ix < tspec->size);
+
+    allctr = tspec->allctr[ix];
+
+    if (allctr->thread_safe)
 	erts_mtx_lock(&allctr->mutex);
-    }
 
     do_erts_alcu_free(type, allctr, p);
-    if (unlock)
+
+    if (allctr->thread_safe)
 	erts_mtx_unlock(&allctr->mutex);
 }
 
 void
-erts_alcu_free_thr_pref(ErtsAlcType_t type, void *unused, void *p)
+erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p)
 {
     if (p) {
-	void *ptr = (void *) (((char *) p) - sizeof(Uint));
-	Allctr_t *allctr = *((Allctr_t **) ptr);
-	erts_mtx_lock(&allctr->mutex);
-	do_erts_alcu_free(type, allctr, ptr);
-	erts_mtx_unlock(&allctr->mutex);
+	Allctr_t *pref_allctr, *used_allctr;
+	void *ptr;
+
+	get_pref_allctr(extra, &pref_allctr);
+	ptr = get_used_allctr(extra, p, &used_allctr, NULL);
+	if (pref_allctr != used_allctr)
+	    enqueue_dealloc_other_instance(type,
+					   used_allctr,
+					   ptr,
+					   (used_allctr->dd.ix
+					    - pref_allctr->dd.ix));
+	else {
+	    if (used_allctr->thread_safe)
+		erts_mtx_lock(&used_allctr->mutex);
+	    ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr);
+	    do_erts_alcu_free(type, used_allctr, ptr);
+	    if (used_allctr->thread_safe)
+		erts_mtx_unlock(&used_allctr->mutex);
+	}
     }
 }
 
 #endif
 
+#endif
+
 /* ------------------------------------------------------------------------- */
 
 static ERTS_INLINE void *
@@ -2698,7 +3676,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
 		     void *extra,
 		     void *p,
 		     Uint size,
-		     Uint flgs)
+		     Uint32 alcu_flgs)
 {
     Allctr_t *allctr = (Allctr_t *) extra; 
     Block_t *blk;
@@ -2708,7 +3686,10 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
 
     ASSERT(allctr);
 
-    ERTS_ALCU_DBG_CHK_THR_SPEC(allctr);
+    ERTS_SMP_LC_ASSERT(!allctr->thread_safe
+		       || erts_lc_mtx_is_locked(&allctr->mutex));
+
+    ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr);
 
     if (!p) {
 	res = do_erts_alcu_alloc(type, extra, size);
@@ -2731,9 +3712,32 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
     
     blk = UMEM2BLK(p);
 
+    if (allctr->sbmbc_threshold > 0) {
+	Uint old_sz, new_sz, lim;
+	lim = allctr->sbmbc_threshold;
+	old_sz = BLK_SZ(blk);
+	new_sz = UMEMSZ2BLKSZ(allctr, size);
+	if ((old_sz < lim && lim <= new_sz)
+	    || (new_sz < lim && lim <= old_sz)) {
+	    /* *Need* to move it... */
+
+	    INC_CC(allctr->calls.this_realloc);
+	    res = do_erts_alcu_alloc(type, extra, size);
+	    DEC_CC(allctr->calls.this_alloc);
+
+	    sys_memcpy(res, p, MIN(size, old_sz - ABLK_HDR_SZ));
+
+	    do_erts_alcu_free(type, extra, p);
+	    DEC_CC(allctr->calls.this_free);
+	    return res;
+	}
+	if (old_sz < lim)
+	    alcu_flgs |= ERTS_ALCU_FLG_SBMBC;
+    }
+
     if (size < allctr->sbc_threshold) {
 	if (IS_MBC_BLK(blk))
-	    res = mbc_realloc(allctr, p, size, flgs);
+	    res = mbc_realloc(allctr, p, size, alcu_flgs);
 	else {
 	    Uint used_sz = allctr->sbc_header_size + ABLK_HDR_SZ + size;
 	    Uint crr_sz;
@@ -2764,7 +3768,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
 	    if (100*diff_sz_val < allctr->sbc_move_threshold*crr_sz_val)
 		/* Data won't be copied into a new carrier... */
 		goto do_carrier_resize;
-	    else if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
+	    else if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
 		return NULL;
 
 	    res = mbc_alloc(allctr, size);
@@ -2778,15 +3782,27 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
     }
     else {
 	Block_t *new_blk;
+#ifdef ERTS_SMP
+	if (allctr->dd.use)
+	    ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1);
+#endif
 	if(IS_SBC_BLK(blk)) {
 	do_carrier_resize:
+#if HALFWORD_HEAP
+	    new_blk = resize_carrier(allctr, blk, size, CFLG_SBC | CFLG_FORCE_MSEG);
+#else
 	    new_blk = resize_carrier(allctr, blk, size, CFLG_SBC);
+#endif
 	    res = new_blk ? BLK2UMEM(new_blk) : NULL;
 	}
-	else if (flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
+	else if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE)
 	    return NULL;
 	else {
+#if HALFWORD_HEAP
+	    new_blk = create_carrier(allctr, size, CFLG_SBC | CFLG_FORCE_MSEG);
+#else
 	    new_blk = create_carrier(allctr, size, CFLG_SBC);
+#endif
 	    if (new_blk) {
 		res = BLK2UMEM(new_blk);
 		sys_memcpy((void *) res,
@@ -2873,30 +3889,29 @@ erts_alcu_realloc_mv_ts(ErtsAlcType_t type, void *extra, void *p, Uint size)
     return res;
 }
 
+#ifdef ERTS_SMP
+
 void *
 erts_alcu_realloc_thr_spec(ErtsAlcType_t type, void *extra,
 			   void *ptr, Uint size)
 {
     ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra;
-    int ix = erts_alc_get_thr_ix();
+    int ix;
     Allctr_t *allctr;
-    int unlock;
     void *res;
 
-    ASSERT(ix > 0);
-    if (ix < tspec->size) {
-	allctr = tspec->allctr[ix];
-	unlock = 0;
-    }
-    else {
-	allctr = tspec->allctr[0];
-	unlock = 1;
+    ix = ERTS_ALC_GET_THR_IX();
+
+    ASSERT(0 <= ix && ix < tspec->size);
+
+    allctr = tspec->allctr[ix];
+
+    if (allctr->thread_safe)
 	erts_mtx_lock(&allctr->mutex);
-    }
 
     res = do_erts_alcu_realloc(type, allctr, ptr, size, 0);
 
-    if (unlock)
+    if (allctr->thread_safe)
 	erts_mtx_unlock(&allctr->mutex);
 
     DEBUG_CHECK_ALIGNMENT(res);
@@ -2909,26 +3924,22 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra,
 			      void *ptr, Uint size)
 {
     ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra;
-    int ix = erts_alc_get_thr_ix();
+    int ix;
     Allctr_t *allctr;
-    int unlock;
     void *res;
 
-    ASSERT(ix > 0);
-    if (ix < tspec->size) {
-	allctr = tspec->allctr[ix];
-	unlock = 0;
-    }
-    else {
-	allctr = tspec->allctr[0];
-	unlock = 1;
-	erts_mtx_lock(&allctr->mutex);
-    }
+    ix = ERTS_ALC_GET_THR_IX();
 
+    ASSERT(0 <= ix && ix < tspec->size);
+
+    allctr = tspec->allctr[ix];
+
+    if (allctr->thread_safe)
+	erts_mtx_lock(&allctr->mutex);
 
     res = do_erts_alcu_alloc(type, allctr, size);
     if (!res) {
-	if (unlock)
+	if (allctr->thread_safe)
 	    erts_mtx_unlock(&allctr->mutex);
 	res = erts_alcu_realloc_thr_spec(type, allctr, ptr, size);
     }
@@ -2942,7 +3953,7 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra,
 	    cpy_size = size;
 	sys_memcpy(res, ptr, cpy_size);
 	do_erts_alcu_free(type, allctr, ptr);
-	if (unlock)
+	if (allctr->thread_safe)
 	    erts_mtx_unlock(&allctr->mutex);
     }
 
@@ -2951,129 +3962,106 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra,
     return res;
 }
 
-void *
-erts_alcu_realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size)
+static ERTS_INLINE void *
+realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size,
+		 int force_move)
 {
-    ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra;
-    int ix;
+    int pref_ix;
     void *ptr, *res;
     Allctr_t *pref_allctr, *used_allctr;
+    UWord old_user_size;
 
     if (!p)
 	return erts_alcu_alloc_thr_pref(type, extra, size);
 
-    ptr = (void *) (((char *) p) - sizeof(Uint));
-    used_allctr = *((Allctr_t **) ptr);
+    pref_ix = get_pref_allctr(extra, &pref_allctr);
+    ptr = get_used_allctr(extra, p, &used_allctr, &old_user_size);
 
-    ix = erts_alc_get_thr_ix();
-    ASSERT(ix > 0);
-    if (ix >= tspec->size)
-	ix = (ix % (tspec->size - 1)) + 1;
-    pref_allctr = tspec->allctr[ix];
     ASSERT(used_allctr && pref_allctr);
 
-    erts_mtx_lock(&used_allctr->mutex);
-    res = do_erts_alcu_realloc(type,
-			       used_allctr,
-			       ptr,
-			       size + sizeof(Uint),
-			       (pref_allctr != used_allctr
-				? ERTS_ALCU_FLG_FAIL_REALLOC_MOVE
-				: 0));
-    erts_mtx_unlock(&used_allctr->mutex);
-    if (res) {
-	ASSERT(used_allctr == *((Allctr_t **) res));
-	res = (void *) (((char *) res) + sizeof(Uint));
-	DEBUG_CHECK_ALIGNMENT(res);
+    if (!force_move && used_allctr == pref_allctr) {
+	if (used_allctr->thread_safe)
+	    erts_mtx_lock(&used_allctr->mutex);
+	ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr);
+	res = do_erts_alcu_realloc(type,
+				   used_allctr,
+				   ptr,
+				   size + sizeof(UWord),
+				   0);
+	if (used_allctr->thread_safe)
+	    erts_mtx_unlock(&used_allctr->mutex);
+	if (res)
+	    res = put_used_allctr(res, pref_ix, size);
     }
     else {
-	erts_mtx_lock(&pref_allctr->mutex);
-	res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(Uint));
-	erts_mtx_unlock(&pref_allctr->mutex);
+	if (pref_allctr->thread_safe)
+	    erts_mtx_lock(&pref_allctr->mutex);
+	res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(UWord));
+	if (pref_allctr->thread_safe && (!force_move
+					 || used_allctr != pref_allctr))
+	    erts_mtx_unlock(&pref_allctr->mutex);
 	if (res) {
 	    Block_t *blk;
 	    size_t cpy_size;
 
-	    *((Allctr_t **) res) = pref_allctr;
-	    res = (void *) (((char *) res) + sizeof(Uint));
+	    res = put_used_allctr(res, pref_ix, size);
 
 	    DEBUG_CHECK_ALIGNMENT(res);
 
-	    erts_mtx_lock(&used_allctr->mutex);
 	    blk = UMEM2BLK(ptr);
-	    cpy_size = BLK_SZ(blk) - ABLK_HDR_SZ - sizeof(Uint);
+	    if (old_user_size != ERTS_AU_PREF_ALLOC_SIZE_MASK)
+		cpy_size = old_user_size;
+	    else {
+		if (used_allctr->thread_safe && (!force_move
+						 || used_allctr != pref_allctr))
+		    erts_mtx_lock(&used_allctr->mutex);
+		ERTS_SMP_LC_ASSERT(!used_allctr->thread_safe ||
+				   erts_lc_mtx_is_locked(&used_allctr->mutex));
+		cpy_size = BLK_SZ(blk);
+		if (used_allctr->thread_safe && (!force_move
+						 || used_allctr != pref_allctr))
+		    erts_mtx_unlock(&used_allctr->mutex);
+		cpy_size -= ABLK_HDR_SZ + sizeof(UWord);
+	    }
 	    if (cpy_size > size)
 		cpy_size = size;
 	    sys_memcpy(res, p, cpy_size);
-	    do_erts_alcu_free(type, used_allctr, ptr);
-	    erts_mtx_unlock(&used_allctr->mutex);
+
+	    if (!force_move || used_allctr != pref_allctr)
+		enqueue_dealloc_other_instance(type,
+					       used_allctr,
+					       ptr,
+					       (used_allctr->dd.ix
+						- pref_allctr->dd.ix));
+	    else {
+		do_erts_alcu_free(type, used_allctr, ptr);
+		ASSERT(pref_allctr == used_allctr);
+		if (pref_allctr->thread_safe)
+		    erts_mtx_unlock(&pref_allctr->mutex);
+	    }
 	}
     }
 
     return res;
 }
 
+void *
+erts_alcu_realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size)
+{
+    return realloc_thr_pref(type, extra, p, size, 0);
+}
 
 void *
 erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra,
 			      void *p, Uint size)
 {
-    ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra;
-    int ix;
-    void *ptr, *res;
-    Allctr_t *pref_allctr, *used_allctr;
-
-    if (!p)
-	return erts_alcu_alloc_thr_pref(type, extra, size);
-
-    ptr = (void *) (((char *) p) - sizeof(Uint));
-    used_allctr = *((Allctr_t **) ptr);
-
-    ix = erts_alc_get_thr_ix();
-    ASSERT(ix > 0);
-    if (ix >= tspec->size)
-	ix = (ix % (tspec->size - 1)) + 1;
-    pref_allctr = tspec->allctr[ix];
-    ASSERT(used_allctr && pref_allctr);
-
-    erts_mtx_lock(&pref_allctr->mutex);
-    res = do_erts_alcu_alloc(type, pref_allctr, size + sizeof(Uint));
-    if (!res) {
-	erts_mtx_unlock(&pref_allctr->mutex);
-	res = erts_alcu_realloc_thr_pref(type, extra, p, size);
-    }
-    else {
-	Block_t *blk;
-	size_t cpy_size;
-	Allctr_t *allctr;
-
-	*((Allctr_t **) res) = pref_allctr;
-	res = (void *) (((char *) res) + sizeof(Uint));
-
-	DEBUG_CHECK_ALIGNMENT(res);
-
-	if (used_allctr == pref_allctr)
-	    allctr = pref_allctr;
-	else {
-	    erts_mtx_unlock(&pref_allctr->mutex);
-	    allctr = used_allctr;
-	    erts_mtx_lock(&allctr->mutex);
-	}
-
-	blk = UMEM2BLK(ptr);
-	cpy_size = BLK_SZ(blk) - ABLK_HDR_SZ - sizeof(Uint);
-	if (cpy_size > size)
-	    cpy_size = size;
-	sys_memcpy(res, p, cpy_size);
-	do_erts_alcu_free(type, allctr, ptr);
-	erts_mtx_unlock(&allctr->mutex);
-    }
-
-    return res;
+    return realloc_thr_pref(type, extra, p, size, 1);
 }
 
 #endif
 
+#endif
+
 /* ------------------------------------------------------------------------- */
 
 int
@@ -3085,19 +4073,23 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 	goto error;
 
 #if HAVE_ERTS_MSEG
-    {
-	ErtsMsegOpt_t mseg_opt = ERTS_MSEG_DEFAULT_OPT_INITIALIZER;
-    
-	sys_memcpy((void *) &allctr->mseg_opt,
-		   (void *) &mseg_opt,
-		   sizeof(ErtsMsegOpt_t));
-    }
+    sys_memcpy((void *) &allctr->mseg_opt,
+	       (void *) &erts_mseg_default_opt,
+	       sizeof(ErtsMsegOpt_t));
+#ifdef ERTS_SMP
+    if (init->tspec || init->tpref)
+	allctr->mseg_opt.sched_spec = 1;
+#endif
+# if HALFWORD_HEAP
+    allctr->mseg_opt.low_mem = init->low_mem;
+# endif
 #endif
 
     allctr->name_prefix			= init->name_prefix;
     if (!allctr->name_prefix)
 	goto error;
 
+    allctr->ix				= init->ix;
     allctr->alloc_no			= init->alloc_no;
     if (allctr->alloc_no < ERTS_ALC_A_MIN
 	|| ERTS_ALC_A_MAX < allctr->alloc_no)
@@ -3138,11 +4130,43 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
     if (allctr->min_block_size < ABLK_HDR_SZ)
 	goto error;
     allctr->min_block_size		= UNIT_CEILING(allctr->min_block_size
-						       + sizeof(Uint));
+						       + sizeof(UWord));
+#if ERTS_SMP
+    if (init->tpref) {
+	Uint sz = sizeof(Block_t);
+	sz += ERTS_ALCU_DD_FIX_TYPE_OFFS*sizeof(UWord);
+	if (init->fix)
+	    sz += sizeof(UWord);
+	sz = UNIT_CEILING(sz);
+	if (sz > allctr->min_block_size)
+	    allctr->min_block_size = sz;
+    }
+#endif
+
+
+
+    allctr->sbmbc_threshold		= init->sbmbct;
+
+    if (!erts_have_sbmbc_alloc
+	|| ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no))
+	allctr->sbmbc_threshold = 0;
+
+    if (!allctr->sbmbc_threshold)
+	allctr->sbmbc_size = 0;
+    else {
+	Uint min_size;
+	allctr->sbmbc_size = init->sbmbcs;
+	min_size = allctr->sbmbc_threshold;
+	min_size += allctr->min_block_size;
+	min_size += allctr->mbc_header_size;
+	if (allctr->sbmbc_size < min_size)
+	    allctr->sbmbc_size = min_size;
+    }
+
 
 #if HAVE_ERTS_MSEG
-    if (allctr->mseg_opt.abs_shrink_th > ~((Uint) 0) / 100)
-	allctr->mseg_opt.abs_shrink_th = ~((Uint) 0) / 100;
+    if (allctr->mseg_opt.abs_shrink_th > ~((UWord) 0) / 100)
+	allctr->mseg_opt.abs_shrink_th = ~((UWord) 0) / 100;
 #endif
 
 #ifdef USE_THREADS
@@ -3151,12 +4175,16 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 	
 #ifdef ERTS_ENABLE_LOCK_COUNT
 	erts_mtx_init_x_opt(&allctr->mutex,
-			"alcu_allocator",
-			make_small(allctr->alloc_no),
-			ERTS_LCNT_LT_ALLOC);
+			    ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no)
+			    ? "sbmbc_alloc"
+			    : "alcu_allocator",
+			    make_small(allctr->alloc_no),
+			    ERTS_LCNT_LT_ALLOC);
 #else
 	erts_mtx_init_x(&allctr->mutex,
-			"alcu_allocator",
+			ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no)
+			? "sbmbc_alloc"
+			: "alcu_allocator",
 			make_small(allctr->alloc_no));
 #endif /*ERTS_ENABLE_LOCK_COUNT*/
 	
@@ -3177,20 +4205,25 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 
     if (allctr->mbc_header_size < sizeof(Carrier_t))
 	goto error;
-#ifdef USE_THREADS
+#ifdef ERTS_SMP
+    allctr->dd.use = 0;
     if (init->tpref) {
 	allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size
 						+ FBLK_FTR_SZ
 						+ ABLK_HDR_SZ
-						+ sizeof(Uint))
+						+ sizeof(UWord))
 				   - ABLK_HDR_SZ
-				   - sizeof(Uint));
+				   - sizeof(UWord));
 	allctr->sbc_header_size = (UNIT_CEILING(sizeof(Carrier_t)
 						+ FBLK_FTR_SZ
 						+ ABLK_HDR_SZ
-						+ sizeof(Uint))
+						+ sizeof(UWord))
 				   - ABLK_HDR_SZ
-				   - sizeof(Uint));
+				   - sizeof(UWord));
+
+	allctr->dd.use = 1;
+	init_dd_queue(&allctr->dd.q);
+	allctr->dd.ix = init->ix;
     }
     else
 #endif
@@ -3208,21 +4241,45 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
     if (allctr->main_carrier_size) {
 	Block_t *blk;
 
+#if HALFWORD_HEAP
+	blk = create_carrier(allctr,
+			     allctr->main_carrier_size,
+			     CFLG_MBC
+			     | CFLG_FORCE_SIZE
+			     | CFLG_FORCE_MSEG
+			     | CFLG_MAIN_CARRIER);
+#else
 	blk = create_carrier(allctr,
 			     allctr->main_carrier_size,
 			     CFLG_MBC
 			     | CFLG_FORCE_SIZE
 			     | CFLG_FORCE_SYS_ALLOC
 			     | CFLG_MAIN_CARRIER);
+#endif
 	if (!blk)
 	    goto error;
 
-	(*allctr->link_free_block)(allctr, blk);
+	(*allctr->link_free_block)(allctr, blk, 0);
 
 	HARD_CHECK_BLK_CARRIER(allctr, blk);
 
     }
 
+    if (init->fix) {
+	int i;
+	allctr->fix = init->fix;
+	allctr->fix_shrink_scheduled = 0;
+	for (i = 0; i < ERTS_ALC_NO_FIXED_SIZES; i++) {
+	    allctr->fix[i].max_used = 0;
+	    allctr->fix[i].limit = 0;
+	    allctr->fix[i].type_size = init->fix_type_size[i];
+	    allctr->fix[i].list_size = 0;
+	    allctr->fix[i].list = NULL;
+	    allctr->fix[i].allocated = 0;
+	    allctr->fix[i].used = 0;
+	}
+    }
+
     return 1;
 
  error:
@@ -3247,6 +4304,8 @@ erts_alcu_stop(Allctr_t *allctr)
 	destroy_carrier(allctr, SBC2BLK(allctr, allctr->sbc_list.first));
     while (allctr->mbc_list.first)
 	destroy_carrier(allctr, MBC2FBLK(allctr, allctr->mbc_list.first));
+    while (allctr->sbmbc_list.first)
+	destroy_sbmbc(allctr, MBC2FBLK(allctr, allctr->sbmbc_list.first));
 
 #ifdef USE_THREADS
     if (allctr->thread_safe)
@@ -3339,6 +4398,40 @@ erts_alcu_test(unsigned long op, unsigned long a1, unsigned long a2)
  * Debug functions                                                           *
 \*                                                                           */
 
+void
+erts_alcu_verify_unused(Allctr_t *allctr)
+{
+    UWord no;
+
+    no = allctr->sbcs.curr.norm.mseg.no;
+    no += allctr->sbcs.curr.norm.sys_alloc.no;
+    no += allctr->mbcs.blocks.curr.no;
+    no += allctr->sbmbcs.blocks.curr.no;
+
+    if (no) {
+	UWord sz = allctr->sbcs.blocks.curr.size;
+	sz += allctr->mbcs.blocks.curr.size;
+	sz += allctr->sbmbcs.blocks.curr.size;
+	erl_exit(ERTS_ABORT_EXIT,
+		 "%salloc() used when expected to be unused!\n"
+		 "Total amount of blocks allocated: %bpu\n"
+		 "Total amount of bytes allocated: %bpu\n",
+		 allctr->name_prefix, no, sz);
+    }
+}
+
+void
+erts_alcu_verify_unused_ts(Allctr_t *allctr)
+{
+#ifdef USE_THREADS
+    erts_mtx_lock(&allctr->mutex);
+#endif
+    erts_alcu_verify_unused(allctr);
+#ifdef USE_THREADS
+    erts_mtx_unlock(&allctr->mutex);
+#endif
+}
+
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
 
 static void
@@ -3409,7 +4502,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
 	    is_free_blk = (int) IS_FREE_BLK(blk);
 	    if(is_free_blk) {
 		if (IS_NOT_LAST_BLK(blk))
-		    ASSERT(*((Uint *) (((char *) blk)+blk_sz-sizeof(Uint)))
+		    ASSERT(*((UWord *) (((char *) blk)+blk_sz-sizeof(UWord)))
 			   == blk_sz);
 	    }
 
@@ -3417,7 +4510,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
 		(*allctr->check_block)(allctr, blk, (int) is_free_blk);
 
 	    if (IS_LAST_BLK(blk)) {
-		carrier_end = ((char *) NXT_BLK(blk)) + sizeof(Uint);
+		carrier_end = ((char *) NXT_BLK(blk));
 		mbc = *((Carrier_t **) NXT_BLK(blk));
 		prev_blk = NULL;
 		blk = MBC2FBLK(allctr, mbc);
@@ -3432,9 +4525,9 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
 	ASSERT(IS_MB_CARRIER(mbc));
 	ASSERT((((char *) mbc)
 		+ allctr->mbc_header_size
-		+ tot_blk_sz
-		+ sizeof(Uint)) == carrier_end);
-	ASSERT(((char *) mbc) + CARRIER_SZ(mbc) == carrier_end);
+		+ tot_blk_sz) == carrier_end);
+	ASSERT(((char *) mbc) + CARRIER_SZ(mbc) - sizeof(Unit_t) <= carrier_end
+	       && carrier_end <= ((char *) mbc) + CARRIER_SZ(mbc));
 
 	if (allctr->check_mbc)
 	    (*allctr->check_mbc)(allctr, mbc);
@@ -3448,6 +4541,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
 	cl = &allctr->mbc_list;
     }
 
+#if 0 /* FIXIT sbmbc */
     if (cl->first == crr) {
 	ASSERT(!crr->prev);
     }
@@ -3462,6 +4556,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk)
 	ASSERT(crr->next);
 	ASSERT(crr->next->prev == crr);
     }
+#endif
 }
 
 #endif
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 10b11661e6..cedf4ccf85 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -1,61 +1,77 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2002-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2002-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
 #ifndef ERL_ALLOC_UTIL__
 #define ERL_ALLOC_UTIL__
 
-#define ERTS_ALCU_VSN_STR "2.2"
+#define ERTS_ALCU_VSN_STR "3.0"
 
 #include "erl_alloc_types.h"
 
+#define ERTS_AU_PREF_ALLOC_BITS 11
+#define ERTS_AU_MAX_PREF_ALLOC_INSTANCES (1 << ERTS_AU_PREF_ALLOC_BITS)
+
 typedef struct Allctr_t_ Allctr_t;
 
 typedef struct {
-    Uint ycs;
-    Uint mmc;
+    UWord ycs;
+    UWord mmc;
 } AlcUInit_t;
 
 typedef struct {
     char *name_prefix;
     ErtsAlcType_t alloc_no;
+    int force;
+    int ix;
     int ts;
     int tspec;
     int tpref;
     int ramv;
-    Uint sbct;
-    Uint asbcst;
-    Uint rsbcst;
-    Uint rsbcmt;
-    Uint rmbcmt;
-    Uint mmbcs;
-    Uint mmsbc;
-    Uint mmmbc;
-    Uint lmbcs;
-    Uint smbcs;
-    Uint mbcgs;
+    int low_mem;   /* HALFWORD only */
+    UWord sbct;
+    UWord asbcst;
+    UWord rsbcst;
+    UWord rsbcmt;
+    UWord rmbcmt;
+    UWord mmbcs;
+    UWord mmsbc;
+    UWord mmmbc;
+    UWord lmbcs;
+    UWord smbcs;
+    UWord mbcgs;
+    UWord sbmbct;
+    UWord sbmbcs;
+
+    void *fix;
+    size_t *fix_type_size;
 } AllctrInit_t;
 
 typedef struct {
-    Uint blocks;
-    Uint carriers;
+    UWord blocks;
+    UWord carriers;
 } AllctrSize_t;
 
+typedef struct {
+    UWord allocated;
+    UWord used;
+} ErtsAlcUFixInfo_t;
+
 #ifndef SMALL_MEMORY
 
 #define ERTS_DEFAULT_ALCU_INIT {                                           \
@@ -66,10 +82,13 @@ typedef struct {
 #define ERTS_DEFAULT_ALLCTR_INIT {                                         \
     NULL,                                                                  \
     ERTS_ALC_A_INVALID,	/* (number) alloc_no: allocator number           */\
+    0,			/* (bool)   force:  force enabled                */\
+    0,			/* (number) ix: instance index                   */\
     1,			/* (bool)   ts:     thread safe                  */\
     0,			/* (bool)   tspec:  thread specific              */\
     0,			/* (bool)   tpref:  thread preferred             */\
     0,			/* (bool)   ramv:   realloc always moves         */\
+    0,			/* (bool)   low_mem: HALFWORD only               */\
     512*1024,		/* (bytes)  sbct:   sbc threshold                */\
     2*1024*2024,	/* (amount) asbcst: abs sbc shrink threshold     */\
     20,			/* (%)      rsbcst: rel sbc shrink threshold     */\
@@ -80,7 +99,12 @@ typedef struct {
     10,			/* (amount) mmmbc:  max mseg mbcs                */\
     10*1024*1024,	/* (bytes)  lmbcs:  largest mbc size             */\
     1024*1024,		/* (bytes)  smbcs:  smallest mbc size            */\
-    10			/* (amount) mbcgs:  mbc growth stages            */\
+    10,			/* (amount) mbcgs:  mbc growth stages            */\
+    256,	       	/* (bytes)  sbmbct:  small block mbc threshold   */\
+    8*1024,		/* (bytes)  sbmbcs:  small block mbc size        */ \
+    /* --- Data not options -------------------------------------------- */\
+    NULL,		/* (ptr)    fix                                  */\
+    NULL		/* (ptr)    fix_type_size                        */\
 }
 
 #else /* if SMALL_MEMORY */
@@ -93,10 +117,13 @@ typedef struct {
 #define ERTS_DEFAULT_ALLCTR_INIT {                                         \
     NULL,                                                                  \
     ERTS_ALC_A_INVALID,	/* (number) alloc_no: allocator number           */\
+    0,			/* (bool)   force:  force enabled                */\
+    0,			/* (number) ix: instance index                   */\
     1,			/* (bool)   ts:     thread safe                  */\
     0,			/* (bool)   tspec:  thread specific              */\
     0,			/* (bool)   tpref:  thread preferred             */\
     0,			/* (bool)   ramv:   realloc always moves         */\
+    0,			/* (bool)   low_mem: HALFWORD only               */\
     64*1024,		/* (bytes)  sbct:   sbc threshold                */\
     2*1024*2024,	/* (amount) asbcst: abs sbc shrink threshold     */\
     20,			/* (%)      rsbcst: rel sbc shrink threshold     */\
@@ -106,7 +133,12 @@ typedef struct {
     10,			/* (amount) mmmbc:  max mseg mbcs                */\
     1024*1024,		/* (bytes)  lmbcs:  largest mbc size             */\
     128*1024,		/* (bytes)  smbcs:  smallest mbc size            */\
-    10			/* (amount) mbcgs:  mbc growth stages            */\
+    10,			/* (amount) mbcgs:  mbc growth stages            */\
+    256,	       	/* (bytes)  sbmbct:  small block mbc threshold   */\
+    8*1024,		/* (bytes)  sbmbcs:  small block mbc size        */ \
+    /* --- Data not options -------------------------------------------- */\
+    NULL,		/* (ptr)    fix                                  */\
+    NULL		/* (ptr)    fix_type_size                        */\
 }
 
 #endif
@@ -120,6 +152,7 @@ void *	erts_alcu_alloc_ts(ErtsAlcType_t, void *, Uint);
 void *	erts_alcu_realloc_ts(ErtsAlcType_t, void *, void *, Uint);
 void *	erts_alcu_realloc_mv_ts(ErtsAlcType_t, void *, void *, Uint);
 void	erts_alcu_free_ts(ErtsAlcType_t, void *, void *);
+#ifdef ERTS_SMP
 void *	erts_alcu_alloc_thr_spec(ErtsAlcType_t, void *, Uint);
 void *	erts_alcu_realloc_thr_spec(ErtsAlcType_t, void *, void *, Uint);
 void *	erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t, void *, void *, Uint);
@@ -129,18 +162,27 @@ void *	erts_alcu_realloc_thr_pref(ErtsAlcType_t, void *, void *, Uint);
 void *	erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t, void *, void *, Uint);
 void	erts_alcu_free_thr_pref(ErtsAlcType_t, void *, void *);
 #endif
+#endif
 Eterm	erts_alcu_au_info_options(int *, void *, Uint **, Uint *);
 Eterm	erts_alcu_info_options(Allctr_t *, int *, void *, Uint **, Uint *);
 Eterm	erts_alcu_sz_info(Allctr_t *, int, int *, void *, Uint **, Uint *);
 Eterm	erts_alcu_info(Allctr_t *, int, int *, void *, Uint **, Uint *);
 void	erts_alcu_init(AlcUInit_t *);
-void    erts_alcu_current_size(Allctr_t *, AllctrSize_t *);
+void    erts_alcu_current_size(Allctr_t *, AllctrSize_t *,
+			       ErtsAlcUFixInfo_t *, int);
+#ifdef ERTS_SMP
+void    erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, ErtsThrPrgrVal *, int *);
+#endif
+erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
 
 #endif
 
 #if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__)
 #define ERL_ALLOC_UTIL_IMPL__
 
+#define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE		(((Uint32) 1) << 0)
+#define ERTS_ALCU_FLG_SBMBC			(((Uint32) 1) << 1)
+
 #ifdef USE_THREADS
 #define ERL_THREADS_EMU_INTERNAL__
 #include "erl_threads.h"
@@ -163,19 +205,19 @@ void    erts_alcu_current_size(Allctr_t *, AllctrSize_t *);
 #define CEILING(X, I)  ((((X) - 1)/(I) + 1)*(I))
 
 #undef  WORD_MASK
-#define INV_WORD_MASK	((Uint) (sizeof(Uint) - 1))
+#define INV_WORD_MASK	((UWord) (sizeof(UWord) - 1))
 #define WORD_MASK	(~INV_WORD_MASK)
 #define WORD_FLOOR(X)	((X) & WORD_MASK)
 #define WORD_CEILING(X)	WORD_FLOOR((X) + INV_WORD_MASK)
 
 #undef  UNIT_MASK
-#define INV_UNIT_MASK	((Uint) (sizeof(Unit_t) - 1))
+#define INV_UNIT_MASK	((UWord) (sizeof(Unit_t) - 1))
 #define UNIT_MASK	(~INV_UNIT_MASK)
 #define UNIT_FLOOR(X)	((X) & UNIT_MASK)
 #define UNIT_CEILING(X)	UNIT_FLOOR((X) + INV_UNIT_MASK)
 
 
-#define SZ_MASK			(~((Uint) 0) << 3)
+#define SZ_MASK			(~((UWord) 0) << 3)
 #define FLG_MASK		(~(SZ_MASK))
 
 
@@ -185,11 +227,13 @@ void    erts_alcu_current_size(Allctr_t *, AllctrSize_t *);
 #define CARRIER_SZ(C) \
   ((C)->chdr & SZ_MASK)
 
-typedef union {char c[8]; long l; double d;} Unit_t;
+extern int erts_have_sbmbc_alloc;
+
+typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t;
 
 typedef struct Carrier_t_ Carrier_t;
 struct Carrier_t_ {
-    Uint chdr;
+    UWord chdr;
     Carrier_t *next;
     Carrier_t *prev;
 };
@@ -199,22 +243,27 @@ typedef struct {
     Carrier_t *last;
 } CarrierList_t;
 
-typedef Uint Block_t;
-typedef Uint FreeBlkFtr_t;
+typedef UWord Block_t;
+typedef UWord FreeBlkFtr_t;
 
 typedef struct {
-    Uint giga_no;
-    Uint no;
+    UWord giga_no;
+    UWord no;
 } CallCounter_t;
 
 typedef struct {
-    Uint		no;
-    Uint		size;
+    UWord		no;
+    UWord		size;
 } StatValues_t;
 
 typedef struct {
-    StatValues_t	curr_mseg;
-    StatValues_t	curr_sys_alloc;
+    union {
+	struct {
+	    StatValues_t	mseg;
+	    StatValues_t	sys_alloc;
+	} norm;
+	StatValues_t	small_block;
+    } curr;
     StatValues_t	max;
     StatValues_t	max_ever;
     struct {
@@ -224,7 +273,74 @@ typedef struct {
     } blocks;
 } CarriersStats_t;
 
+#ifdef ERTS_SMP
+
+typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t;
+
+union ErtsAllctrDDBlock_t_ {
+    erts_atomic_t atmc_next;
+    ErtsAllctrDDBlock_t *ptr_next;
+};
+
+typedef struct {
+    ErtsAllctrDDBlock_t marker;
+    erts_atomic_t last;
+    erts_atomic_t um_refc[2];
+    erts_atomic32_t um_refc_ix;
+} ErtsDDTail_t;
+
+typedef struct {
+    /*
+     * This structure needs to be cache line aligned for best
+     * performance.
+     */
+    union {
+	/* Modified by threads returning memory to this allocator */
+	ErtsDDTail_t data;
+	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsDDTail_t))];
+    } tail;
+    /*
+     * Everything below this point is *only* accessed by the
+     * thread owning the allocator.
+     */
+    struct {
+	ErtsAllctrDDBlock_t *first;
+	ErtsAllctrDDBlock_t *unref_end;
+	struct {
+	    ErtsThrPrgrVal thr_progress;
+	    int thr_progress_reached;
+	    int um_refc_ix;
+	    ErtsAllctrDDBlock_t *unref_end;
+	} next;
+	int used_marker;
+    } head;
+} ErtsAllctrDDQueue_t;
+
+#endif
+
+typedef struct {
+    size_t type_size;
+    SWord list_size;
+    void *list;
+    SWord max_used;
+    SWord limit;
+    SWord allocated;
+    SWord used;
+} ErtsAlcFixList_t;
+
 struct Allctr_t_ {
+#ifdef ERTS_SMP
+    struct {
+	/*
+	 * We want the queue at the beginning of
+	 * the Allctr_t struct, due to cache line
+	 * alignment reasons.
+	 */
+	ErtsAllctrDDQueue_t q;
+	int		use;
+	int		ix;
+    } dd;
+#endif
 
     /* Allocator name prefix */
     char *		name_prefix;
@@ -232,6 +348,9 @@ struct Allctr_t_ {
     /* Allocator number */
     ErtsAlcType_t	alloc_no;
 
+    /* Instance index */
+    int			ix;
+
     /* Alloc, realloc and free names as atoms */
     struct {
 	Eterm		alloc;
@@ -254,6 +373,9 @@ struct Allctr_t_ {
     Uint		largest_mbc_size;
     Uint		smallest_mbc_size;
     Uint		mbc_growth_stages;
+    Uint		sbmbc_threshold;
+    Uint		sbmbc_size;
+
 #if HAVE_ERTS_MSEG
     ErtsMsegOpt_t	mseg_opt;
 #endif
@@ -266,6 +388,7 @@ struct Allctr_t_ {
     Uint		min_block_size;
 
     /* Carriers */
+    CarrierList_t	sbmbc_list;
     CarrierList_t	mbc_list;
     CarrierList_t	sbc_list;
 
@@ -274,15 +397,15 @@ struct Allctr_t_ {
 
     /* Callback functions (first 4 are mandatory) */
     Block_t *		(*get_free_block)	(Allctr_t *, Uint,
-						 Block_t *, Uint);
-    void		(*link_free_block)	(Allctr_t *, Block_t *);
-    void		(*unlink_free_block)	(Allctr_t *, Block_t *);
+						 Block_t *, Uint, Uint32);
+    void		(*link_free_block)	(Allctr_t *, Block_t *, Uint32);
+    void		(*unlink_free_block)	(Allctr_t *, Block_t *, Uint32);
     Eterm		(*info_options)		(Allctr_t *, char *, int *,
 						 void *, Uint **, Uint *);
 
     Uint		(*get_next_mbc_size)	(Allctr_t *);
-    void		(*creating_mbc)		(Allctr_t *, Carrier_t *);
-    void		(*destroying_mbc)	(Allctr_t *, Carrier_t *);
+    void		(*creating_mbc)		(Allctr_t *, Carrier_t *, Uint32);
+    void		(*destroying_mbc)	(Allctr_t *, Carrier_t *, Uint32);
     void		(*init_atoms)		(void);
 
 #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
@@ -290,6 +413,10 @@ struct Allctr_t_ {
     void		(*check_mbc)		(Allctr_t *, Carrier_t *);
 #endif
 
+    int			fix_n_base;
+    int			fix_shrink_scheduled;
+    ErtsAlcFixList_t	*fix;
+
 #ifdef USE_THREADS
     /* Mutex for this allocator */
     erts_mtx_t		mutex;
@@ -298,6 +425,7 @@ struct Allctr_t_ {
 	Allctr_t	*prev;
 	Allctr_t	*next;
     } ts_list;
+
 #endif
 
     int			atoms_initialized;
@@ -309,6 +437,8 @@ struct Allctr_t_ {
 	CallCounter_t	this_alloc;
 	CallCounter_t	this_free;
 	CallCounter_t	this_realloc;
+	CallCounter_t	sbmbc_alloc;
+	CallCounter_t	sbmbc_free;
 	CallCounter_t	mseg_alloc;
 	CallCounter_t	mseg_dealloc;
 	CallCounter_t	mseg_realloc;
@@ -319,6 +449,7 @@ struct Allctr_t_ {
 
     CarriersStats_t	sbcs;
     CarriersStats_t	mbcs;
+    CarriersStats_t	sbmbcs;
 
 #ifdef DEBUG
 #ifdef USE_THREADS
@@ -333,6 +464,9 @@ struct Allctr_t_ {
 int	erts_alcu_start(Allctr_t *, AllctrInit_t *);
 void	erts_alcu_stop(Allctr_t *);
 
+void	erts_alcu_verify_unused(Allctr_t *);
+void	erts_alcu_verify_unused_ts(Allctr_t *allctr);
+
 unsigned long	erts_alcu_test(unsigned long, unsigned long, unsigned long);
 
 
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c
new file mode 100644
index 0000000000..5bdb752d3a
--- /dev/null
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c
@@ -0,0 +1,976 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ * 
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ * 
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+
+/*
+ * Description:	An "address order first fit" allocator
+ *              based on a Red-Black (binary search) Tree. The search,
+ *              insert, and delete operations are all O(log n) operations
+ *              on a Red-Black Tree.
+ *              Red-Black Trees are described in "Introduction to Algorithms",
+ *              by Thomas H. Cormen, Charles E. Leiserson, and Ronald L. Riverest.
+ *
+ *              This module is a callback-module for erl_alloc_util.c
+ *
+ * Algorithm:   The tree nodes are free-blocks ordered in address order.
+ *              Every node also keeps the size of the largest block in its
+ *              sub-tree ('max_size'). By that we can start from root and keep
+ *              left (for low addresses) while dismissing entire sub-trees with
+ *              too small blocks.
+ *
+ * Authors: 	Rickard Green/Sverker Eriksson
+ */
+
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+#include "global.h"
+#define GET_ERL_AOFF_ALLOC_IMPL
+#include "erl_ao_firstfit_alloc.h"
+
+#ifdef DEBUG
+#if 0
+#define HARD_DEBUG
+#endif
+#else
+#undef HARD_DEBUG
+#endif
+
+#define MIN_MBC_SZ		(16*1024)
+#define MIN_MBC_FIRST_FREE_SZ	(4*1024)
+
+#define TREE_NODE_FLG		(((Uint) 1) << 0)
+#define RED_FLG			(((Uint) 1) << 1)
+#ifdef HARD_DEBUG
+#  define LEFT_VISITED_FLG	(((Uint) 1) << 2)
+#  define RIGHT_VISITED_FLG	(((Uint) 1) << 3)
+#endif
+
+#define IS_RED(N)		(((AOFF_RBTree_t *) (N)) \
+				 && ((AOFF_RBTree_t *) (N))->flags & RED_FLG)
+#define IS_BLACK(N)		(!IS_RED(((AOFF_RBTree_t *) (N))))
+
+#define SET_RED(N)		(((AOFF_RBTree_t *) (N))->flags |= RED_FLG)
+#define SET_BLACK(N)		(((AOFF_RBTree_t *) (N))->flags &= ~RED_FLG)
+
+#undef ASSERT
+#define ASSERT ASSERT_EXPR
+
+#if 1
+#define RBT_ASSERT	ASSERT
+#else
+#define RBT_ASSERT(x)
+#endif
+
+
+/* Types... */
+typedef struct AOFF_RBTree_t_ AOFF_RBTree_t;
+
+struct AOFF_RBTree_t_ {
+    Block_t hdr;
+    Uint flags;
+    AOFF_RBTree_t *parent;
+    AOFF_RBTree_t *left;
+    AOFF_RBTree_t *right;
+    Uint max_sz;  /* of all blocks in this sub-tree */
+};
+
+#ifdef HARD_DEBUG
+static AOFF_RBTree_t * check_tree(AOFF_RBTree_t* root, Uint);
+#endif
+
+
+/* Calculate 'max_size' of tree node x by only looking at the direct children
+ * of x and x itself.
+ */
+static ERTS_INLINE Uint node_max_size(AOFF_RBTree_t *x)
+{
+    Uint sz = BLK_SZ(x);
+    if (x->left && x->left->max_sz > sz) {
+	sz = x->left->max_sz;
+    }
+    if (x->right && x->right->max_sz > sz) {
+	sz = x->right->max_sz;
+    }
+    return sz;
+}
+
+/* Set new possibly lower 'max_size' of node and propagate change toward root
+*/
+static ERTS_INLINE void lower_max_size(AOFF_RBTree_t *node,
+				       AOFF_RBTree_t* stop_at)
+{
+    AOFF_RBTree_t* x = node;    
+    Uint old_max = x->max_sz;
+    Uint new_max = node_max_size(x);
+
+    if (new_max < old_max) {
+	x->max_sz = new_max;
+	while ((x=x->parent) != stop_at && x->max_sz == old_max) {		
+	    x->max_sz = node_max_size(x);
+	}
+	ASSERT(x == stop_at || x->max_sz > old_max);
+    }
+    else ASSERT(new_max == old_max);
+}
+
+
+/* Prototypes of callback functions */
+static Block_t*	aoff_get_free_block(Allctr_t *, Uint, Block_t *, Uint, Uint32 flags);
+static void aoff_link_free_block(Allctr_t *, Block_t*, Uint32 flags);
+static void aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags);
+
+static Eterm info_options(Allctr_t *, char *, int *, void *, Uint **, Uint *);
+static void init_atoms(void);
+
+
+
+#ifdef DEBUG
+
+/* Destroy all tree fields */
+#define DESTROY_TREE_NODE(N)						\
+  sys_memset((void *) (((Block_t *) (N)) + 1),				\
+	     0xff,							\
+	     (sizeof(AOFF_RBTree_t) - sizeof(Block_t)))
+
+#else
+
+#define DESTROY_TREE_NODE(N)
+
+#endif
+
+
+static int atoms_initialized = 0;
+
+void
+erts_aoffalc_init(void)
+{
+    atoms_initialized = 0;
+}
+
+Allctr_t *
+erts_aoffalc_start(AOFFAllctr_t *alc,
+		   AOFFAllctrInit_t* aoffinit,
+		   AllctrInit_t *init)
+{
+    struct {
+	int dummy;
+	AOFFAllctr_t allctr;
+    } zero = {0};
+    /* The struct with a dummy element first is used in order to avoid (an
+       incorrect) gcc warning. gcc warns if {0} is used as initializer of
+       a struct when the first member is a struct (not if, for example,
+       the third member is a struct). */
+
+    Allctr_t *allctr = (Allctr_t *) alc;
+
+    sys_memcpy((void *) alc, (void *) &zero.allctr, sizeof(AOFFAllctr_t));
+
+    allctr->mbc_header_size		= sizeof(Carrier_t);
+    allctr->min_mbc_size		= MIN_MBC_SZ;
+    allctr->min_mbc_first_free_size	= MIN_MBC_FIRST_FREE_SZ;
+    allctr->min_block_size		= sizeof(AOFF_RBTree_t);
+
+    allctr->vsn_str			= ERTS_ALC_AOFF_ALLOC_VSN_STR;
+
+
+    /* Callback functions */
+
+    allctr->get_free_block		= aoff_get_free_block;
+    allctr->link_free_block		= aoff_link_free_block;
+    allctr->unlink_free_block	        = aoff_unlink_free_block;
+    allctr->info_options		= info_options;
+
+    allctr->get_next_mbc_size		= NULL;
+    allctr->creating_mbc		= NULL;
+    allctr->destroying_mbc		= NULL;
+    allctr->init_atoms			= init_atoms;
+
+#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
+    allctr->check_block			= NULL;
+    allctr->check_mbc			= NULL;
+#endif
+
+    allctr->atoms_initialized		= 0;
+
+    if (!erts_alcu_start(allctr, init))
+	return NULL;
+
+    return allctr;
+}
+
+/*
+ * Red-Black Tree operations needed
+ */
+
+static ERTS_INLINE void
+left_rotate(AOFF_RBTree_t **root, AOFF_RBTree_t *x)
+{
+    AOFF_RBTree_t *y = x->right;
+    x->right = y->left;
+    if (y->left)
+	y->left->parent = x;
+    y->parent = x->parent;
+    if (!y->parent) {
+	RBT_ASSERT(*root == x);
+	*root = y;
+    }
+    else if (x == x->parent->left)
+	x->parent->left = y;
+    else {
+	RBT_ASSERT(x == x->parent->right);
+	x->parent->right = y;
+    }
+    y->left = x;
+    x->parent = y;
+
+    y->max_sz = x->max_sz;
+    x->max_sz = node_max_size(x); 
+    ASSERT(y->max_sz >= x->max_sz);
+}
+
+static ERTS_INLINE void
+right_rotate(AOFF_RBTree_t **root, AOFF_RBTree_t *x)
+{
+    AOFF_RBTree_t *y = x->left;
+    x->left = y->right;
+    if (y->right)
+	y->right->parent = x;
+    y->parent = x->parent;
+    if (!y->parent) {
+	RBT_ASSERT(*root == x);
+	*root = y;
+    }
+    else if (x == x->parent->right)
+	x->parent->right = y;
+    else {
+	RBT_ASSERT(x == x->parent->left);
+	x->parent->left = y;
+    }
+    y->right = x;
+    x->parent = y;
+    y->max_sz = x->max_sz;
+    x->max_sz = node_max_size(x);    
+    ASSERT(y->max_sz >= x->max_sz);
+}
+
+
+/*
+ * Replace node x with node y
+ * NOTE: block header of y is not changed
+ */
+static ERTS_INLINE void
+replace(AOFF_RBTree_t **root, AOFF_RBTree_t *x, AOFF_RBTree_t *y)
+{
+
+    if (!x->parent) {
+	RBT_ASSERT(*root == x);
+	*root = y;
+    }
+    else if (x == x->parent->left)
+	x->parent->left = y;
+    else {
+	RBT_ASSERT(x == x->parent->right);
+	x->parent->right = y;
+    }
+    if (x->left) {
+	RBT_ASSERT(x->left->parent == x);
+	x->left->parent = y;
+    }
+    if (x->right) {
+	RBT_ASSERT(x->right->parent == x);
+	x->right->parent = y;
+    }
+
+    y->flags	= x->flags;
+    y->parent	= x->parent;
+    y->right	= x->right;
+    y->left	= x->left;
+
+    y->max_sz = x->max_sz;
+    lower_max_size(y, NULL);
+    DESTROY_TREE_NODE(x);
+}
+
+static void
+tree_insert_fixup(AOFF_RBTree_t** root, AOFF_RBTree_t *blk)
+{
+    AOFF_RBTree_t *x = blk, *y;
+
+    /*
+     * Rearrange the tree so that it satisfies the Red-Black Tree properties
+     */
+
+    RBT_ASSERT(x != *root && IS_RED(x->parent));
+    do {
+
+	/*
+	 * x and its parent are both red. Move the red pair up the tree
+	 * until we get to the root or until we can separate them.
+	 */
+
+	RBT_ASSERT(IS_RED(x));
+	RBT_ASSERT(IS_BLACK(x->parent->parent));
+	RBT_ASSERT(x->parent->parent);
+
+	if (x->parent == x->parent->parent->left) {
+	    y = x->parent->parent->right;
+	    if (IS_RED(y)) {
+		SET_BLACK(y);
+		x = x->parent;
+		SET_BLACK(x);
+		x = x->parent;
+		SET_RED(x);
+	    }
+	    else {
+
+		if (x == x->parent->right) {
+		    x = x->parent;
+		    left_rotate(root, x);
+		}
+
+		RBT_ASSERT(x == x->parent->parent->left->left);
+		RBT_ASSERT(IS_RED(x));
+		RBT_ASSERT(IS_RED(x->parent));
+		RBT_ASSERT(IS_BLACK(x->parent->parent));
+		RBT_ASSERT(IS_BLACK(y));
+
+		SET_BLACK(x->parent);
+		SET_RED(x->parent->parent);
+		right_rotate(root, x->parent->parent);
+
+		RBT_ASSERT(x == x->parent->left);
+		RBT_ASSERT(IS_RED(x));
+		RBT_ASSERT(IS_RED(x->parent->right));
+		RBT_ASSERT(IS_BLACK(x->parent));
+		break;
+	    }
+	}
+	else {
+	    RBT_ASSERT(x->parent == x->parent->parent->right);
+	    y = x->parent->parent->left;
+	    if (IS_RED(y)) {
+		SET_BLACK(y);
+		x = x->parent;
+		SET_BLACK(x);
+		x = x->parent;
+		SET_RED(x);
+	    }
+	    else {
+
+		if (x == x->parent->left) {
+		    x = x->parent;
+		    right_rotate(root, x);
+		}
+
+		RBT_ASSERT(x == x->parent->parent->right->right);
+		RBT_ASSERT(IS_RED(x));
+		RBT_ASSERT(IS_RED(x->parent));
+		RBT_ASSERT(IS_BLACK(x->parent->parent));
+		RBT_ASSERT(IS_BLACK(y));
+
+		SET_BLACK(x->parent);
+		SET_RED(x->parent->parent);
+		left_rotate(root, x->parent->parent);
+
+		RBT_ASSERT(x == x->parent->right);
+		RBT_ASSERT(IS_RED(x));
+		RBT_ASSERT(IS_RED(x->parent->left));
+		RBT_ASSERT(IS_BLACK(x->parent));
+		break;
+	    }
+	}
+    } while (x != *root && IS_RED(x->parent));
+
+    SET_BLACK(*root);
+}
+
+static void
+aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags)
+{
+    AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr;
+    AOFF_RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC)
+			    ? &alc->sbmbc_root : &alc->mbc_root);
+    Uint spliced_is_black;
+    AOFF_RBTree_t *x, *y, *z = (AOFF_RBTree_t *) del;
+    AOFF_RBTree_t null_x; /* null_x is used to get the fixup started when we
+			splice out a node without children. */
+
+    null_x.parent = NULL;
+
+#ifdef HARD_DEBUG
+    check_tree(*root, 0);
+#endif
+
+    /* Remove node from tree... */
+
+    /* Find node to splice out */
+    if (!z->left || !z->right)
+	y = z;
+    else
+	/* Set y to z:s successor */
+	for(y = z->right; y->left; y = y->left);
+    /* splice out y */
+    x = y->left ? y->left : y->right;
+    spliced_is_black = IS_BLACK(y);
+    if (x) {
+	x->parent = y->parent;
+    }
+    else if (spliced_is_black) {
+	x = &null_x;
+	x->flags = 0;
+	SET_BLACK(x);
+	x->right = x->left = NULL;
+	x->max_sz = 0;
+	x->parent = y->parent;
+	y->left = x;
+    }
+
+    if (!y->parent) {
+	RBT_ASSERT(*root == y);
+	*root = x;
+    }
+    else {
+	if (y == y->parent->left) {
+	    y->parent->left = x;
+	}
+	else {
+	    RBT_ASSERT(y == y->parent->right);
+	    y->parent->right = x;
+	}
+	if (y->parent != z) {
+	    lower_max_size(y->parent, (y==z ? NULL : z));
+	}
+    }
+    if (y != z) {
+	/* We spliced out the successor of z; replace z by the successor */
+	replace(root, z, y);
+    }
+
+    if (spliced_is_black) {
+	/* We removed a black node which makes the resulting tree
+	   violate the Red-Black Tree properties. Fixup tree... */
+
+	while (IS_BLACK(x) && x->parent) {
+
+	    /*
+	     * x has an "extra black" which we move up the tree
+	     * until we reach the root or until we can get rid of it.
+	     *
+	     * y is the sibbling of x
+	     */
+
+	    if (x == x->parent->left) {
+		y = x->parent->right;
+		RBT_ASSERT(y);
+		if (IS_RED(y)) {
+		    RBT_ASSERT(y->right);
+		    RBT_ASSERT(y->left);
+		    SET_BLACK(y);
+		    RBT_ASSERT(IS_BLACK(x->parent));
+		    SET_RED(x->parent);
+		    left_rotate(root, x->parent);
+		    y = x->parent->right;
+		}
+		RBT_ASSERT(y);
+		RBT_ASSERT(IS_BLACK(y));
+		if (IS_BLACK(y->left) && IS_BLACK(y->right)) {
+		    SET_RED(y);
+		    x = x->parent;
+		}
+		else {
+		    if (IS_BLACK(y->right)) {
+			SET_BLACK(y->left);
+			SET_RED(y);
+			right_rotate(root, y);
+			y = x->parent->right;
+		    }
+		    RBT_ASSERT(y);
+		    if (IS_RED(x->parent)) {
+
+			SET_BLACK(x->parent);
+			SET_RED(y);
+		    }
+		    RBT_ASSERT(y->right);
+		    SET_BLACK(y->right);
+		    left_rotate(root, x->parent);
+		    x = *root;
+		    break;
+		}
+	    }
+	    else {
+		RBT_ASSERT(x == x->parent->right);
+		y = x->parent->left;
+		RBT_ASSERT(y);
+		if (IS_RED(y)) {
+		    RBT_ASSERT(y->right);
+		    RBT_ASSERT(y->left);
+		    SET_BLACK(y);
+		    RBT_ASSERT(IS_BLACK(x->parent));
+		    SET_RED(x->parent);
+		    right_rotate(root, x->parent);
+		    y = x->parent->left;
+		}
+		RBT_ASSERT(y);
+		RBT_ASSERT(IS_BLACK(y));
+		if (IS_BLACK(y->right) && IS_BLACK(y->left)) {
+		    SET_RED(y);
+		    x = x->parent;
+		}
+		else {
+		    if (IS_BLACK(y->left)) {
+			SET_BLACK(y->right);
+			SET_RED(y);
+			left_rotate(root, y);
+			y = x->parent->left;
+		    }
+		    RBT_ASSERT(y);
+		    if (IS_RED(x->parent)) {
+			SET_BLACK(x->parent);
+			SET_RED(y);
+		    }
+		    RBT_ASSERT(y->left);
+		    SET_BLACK(y->left);
+		    right_rotate(root, x->parent);
+		    x = *root;
+		    break;
+		}
+	    }
+	}
+	SET_BLACK(x);
+
+	if (null_x.parent) {
+	    if (null_x.parent->left == &null_x)
+		null_x.parent->left = NULL;
+	    else {
+		RBT_ASSERT(null_x.parent->right == &null_x);
+		null_x.parent->right = NULL;
+	    }
+	    RBT_ASSERT(!null_x.left);
+	    RBT_ASSERT(!null_x.right);
+	}
+	else if (*root == &null_x) {
+	    *root = NULL;
+	    RBT_ASSERT(!null_x.left);
+	    RBT_ASSERT(!null_x.right);
+	}
+    }
+
+    DESTROY_TREE_NODE(del);
+
+#ifdef HARD_DEBUG
+    check_tree(*root, 0);
+#endif
+}
+
+static void
+aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
+{
+    AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr;
+    AOFF_RBTree_t *blk = (AOFF_RBTree_t *) block;
+    AOFF_RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC)
+			    ? &alc->sbmbc_root : &alc->mbc_root);
+    Uint blk_sz = BLK_SZ(blk);
+
+#ifdef HARD_DEBUG
+    check_tree(*root, 0);
+#endif
+
+    blk->flags	= 0;
+    blk->left	= NULL;
+    blk->right	= NULL;
+    blk->max_sz = blk_sz;
+
+    if (!*root) {
+	blk->parent = NULL;
+	SET_BLACK(blk);
+	*root = blk;
+    }
+    else {
+	AOFF_RBTree_t *x = *root;
+	while (1) {
+	    if (x->max_sz < blk_sz) {
+		x->max_sz = blk_sz;
+	    }
+	    if (blk < x) {
+		if (!x->left) {
+		    blk->parent = x;
+		    x->left = blk;
+		    break;
+		}
+		x = x->left;
+	    }
+	    else {
+		if (!x->right) {
+		    blk->parent = x;
+		    x->right = blk;
+		    break;
+		}
+		x = x->right;
+	    }
+
+	}
+
+	/* Insert block into size tree */
+	RBT_ASSERT(blk->parent);
+
+	SET_RED(blk);
+	if (IS_RED(blk->parent))
+	    tree_insert_fixup(root, blk);
+    }
+
+#ifdef HARD_DEBUG
+    check_tree(*root, 0);
+#endif
+}
+
+static Block_t *
+aoff_get_free_block(Allctr_t *allctr, Uint size,
+		    Block_t *cand_blk, Uint cand_size, Uint32 flags)
+{
+    AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr;
+    AOFF_RBTree_t *x = ((flags & ERTS_ALCU_FLG_SBMBC)
+		       ? alc->sbmbc_root : alc->mbc_root);
+    AOFF_RBTree_t *blk = NULL;
+#ifdef HARD_DEBUG
+    AOFF_RBTree_t* dbg_blk = check_tree(x, size);
+#endif
+
+    ASSERT(!cand_blk || cand_size >= size);
+
+    while (x) {
+	if (x->left && x->left->max_sz >= size) {
+	    x = x->left;
+	}
+	else if (BLK_SZ(x) >= size) {
+	    blk = x;
+	    break;
+	}
+	else {
+	    x = x->right;
+	}
+    }
+
+#ifdef HARD_DEBUG
+    ASSERT(blk == dbg_blk);
+#endif
+
+    if (!blk)
+	return NULL;
+
+    if (cand_blk && cand_blk < &blk->hdr) {
+	return NULL; /* cand_blk was better */
+    }
+
+    aoff_unlink_free_block(allctr, (Block_t *) blk, flags);
+
+    return (Block_t *) blk;
+}
+
+
+/*
+ * info_options()
+ */
+
+static struct {
+    Eterm as;
+    Eterm aoff;
+#ifdef DEBUG
+    Eterm end_of_atoms;
+#endif
+} am;
+
+static void ERTS_INLINE atom_init(Eterm *atom, char *name)
+{
+    *atom = am_atom_put(name, strlen(name));
+}
+#define AM_INIT(AM) atom_init(&am.AM, #AM)
+
+static void
+init_atoms(void)
+{
+#ifdef DEBUG
+    Eterm *atom;
+#endif
+
+    if (atoms_initialized)
+	return;
+
+#ifdef DEBUG
+    for (atom = (Eterm *) &am; atom <= &am.end_of_atoms; atom++) {
+	*atom = THE_NON_VALUE;
+    }
+#endif
+    AM_INIT(as);
+    AM_INIT(aoff);
+
+#ifdef DEBUG
+    for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) {
+	ASSERT(*atom != THE_NON_VALUE);
+    }
+#endif
+
+    atoms_initialized = 1;
+}
+
+
+#define bld_uint	erts_bld_uint
+#define bld_cons	erts_bld_cons
+#define bld_tuple	erts_bld_tuple
+
+static ERTS_INLINE void
+add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2)
+{
+    *lp = bld_cons(hpp, szp, bld_tuple(hpp, szp, 2, el1, el2), *lp);
+}
+
+static Eterm
+info_options(Allctr_t *allctr,
+	     char *prefix,
+	     int *print_to_p,
+	     void *print_to_arg,
+	     Uint **hpp,
+	     Uint *szp)
+{
+    Eterm res = THE_NON_VALUE;
+
+    if (print_to_p) {
+	erts_print(*print_to_p,
+		   print_to_arg,
+		   "%sas: %s\n",
+		   prefix,
+		   "aoff");
+    }
+
+    if (hpp || szp) {
+	
+	if (!atoms_initialized)
+	    erl_exit(1, "%s:%d: Internal error: Atoms not initialized",
+		     __FILE__, __LINE__);;
+
+	res = NIL;
+	add_2tup(hpp, szp, &res, am.as, am.aoff);
+    }
+
+    return res;
+}
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * NOTE:  erts_aoffalc_test() is only supposed to be used for testing.       *
+ *                                                                           *
+ * Keep alloc_SUITE_data/allocator_test.h updated if changes are made        *
+ * to erts_aoffalc_test()                                                    *
+\*                                                                           */
+
+unsigned long
+erts_aoffalc_test(unsigned long op, unsigned long a1, unsigned long a2)
+{
+    switch (op) {
+    case 0x500:	return (unsigned long) 0; /* IS_AOBF */
+    case 0x501:	return (unsigned long) ((AOFFAllctr_t *) a1)->mbc_root;
+    case 0x502:	return (unsigned long) ((AOFF_RBTree_t *) a1)->parent;
+    case 0x503:	return (unsigned long) ((AOFF_RBTree_t *) a1)->left;
+    case 0x504:	return (unsigned long) ((AOFF_RBTree_t *) a1)->right;
+    case 0x506:	return (unsigned long) IS_BLACK((AOFF_RBTree_t *) a1);
+    case 0x508: return (unsigned long) 1; /* IS_AOFF */
+    case 0x509: return (unsigned long) ((AOFF_RBTree_t *) a1)->max_sz;
+    default:	ASSERT(0); return ~((unsigned long) 0);
+    }
+}
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Debug functions                                                           *
+\*                                                                           */
+
+
+#ifdef HARD_DEBUG
+
+#define IS_LEFT_VISITED(FB)	((FB)->flags & LEFT_VISITED_FLG)
+#define IS_RIGHT_VISITED(FB)	((FB)->flags & RIGHT_VISITED_FLG)
+
+#define SET_LEFT_VISITED(FB)	((FB)->flags |= LEFT_VISITED_FLG)
+#define SET_RIGHT_VISITED(FB)	((FB)->flags |= RIGHT_VISITED_FLG)
+
+#define UNSET_LEFT_VISITED(FB)	((FB)->flags &= ~LEFT_VISITED_FLG)
+#define UNSET_RIGHT_VISITED(FB)	((FB)->flags &= ~RIGHT_VISITED_FLG)
+
+
+#if 0
+#  define PRINT_TREE
+#else
+#  undef PRINT_TREE
+#endif
+
+#ifdef PRINT_TREE
+static void print_tree(AOFF_RBTree_t*);
+#endif
+
+/*
+ * Checks that the order between parent and children are correct,
+ * and that the Red-Black Tree properies are satisfied. if size > 0,
+ * check_tree() returns the node that satisfies "address order first fit"
+ *
+ * The Red-Black Tree properies are:
+ *   1. Every node is either red or black.
+ *   2. Every leaf (NIL) is black.
+ *   3. If a node is red, then both its children are black.
+ *   4. Every simple path from a node to a descendant leaf
+ *      contains the same number of black nodes.
+ *
+ *   + own.max_size == MAX(own.size, left.max_size, right.max_size)
+ */
+
+static AOFF_RBTree_t *
+check_tree(AOFF_RBTree_t* root, Uint size)
+{
+    AOFF_RBTree_t *res = NULL;
+    Sint blacks;
+    Sint curr_blacks;
+    AOFF_RBTree_t *x;
+
+#ifdef PRINT_TREE
+    print_tree(root);
+#endif
+
+    if (!root)
+	return res;
+
+    x = root;
+    ASSERT(IS_BLACK(x));
+    ASSERT(!x->parent);
+    curr_blacks = 1;
+    blacks = -1;
+
+    while (x) {
+	if (!IS_LEFT_VISITED(x)) {
+	    SET_LEFT_VISITED(x);
+	    if (x->left) {
+		x = x->left;
+		if (IS_BLACK(x))
+		    curr_blacks++;
+		continue;
+	    }
+	    else {
+		if (blacks < 0)
+		    blacks = curr_blacks;
+		ASSERT(blacks == curr_blacks);
+	    }
+	}
+
+	if (!IS_RIGHT_VISITED(x)) {
+	    SET_RIGHT_VISITED(x);
+	    if (x->right) {
+		x = x->right;
+		if (IS_BLACK(x))
+		    curr_blacks++;
+		continue;
+	    }
+	    else {
+		if (blacks < 0)
+		    blacks = curr_blacks;
+		ASSERT(blacks == curr_blacks);
+	    }
+	}
+
+
+	if (IS_RED(x)) {
+	    ASSERT(IS_BLACK(x->right));
+	    ASSERT(IS_BLACK(x->left));
+	}
+
+	ASSERT(x->parent || x == root);
+
+	if (x->left) {
+	    ASSERT(x->left->parent == x);
+	    ASSERT(x->left < x);
+	    ASSERT(x->left->max_sz <= x->max_sz);	    
+	}
+
+	if (x->right) {
+	    ASSERT(x->right->parent == x);
+	    ASSERT(x->right > x);
+	    ASSERT(x->right->max_sz <= x->max_sz);	    
+	}
+	ASSERT(x->max_sz >= BLK_SZ(x));
+	ASSERT(x->max_sz == BLK_SZ(x)
+	       || x->max_sz == (x->left ? x->left->max_sz : 0)
+	       || x->max_sz == (x->right ? x->right->max_sz : 0));
+
+	if (size && BLK_SZ(x) >= size) {
+	    if (!res || x < res) {
+		res = x;
+	    }
+	}
+
+	UNSET_LEFT_VISITED(x);
+	UNSET_RIGHT_VISITED(x);
+	if (IS_BLACK(x))
+	    curr_blacks--;
+	x = x->parent;
+
+    }
+
+    ASSERT(curr_blacks == 0);
+
+    UNSET_LEFT_VISITED(root);
+    UNSET_RIGHT_VISITED(root);
+
+    return res;
+
+}
+
+
+#ifdef PRINT_TREE
+#define INDENT_STEP 2
+
+#include <stdio.h>
+
+static void
+print_tree_aux(AOFF_RBTree_t *x, int indent)
+{
+    int i;
+
+    if (x) {
+	print_tree_aux(x->right, indent + INDENT_STEP);
+	for (i = 0; i < indent; i++) {
+	    putc(' ', stderr);
+	}
+	fprintf(stderr, "%s: sz=%lu addr=0x%lx max_size=%lu\r\n",
+		IS_BLACK(x) ? "BLACK" : "RED",
+		BLK_SZ(x), (Uint)x, x->max_sz);
+	print_tree_aux(x->left,  indent + INDENT_STEP);
+    }
+}
+
+
+static void
+print_tree(AOFF_RBTree_t* root)
+{
+    fprintf(stderr, " --- AOFF tree begin ---\r\n");
+    print_tree_aux(root, 0);
+    fprintf(stderr, " --- AOFF tree end ---\r\n");
+}
+
+#endif /* PRINT_TREE */
+
+#endif /* HARD_DEBUG */
+
diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h
new file mode 100644
index 0000000000..6fa626f723
--- /dev/null
+++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h
@@ -0,0 +1,60 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ * 
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ * 
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+
+#ifndef ERL_AO_FIRSTFIT_ALLOC__
+#define ERL_AO_FIRSTFIT_ALLOC__
+
+#include "erl_alloc_util.h"
+
+#define ERTS_ALC_AOFF_ALLOC_VSN_STR "0.9"
+
+typedef struct AOFFAllctr_t_ AOFFAllctr_t;
+
+typedef struct {
+    int dummy;
+} AOFFAllctrInit_t;
+
+#define ERTS_DEFAULT_AOFF_ALLCTR_INIT {0/*dummy*/}
+
+void erts_aoffalc_init(void);
+Allctr_t *erts_aoffalc_start(AOFFAllctr_t *, AOFFAllctrInit_t*, AllctrInit_t *);
+
+#endif /* #ifndef ERL_AO_FIRSTFIT_ALLOC__ */
+
+
+
+#if defined(GET_ERL_AOFF_ALLOC_IMPL) && !defined(ERL_AOFF_ALLOC_IMPL__)
+#define ERL_AOFF_ALLOC_IMPL__
+
+#define GET_ERL_ALLOC_UTIL_IMPL
+#include "erl_alloc_util.h"
+
+
+struct AOFFAllctr_t_ {
+    Allctr_t		allctr; /* Has to be first! */
+
+    struct AOFF_RBTree_t_* mbc_root;
+    struct AOFF_RBTree_t_* sbmbc_root;
+};
+
+unsigned long erts_aoffalc_test(unsigned long, unsigned long, unsigned long);
+
+#endif /* #if defined(GET_ERL_AOFF_ALLOC_IMPL)
+	      && !defined(ERL_AOFF_ALLOC_IMPL__) */
diff --git a/erts/emulator/beam/erl_arith.c b/erts/emulator/beam/erl_arith.c
index 126ec7cc73..5150a8a507 100644
--- a/erts/emulator/beam/erl_arith.c
+++ b/erts/emulator/beam/erl_arith.c
@@ -41,6 +41,16 @@
 #  define MAX(x, y) (((x) > (y)) ? (x) : (y))
 #endif
 
+#if !HEAP_ON_C_STACK
+#  define DECLARE_TMP(VariableName,N,P)  \
+     Eterm *VariableName = ((ERTS_PROC_GET_SCHDATA(P)->erl_arith_tmp_heap) + (2 * N))
+#else
+#  define DECLARE_TMP(VariableName,N,P) \
+     Eterm VariableName[2]
+#endif
+#  define ARG_IS_NOT_TMP(Arg,Tmp) ((Arg) != make_big((Tmp)))
+
+
 static Eterm shift(Process* p, Eterm arg1, Eterm arg2, int right);
 
 static ERTS_INLINE void maybe_shrink(Process* p, Eterm* hp, Eterm res, Uint alloc)
@@ -154,14 +164,14 @@ BIF_RETTYPE bxor_2(BIF_ALIST_2)
     BIF_RET(erts_bxor(BIF_P, BIF_ARG_1, BIF_ARG_2));
 } 
 
-BIF_RETTYPE bsl_2(Process* p, Eterm arg1, Eterm arg2)
+BIF_RETTYPE bsl_2(BIF_ALIST_2)
 {
-    BIF_RET(shift(p, arg1, arg2, 0));
+    BIF_RET(shift(BIF_P, BIF_ARG_1, BIF_ARG_2, 0));
 } 
 
-BIF_RETTYPE bsr_2(Process* p, Eterm arg1, Eterm arg2)
+BIF_RETTYPE bsr_2(BIF_ALIST_2)
 {
-    BIF_RET(shift(p, arg1, arg2, 1));
+    BIF_RET(shift(BIF_P, BIF_ARG_1, BIF_ARG_2, 1));
 } 
 
 static Eterm
@@ -169,7 +179,7 @@ shift(Process* p, Eterm arg1, Eterm arg2, int right)
 {
     Sint i;
     Sint ires;
-    Eterm tmp_big1[2];
+    DECLARE_TMP(tmp_big1,0,p);
     Eterm* bigp;
     Uint need;
 
@@ -312,8 +322,8 @@ BIF_RETTYPE bnot_1(BIF_ALIST_1)
 Eterm
 erts_mixed_plus(Process* p, Eterm arg1, Eterm arg2)
 {
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     Eterm res;
     Eterm hdr;
     FloatDef f1, f2;
@@ -458,8 +468,8 @@ erts_mixed_plus(Process* p, Eterm arg1, Eterm arg2)
 Eterm
 erts_mixed_minus(Process* p, Eterm arg1, Eterm arg2)
 {
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     Eterm hdr;
     Eterm res;
     FloatDef f1, f2;
@@ -602,8 +612,8 @@ erts_mixed_minus(Process* p, Eterm arg1, Eterm arg2)
 Eterm
 erts_mixed_times(Process* p, Eterm arg1, Eterm arg2)
 {
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     Eterm hdr;
     Eterm res;
     FloatDef f1, f2;
@@ -627,8 +637,8 @@ erts_mixed_times(Process* p, Eterm arg1, Eterm arg2)
 		    } else if (arg2 == SMALL_ONE) {
 			return(arg1);
 		    } else {
-			Eterm big_res[3];
-
+			DeclareTmpHeap(big_res,3,p);
+			UseTmpHeap(3,p);
 			/*
 			 * The following code is optimized for the case that
 			 * result is small (which should be the most common case
@@ -636,6 +646,7 @@ erts_mixed_times(Process* p, Eterm arg1, Eterm arg2)
 			 */
 			res = small_times(signed_val(arg1), signed_val(arg2), big_res);
 			if (is_small(res)) {
+			    UnUseTmpHeap(3,p);
 			    return res;
 			} else {
 			    /*
@@ -657,6 +668,7 @@ erts_mixed_times(Process* p, Eterm arg1, Eterm arg2)
 			    if (arity > 1) {
 				*hp = big_res[2];
 			    }
+			    UnUseTmpHeap(3,p);
 			    return res;
 			}
 		    }
@@ -915,8 +927,8 @@ erts_mixed_div(Process* p, Eterm arg1, Eterm arg2)
 Eterm
 erts_int_div(Process* p, Eterm arg1, Eterm arg2)
 {
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     int ires;
 
     switch (NUMBER_CODE(arg1, arg2)) {
@@ -967,8 +979,8 @@ erts_int_div(Process* p, Eterm arg1, Eterm arg2)
 Eterm
 erts_int_rem(Process* p, Eterm arg1, Eterm arg2)
 {
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     int ires;
 
     switch (NUMBER_CODE(arg1, arg2)) {
@@ -979,7 +991,8 @@ erts_int_rem(Process* p, Eterm arg1, Eterm arg2)
 	if (arg1 != make_small(MIN_SMALL)) {
 	    return arg1;
 	} else {
-	    Eterm tmp = small_to_big(signed_val(arg1), tmp_big1);
+	    Eterm tmp;
+	    tmp = small_to_big(signed_val(arg1), tmp_big1);
 	    if ((ires = big_ucomp(tmp, arg2)) == 0) {
 		return SMALL_ZERO;
 	    } else {
@@ -1013,8 +1026,8 @@ erts_int_rem(Process* p, Eterm arg1, Eterm arg2)
 
 Eterm erts_band(Process* p, Eterm arg1, Eterm arg2)
 {
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     Eterm* hp;
     int need;
 
@@ -1041,8 +1054,8 @@ Eterm erts_band(Process* p, Eterm arg1, Eterm arg2)
 
 Eterm erts_bor(Process* p, Eterm arg1, Eterm arg2)
 {
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     Eterm* hp;
     int need;
 
@@ -1069,8 +1082,8 @@ Eterm erts_bor(Process* p, Eterm arg1, Eterm arg2)
 
 Eterm erts_bxor(Process* p, Eterm arg1, Eterm arg2)
 {
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     Eterm* hp;
     int need;
 
@@ -1148,8 +1161,8 @@ erts_gc_mixed_plus(Process* p, Eterm* reg, Uint live)
 {
     Eterm arg1;
     Eterm arg2;
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     Eterm res;
     Eterm hdr;
     FloatDef f1, f2;
@@ -1237,10 +1250,10 @@ erts_gc_mixed_plus(Process* p, Eterm* reg, Uint live)
 		    need_heap = BIG_NEED_SIZE(sz);
 		    if (ERTS_NEED_GC(p, need_heap)) {
 			erts_garbage_collect(p, need_heap, reg, live+2);
-			if (arg1 != make_big(tmp_big1)) {
+			if (ARG_IS_NOT_TMP(arg1,tmp_big1)) {
 			    arg1 = reg[live];
 			}
-			if (arg2 != make_big(tmp_big2)) {
+			if (ARG_IS_NOT_TMP(arg2,tmp_big2)) {
 			    arg2 = reg[live+1];
 			}
 		    }
@@ -1316,8 +1329,8 @@ erts_gc_mixed_minus(Process* p, Eterm* reg, Uint live)
 {
     Eterm arg1;
     Eterm arg2;
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     Eterm hdr;
     Eterm res;
     FloatDef f1, f2;
@@ -1394,10 +1407,10 @@ erts_gc_mixed_minus(Process* p, Eterm* reg, Uint live)
 		    need_heap = BIG_NEED_SIZE(sz);
 		    if (ERTS_NEED_GC(p, need_heap)) {
 			erts_garbage_collect(p, need_heap, reg, live+2);
-			if (arg1 != make_big(tmp_big1)) {
+			if (ARG_IS_NOT_TMP(arg1,tmp_big1)) {
 			    arg1 = reg[live];
 			}
-			if (arg2 != make_big(tmp_big2)) {
+			if (ARG_IS_NOT_TMP(arg2,tmp_big2)) {
 			    arg2 = reg[live+1];
 			}
 		    }
@@ -1482,8 +1495,8 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live)
 {
     Eterm arg1;
     Eterm arg2;
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     Eterm hdr;
     Eterm res;
     FloatDef f1, f2;
@@ -1509,7 +1522,8 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live)
 		    } else if (arg2 == SMALL_ONE) {
 			return(arg1);
 		    } else {
-			Eterm big_res[3];
+			DeclareTmpHeap(big_res,3,p);
+			UseTmpHeap(3,p);
 
 			/*
 			 * The following code is optimized for the case that
@@ -1519,6 +1533,7 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live)
 			res = small_times(signed_val(arg1), signed_val(arg2),
 					  big_res);
 			if (is_small(res)) {
+			    UnUseTmpHeap(3,p);
 			    return res;
 			} else {
 			    /*
@@ -1546,6 +1561,7 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live)
 			    if (arity > 1) {
 				*hp = big_res[2];
 			    }
+			    UnUseTmpHeap(3,p);
 			    return res;
 			}
 		    }
@@ -1609,17 +1625,17 @@ erts_gc_mixed_times(Process* p, Eterm* reg, Uint live)
 		    need_heap = BIG_NEED_SIZE(sz);
 		    if (ERTS_NEED_GC(p, need_heap)) {
 			erts_garbage_collect(p, need_heap, reg, live+2);
-			if (arg1 != make_big(tmp_big1)) {
+			if (ARG_IS_NOT_TMP(arg1,tmp_big1)) {
 			    arg1 = reg[live];
 			}
-			if (arg2 != make_big(tmp_big2)) {
+			if (ARG_IS_NOT_TMP(arg2,tmp_big2)) {
 			    arg2 = reg[live+1];
 			}
 		    }
 		    hp = p->htop;
 		    p->htop += need_heap;
 		    res = big_times(arg1, arg2, hp);
-                    trim_heap(p, hp, res);
+		    trim_heap(p, hp, res);
 
 		    /*
 		     * Note that the result must be big in this case, since
@@ -1828,8 +1844,8 @@ erts_gc_int_div(Process* p, Eterm* reg, Uint live)
 {
     Eterm arg1;
     Eterm arg2;
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     int ires;
 
     arg1 = reg[live];
@@ -1866,10 +1882,10 @@ erts_gc_int_div(Process* p, Eterm* reg, Uint live)
 	    need = BIG_NEED_SIZE(i-ires+1) + BIG_NEED_SIZE(i);
 	    if (ERTS_NEED_GC(p, need)) {
 		erts_garbage_collect(p, need, reg, live+2);
-		if (arg1 != make_big(tmp_big1)) {
+		if (ARG_IS_NOT_TMP(arg1,tmp_big1)) {
 		    arg1 = reg[live];
 		}
-		if (arg2 != make_big(tmp_big2)) {
+		if (ARG_IS_NOT_TMP(arg2,tmp_big2)) {
 		    arg2 = reg[live+1];
 		}
 	    }
@@ -1894,8 +1910,8 @@ erts_gc_int_rem(Process* p, Eterm* reg, Uint live)
 {
     Eterm arg1;
     Eterm arg2;
-    Eterm tmp_big1[2];
-    Eterm tmp_big2[2];
+    DECLARE_TMP(tmp_big1,0,p);
+    DECLARE_TMP(tmp_big2,1,p);
     int ires;
 
     arg1 = reg[live];
@@ -1908,7 +1924,8 @@ erts_gc_int_rem(Process* p, Eterm* reg, Uint live)
 	if (arg1 != make_small(MIN_SMALL)) {
 	    return arg1;
 	} else {
-	    Eterm tmp = small_to_big(signed_val(arg1), tmp_big1);
+	    Eterm tmp;
+	    tmp = small_to_big(signed_val(arg1), tmp_big1);
 	    if ((ires = big_ucomp(tmp, arg2)) == 0) {
 		return SMALL_ZERO;
 	    } else {
@@ -1928,10 +1945,10 @@ erts_gc_int_rem(Process* p, Eterm* reg, Uint live)
 
 	    if (ERTS_NEED_GC(p, need)) {
 		erts_garbage_collect(p, need, reg, live+2);
-		if (arg1 != make_big(tmp_big1)) {
+		if (ARG_IS_NOT_TMP(arg1,tmp_big1)) {
 		    arg1 = reg[live];
 		}
-		if (arg2 != make_big(tmp_big2)) {
+		if (ARG_IS_NOT_TMP(arg2,tmp_big2)) {
 		    arg2 = reg[live+1];
 		}
 	    }
@@ -1956,8 +1973,8 @@ Eterm erts_gc_##func(Process* p, Eterm* reg, Uint live)				\
 {										\
     Eterm arg1;									\
     Eterm arg2;									\
-    Eterm tmp_big1[2];								\
-    Eterm tmp_big2[2];								\
+    DECLARE_TMP(tmp_big1,0,p);							\
+    DECLARE_TMP(tmp_big2,1,p);							\
     Eterm* hp;									\
     int need;									\
 										\
diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c
index b090564649..cb975d64b0 100644
--- a/erts/emulator/beam/erl_async.c
+++ b/erts/emulator/beam/erl_async.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2000-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2000-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 #ifdef HAVE_CONFIG_H
@@ -24,10 +24,19 @@
 #include "erl_sys_driver.h"
 #include "global.h"
 #include "erl_threads.h"
+#include "erl_thr_queue.h"
+#include "erl_async.h"
+#include "dtrace-wrapper.h"
+
+#define ERTS_MAX_ASYNC_READY_CALLS_IN_SEQ 20
+
+#define ERTS_ASYNC_PRINT_JOB 0
+
+#if !defined(ERTS_SMP) && defined(USE_THREADS) && !ERTS_USE_ASYNC_READY_Q
+#  error "Need async ready queue in non-smp case"
+#endif
 
 typedef struct _erl_async {
-    struct _erl_async* next;
-    struct _erl_async* prev;
     DE_Handle*         hndl;   /* The DE_Handle is needed when port is gone */
     Eterm              port;
     long               async_id;
@@ -35,346 +44,534 @@ typedef struct _erl_async {
     ErlDrvPDL          pdl;
     void (*async_invoke)(void*);
     void (*async_free)(void*);
-} ErlAsync;
+#if ERTS_USE_ASYNC_READY_Q
+    Uint               sched_id;
+    union {
+	ErtsThrQPrepEnQ_t *prep_enq;
+	ErtsThrQFinDeQ_t   fin_deq;
+    } q;
+#endif
+} ErtsAsync;
+
+#if ERTS_USE_ASYNC_READY_Q
+
+/*
+ * We can do without the enqueue mutex since it isn't needed for
+ * thread safety. Its only purpose is to put async threads to sleep
+ * during a blast of ready async jobs. This in order to reduce
+ * contention on the enqueue end of the async ready queues. During
+ * such a blast without the enqueue mutex much cpu time is consumed
+ * by the async threads without them doing much progress which in turn
+ * slow down progress of scheduler threads.
+ */
+#define ERTS_USE_ASYNC_READY_ENQ_MTX 1
+
+#if ERTS_USE_ASYNC_READY_ENQ_MTX
 
 typedef struct {
-    erts_mtx_t mtx;
-    erts_cnd_t cv;
-    erts_tid_t thr;
-    int   len;
-#ifndef ERTS_SMP
-    int   hndl;
+    erts_mtx_t enq_mtx;
+} ErtsAsyncReadyQXData;
+
 #endif
-    ErlAsync* head;
-    ErlAsync* tail;
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    int no;
+
+typedef struct {
+#if ERTS_USE_ASYNC_READY_ENQ_MTX
+    union {
+	ErtsAsyncReadyQXData data;
+	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(
+		sizeof(ErtsAsyncReadyQXData))];
+    } x;
 #endif
-} AsyncQueue;
+    ErtsThrQ_t thr_q;
+    ErtsThrQFinDeQ_t fin_deq;
+} ErtsAsyncReadyQ;
+
 
-static erts_smp_spinlock_t async_id_lock;
-static long async_id = 0;
+typedef union {
+    ErtsAsyncReadyQ arq;
+    char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncReadyQ))];
+} ErtsAlgndAsyncReadyQ;
 
+#endif /* ERTS_USE_ASYNC_READY_Q */
 
-#ifndef ERTS_SMP
+typedef struct {
+    ErtsThrQ_t thr_q;
+    erts_tid_t thr_id;
+} ErtsAsyncQ;
 
-erts_mtx_t async_ready_mtx;
-static ErlAsync* async_ready_list = NULL;
+typedef union {
+    ErtsAsyncQ aq;
+    char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncQ))];
+} ErtsAlgndAsyncQ;
 
+typedef struct {
+    int no_initialized;
+    erts_mtx_t mtx;
+    erts_cnd_t cnd;
+    erts_atomic_t id;
+} ErtsAsyncInit;
+
+typedef struct {
+    union {
+	ErtsAsyncInit data;
+	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncInit))];
+    } init;
+    ErtsAlgndAsyncQ *queue;
+#if ERTS_USE_ASYNC_READY_Q
+    ErtsAlgndAsyncReadyQ *ready_queue;
 #endif
+} ErtsAsyncData;
 
 /*
-** Initialize worker threads (if supported)
-*/
+ * Some compilers, e.g. GCC 4.2.1 and -O3, will optimize away DTrace
+ * calls if they're the last thing in the function.  :-(
+ * Many thanks to Trond Norbye, via:
+ * https://github.com/memcached/memcached/commit/6298b3978687530bc9d219b6ac707a1b681b2a46
+ */
+static unsigned gcc_optimizer_hack = 0;
 
-/* Detach from driver */
-static void async_detach(DE_Handle* dh)
-{
-    /* XXX:PaN what should happen here? we want to unload the driver or??? */
-    return;
-}
+int erts_async_max_threads; /* Initialized by erl_init.c */
+int erts_async_thread_suggested_stack_size; /* Initialized by erl_init.c */
 
+static ErtsAsyncData *async;
 
-#ifdef USE_THREADS
+#ifndef USE_THREADS
 
-static AsyncQueue* async_q;
+void
+erts_init_async(void)
+{
 
-static void* async_main(void*);
-static void async_add(ErlAsync*, AsyncQueue*);
+}
 
-#ifndef ERTS_SMP
-typedef struct ErtsAsyncReadyCallback_ ErtsAsyncReadyCallback;
-struct ErtsAsyncReadyCallback_ {
-    struct ErtsAsyncReadyCallback_ *next;
-    void (*callback)(void);
-};
+#else
 
-static ErtsAsyncReadyCallback *callbacks;
-static int async_handle;
+static void *async_main(void *);
 
-int erts_register_async_ready_callback(void (*funcp)(void))
+static ERTS_INLINE ErtsAsyncQ *
+async_q(int i)
 {
-    ErtsAsyncReadyCallback *cb = erts_alloc(ERTS_ALC_T_ARCALLBACK,
-					    sizeof(ErtsAsyncReadyCallback));
-    cb->next = callbacks;
-    cb->callback = funcp;
-    erts_mtx_lock(&async_ready_mtx);
-    callbacks = cb;
-    erts_mtx_unlock(&async_ready_mtx);
-    return async_handle;
+    return &async->queue[i].aq;
 }
-#endif
 
-int init_async(int hndl)
-{
-    erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER;
-    AsyncQueue* q;
-    int i;
+#if ERTS_USE_ASYNC_READY_Q
 
-    thr_opts.detached = 0;
-    thr_opts.suggested_stack_size = erts_async_thread_suggested_stack_size;
-
-#ifndef ERTS_SMP
-    callbacks = NULL;
-    async_handle = hndl;
-    erts_mtx_init(&async_ready_mtx, "async_ready");
-    async_ready_list = NULL;
-#endif
-
-    async_id = 0;
-    erts_smp_spinlock_init(&async_id_lock, "async_id");
-
-    async_q = q = (AsyncQueue*)
-	(erts_async_max_threads
-	 ? erts_alloc(ERTS_ALC_T_ASYNC_Q,
-		      erts_async_max_threads * sizeof(AsyncQueue))
-	 : NULL);
-    for (i = 0; i < erts_async_max_threads; i++) {
-	q->head = NULL;
-	q->tail = NULL;
-	q->len = 0;
-#ifndef ERTS_SMP
-	q->hndl = hndl;
-#endif
-#ifdef ERTS_ENABLE_LOCK_CHECK
-	q->no = i;
-#endif
-	erts_mtx_init(&q->mtx, "asyncq");
-	erts_cnd_init(&q->cv);
-	erts_thr_create(&q->thr, async_main, (void*)q, &thr_opts);
-	q++;
-    }
-    return 0;
+static ERTS_INLINE ErtsAsyncReadyQ *
+async_ready_q(Uint sched_id)
+{
+    return &async->ready_queue[((int)sched_id)-1].arq;
 }
 
+#endif
 
-int exit_async()
+void
+erts_init_async(void)
 {
-    int i;
+    async = NULL;
+    if (erts_async_max_threads > 0) {
+#if ERTS_USE_ASYNC_READY_Q
+	ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT;
+#endif
+	erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER;
+	char *ptr;
+	size_t tot_size = 0;
+	int i;
+
+	tot_size += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncData));
+	tot_size += sizeof(ErtsAlgndAsyncQ)*erts_async_max_threads;
+#if ERTS_USE_ASYNC_READY_Q
+	tot_size += sizeof(ErtsAlgndAsyncReadyQ)*erts_no_schedulers;
+#endif
 
-    /* terminate threads */
-    for (i = 0; i < erts_async_max_threads; i++) {
-	ErlAsync* a = (ErlAsync*) erts_alloc(ERTS_ALC_T_ASYNC,
-					     sizeof(ErlAsync));
-	a->port = NIL;
-	async_add(a, &async_q[i]);
-    }
+	ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_ASYNC_DATA,
+						 tot_size);
 
-    for (i = 0; i < erts_async_max_threads; i++) {
-	erts_thr_join(async_q[i].thr, NULL);
-	erts_mtx_destroy(&async_q[i].mtx);
-	erts_cnd_destroy(&async_q[i].cv);
-    }
-#ifndef ERTS_SMP
-    erts_mtx_destroy(&async_ready_mtx);
+	async = (ErtsAsyncData *) ptr;
+	ptr += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncData));
+
+	async->init.data.no_initialized = 0;
+	erts_mtx_init(&async->init.data.mtx, "async_init_mtx");
+	erts_cnd_init(&async->init.data.cnd);
+	erts_atomic_init_nob(&async->init.data.id, 0);
+
+	async->queue = (ErtsAlgndAsyncQ *) ptr;
+	ptr += sizeof(ErtsAlgndAsyncQ)*erts_async_max_threads;
+
+#if ERTS_USE_ASYNC_READY_Q
+
+	qinit.live.queue = ERTS_THR_Q_LIVE_LONG;
+	qinit.live.objects = ERTS_THR_Q_LIVE_SHORT;
+	qinit.notify = erts_notify_check_async_ready_queue;
+
+	async->ready_queue = (ErtsAlgndAsyncReadyQ *) ptr;
+	ptr += sizeof(ErtsAlgndAsyncReadyQ)*erts_no_schedulers;
+
+	for (i = 1; i <= erts_no_schedulers; i++) {
+	    ErtsAsyncReadyQ *arq = async_ready_q(i);
+#if ERTS_USE_ASYNC_READY_ENQ_MTX
+	    erts_mtx_init(&arq->x.data.enq_mtx, "async_enq_mtx");
 #endif
-    if (async_q)
-	erts_free(ERTS_ALC_T_ASYNC_Q, (void *) async_q);
-    return 0;
+	    erts_thr_q_finalize_dequeue_state_init(&arq->fin_deq);
+	    qinit.arg = (void *) (SWord) i;
+	    erts_thr_q_initialize(&arq->thr_q, &qinit);
+	}
+
+#endif
+
+	/* Create async threads... */
+
+	thr_opts.detached = 0;
+	thr_opts.suggested_stack_size
+	    = erts_async_thread_suggested_stack_size;
+
+	for (i = 0; i < erts_async_max_threads; i++) {
+	    ErtsAsyncQ *aq = async_q(i);
+	    erts_thr_create(&aq->thr_id, async_main, (void*) aq, &thr_opts);
+	}
+
+	/* Wait for async threads to initialize... */
+
+	erts_mtx_lock(&async->init.data.mtx);
+	while (async->init.data.no_initialized != erts_async_max_threads)
+	    erts_cnd_wait(&async->init.data.cnd, &async->init.data.mtx);
+	erts_mtx_unlock(&async->init.data.mtx);
+
+	erts_mtx_destroy(&async->init.data.mtx);
+	erts_cnd_destroy(&async->init.data.cnd);
+
+    }
 }
 
+#if ERTS_USE_ASYNC_READY_Q
 
-static void async_add(ErlAsync* a, AsyncQueue* q)
+void *
+erts_get_async_ready_queue(Uint sched_id)
 {
-    /* XXX:PaN Is this still necessary when ports lock drivers? */
+    return (void *) async ? async_ready_q(sched_id) : NULL;
+}
+
+#endif
+
+static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q)
+{
+#ifdef USE_VM_PROBES
+    int len;
+#endif
+
     if (is_internal_port(a->port)) {
-	ERTS_LC_ASSERT(erts_drvportid2port(a->port));
+#if ERTS_USE_ASYNC_READY_Q
+	ErtsAsyncReadyQ *arq = async_ready_q(a->sched_id);
+	a->q.prep_enq = erts_thr_q_prepare_enqueue(&arq->thr_q);
+#endif
 	/* make sure the driver will stay around */
-	driver_lock_driver(internal_port_index(a->port));
+	if (a->hndl)
+	    erts_ddll_reference_referenced_driver(a->hndl);
     }
 
-    erts_mtx_lock(&q->mtx);
+#if ERTS_ASYNC_PRINT_JOB
+    erts_fprintf(stderr, "-> %ld\n", a->async_id);
+#endif
 
-    if (q->len == 0) {
-	q->head = a;
-	q->tail = a;
-	q->len = 1;
-	erts_cnd_signal(&q->cv);
-    }
-    else { /* no need to signal (since the worker is working) */
-	a->next = q->head;
-	q->head->prev = a;
-	q->head = a;
-	q->len++;
+    erts_thr_q_enqueue(&q->thr_q, a);
+#ifdef USE_VM_PROBES
+    if (DTRACE_ENABLED(aio_pool_add)) {
+        DTRACE_CHARBUF(port_str, 16);
+
+        erts_snprintf(port_str, sizeof(port_str), "%T", a->port);
+        /* DTRACE TODO: Get the queue length from erts_thr_q_enqueue() ? */
+        len = -1;
+        DTRACE2(aio_pool_add, port_str, len);
     }
-    erts_mtx_unlock(&q->mtx);
+#endif
+    gcc_optimizer_hack++;
 }
 
-static ErlAsync* async_get(AsyncQueue* q)
+static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q,
+					erts_tse_t *tse,
+					ErtsThrQPrepEnQ_t **prep_enq)
 {
-    ErlAsync* a;
+#if ERTS_USE_ASYNC_READY_Q
+    int saved_fin_deq = 0;
+    ErtsThrQFinDeQ_t fin_deq;
+#endif
+#ifdef USE_VM_PROBES
+    int len;
+#endif
 
-    erts_mtx_lock(&q->mtx);
-    while((a = q->tail) == NULL) {
-	erts_cnd_wait(&q->cv, &q->mtx);
-    }
+    while (1) {
+	ErtsAsync *a = (ErtsAsync *) erts_thr_q_dequeue(q);
+	if (a) {
+
+#if ERTS_USE_ASYNC_READY_Q
+	    *prep_enq = a->q.prep_enq;
+	    erts_thr_q_get_finalize_dequeue_data(q, &a->q.fin_deq);
+	    if (saved_fin_deq)
+		erts_thr_q_append_finalize_dequeue_data(&a->q.fin_deq, &fin_deq);
+#endif
+#ifdef USE_VM_PROBES
+            if (DTRACE_ENABLED(aio_pool_get)) {
+                DTRACE_CHARBUF(port_str, 16);
+
+                erts_snprintf(port_str, sizeof(port_str), "%T", a->port);
+                /* DTRACE TODO: Get the length from erts_thr_q_dequeue() ? */
+                len = -1;
+                DTRACE2(aio_pool_get, port_str, len);
+            }
+#endif
+	    return a;
+	}
+
+	if (ERTS_THR_Q_DIRTY != erts_thr_q_clean(q)) {
+	    ErtsThrQFinDeQ_t tmp_fin_deq;
+
+	    erts_tse_reset(tse);
+
+#if ERTS_USE_ASYNC_READY_Q
+	chk_fin_deq:
+	    if (erts_thr_q_get_finalize_dequeue_data(q, &tmp_fin_deq)) {
+		if (!saved_fin_deq) {
+		    erts_thr_q_finalize_dequeue_state_init(&fin_deq);
+		    saved_fin_deq = 1;
+		}
+		erts_thr_q_append_finalize_dequeue_data(&fin_deq,
+							&tmp_fin_deq);
+	    }
+#endif
+
+	    switch (erts_thr_q_inspect(q, 1)) {
+	    case ERTS_THR_Q_DIRTY:
+		break;
+	    case ERTS_THR_Q_NEED_THR_PRGR:
 #ifdef ERTS_SMP
-    ASSERT(a && q->tail == a);
+	    {
+		ErtsThrPrgrVal prgr = erts_thr_q_need_thr_progress(q);
+		erts_thr_progress_wakeup(NULL, prgr);
+		/*
+		 * We do no dequeue finalizing in hope that a new async
+		 * job will arrive before we are woken due to thread
+		 * progress...
+		 */
+		erts_tse_wait(tse);
+		break;
+	    }
 #endif
-    if (q->head == q->tail) {
-	q->head = q->tail = NULL;
-	q->len = 0;
-    }
-    else {
-	q->tail->prev->next = NULL;
-	q->tail = q->tail->prev;
-	q->len--;
+	    case ERTS_THR_Q_CLEAN:
+
+#if ERTS_USE_ASYNC_READY_Q
+		if (saved_fin_deq) {
+		    if (erts_thr_q_finalize_dequeue(&fin_deq))
+			goto chk_fin_deq;
+		    else
+			saved_fin_deq = 0;
+		}
+#endif
+
+		erts_tse_wait(tse);
+		break;
+
+	    default:
+		ASSERT(0);
+		break;
+	    }
+
+	}
     }
-    erts_mtx_unlock(&q->mtx);
-    return a;
 }
 
-
-static int async_del(long id)
+static ERTS_INLINE void call_async_ready(ErtsAsync *a)
 {
-    int i;
-    /* scan all queue for an entry with async_id == 'id' */
-
-    for (i = 0; i < erts_async_max_threads; i++) {
-	ErlAsync* a;
-	erts_mtx_lock(&async_q[i].mtx);
-	
-	a = async_q[i].head;
-	while(a != NULL) {
-	    if (a->async_id == id) {
-		if (a->prev != NULL)
-		    a->prev->next = a->next;
-		else
-		    async_q[i].head = a->next;
-		if (a->next != NULL)
-		    a->next->prev = a->prev;
-		else
-		    async_q[i].tail = a->prev;
-		async_q[i].len--;
-		erts_mtx_unlock(&async_q[i].mtx);
-		if (a->async_free != NULL)
-		    a->async_free(a->async_data);
-		async_detach(a->hndl);
-		erts_free(ERTS_ALC_T_ASYNC, a);
-		return 1;
-	    }
+    Port *p = erts_id2port_sflgs(a->port,
+				 NULL,
+				 0,
+				 ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP);
+    if (!p) {
+	if (a->async_free)
+	    a->async_free(a->async_data);
+    }
+    else {
+	if (async_ready(p, a->async_data)) {
+	    if (a->async_free)
+		a->async_free(a->async_data);
 	}
-	erts_mtx_unlock(&async_q[i].mtx);
+	erts_port_release(p);
     }
-    return 0;
+    if (a->hndl)
+	erts_ddll_dereference_driver(a->hndl);
 }
 
-static void* async_main(void* arg)
+static ERTS_INLINE void async_reply(ErtsAsync *a, ErtsThrQPrepEnQ_t *prep_enq)
 {
-    AsyncQueue* q = (AsyncQueue*) arg;
+#if ERTS_USE_ASYNC_READY_Q
+    ErtsAsyncReadyQ *arq;
 
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    {
-	char buf[27];
-	erts_snprintf(&buf[0], 27, "async %d", q->no);
-	erts_lc_set_thread_name(&buf[0]);
-    }
+    if (a->pdl)
+	driver_pdl_dec_refc(a->pdl);
+
+#if ERTS_ASYNC_PRINT_JOB
+    erts_fprintf(stderr, "=>> %ld\n", a->async_id);
 #endif
 
-    while(1) {
-	ErlAsync* a = async_get(q);
+    arq = async_ready_q(a->sched_id);
 
-	if (a->port == NIL) { /* TIME TO DIE SIGNAL */
-	    erts_free(ERTS_ALC_T_ASYNC, (void *) a);
-	    break;
-	}
-	else {
-	    (*a->async_invoke)(a->async_data);
-	    /* Major problem if the code for async_invoke
-	       or async_free is removed during a blocking operation */
+#if ERTS_USE_ASYNC_READY_ENQ_MTX
+	erts_mtx_lock(&arq->x.data.enq_mtx);
+#endif
+
+	erts_thr_q_enqueue_prepared(&arq->thr_q, (void *) a, prep_enq);
+
+#if ERTS_USE_ASYNC_READY_ENQ_MTX
+	erts_mtx_unlock(&arq->x.data.enq_mtx);
+#endif
+
+#else /* ERTS_USE_ASYNC_READY_Q */
+
+	call_async_ready(a);
+	if (a->pdl)
+	    driver_pdl_dec_refc(a->pdl);
+	erts_free(ERTS_ALC_T_ASYNC, (void *) a);
+
+#endif /* ERTS_USE_ASYNC_READY_Q */
+}
+
+
+static void
+async_wakeup(void *vtse)
+{
+    erts_tse_set((erts_tse_t *) vtse);
+}
+
+static erts_tse_t *async_thread_init(ErtsAsyncQ *aq)
+{
+    ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT;
+    erts_tse_t *tse = erts_tse_fetch();
 #ifdef ERTS_SMP
-	    {
-		Port *p;
-		p = erts_id2port_sflgs(a->port,
-				       NULL,
-				       0,
-				       ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP);
-		if (!p) {
-		    if (a->async_free)
-			(*a->async_free)(a->async_data);
-		}
-		else {
-		    if (async_ready(p, a->async_data)) {
-			if (a->async_free)
-			    (*a->async_free)(a->async_data);
-		    }
-		    async_detach(a->hndl);
-		    erts_port_release(p);
-		}
-		if (a->pdl) {
-		    driver_pdl_dec_refc(a->pdl);
-		}
-		erts_free(ERTS_ALC_T_ASYNC, (void *) a);
-	    }
-#else
-	    if (a->pdl) {
-		driver_pdl_dec_refc(a->pdl);
-	    }
-	    erts_mtx_lock(&async_ready_mtx);
-	    a->next = async_ready_list;
-	    async_ready_list = a;
-	    erts_mtx_unlock(&async_ready_mtx);
-	    sys_async_ready(q->hndl);
+    ErtsThrPrgrCallbacks callbacks;
+
+    callbacks.arg = (void *) tse;
+    callbacks.wakeup = async_wakeup;
+    callbacks.prepare_wait = NULL;
+    callbacks.wait = NULL;
+
+    erts_thr_progress_register_unmanaged_thread(&callbacks);
 #endif
-	}
-    }
 
-    return NULL;
+    qinit.live.queue = ERTS_THR_Q_LIVE_LONG;
+    qinit.live.objects = ERTS_THR_Q_LIVE_SHORT;
+    qinit.arg = (void *) tse;
+    qinit.notify = async_wakeup;
+#if ERTS_USE_ASYNC_READY_Q
+    qinit.auto_finalize_dequeue = 0;
+#endif
+
+    erts_thr_q_initialize(&aq->thr_q, &qinit);
+
+    /* Inform main thread that we are done initializing... */
+    erts_mtx_lock(&async->init.data.mtx);
+    async->init.data.no_initialized++;
+    erts_cnd_signal(&async->init.data.cnd);
+    erts_mtx_unlock(&async->init.data.mtx);
+
+    return tse;
 }
 
+static void *async_main(void* arg)
+{
+    ErtsAsyncQ *aq = (ErtsAsyncQ *) arg;
+    erts_tse_t *tse = async_thread_init(aq);
 
+    while (1) {
+	ErtsThrQPrepEnQ_t *prep_enq;
+	ErtsAsync *a = async_get(&aq->thr_q, tse, &prep_enq);
+	if (is_nil(a->port))
+	    break; /* Time to die */
+
+#if ERTS_ASYNC_PRINT_JOB
+	erts_fprintf(stderr, "<- %ld\n", a->async_id);
 #endif
 
-#ifndef ERTS_SMP
+	a->async_invoke(a->async_data);
+
+	async_reply(a, prep_enq);
+    }
+
+    return NULL;
+}
+
+#endif /* USE_THREADS */
 
-int check_async_ready(void)
+void
+erts_exit_flush_async(void)
 {
 #ifdef USE_THREADS
-    ErtsAsyncReadyCallback *cbs;
+    int i;
+    ErtsAsync a;
+    a.port = NIL;
+    /*
+     * Terminate threads in order to flush queues. We do not
+     * bother to clean everything up since we are about to
+     * terminate the runtime system and a cleanup would only
+     * delay the termination.
+     */
+    for (i = 0; i < erts_async_max_threads; i++)
+	async_add(&a, async_q(i));
+    for (i = 0; i < erts_async_max_threads; i++)
+	erts_thr_join(async->queue[i].aq.thr_id, NULL);
 #endif
-    ErlAsync* a;
-    int count = 0;
+}
 
-    erts_mtx_lock(&async_ready_mtx);
-    a = async_ready_list;
-    async_ready_list = NULL;
-#ifdef USE_THREADS
-    cbs = callbacks;
-#endif
-    erts_mtx_unlock(&async_ready_mtx);
-
-    while(a != NULL) {
-	ErlAsync* a_next = a->next;
-	/* Every port not dead */
-	Port *p = erts_id2port_sflgs(a->port,
-				     NULL,
-				     0,
-				     ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP);
-	if (!p) {
-	    if (a->async_free)
-		(*a->async_free)(a->async_data);
-	}
-	else {
-	    count++;
-	    if (async_ready(p, a->async_data)) {
-		if (a->async_free != NULL)
-		    (*a->async_free)(a->async_data);
-	    }
-	    async_detach(a->hndl);
-	    erts_port_release(p);
+#if defined(USE_THREADS) && ERTS_USE_ASYNC_READY_Q
+
+int erts_check_async_ready(void *varq)
+{
+    ErtsAsyncReadyQ *arq = (ErtsAsyncReadyQ *) varq;
+    int res = 1;
+    int i;
+
+    for (i = 0; i < ERTS_MAX_ASYNC_READY_CALLS_IN_SEQ; i++) {
+	ErtsAsync *a = (ErtsAsync *) erts_thr_q_dequeue(&arq->thr_q);
+	if (!a) {
+	    res = 0;
+	    break;
 	}
+
+#if ERTS_ASYNC_PRINT_JOB
+	erts_fprintf(stderr, "<<= %ld\n", a->async_id);
+#endif
+	erts_thr_q_append_finalize_dequeue_data(&arq->fin_deq, &a->q.fin_deq);
+	call_async_ready(a);
 	erts_free(ERTS_ALC_T_ASYNC, (void *) a);
-	a = a_next;
     }
-#ifdef USE_THREADS
-    for (; cbs; cbs = cbs->next)
-	(*cbs->callback)();
-#endif
-    return count;
+
+    erts_thr_q_finalize_dequeue(&arq->fin_deq);
+    
+    return res;
 }
 
+int erts_async_ready_clean(void *varq, void *val)
+{
+    ErtsAsyncReadyQ *arq = (ErtsAsyncReadyQ *) varq;
+    ErtsThrQCleanState_t cstate;
+
+    cstate = erts_thr_q_clean(&arq->thr_q);
+
+    if (erts_thr_q_finalize_dequeue(&arq->fin_deq))
+	return ERTS_ASYNC_READY_DIRTY;
+
+    switch (cstate) {
+    case ERTS_THR_Q_DIRTY:
+	return ERTS_ASYNC_READY_DIRTY;
+    case ERTS_THR_Q_NEED_THR_PRGR:
+#ifdef ERTS_SMP
+	*((ErtsThrPrgrVal *) val)
+	    = erts_thr_q_need_thr_progress(&arq->thr_q);
+	return ERTS_ASYNC_READY_NEED_THR_PRGR;
 #endif
+    case ERTS_THR_Q_CLEAN:
+	break;
+    }
+    return ERTS_ASYNC_READY_CLEAN;
+}
 
+#endif
 
 /*
 ** Schedule async_invoke on a worker thread
@@ -394,19 +591,29 @@ long driver_async(ErlDrvPort ix, unsigned int* key,
 		  void (*async_invoke)(void*), void* async_data,
 		  void (*async_free)(void*))
 {
-    ErlAsync* a = (ErlAsync*) erts_alloc(ERTS_ALC_T_ASYNC, sizeof(ErlAsync));
-    Port* prt = erts_drvport2port(ix);
+    ErtsAsync* a;
+    Port* prt;
     long id;
     unsigned int qix;
+#if ERTS_USE_ASYNC_READY_Q
+    Uint sched_id;
 
+    sched_id = erts_get_scheduler_id();
+    if (!sched_id)
+	sched_id = 1;
+#endif
 
+    prt = erts_drvport2port(ix);
     if (!prt)
 	return -1;
 
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
 
-    a->next = NULL;
-    a->prev = NULL;
+    a = (ErtsAsync*) erts_alloc(ERTS_ALC_T_ASYNC, sizeof(ErtsAsync));
+
+#if ERTS_USE_ASYNC_READY_Q
+    a->sched_id = sched_id;
+#endif
     a->hndl = (DE_Handle*)prt->drv_ptr->handle;
     a->port = prt->id;
     a->pdl = NULL;
@@ -414,12 +621,16 @@ long driver_async(ErlDrvPort ix, unsigned int* key,
     a->async_invoke = async_invoke;
     a->async_free = async_free;
 
-    erts_smp_spin_lock(&async_id_lock);
-    async_id = (async_id + 1) & 0x7fffffff;
-    if (async_id == 0)
-	async_id++;
-    id = async_id;
-    erts_smp_spin_unlock(&async_id_lock);
+    if (!async)
+	id = 0;
+    else {
+	do {
+	    id = erts_atomic_inc_read_nob(&async->init.data.id);
+	} while (id == 0);
+	if (id < 0)
+	    id *= -1;
+	ASSERT(id > 0);
+    }
 
     a->async_id = id;
 
@@ -438,7 +649,7 @@ long driver_async(ErlDrvPort ix, unsigned int* key,
 	    driver_pdl_inc_refc(prt->port_data_lock);
 	    a->pdl = prt->port_data_lock;
 	}
-	async_add(a, &async_q[qix]);
+	async_add(a, async_q(qix));
 	return id;
     }
 #endif
@@ -456,10 +667,16 @@ long driver_async(ErlDrvPort ix, unsigned int* key,
 
 int driver_async_cancel(unsigned int id)
 {
-#ifdef USE_THREADS
-    if (erts_async_max_threads > 0)
-	return async_del(id);
-#endif
+    /*
+     * Not supported anymore. Always fail (which is backward
+     * compatible).
+     *
+     * This functionality could be implemented again. However,
+     * it is (and always has been) completely useless since
+     * it doesn't give you any guarantees whatsoever. The user
+     * needs to (and always have had to) synchronize in his/her
+     * own code in order to get any guarantees.
+     */
     return 0;
 }
 
diff --git a/erts/emulator/beam/erl_async.h b/erts/emulator/beam/erl_async.h
new file mode 100644
index 0000000000..95374a8fc9
--- /dev/null
+++ b/erts/emulator/beam/erl_async.h
@@ -0,0 +1,66 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifndef ERL_ASYNC_H__
+#define ERL_ASYNC_H__
+
+#define ERTS_MAX_NO_OF_ASYNC_THREADS 1024
+extern int erts_async_max_threads;
+#define ERTS_ASYNC_THREAD_MIN_STACK_SIZE 16	/* Kilo words */
+#define ERTS_ASYNC_THREAD_MAX_STACK_SIZE 8192	/* Kilo words */
+extern int erts_async_thread_suggested_stack_size;
+
+#ifdef USE_THREADS
+
+#ifdef ERTS_SMP
+/*
+ * With smp support we can choose to have, or not to
+ * have an async ready queue.
+ */
+#define ERTS_USE_ASYNC_READY_Q 1
+#endif
+
+#ifndef ERTS_SMP
+/* In non-smp case we *need* the async ready queue */
+#  undef ERTS_USE_ASYNC_READY_Q
+#  define ERTS_USE_ASYNC_READY_Q 1
+#endif
+
+#ifndef ERTS_USE_ASYNC_READY_Q
+#  define ERTS_USE_ASYNC_READY_Q 0
+#endif
+
+#if ERTS_USE_ASYNC_READY_Q
+int erts_check_async_ready(void *);
+int erts_async_ready_clean(void *, void *);
+void *erts_get_async_ready_queue(Uint sched_id);
+#define ERTS_ASYNC_READY_CLEAN 0
+#define ERTS_ASYNC_READY_DIRTY 1
+#ifdef ERTS_SMP
+#define ERTS_ASYNC_READY_NEED_THR_PRGR 2
+#endif
+#endif /* ERTS_USE_ASYNC_READY_Q */
+
+#endif /* USE_THREADS */
+
+void erts_init_async(void);
+void erts_exit_flush_async(void);
+
+
+#endif /* ERL_ASYNC_H__ */
diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c
index 3035e5df16..c50fdeb4e8 100644
--- a/erts/emulator/beam/erl_bestfit_alloc.c
+++ b/erts/emulator/beam/erl_bestfit_alloc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -84,24 +84,24 @@
 
 
 #ifdef HARD_DEBUG
-static RBTree_t * check_tree(BFAllctr_t *, Uint);
+static RBTree_t * check_tree(RBTree_t, int, Uint);
 #endif
 
-static void tree_delete(Allctr_t *allctr, Block_t *del);
+static void tree_delete(Allctr_t *allctr, Block_t *del, Uint32 flags);
 
 /* Prototypes of callback functions */
 
 /* "address order best fit" specific callback functions */
 static Block_t *	aobf_get_free_block	(Allctr_t *, Uint,
-						 Block_t *, Uint);
-static void		aobf_link_free_block	(Allctr_t *, Block_t *);
+						 Block_t *, Uint, Uint32);
+static void		aobf_link_free_block	(Allctr_t *, Block_t *, Uint32);
 #define			aobf_unlink_free_block	tree_delete
 
 /* "best fit" specific callback functions */
 static Block_t *	bf_get_free_block	(Allctr_t *, Uint,
-						 Block_t *, Uint);
-static void		bf_link_free_block	(Allctr_t *, Block_t *);
-static ERTS_INLINE void	bf_unlink_free_block	(Allctr_t *, Block_t *);
+						 Block_t *, Uint, Uint32);
+static void		bf_link_free_block	(Allctr_t *, Block_t *, Uint32);
+static ERTS_INLINE void	bf_unlink_free_block	(Allctr_t *, Block_t *, Uint32);
 
 
 static Eterm		info_options		(Allctr_t *, char *, int *,
@@ -161,14 +161,18 @@ erts_bfalc_start(BFAllctr_t *bfallctr,
 		 BFAllctrInit_t *bfinit,
 		 AllctrInit_t *init)
 {
-    BFAllctr_t nulled_state = {{0}};
-    /* {{0}} is used instead of {0}, in order to avoid (an incorrect) gcc
-       warning. gcc warns if {0} is used as initializer of a struct when
-       the first member is a struct (not if, for example, the third member
-       is a struct). */
+    struct {
+	int dummy;
+	BFAllctr_t allctr;
+    } zero = {0};
+    /* The struct with a dummy element first is used in order to avoid (an
+       incorrect) gcc warning. gcc warns if {0} is used as initializer of
+       a struct when the first member is a struct (not if, for example,
+       the third member is a struct). */
+
     Allctr_t *allctr = (Allctr_t *) bfallctr;
 
-    sys_memcpy((void *) bfallctr, (void *) &nulled_state, sizeof(BFAllctr_t));
+    sys_memcpy((void *) bfallctr, (void *) &zero.allctr, sizeof(BFAllctr_t));
 
     bfallctr->address_order		= bfinit->ao;
 
@@ -303,7 +307,7 @@ replace(RBTree_t **root, RBTree_t *x, RBTree_t *y)
 }
 
 static void
-tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk)
+tree_insert_fixup(RBTree_t **root, RBTree_t *blk)
 {
     RBTree_t *x = blk, *y;
 
@@ -311,7 +315,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk)
      * Rearrange the tree so that it satisfies the Red-Black Tree properties
      */
 
-    RBT_ASSERT(x != bfallctr->root && IS_RED(x->parent));
+    RBT_ASSERT(x != *root && IS_RED(x->parent));
     do {
 
 	/*
@@ -336,7 +340,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk)
 
 		if (x == x->parent->right) {
 		    x = x->parent;
-		    left_rotate(&bfallctr->root, x);
+		    left_rotate(root, x);
 		}
 
 		RBT_ASSERT(x == x->parent->parent->left->left);
@@ -347,7 +351,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk)
 
 		SET_BLACK(x->parent);
 		SET_RED(x->parent->parent);
-		right_rotate(&bfallctr->root, x->parent->parent);
+		right_rotate(root, x->parent->parent);
 
 		RBT_ASSERT(x == x->parent->left);
 		RBT_ASSERT(IS_RED(x));
@@ -370,7 +374,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk)
 
 		if (x == x->parent->left) {
 		    x = x->parent;
-		    right_rotate(&bfallctr->root, x);
+		    right_rotate(root, x);
 		}
 
 		RBT_ASSERT(x == x->parent->parent->right->right);
@@ -381,7 +385,7 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk)
 
 		SET_BLACK(x->parent);
 		SET_RED(x->parent->parent);
-		left_rotate(&bfallctr->root, x->parent->parent);
+		left_rotate(root, x->parent->parent);
 
 		RBT_ASSERT(x == x->parent->right);
 		RBT_ASSERT(IS_RED(x));
@@ -390,9 +394,9 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk)
 		break;
 	    }
 	}
-    } while (x != bfallctr->root && IS_RED(x->parent));
+    } while (x != *root && IS_RED(x->parent));
 
-    SET_BLACK(bfallctr->root);
+    SET_BLACK(*root);
 
 }
 
@@ -402,18 +406,22 @@ tree_insert_fixup(BFAllctr_t *bfallctr, RBTree_t *blk)
  * callback function in the address order case.
  */
 static void
-tree_delete(Allctr_t *allctr, Block_t *del)
+tree_delete(Allctr_t *allctr, Block_t *del, Uint32 flags)
 {
     BFAllctr_t *bfallctr = (BFAllctr_t *) allctr;
     Uint spliced_is_black;
+    RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC)
+		       ? &bfallctr->sbmbc_root
+		       : &bfallctr->mbc_root);
     RBTree_t *x, *y, *z = (RBTree_t *) del;
     RBTree_t null_x; /* null_x is used to get the fixup started when we
 			splice out a node without children. */
 
     null_x.parent = NULL;
 
+
 #ifdef HARD_DEBUG
-    check_tree(bfallctr, 0);
+    check_tree(*root, bfallctr->address_order, 0);
 #endif
 
     /* Remove node from tree... */
@@ -440,8 +448,8 @@ tree_delete(Allctr_t *allctr, Block_t *del)
     }
 
     if (!y->parent) {
-	RBT_ASSERT(bfallctr->root == y);
-	bfallctr->root = x;
+	RBT_ASSERT(*root == y);
+	*root = x;
     }
     else if (y == y->parent->left)
 	y->parent->left = x;
@@ -451,7 +459,7 @@ tree_delete(Allctr_t *allctr, Block_t *del)
     }
     if (y != z) {
 	/* We spliced out the successor of z; replace z by the successor */
-	replace(&bfallctr->root, z, y);
+	replace(root, z, y);
     }
 
     if (spliced_is_black) {
@@ -476,7 +484,7 @@ tree_delete(Allctr_t *allctr, Block_t *del)
 		    SET_BLACK(y);
 		    RBT_ASSERT(IS_BLACK(x->parent));
 		    SET_RED(x->parent);
-		    left_rotate(&bfallctr->root, x->parent);
+		    left_rotate(root, x->parent);
 		    y = x->parent->right;
 		}
 		RBT_ASSERT(y);
@@ -489,7 +497,7 @@ tree_delete(Allctr_t *allctr, Block_t *del)
 		    if (IS_BLACK(y->right)) {
 			SET_BLACK(y->left);
 			SET_RED(y);
-			right_rotate(&bfallctr->root, y);
+			right_rotate(root, y);
 			y = x->parent->right;
 		    }
 		    RBT_ASSERT(y);
@@ -500,8 +508,8 @@ tree_delete(Allctr_t *allctr, Block_t *del)
 		    }
 		    RBT_ASSERT(y->right);
 		    SET_BLACK(y->right);
-		    left_rotate(&bfallctr->root, x->parent);
-		    x = bfallctr->root;
+		    left_rotate(root, x->parent);
+		    x = *root;
 		    break;
 		}
 	    }
@@ -515,7 +523,7 @@ tree_delete(Allctr_t *allctr, Block_t *del)
 		    SET_BLACK(y);
 		    RBT_ASSERT(IS_BLACK(x->parent));
 		    SET_RED(x->parent);
-		    right_rotate(&bfallctr->root, x->parent);
+		    right_rotate(root, x->parent);
 		    y = x->parent->left;
 		}
 		RBT_ASSERT(y);
@@ -528,7 +536,7 @@ tree_delete(Allctr_t *allctr, Block_t *del)
 		    if (IS_BLACK(y->left)) {
 			SET_BLACK(y->right);
 			SET_RED(y);
-			left_rotate(&bfallctr->root, y);
+			left_rotate(root, y);
 			y = x->parent->left;
 		    }
 		    RBT_ASSERT(y);
@@ -538,8 +546,8 @@ tree_delete(Allctr_t *allctr, Block_t *del)
 		    }
 		    RBT_ASSERT(y->left);
 		    SET_BLACK(y->left);
-		    right_rotate(&bfallctr->root, x->parent);
-		    x = bfallctr->root;
+		    right_rotate(root, x->parent);
+		    x = *root;
 		    break;
 		}
 	    }
@@ -556,8 +564,8 @@ tree_delete(Allctr_t *allctr, Block_t *del)
 	    RBT_ASSERT(!null_x.left);
 	    RBT_ASSERT(!null_x.right);
 	}
-	else if (bfallctr->root == &null_x) {
-	    bfallctr->root = NULL;
+	else if (*root == &null_x) {
+	    *root = NULL;
 	    RBT_ASSERT(!null_x.left);
 	    RBT_ASSERT(!null_x.right);
 	}
@@ -567,7 +575,7 @@ tree_delete(Allctr_t *allctr, Block_t *del)
     DESTROY_TREE_NODE(del);
 
 #ifdef HARD_DEBUG
-    check_tree(bfallctr, 0);
+    check_tree(root, bfallctr->address_order, 0);
 #endif
 
 }
@@ -577,23 +585,28 @@ tree_delete(Allctr_t *allctr, Block_t *del)
 \*                                                                           */
 
 static void
-aobf_link_free_block(Allctr_t *allctr, Block_t *block)
+aobf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
 {
     BFAllctr_t *bfallctr = (BFAllctr_t *) allctr;
+    RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC)
+		       ? &bfallctr->sbmbc_root
+		       : &bfallctr->mbc_root);
     RBTree_t *blk = (RBTree_t *) block;
     Uint blk_sz = BLK_SZ(blk);
 
+    
+
     blk->flags	= 0;
     blk->left	= NULL;
     blk->right	= NULL;
 
-    if (!bfallctr->root) {
+    if (!*root) {
 	blk->parent = NULL;
 	SET_BLACK(blk);
-	bfallctr->root = blk;
+	*root = blk;
     }
     else {
-	RBTree_t *x = bfallctr->root;
+	RBTree_t *x = *root;
 	while (1) {
 	    Uint size;
 
@@ -623,28 +636,32 @@ aobf_link_free_block(Allctr_t *allctr, Block_t *block)
 
 	SET_RED(blk);
 	if (IS_RED(blk->parent))
-	    tree_insert_fixup(bfallctr, blk);
+	    tree_insert_fixup(root, blk);
     }
 
 #ifdef HARD_DEBUG
-    check_tree(bfallctr, 0);
+    check_tree(root, 1, 0);
 #endif
 }
 
 #if 0 /* tree_delete() is directly used instead */
 static void
-aobf_unlink_free_block(Allctr_t *allctr, Block_t *block)
+aobf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
 {
-    tree_delete(allctr, block);
+    tree_delete(allctr, block, flags);
 }
 #endif
 
 static Block_t *
 aobf_get_free_block(Allctr_t *allctr, Uint size,
-		    Block_t *cand_blk, Uint cand_size)
+		    Block_t *cand_blk, Uint cand_size,
+		    Uint32 flags)
 {
     BFAllctr_t *bfallctr = (BFAllctr_t *) allctr;
-    RBTree_t *x = bfallctr->root;
+    RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC)
+		       ? &bfallctr->sbmbc_root
+		       : &bfallctr->mbc_root);
+    RBTree_t *x = *root;
     RBTree_t *blk = NULL;
     Uint blk_sz;
 
@@ -665,7 +682,7 @@ aobf_get_free_block(Allctr_t *allctr, Uint size,
 	return NULL;
 
 #ifdef HARD_DEBUG
-    ASSERT(blk == check_tree(bfallctr, size));
+    ASSERT(blk == check_tree(root, 1, size));
 #endif
 
     if (cand_blk) {
@@ -676,7 +693,7 @@ aobf_get_free_block(Allctr_t *allctr, Uint size,
 	    return NULL; /* cand_blk was better */
     }
 
-    aobf_unlink_free_block(allctr, (Block_t *) blk);
+    aobf_unlink_free_block(allctr, (Block_t *) blk, flags);
 
     return (Block_t *) blk;
 }
@@ -687,9 +704,12 @@ aobf_get_free_block(Allctr_t *allctr, Uint size,
 \*                                                                           */
 
 static void
-bf_link_free_block(Allctr_t *allctr, Block_t *block)
+bf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
 {
     BFAllctr_t *bfallctr = (BFAllctr_t *) allctr;
+    RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC)
+		       ? &bfallctr->sbmbc_root
+		       : &bfallctr->mbc_root);
     RBTree_t *blk = (RBTree_t *) block;
     Uint blk_sz = BLK_SZ(blk);
 
@@ -700,13 +720,13 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block)
     blk->left	= NULL;
     blk->right	= NULL;
 
-    if (!bfallctr->root) {
+    if (!*root) {
 	blk->parent = NULL;
 	SET_BLACK(blk);
-	bfallctr->root = blk;
+	*root = blk;
     }
     else {
-	RBTree_t *x = bfallctr->root;
+	RBTree_t *x = *root;
 	while (1) {
 	    Uint size;
 
@@ -745,7 +765,7 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block)
 
 	SET_RED(blk);
 	if (IS_RED(blk->parent))
-	    tree_insert_fixup(bfallctr, blk);
+	    tree_insert_fixup(root, blk);
 
     }
 
@@ -753,14 +773,17 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block)
     LIST_NEXT(blk) = NULL;
 
 #ifdef HARD_DEBUG
-    check_tree(bfallctr, 0);
+    check_tree(root, 0, 0);
 #endif
 }
 
 static ERTS_INLINE void
-bf_unlink_free_block(Allctr_t *allctr, Block_t *block)
+bf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
 {
     BFAllctr_t *bfallctr = (BFAllctr_t *) allctr;
+    RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC)
+		       ? &bfallctr->sbmbc_root
+		       : &bfallctr->mbc_root);
     RBTree_t *x = (RBTree_t *) block;
 
     if (IS_LIST_ELEM(x)) {
@@ -778,9 +801,9 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block)
 	ASSERT(IS_LIST_ELEM(LIST_NEXT(x)));
 
 #ifdef HARD_DEBUG
-	check_tree(bfallctr, 0);
+	check_tree(root, 0, 0);
 #endif
-	replace(&bfallctr->root, x, LIST_NEXT(x));
+	replace(root, x, LIST_NEXT(x));
 
 #ifdef HARD_DEBUG
 	check_tree(bfallctr, 0);
@@ -788,7 +811,7 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block)
     }
     else {
 	/* Remove from tree */
-	tree_delete(allctr, block);
+	tree_delete(allctr, block, flags);
     }
 
     DESTROY_LIST_ELEM(x);
@@ -797,10 +820,14 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block)
 
 static Block_t *
 bf_get_free_block(Allctr_t *allctr, Uint size,
-		  Block_t *cand_blk, Uint cand_size)
+		  Block_t *cand_blk, Uint cand_size,
+		  Uint32 flags)
 {
     BFAllctr_t *bfallctr = (BFAllctr_t *) allctr;
-    RBTree_t *x = bfallctr->root;
+    RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC)
+		       ? &bfallctr->sbmbc_root
+		       : &bfallctr->mbc_root);
+    RBTree_t *x = *root;
     RBTree_t *blk = NULL;
     Uint blk_sz;
 
@@ -827,7 +854,7 @@ bf_get_free_block(Allctr_t *allctr, Uint size,
 
 #ifdef HARD_DEBUG
     {
-	RBTree_t *ct_blk = check_tree(bfallctr, size);
+	RBTree_t *ct_blk = check_tree(root, 0, size);
 	ASSERT(BLK_SZ(ct_blk) == BLK_SZ(blk));
     }
 #endif
@@ -839,7 +866,7 @@ bf_get_free_block(Allctr_t *allctr, Uint size,
        the tree node */
     blk = LIST_NEXT(blk) ? LIST_NEXT(blk) : blk;
 
-    bf_unlink_free_block(allctr, (Block_t *) blk);
+    bf_unlink_free_block(allctr, (Block_t *) blk, flags);
     return (Block_t *) blk;
 }
 
@@ -949,13 +976,14 @@ erts_bfalc_test(unsigned long op, unsigned long a1, unsigned long a2)
 {
     switch (op) {
     case 0x200:	return (unsigned long) ((BFAllctr_t *) a1)->address_order;
-    case 0x201:	return (unsigned long) ((BFAllctr_t *) a1)->root;
+    case 0x201:	return (unsigned long) ((BFAllctr_t *) a1)->mbc_root;
     case 0x202:	return (unsigned long) ((RBTree_t *) a1)->parent;
     case 0x203:	return (unsigned long) ((RBTree_t *) a1)->left;
     case 0x204:	return (unsigned long) ((RBTree_t *) a1)->right;
     case 0x205:	return (unsigned long) ((RBTreeList_t *) a1)->next;
     case 0x206:	return (unsigned long) IS_BLACK((RBTree_t *) a1);
     case 0x207:	return (unsigned long) IS_TREE_NODE((RBTree_t *) a1);
+    case 0x208:	return (unsigned long) 0; /* IS_AOFF */
     default:	ASSERT(0); return ~((unsigned long) 0);
     }
 }
@@ -985,7 +1013,7 @@ erts_bfalc_test(unsigned long op, unsigned long a1, unsigned long a2)
 #endif
 
 #ifdef PRINT_TREE
-static void print_tree(BFAllctr_t *);
+static void print_tree(RBTree_t *, int);
 #endif
 
 /*
@@ -1003,7 +1031,7 @@ static void print_tree(BFAllctr_t *);
  */
 
 static RBTree_t *
-check_tree(BFAllctr_t *bfallctr, Uint size)
+check_tree(RBTree_t *root, int ao, Uint size)
 {
     RBTree_t *res = NULL;
     Sint blacks;
@@ -1011,13 +1039,13 @@ check_tree(BFAllctr_t *bfallctr, Uint size)
     RBTree_t *x;
 
 #ifdef PRINT_TREE
-    print_tree(bfallctr);
+    print_tree(root, ao);
 #endif
 
-    if (!bfallctr->root)
+    if (!root)
 	return res;
 
-    x = bfallctr->root;
+    x = root;
     ASSERT(IS_BLACK(x));
     ASSERT(!x->parent);
     curr_blacks = 1;
@@ -1060,11 +1088,11 @@ check_tree(BFAllctr_t *bfallctr, Uint size)
 	    ASSERT(IS_BLACK(x->left));
 	}
 
-	ASSERT(x->parent || x == bfallctr->root);
+	ASSERT(x->parent || x == root);
 
 	if (x->left) {
 	    ASSERT(x->left->parent == x);
-	    if (bfallctr->address_order) {
+	    if (ao) {
 		ASSERT(BLK_SZ(x->left) < BLK_SZ(x)
 		       || (BLK_SZ(x->left) == BLK_SZ(x) && x->left < x));
 	    }
@@ -1076,7 +1104,7 @@ check_tree(BFAllctr_t *bfallctr, Uint size)
 
 	if (x->right) {
 	    ASSERT(x->right->parent == x);
-	    if (bfallctr->address_order) {
+	    if (ao) {
 		ASSERT(BLK_SZ(x->right) > BLK_SZ(x)
 		       || (BLK_SZ(x->right) == BLK_SZ(x) && x->right > x));
 	    }
@@ -1087,7 +1115,7 @@ check_tree(BFAllctr_t *bfallctr, Uint size)
 	}
 
 	if (size && BLK_SZ(x) >= size) {
-	    if (bfallctr->address_order) {
+	    if (ao) {
 		if (!res
 		    || BLK_SZ(x) < BLK_SZ(res)
 		    || (BLK_SZ(x) == BLK_SZ(res) && x < res))
@@ -1109,8 +1137,8 @@ check_tree(BFAllctr_t *bfallctr, Uint size)
 
     ASSERT(curr_blacks == 0);
 
-    UNSET_LEFT_VISITED(bfallctr->root);
-    UNSET_RIGHT_VISITED(bfallctr->root);
+    UNSET_LEFT_VISITED(root);
+    UNSET_RIGHT_VISITED(root);
 
     return res;
 
@@ -1148,11 +1176,11 @@ print_tree_aux(RBTree_t *x, int indent)
 
 
 static void
-print_tree(BFAllctr_t *bfallctr)
+print_tree(RBTree_t *root, int ao)
 {
-    char *type = bfallctr->address_order ? "Size-Adress" : "Size";
+    char *type = ao ? "Size-Adress" : "Size";
     fprintf(stderr, " --- %s tree begin ---\r\n", type);
-    print_tree_aux(bfallctr->root, 0);
+    print_tree_aux(root, 0);
     fprintf(stderr, " --- %s tree end ---\r\n", type);
 }
 
diff --git a/erts/emulator/beam/erl_bestfit_alloc.h b/erts/emulator/beam/erl_bestfit_alloc.h
index cb35e21e57..0c29662852 100644
--- a/erts/emulator/beam/erl_bestfit_alloc.h
+++ b/erts/emulator/beam/erl_bestfit_alloc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -54,7 +54,8 @@ typedef struct RBTree_t_ RBTree_t;
 struct BFAllctr_t_ {
     Allctr_t		allctr; /* Has to be first! */
 
-    RBTree_t *		root;
+    RBTree_t *		mbc_root;
+    RBTree_t *		sbmbc_root;
     int 		address_order;
 };
 
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
new file mode 100644
index 0000000000..cc4f2be8eb
--- /dev/null
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -0,0 +1,2955 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * NOTE: This file contains the BIF's for the *module* binary in stdlib.
+ * other BIF's concerning binaries are in binary.c.
+ */
+
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "sys.h"
+#include "erl_vm.h"
+#include "global.h"
+#include "erl_process.h"
+#include "error.h"
+#include "bif.h"
+#include "big.h"
+#include "erl_binary.h"
+#include "erl_bits.h"
+
+
+/*
+ * The native implementation functions for the module binary.
+ * Searching is implemented using either Boyer-Moore or Aho-Corasick
+ * depending on number of searchstrings (BM if one, AC if more than one).
+ * Native implementation is mostly for efficiency, nothing
+ * (except binary:referenced_byte_size) really *needs* to be implemented
+ * in native code.
+ */
+
+/* #define HARDDEBUG */
+
+/* Init and local variables */
+
+static Export binary_match_trap_export;
+static BIF_RETTYPE binary_match_trap(BIF_ALIST_3);
+static Export binary_matches_trap_export;
+static BIF_RETTYPE binary_matches_trap(BIF_ALIST_3);
+static Export binary_longest_prefix_trap_export;
+static BIF_RETTYPE binary_longest_prefix_trap(BIF_ALIST_3);
+static Export binary_longest_suffix_trap_export;
+static BIF_RETTYPE binary_longest_suffix_trap(BIF_ALIST_3);
+static Export binary_bin_to_list_trap_export;
+static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3);
+static Export binary_copy_trap_export;
+static BIF_RETTYPE binary_copy_trap(BIF_ALIST_2);
+static Uint max_loop_limit;
+
+static BIF_RETTYPE
+binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3);
+static BIF_RETTYPE
+binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3);
+
+void erts_init_bif_binary(void)
+{
+    sys_memset((void *) &binary_match_trap_export, 0, sizeof(Export));
+    binary_match_trap_export.address = &binary_match_trap_export.code[3];
+    binary_match_trap_export.code[0] = am_erlang;
+    binary_match_trap_export.code[1] = am_binary_match_trap;
+    binary_match_trap_export.code[2] = 3;
+    binary_match_trap_export.code[3] = (BeamInstr) em_apply_bif;
+    binary_match_trap_export.code[4] = (BeamInstr) &binary_match_trap;
+
+    sys_memset((void *) &binary_matches_trap_export, 0, sizeof(Export));
+    binary_matches_trap_export.address = &binary_matches_trap_export.code[3];
+    binary_matches_trap_export.code[0] = am_erlang;
+    binary_matches_trap_export.code[1] = am_binary_matches_trap;
+    binary_matches_trap_export.code[2] = 3;
+    binary_matches_trap_export.code[3] = (BeamInstr) em_apply_bif;
+    binary_matches_trap_export.code[4] = (BeamInstr) &binary_matches_trap;
+
+    sys_memset((void *) &binary_longest_prefix_trap_export, 0, sizeof(Export));
+    binary_longest_prefix_trap_export.address = &binary_longest_prefix_trap_export.code[3];
+    binary_longest_prefix_trap_export.code[0] = am_erlang;
+    binary_longest_prefix_trap_export.code[1] = am_binary_longest_prefix_trap;
+    binary_longest_prefix_trap_export.code[2] = 3;
+    binary_longest_prefix_trap_export.code[3] = (BeamInstr) em_apply_bif;
+    binary_longest_prefix_trap_export.code[4] = (BeamInstr) &binary_longest_prefix_trap;
+
+    sys_memset((void *) &binary_longest_suffix_trap_export, 0, sizeof(Export));
+    binary_longest_suffix_trap_export.address = &binary_longest_suffix_trap_export.code[3];
+    binary_longest_suffix_trap_export.code[0] = am_erlang;
+    binary_longest_suffix_trap_export.code[1] = am_binary_longest_suffix_trap;
+    binary_longest_suffix_trap_export.code[2] = 3;
+    binary_longest_suffix_trap_export.code[3] = (BeamInstr) em_apply_bif;
+    binary_longest_suffix_trap_export.code[4] = (BeamInstr) &binary_longest_suffix_trap;
+
+    sys_memset((void *) &binary_bin_to_list_trap_export, 0, sizeof(Export));
+    binary_bin_to_list_trap_export.address = &binary_bin_to_list_trap_export.code[3];
+    binary_bin_to_list_trap_export.code[0] = am_erlang;
+    binary_bin_to_list_trap_export.code[1] = am_binary_bin_to_list_trap;
+    binary_bin_to_list_trap_export.code[2] = 3;
+    binary_bin_to_list_trap_export.code[3] = (BeamInstr) em_apply_bif;
+    binary_bin_to_list_trap_export.code[4] = (BeamInstr) &binary_bin_to_list_trap;
+    sys_memset((void *) &binary_copy_trap_export, 0, sizeof(Export));
+    binary_copy_trap_export.address = &binary_copy_trap_export.code[3];
+    binary_copy_trap_export.code[0] = am_erlang;
+    binary_copy_trap_export.code[1] = am_binary_copy_trap;
+    binary_copy_trap_export.code[2] = 2;
+    binary_copy_trap_export.code[3] = (BeamInstr) em_apply_bif;
+    binary_copy_trap_export.code[4] = (BeamInstr) &binary_copy_trap;
+
+    max_loop_limit = 0;
+    return;
+}
+
+/*
+ * Setting the loop_limit for searches for debugging
+ */
+Sint erts_binary_set_loop_limit(Sint limit)
+{
+    Sint save = (Sint) max_loop_limit;
+    if (limit <= 0) {
+	max_loop_limit = 0;
+    } else {
+	max_loop_limit = (Uint) limit;
+    }
+
+    return save;
+}
+
+static Uint get_reds(Process *p, int loop_factor)
+{
+    Uint reds = ERTS_BIF_REDS_LEFT(p) * loop_factor;
+    Uint tmp = max_loop_limit;
+    if (tmp != 0 && tmp < reds) {
+	return tmp;
+    }
+    if (!reds) {
+	reds = 1;
+    }
+    return reds;
+}
+
+/*
+ * A micro allocator used when building search structures, just a convenience
+ * for building structures inside a pre-allocated magic binary using
+ * conventional malloc-like interface.
+ */
+
+#define MYALIGN(Size) (SIZEOF_VOID_P * (((Size) / SIZEOF_VOID_P) + \
+                       !!(((Size) % SIZEOF_VOID_P))))
+
+#ifdef DEBUG
+#define CHECK_ALLOCATOR(My) ASSERT((My).current <= ((My).mem + (My).size))
+#else
+#define CHECK_ALLOCATOR(My) /* nothing */
+#endif
+
+typedef struct _my_allocator {
+    Uint size;
+    byte *current;
+    byte *mem;
+} MyAllocator;
+
+static void init_my_allocator(MyAllocator *my, Uint siz, byte *array)
+{
+    ASSERT((siz % SIZEOF_VOID_P) == 0);
+    my->size = siz;
+    my->mem = array;
+    my->current = my->mem;
+}
+
+static void *my_alloc(MyAllocator *my, Uint size)
+{
+    void *ptr = my->current;
+    my->current += MYALIGN(size);
+    return ptr;
+}
+
+/*
+ * The search functionality.
+ *
+ * The search is byte oriented, which works nicely for UTF-8 as well as
+ * latin1 data
+ */
+
+#define ALPHABET_SIZE 256
+
+typedef struct _ac_node {
+#ifdef HARDDEBUG
+    Uint32 id;                        /* To identify h pointer targets when
+					 dumping */
+#endif
+    Uint32 d;                         /* Depth in trie, also represents the
+					 length (-1) of the matched string if
+					 in final set */
+    Sint32 final;                     /* Members in final set represent
+				       * matches.
+				       * The set representation is scattered
+				       * among the nodes in this way:
+				       * >0 -> this represents a member of
+				       * the final set, <0 -> member of
+				       * final set somewhere in the failure
+				       * chain,
+				       * 0 -> not member of the final set */
+    struct _ac_node *h;                /* h(Hode) is the failure function */
+    struct _ac_node *g[ALPHABET_SIZE]; /* g(Node,Character) is the
+					  transition function */
+} ACNode;
+
+typedef struct _ac_trie {
+#ifdef HARDDEBUG
+    Uint32 idc;
+#endif
+    Uint32 counter;                    /* Number of added patterns */
+    ACNode *root;                      /* pointer to the root state */
+} ACTrie;
+
+typedef struct _bm_data {
+    byte *x;
+    Sint len;
+    Sint *goodshift;
+    Sint badshift[ALPHABET_SIZE];
+} BMData;
+
+#ifdef HARDDEBUG
+static void dump_bm_data(BMData *bm);
+static void dump_ac_trie(ACTrie *act);
+static void dump_ac_node(ACNode *node, int indent, int ch);
+#endif
+
+/*
+ * The needed size of binary data for a search structure - given the
+ * accumulated string lengths.
+ */
+#define BM_SIZE(StrLen) 	      /* StrLen: length of searchstring */ \
+((MYALIGN(sizeof(Sint) * (StrLen))) + /* goodshift array */                \
+ MYALIGN(StrLen) +                    /* searchstring saved */             \
+ (MYALIGN(sizeof(BMData))))           /* Structure */
+
+#define AC_SIZE(StrLens)       /* StrLens: sum of all searchstring lengths */ \
+((MYALIGN(sizeof(ACNode)) *                                                   \
+((StrLens)+1)) + 	       /* The actual nodes (including rootnode) */    \
+ MYALIGN(sizeof(ACTrie)))      /* Structure */
+
+
+#ifndef MAX
+#define MAX(A,B) (((A) > (B)) ? (A) : (B))
+#endif
+
+#ifndef MIN
+#define MIN(A,B) (((A) > (B)) ? (B) : (A))
+#endif
+/*
+ * Callback for the magic binary
+ */
+static void cleanup_my_data_ac(Binary *bp)
+{
+    return;
+}
+static void cleanup_my_data_bm(Binary *bp)
+{
+    return;
+}
+
+/*
+ * Initiate a (allocated) micro allocator and fill in the base
+ * for an Aho-Corasick search trie, given the accumulated length of the search
+ * strings.
+ */
+static ACTrie *create_acdata(MyAllocator *my, Uint len,
+			     ACNode ***qbuff /* out */,
+			     Binary **the_bin /* out */)
+{
+    Uint datasize = AC_SIZE(len);
+    ACTrie *act;
+    ACNode *acn;
+    Binary *mb = erts_create_magic_binary(datasize,cleanup_my_data_ac);
+    byte *data = ERTS_MAGIC_BIN_DATA(mb);
+
+    init_my_allocator(my, datasize, data);
+    act = my_alloc(my, sizeof(ACTrie)); /* Important that this is the first
+					   allocation */
+    act->counter = 0;
+    act->root = acn = my_alloc(my, sizeof(ACNode));
+    acn->d = 0;
+    acn->final = 0;
+    acn->h = NULL;
+    memset(acn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE);
+#ifdef HARDDEBUG
+    act->idc = 0;
+    acn->id = 0;
+#endif
+    *qbuff = erts_alloc(ERTS_ALC_T_TMP, sizeof(ACNode *) * len);
+    *the_bin = mb;
+    return act;
+}
+
+/*
+ * The same initialization of allocator and basic data for Boyer-Moore.
+ */
+static BMData *create_bmdata(MyAllocator *my, byte *x, Uint len,
+			     Binary **the_bin /* out */)
+{
+    Uint datasize = BM_SIZE(len);
+    BMData *bmd;
+    Binary *mb = erts_create_magic_binary(datasize,cleanup_my_data_bm);
+    byte *data = ERTS_MAGIC_BIN_DATA(mb);
+    init_my_allocator(my, datasize, data);
+    bmd = my_alloc(my, sizeof(BMData));
+    bmd->x = my_alloc(my,len);
+    memcpy(bmd->x,x,len);
+    bmd->len = len;
+    bmd->goodshift = my_alloc(my,sizeof(Uint) * len);
+    *the_bin = mb;
+    return bmd;
+}
+
+/*
+ * Compilation of search structures
+ */
+
+/*
+ * Aho Corasick - Build a Trie and fill in the failure functions
+ * when all strings are added.
+ * The algorithm is nicely described by Dieter B�hler of University of
+ * T�bingen:
+ * http://www-sr.informatik.uni-tuebingen.de/~buehler/AC/AC.html
+ */
+
+/*
+ * Helper called once for each search pattern
+ */
+static void ac_add_one_pattern(MyAllocator *my, ACTrie *act, byte *x, Uint len)
+{
+    ACNode *acn = act->root;
+    Uint32 n = ++act->counter; /* Always increase counter, even if it's a
+				  duplicate as this may identify the pattern
+				  in the final set (not in current interface
+				  though) */
+    Uint i = 0;
+
+    while(i < len) {
+	if (acn->g[x[i]] != NULL) {
+	    /* node exists, continue */
+	    acn = acn->g[x[i]];
+	    ++i;
+	} else {
+	    /* allocate a new node */
+	    ACNode *nn = my_alloc(my,sizeof(ACNode));
+#ifdef HARDDEBUG
+	    nn->id = ++(act->idc);
+#endif
+	    nn->d = i+1;
+	    nn->h = act->root;
+	    nn->final = 0;
+	    memset(nn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE);
+	    acn->g[x[i]] = nn;
+	    ++i;
+	    acn = nn;
+	}
+    }
+    if (acn->final == 0) { /* New pattern, add to final set */
+	acn->final = n;
+    }
+}
+
+/*
+ * Called when all search patterns are added.
+ */
+static void ac_compute_failure_functions(ACTrie *act, ACNode **qbuff)
+{
+    ACNode *root = act->root;
+    ACNode *parent;
+    int i;
+    int qh = 0,qt = 0;
+    ACNode *child, *r;
+
+    /* Set all children of the root to have the root as failure function */
+    for (i = 0; i < ALPHABET_SIZE; ++i) {
+	if (root->g[i] != NULL) {
+	    root->g[i]->h = root;
+	    /* Add to que for later traversal */
+	    qbuff[qt++] = root->g[i];
+	}
+    }
+
+    /* So, now we've handled children of the root state, traverse the
+       rest of the trie BF... */
+    while (qh < qt) {
+	parent = qbuff[qh++];
+	for (i = 0; i < ALPHABET_SIZE; ++ i) {
+	    if ((child = parent->g[i]) != NULL) {
+		/* Visit this node to */
+		qbuff[qt++] = child;
+		/* Search for correct failure function, follow the parent's
+		   failure function until you find a similar transition
+		   funtion to this child's */
+		r =  parent->h;
+		while (r != NULL && r->g[i] == NULL) {
+		    r = r->h;
+		}
+		if (r == NULL) {
+		    /* Replace NULL failures with the root as we go */
+		    child->h = (root->g[i] == NULL) ? root : root->g[i];
+		} else {
+		    child->h = r->g[i];
+		    /*
+		     * The "final" set is scattered among the nodes. When
+		     * the failure function points to a member of the final
+		     * set, we have a match, but we might not see it in the
+		     * current node if we dont mark it as a special type of
+		     * final, i.e. foolow the failure function and you will
+		     * find a real member of final set. This is marked with
+		     * a negative string id and only done if this node does
+		     * not represent a member in the final set.
+		     */
+		    if (!(child->final) && (child->h->final)) {
+			child->final = -1;
+		    }
+		}
+	    }
+	}
+    }
+    /* Finally the failure function of the root should point to itself */
+    root->h = root;
+}
+
+
+/*
+ * The actual searching for needles in the haystack...
+ * Find first match using Aho-Coracick Trie
+ * return pattern number and fill in mpos + mlen if found, otherwise return 0
+ * Return the matching pattern that *starts* first, and ends
+ * last (difference when overlapping), hence the candidate thing.
+ * Basic AC finds the first end before the first start...
+ *
+ */
+typedef struct {
+    ACNode *q;
+    Uint pos;
+    Uint len;
+    ACNode *candidate;
+    Uint candidate_start;
+} ACFindFirstState;
+
+
+static void ac_init_find_first_match(ACFindFirstState *state, ACTrie *act, Sint startpos, Uint len)
+{
+    state->q = act->root;
+    state->pos = startpos;
+    state->len = len;
+    state->candidate = NULL;
+    state->candidate_start = 0;
+}
+#define AC_OK 0
+#define AC_NOT_FOUND -1
+#define AC_RESTART -2
+
+#define AC_LOOP_FACTOR 10
+
+static int ac_find_first_match(ACFindFirstState *state, byte *haystack,
+				Uint *mpos, Uint *mlen, Uint *reductions)
+{
+    ACNode *q = state->q;
+    Uint i = state->pos;
+    ACNode *candidate = state->candidate, *r;
+    Uint len = state->len;
+    Uint candidate_start = state->candidate_start;
+    Uint rstart;
+    register Uint reds = *reductions;
+
+    while (i < len) {
+	if (--reds == 0) {
+	    state->q = q;
+	    state->pos = i;
+	    state->len = len;
+	    state->candidate = candidate;
+	    state->candidate_start = candidate_start;
+	    return AC_RESTART;
+	}
+
+	while (q->g[haystack[i]] == NULL && q->h != q) {
+	    q = q->h;
+	}
+	if (q->g[haystack[i]] != NULL) {
+	    q = q->g[haystack[i]];
+	}
+#ifdef HARDDEBUG
+	erts_printf("ch = %c, Current: %u\n", (int) haystack[i], (unsigned) q->id);
+#endif
+	++i;
+	if (candidate != NULL && (i - q->d) > candidate_start) {
+	    break;
+	}
+	if (q->final) {
+	    r = q;
+	    while (r->final < 0)
+		r = r->h;
+	    rstart = i - r->d;
+	    if (candidate == NULL || rstart < candidate_start ||
+		(rstart == candidate_start && candidate->d < q->d)) {
+		candidate_start = rstart;
+		candidate = r;
+	    }
+	}
+    }
+    *reductions = reds;
+    if (!candidate) {
+	return AC_NOT_FOUND;
+    }
+#ifdef HARDDEBUG
+    dump_ac_node(candidate,0,'?');
+#endif
+    *mpos = candidate_start;
+    *mlen = candidate->d;
+    return AC_OK;
+}
+
+typedef struct _findall_data {
+    Uint pos;
+    Uint len;
+#ifdef HARDDEBUG
+    Uint id;
+#endif
+    Eterm epos;
+    Eterm elen;
+} FindallData;
+
+typedef struct {
+    ACNode *q;
+    Uint pos;
+    Uint len;
+    Uint m;
+    Uint allocated;
+    FindallData *out;
+} ACFindAllState;
+
+static void ac_init_find_all(ACFindAllState *state, ACTrie *act, Sint startpos, Uint len)
+{
+    state->q = act->root;
+    state->pos = startpos;
+    state->len = len;
+    state->m = 0;
+    state->allocated = 0;
+    state->out = NULL;
+}
+
+static void ac_restore_find_all(ACFindAllState *state, char *buff)
+{
+    memcpy(state,buff,sizeof(ACFindAllState));
+    if (state->allocated > 0) {
+	state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * (state->allocated));
+	memcpy(state->out,buff+sizeof(ACFindAllState),sizeof(FindallData)*state->m);
+    } else {
+	state->out = NULL;
+    }
+}
+
+static void ac_serialize_find_all(ACFindAllState *state, char *buff)
+{
+    memcpy(buff,state,sizeof(ACFindAllState));
+    memcpy(buff+sizeof(ACFindAllState),state->out,sizeof(FindallData)*state->m);
+}
+
+static void ac_clean_find_all(ACFindAllState *state)
+{
+    if (state->out != NULL) {
+	erts_free(ERTS_ALC_T_TMP, state->out);
+    }
+#ifdef HARDDEBUG
+    state->out = NULL;
+    state->allocated = 0;
+#endif
+}
+
+#define SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(S) \
+          (sizeof(ACFindAllState)+(sizeof(FindallData)*(S).m))
+
+/*
+ * Differs to the find_first function in that it stores all matches and the values
+ * arte returned only in the state.
+ */
+static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack,
+				       Uint *reductions)
+{
+    ACNode *q = state->q;
+    Uint i = state->pos;
+    Uint rstart;
+    ACNode *r;
+    Uint len = state->len;
+    Uint m = state->m, save_m;
+    Uint allocated = state->allocated;
+    FindallData *out = state->out;
+    register Uint reds = *reductions;
+
+
+    while (i < len) {
+	if (--reds == 0) {
+	    state->q = q;
+	    state->pos = i;
+	    state->len = len;
+	    state->m = m;
+	    state->allocated = allocated;
+	    state->out = out;
+	    return AC_RESTART;
+	}
+	while (q->g[haystack[i]] == NULL && q->h != q) {
+	    q = q->h;
+	}
+	if (q->g[haystack[i]] != NULL) {
+	    q = q->g[haystack[i]];
+	}
+	++i;
+	if (q->final) {
+	    r = q;
+	    while (r->final) {
+		while (r->final < 0)
+		    r = r->h;
+#ifdef HARDDEBUG
+		erts_printf("Trying to add %u\n",(unsigned) r->final);
+#endif
+		rstart = i - r->d;
+		save_m = m;
+		while (m > 0 && (out[m-1].pos > rstart ||
+				 (out[m-1].pos == rstart &&
+				  out[m-1].len < r->d))) {
+#ifdef HARDDEBUG
+		    erts_printf("Popping %u\n",(unsigned) out[m-1].id);
+#endif
+		    --m;
+		}
+#ifdef HARDDEBUG
+		if (m > 0) {
+		    erts_printf("Pos %u\n",out[m-1].pos);
+		    erts_printf("Len %u\n",out[m-1].len);
+		}
+		erts_printf("Rstart %u\n",rstart);
+#endif
+		if (m == 0 ||  out[m-1].pos + out[m-1].len <= rstart) {
+		    if (m >= allocated) {
+			if (!allocated) {
+			    allocated = 10;
+			    out = erts_alloc(ERTS_ALC_T_TMP,
+					     sizeof(FindallData) * allocated);
+			} else {
+			    allocated *= 2;
+			    out = erts_realloc(ERTS_ALC_T_TMP, out,
+					       sizeof(FindallData) *
+					       allocated);
+			}
+		    }
+		    out[m].pos = rstart;
+		    out[m].len = r->d;
+#ifdef HARDDEBUG
+		    out[m].id = r->final;
+#endif
+		    ++m;
+#ifdef HARDDEBUG
+		    erts_printf("Pushing %u\n",(unsigned) out[m-1].id);
+#endif
+		} else {
+#ifdef HARDDEBUG
+		    erts_printf("Backtracking %d steps\n",save_m - m);
+#endif
+		    m = save_m;
+		}
+		r = r->h;
+	    }
+	}
+    }
+    *reductions = reds;
+    state->m = m;
+    state->out = out;
+    return (m == 0) ? AC_NOT_FOUND : AC_OK;
+}
+
+/*
+ * Boyer Moore - most obviously implemented more or less exactly as
+ * Christian Charras and Thierry Lecroq describe it in "Handbook of
+ * Exact String-Matching Algorithms"
+ * http://www-igm.univ-mlv.fr/~lecroq/string/
+ */
+
+/*
+ * Call this to compute badshifts array
+ */
+static void compute_badshifts(BMData *bmd)
+{
+    Sint i;
+    Sint m = bmd->len;
+
+    for (i = 0; i < ALPHABET_SIZE; ++i) {
+	bmd->badshift[i] = m;
+    }
+    for (i = 0; i < m - 1; ++i) {
+	bmd->badshift[bmd->x[i]] = m - i - 1;
+    }
+}
+
+/* Helper for "compute_goodshifts" */
+static void compute_suffixes(byte *x, Sint m, Sint *suffixes)
+{
+    int f,g,i;
+
+    suffixes[m - 1] = m;
+
+    f = 0; /* To avoid use before set warning */
+
+    g = m - 1;
+
+    for (i = m - 2; i >= 0; --i) {
+	if (i > g && suffixes[i + m - 1 - f] < i - g) {
+	    suffixes[i] = suffixes[i + m - 1 - f];
+	} else {
+	    if (i < g) {
+		g = i;
+	    }
+	    f = i;
+	    while ( g >= 0 && x[g] == x[g + m - 1 - f] ) {
+		--g;
+	    }
+	    suffixes[i] = f - g;
+	}
+    }
+}
+
+/*
+ * Call this to compute goodshift array
+ */
+static void compute_goodshifts(BMData *bmd)
+{
+    Sint m = bmd->len;
+    byte *x = bmd->x;
+    Sint i, j;
+    Sint *suffixes = erts_alloc(ERTS_ALC_T_TMP, m * sizeof(Sint));
+
+    compute_suffixes(x, m, suffixes);
+
+    for (i = 0; i < m; ++i) {
+	bmd->goodshift[i] = m;
+    }
+
+    j = 0;
+
+    for (i = m - 1; i >= -1; --i) {
+	if (i == -1 || suffixes[i] == i + 1) {
+	    while (j < m - 1 - i) {
+		if (bmd->goodshift[j] == m) {
+		    bmd->goodshift[j] = m - 1 - i;
+		}
+		++j;
+	    }
+	}
+    }
+    for (i = 0; i <= m - 2; ++i) {
+	bmd->goodshift[m - 1 - suffixes[i]] = m - 1 - i;
+    }
+    erts_free(ERTS_ALC_T_TMP, suffixes);
+}
+
+typedef struct {
+    Sint pos;
+    Sint len;
+} BMFindFirstState;
+
+#define BM_OK 0 /* used only for find_all */
+#define BM_NOT_FOUND -1
+#define BM_RESTART -2
+#define BM_LOOP_FACTOR 10 /* Should we have a higher value? */
+
+static void bm_init_find_first_match(BMFindFirstState *state, Sint startpos,
+				     Uint len)
+{
+    state->pos = startpos;
+    state->len = (Sint) len;
+}
+
+
+static Sint bm_find_first_match(BMFindFirstState *state, BMData *bmd,
+				byte *haystack, Uint *reductions)
+{
+    Sint blen = bmd->len;
+    Sint len = state->len;
+    Sint *gs = bmd->goodshift;
+    Sint *bs = bmd->badshift;
+    byte *needle = bmd->x;
+    Sint i;
+    Sint j = state->pos;
+    register Uint reds = *reductions;
+
+    while (j <= len - blen) {
+	if (--reds == 0) {
+	    state->pos = j;
+	    return BM_RESTART;
+	}
+	for (i = blen - 1; i >= 0 && needle[i] == haystack[i + j]; --i)
+	    ;
+	if (i < 0) { /* found */
+	    *reductions = reds;
+	    return j;
+	}
+	j += MAX(gs[i],bs[haystack[i+j]] - blen + 1 + i);
+    }
+    *reductions = reds;
+    return BM_NOT_FOUND;
+}
+
+typedef struct {
+    Sint pos;
+    Sint len;
+    Uint m;
+    Uint allocated;
+    FindallData *out;
+} BMFindAllState;
+
+static void bm_init_find_all(BMFindAllState *state, Sint startpos, Uint len)
+{
+    state->pos = startpos;
+    state->len = (Sint) len;
+    state->m = 0;
+    state->allocated = 0;
+    state->out = NULL;
+}
+
+static void bm_restore_find_all(BMFindAllState *state, char *buff)
+{
+    memcpy(state,buff,sizeof(BMFindAllState));
+    if (state->allocated > 0) {
+	state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) *
+				(state->allocated));
+	memcpy(state->out,buff+sizeof(BMFindAllState),
+	       sizeof(FindallData)*state->m);
+    } else {
+	state->out = NULL;
+    }
+}
+
+static void bm_serialize_find_all(BMFindAllState *state, char *buff)
+{
+    memcpy(buff,state,sizeof(BMFindAllState));
+    memcpy(buff+sizeof(BMFindAllState),state->out,
+	   sizeof(FindallData)*state->m);
+}
+
+static void bm_clean_find_all(BMFindAllState *state)
+{
+    if (state->out != NULL) {
+	erts_free(ERTS_ALC_T_TMP, state->out);
+    }
+#ifdef HARDDEBUG
+    state->out = NULL;
+    state->allocated = 0;
+#endif
+}
+
+#define SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(S) \
+          (sizeof(BMFindAllState)+(sizeof(FindallData)*(S).m))
+
+/*
+ * Differs to the find_first function in that it stores all matches and the
+ * values are returned only in the state.
+ */
+static Sint bm_find_all_non_overlapping(BMFindAllState *state,
+					BMData *bmd, byte *haystack,
+					Uint *reductions)
+{
+    Sint blen = bmd->len;
+    Sint len = state->len;
+    Sint *gs = bmd->goodshift;
+    Sint *bs = bmd->badshift;
+    byte *needle = bmd->x;
+    Sint i;
+    Sint j = state->pos;
+    Uint m = state->m;
+    Uint allocated = state->allocated;
+    FindallData *out = state->out;
+    register Uint reds = *reductions;
+
+    while (j <= len - blen) {
+	if (--reds == 0) {
+	    state->pos = j;
+	    state->m = m;
+	    state->allocated = allocated;
+	    state->out = out;
+	    return BM_RESTART;
+	}
+	for (i = blen - 1; i >= 0 && needle[i] == haystack[i + j]; --i)
+	    ;
+	if (i < 0) { /* found */
+	    if (m >= allocated) {
+		if (!allocated) {
+		    allocated = 10;
+		    out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * allocated);
+		} else {
+		    allocated *= 2;
+		    out = erts_realloc(ERTS_ALC_T_TMP, out,
+				       sizeof(FindallData) * allocated);
+		}
+	    }
+	    out[m].pos = j;
+	    out[m].len = blen;
+	    ++m;
+	    j += blen;
+	} else {
+	    j += MAX(gs[i],bs[haystack[i+j]] - blen + 1 + i);
+	}
+    }
+    state->m = m;
+    state->out = out;
+    *reductions = reds;
+    return (m == 0) ? BM_NOT_FOUND : BM_OK;
+}
+
+/*
+ * Interface functions (i.e. "bif's")
+ */
+
+/*
+ * Search functionality interfaces
+ */
+
+static int do_binary_match_compile(Eterm argument, Eterm *tag, Binary **binp)
+{
+    Eterm t, b, comp_term = NIL;
+    Uint characters;
+    Uint words;
+
+    characters = 0;
+    words = 0;
+
+    if (is_list(argument)) {
+	t = argument;
+	while (is_list(t)) {
+	    b = CAR(list_val(t));
+	    t = CDR(list_val(t));
+	    if (!is_binary(b)) {
+		goto badarg;
+	    }
+	    if (binary_bitsize(b) != 0) {
+		goto badarg;
+	    }
+	    ++words;
+	    characters += binary_size(b);
+	}
+	if (is_not_nil(t)) {
+	    goto badarg;
+	}
+	if (words > 1) {
+	    comp_term = argument;
+	} else {
+	    comp_term = CAR(list_val(argument));
+	}
+    } else if (is_binary(argument)) {
+	if (binary_bitsize(argument) != 0) {
+	    goto badarg;
+	}
+	words = 1;
+	comp_term = argument;
+	characters = binary_size(argument);
+    }
+
+    if (characters == 0) {
+	goto badarg;
+    }
+    ASSERT(words > 0);
+
+    if (words == 1) {
+	byte *bytes;
+	Uint bitoffs, bitsize;
+	byte *temp_alloc = NULL;
+	MyAllocator my;
+	BMData *bmd;
+	Binary *bin;
+
+	ERTS_GET_BINARY_BYTES(comp_term, bytes, bitoffs, bitsize);
+	if (bitoffs != 0) {
+	    bytes = erts_get_aligned_binary_bytes(comp_term, &temp_alloc);
+	}
+	bmd = create_bmdata(&my, bytes, characters, &bin);
+	compute_badshifts(bmd);
+	compute_goodshifts(bmd);
+	erts_free_aligned_binary_bytes(temp_alloc);
+	CHECK_ALLOCATOR(my);
+	*tag = am_bm;
+	*binp = bin;
+	return 0;
+    } else {
+	ACTrie *act;
+	MyAllocator my;
+	ACNode **qbuff;
+	Binary *bin;
+
+	act = create_acdata(&my, characters, &qbuff, &bin);
+	t = comp_term;
+	while (is_list(t)) {
+	    byte *bytes;
+	    Uint bitoffs, bitsize;
+	    byte *temp_alloc = NULL;
+	    b = CAR(list_val(t));
+	    t = CDR(list_val(t));
+	    ERTS_GET_BINARY_BYTES(b, bytes, bitoffs, bitsize);
+	    if (bitoffs != 0) {
+		bytes = erts_get_aligned_binary_bytes(b, &temp_alloc);
+	    }
+	    ac_add_one_pattern(&my,act,bytes,binary_size(b));
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	}
+	ac_compute_failure_functions(act,qbuff);
+	CHECK_ALLOCATOR(my);
+	erts_free(ERTS_ALC_T_TMP,qbuff);
+	*tag = am_ac;
+	*binp = bin;
+	return 0;
+    }
+ badarg:
+    return -1;
+}
+
+BIF_RETTYPE binary_compile_pattern_1(BIF_ALIST_1)
+{
+    Binary *bin;
+    Eterm tag, ret;
+    Eterm *hp;
+
+    if (do_binary_match_compile(BIF_ARG_1,&tag,&bin)) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    hp = HAlloc(BIF_P, PROC_BIN_SIZE+3);
+    ret = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), bin);
+    ret = TUPLE2(hp, tag, ret);
+    BIF_RET(ret);
+}
+
+#define DO_BIN_MATCH_OK 0
+#define DO_BIN_MATCH_BADARG -1
+#define DO_BIN_MATCH_RESTART -2
+
+static int do_binary_match(Process *p, Eterm subject, Uint hsstart, Uint hsend,
+			   Eterm type, Binary *bin, Eterm state_term,
+			   Eterm *res_term)
+{
+    byte *bytes;
+    Uint bitoffs, bitsize;
+    byte *temp_alloc = NULL;
+
+    ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize);
+    if (bitsize != 0) {
+	goto badarg;
+    }
+    if (bitoffs != 0) {
+	bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc);
+    }
+    if (state_term != NIL) {
+	Eterm *ptr = big_val(state_term);
+	type = ptr[1];
+    }
+
+    if (type == am_bm) {
+	BMData *bm;
+	Sint pos;
+	Eterm ret;
+	Eterm *hp;
+	BMFindFirstState state;
+	Uint reds = get_reds(p, BM_LOOP_FACTOR);
+	Uint save_reds = reds;
+
+	bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin);
+#ifdef HARDDEBUG
+	dump_bm_data(bm);
+#endif
+	if (state_term == NIL) {
+	    bm_init_find_first_match(&state, hsstart, hsend);
+	} else {
+	    Eterm *ptr = big_val(state_term);
+	    memcpy(&state,ptr+2,sizeof(state));
+	}
+#ifdef HARDDEBUG
+	erts_printf("(bm) state->pos = %ld, state->len = %lu\n",state.pos,
+		    state.len);
+#endif
+	pos = bm_find_first_match(&state, bm, bytes, &reds);
+	if (pos == BM_NOT_FOUND) {
+	    ret = am_nomatch;
+	} else if (pos == BM_RESTART) {
+	    int x = (sizeof(BMFindFirstState) / sizeof(Eterm)) +
+		!!(sizeof(BMFindFirstState) % sizeof(Eterm));
+#ifdef HARDDEBUG
+	    erts_printf("Trap bm!\n");
+#endif
+	    hp = HAlloc(p,x+2);
+	    hp[0] = make_pos_bignum_header(x+1);
+	    hp[1] = type;
+	    memcpy(hp+2,&state,sizeof(state));
+	    *res_term = make_big(hp);
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    return DO_BIN_MATCH_RESTART;
+	} else {
+	    Eterm erlen = erts_make_integer((Uint) bm->len, p);
+	    ret = erts_make_integer(pos,p);
+	    hp = HAlloc(p,3);
+	    ret = TUPLE2(hp, ret, erlen);
+	}
+	erts_free_aligned_binary_bytes(temp_alloc);
+	BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
+	*res_term = ret;
+	return DO_BIN_MATCH_OK;
+    } else if (type == am_ac) {
+	ACTrie *act;
+	Uint pos, rlen;
+	int acr;
+	ACFindFirstState state;
+	Eterm ret;
+	Eterm *hp;
+	Uint reds = get_reds(p, AC_LOOP_FACTOR);
+	Uint save_reds = reds;
+
+	act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin);
+#ifdef HARDDEBUG
+	dump_ac_trie(act);
+#endif
+	if (state_term == NIL) {
+	    ac_init_find_first_match(&state, act, hsstart, hsend);
+	} else {
+	    Eterm *ptr = big_val(state_term);
+	    memcpy(&state,ptr+2,sizeof(state));
+	}
+	acr = ac_find_first_match(&state, bytes, &pos, &rlen, &reds);
+	if (acr == AC_NOT_FOUND) {
+	    ret = am_nomatch;
+	} else if (acr == AC_RESTART) {
+	    int x = (sizeof(state) / sizeof(Eterm)) +
+		!!(sizeof(ACFindFirstState) % sizeof(Eterm));
+#ifdef HARDDEBUG
+	    erts_printf("Trap ac!\n");
+#endif
+	    hp = HAlloc(p,x+2);
+	    hp[0] = make_pos_bignum_header(x+1);
+	    hp[1] = type;
+	    memcpy(hp+2,&state,sizeof(state));
+	    *res_term = make_big(hp);
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    return DO_BIN_MATCH_RESTART;
+	} else {
+	    Eterm epos = erts_make_integer(pos,p);
+	    Eterm erlen = erts_make_integer(rlen,p);
+	    hp = HAlloc(p,3);
+	    ret = TUPLE2(hp, epos, erlen);
+	}
+	erts_free_aligned_binary_bytes(temp_alloc);
+	BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
+	*res_term = ret;
+	return DO_BIN_MATCH_OK;
+    }
+ badarg:
+    return DO_BIN_MATCH_BADARG;
+}
+
+static int do_binary_matches(Process *p, Eterm subject, Uint hsstart,
+			     Uint hsend, Eterm type, Binary *bin,
+			     Eterm state_term, Eterm *res_term)
+{
+    byte *bytes;
+    Uint bitoffs, bitsize;
+    byte *temp_alloc = NULL;
+
+    ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize);
+    if (bitsize != 0) {
+	goto badarg;
+    }
+    if (bitoffs != 0) {
+	bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc);
+    }
+    if (state_term != NIL) {
+	Eterm *ptr = big_val(state_term);
+	type = ptr[1];
+    }
+
+    if (type == am_bm) {
+	BMData *bm;
+	Sint pos;
+	Eterm ret,tpl;
+	Eterm *hp;
+	BMFindAllState state;
+	Uint reds = get_reds(p, BM_LOOP_FACTOR);
+	Uint save_reds = reds;
+
+	bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin);
+#ifdef HARDDEBUG
+	dump_bm_data(bm);
+#endif
+	if (state_term == NIL) {
+	    bm_init_find_all(&state, hsstart, hsend);
+	} else {
+	    Eterm *ptr = big_val(state_term);
+	    bm_restore_find_all(&state,(char *) (ptr+2));
+	}
+
+	pos = bm_find_all_non_overlapping(&state, bm, bytes, &reds);
+	if (pos == BM_NOT_FOUND) {
+	    ret = NIL;
+	} else if (pos == BM_RESTART) {
+	    int x =
+		(SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) +
+		!!(SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm));
+#ifdef HARDDEBUG
+	    erts_printf("Trap bm!\n");
+#endif
+	    hp = HAlloc(p,x+2);
+	    hp[0] = make_pos_bignum_header(x+1);
+	    hp[1] = type;
+	    bm_serialize_find_all(&state, (char *) (hp+2));
+	    *res_term = make_big(hp);
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    bm_clean_find_all(&state);
+	    return DO_BIN_MATCH_RESTART;
+	} else {
+	    FindallData *fad = state.out;
+	    int i;
+	    for (i = 0; i < state.m; ++i) {
+		fad[i].epos = erts_make_integer(fad[i].pos,p);
+		fad[i].elen = erts_make_integer(fad[i].len,p);
+	    }
+	    hp = HAlloc(p,state.m * (3 + 2));
+	    ret = NIL;
+	    for (i = state.m - 1; i >= 0; --i) {
+		tpl = TUPLE2(hp, fad[i].epos, fad[i].elen);
+		hp +=3;
+		ret = CONS(hp,tpl,ret);
+		hp += 2;
+	    }
+	}
+	erts_free_aligned_binary_bytes(temp_alloc);
+	bm_clean_find_all(&state);
+	BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR);
+	*res_term = ret;
+	return DO_BIN_MATCH_OK;
+    } else if (type == am_ac) {
+	ACTrie *act;
+	int acr;
+	ACFindAllState state;
+	Eterm ret,tpl;
+	Eterm *hp;
+	Uint reds = get_reds(p, AC_LOOP_FACTOR);
+	Uint save_reds = reds;
+
+	act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin);
+#ifdef HARDDEBUG
+	dump_ac_trie(act);
+#endif
+	if (state_term == NIL) {
+	    ac_init_find_all(&state, act, hsstart, hsend);
+	} else {
+	    Eterm *ptr = big_val(state_term);
+	    ac_restore_find_all(&state,(char *) (ptr+2));
+	}
+	acr = ac_find_all_non_overlapping(&state, bytes, &reds);
+	if (acr == AC_NOT_FOUND) {
+	    ret = NIL;
+	} else if (acr == AC_RESTART) {
+	    int x =
+		(SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) +
+		!!(SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm));
+#ifdef HARDDEBUG
+	    erts_printf("Trap ac!\n");
+#endif
+	    hp = HAlloc(p,x+2);
+	    hp[0] = make_pos_bignum_header(x+1);
+	    hp[1] = type;
+	    ac_serialize_find_all(&state, (char *) (hp+2));
+	    *res_term = make_big(hp);
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    ac_clean_find_all(&state);
+	    return DO_BIN_MATCH_RESTART;
+	} else {
+	    FindallData *fad = state.out;
+	    int i;
+	    for (i = 0; i < state.m; ++i) {
+		fad[i].epos = erts_make_integer(fad[i].pos,p);
+		fad[i].elen = erts_make_integer(fad[i].len,p);
+	    }
+	    hp = HAlloc(p,state.m * (3 + 2));
+	    ret = NIL;
+	    for (i = state.m - 1; i >= 0; --i) {
+		tpl = TUPLE2(hp, fad[i].epos, fad[i].elen);
+		hp +=3;
+		ret = CONS(hp,tpl,ret);
+		hp += 2;
+	    }
+	}
+	erts_free_aligned_binary_bytes(temp_alloc);
+	ac_clean_find_all(&state);
+	BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR);
+	*res_term = ret;
+	return DO_BIN_MATCH_OK;
+    }
+ badarg:
+    return DO_BIN_MATCH_BADARG;
+}
+
+static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp)
+{
+    Eterm *tp;
+    Uint pos;
+    Sint len;
+    if (l == ((Eterm) 0) || l == NIL) {
+	/* Invalid term or NIL, we're called from binary_match(es)_2 or
+	   have no options*/
+	*posp = 0;
+	*endp = binary_size(bin);
+	return 0;
+    } else if (is_list(l)) {
+	while(is_list(l)) {
+	    Eterm t = CAR(list_val(l));
+	    Uint orig_size;
+	    if (!is_tuple(t)) {
+		goto badarg;
+	    }
+	    tp = tuple_val(t);
+	    if (arityval(*tp) != 2) {
+		goto badarg;
+	    }
+	    if (tp[1] != am_scope || is_not_tuple(tp[2])) {
+		goto badarg;
+	    }
+	    tp = tuple_val(tp[2]);
+	    if (arityval(*tp) != 2) {
+		goto badarg;
+	    }
+	    if (!term_to_Uint(tp[1], &pos)) {
+		goto badarg;
+	    }
+	    if (!term_to_Sint(tp[2], &len)) {
+		goto badarg;
+	    }
+	    if (len < 0) {
+		Sint lentmp = -len;
+		/* overflow */
+		if (lentmp == len || lentmp < 0 || -lentmp != len) {
+		    goto badarg;
+		}
+		len = lentmp;
+		pos -= len;
+	    }
+	    /* overflow */
+	    if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) {
+		goto badarg;
+	    }
+	    *endp = len + pos;
+	    *posp = pos;
+	    if ((orig_size = binary_size(bin)) < pos ||
+		orig_size < (*endp)) {
+		goto badarg;
+	    }
+	    l = CDR(list_val(l));
+	}
+	return 0;
+    } else {
+    badarg:
+	return 1;
+    }
+}
+
+static BIF_RETTYPE binary_match_trap(BIF_ALIST_3)
+{
+    int runres;
+    Eterm result;
+    Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val;
+    runres = do_binary_match(BIF_P,BIF_ARG_1,0,0,NIL,bin,BIF_ARG_2,&result);
+    if (runres == DO_BIN_MATCH_OK) {
+	BIF_RET(result);
+    } else {
+	BUMP_ALL_REDS(BIF_P);
+	BIF_TRAP3(&binary_match_trap_export, BIF_P, BIF_ARG_1, result,
+		  BIF_ARG_3);
+    }
+}
+
+static BIF_RETTYPE binary_matches_trap(BIF_ALIST_3)
+{
+    int runres;
+    Eterm result;
+    Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val;
+    runres = do_binary_matches(BIF_P,BIF_ARG_1,0,0,NIL,bin,BIF_ARG_2,&result);
+    if (runres == DO_BIN_MATCH_OK) {
+	BIF_RET(result);
+    } else {
+	BUMP_ALL_REDS(BIF_P);
+	BIF_TRAP3(&binary_matches_trap_export, BIF_P, BIF_ARG_1, result,
+		  BIF_ARG_3);
+    }
+}
+
+BIF_RETTYPE binary_match_3(BIF_ALIST_3)
+{
+    return binary_match(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+}
+
+static BIF_RETTYPE
+binary_match(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
+{
+    Uint hsstart;
+    Uint hsend;
+    Eterm *tp;
+    Eterm type;
+    Binary *bin;
+    Eterm bin_term = NIL;
+    int runres;
+    Eterm result;
+
+    if (is_not_binary(arg1)) {
+	goto badarg;
+    }
+    if (parse_match_opts_list(arg3,arg1,&hsstart,&hsend)) {
+	goto badarg;
+    }
+    if (hsend == 0) {
+	BIF_RET(am_nomatch);
+    }
+    if (is_tuple(arg2)) {
+	tp = tuple_val(arg2);
+	if (arityval(*tp) != 2 || is_not_atom(tp[1])) {
+	    goto badarg;
+	}
+	if (((tp[1] != am_bm) && (tp[1] != am_ac)) ||
+	    !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) {
+	    goto badarg;
+	}
+	type = tp[1];
+	bin = ((ProcBin *) binary_val(tp[2]))->val;
+	if (type == am_bm &&
+	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) {
+	    goto badarg;
+	}
+	if (type == am_ac &&
+	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) {
+	    goto badarg;
+	}
+	bin_term = tp[2];
+    } else if (do_binary_match_compile(arg2,&type,&bin)) {
+	goto badarg;
+    }
+    runres = do_binary_match(p,arg1,hsstart,hsend,type,bin,NIL,&result);
+    if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) {
+	Eterm *hp = HAlloc(p, PROC_BIN_SIZE);
+	bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), bin);
+    } else if (bin_term == NIL) {
+	erts_bin_free(bin);
+    }
+    switch (runres) {
+    case DO_BIN_MATCH_OK:
+	BIF_RET(result);
+    case DO_BIN_MATCH_RESTART:
+	BUMP_ALL_REDS(p);
+	BIF_TRAP3(&binary_match_trap_export, p, arg1, result, bin_term);
+    default:
+	goto badarg;
+    }
+ badarg:
+    BIF_ERROR(p,BADARG);
+}
+
+BIF_RETTYPE binary_matches_3(BIF_ALIST_3)
+{
+    return binary_matches(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+}
+
+static BIF_RETTYPE
+binary_matches(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
+{
+    Uint hsstart, hsend;
+    Eterm *tp;
+    Eterm type;
+    Binary *bin;
+    Eterm bin_term = NIL;
+    int runres;
+    Eterm result;
+
+    if (is_not_binary(arg1)) {
+	goto badarg;
+    }
+    if (parse_match_opts_list(arg3,arg1,&hsstart,&hsend)) {
+	goto badarg;
+    }
+    if (hsend == 0) {
+	BIF_RET(NIL);
+    }
+    if (is_tuple(arg2)) {
+	tp = tuple_val(arg2);
+	if (arityval(*tp) != 2 || is_not_atom(tp[1])) {
+	    goto badarg;
+	}
+	if (((tp[1] != am_bm) && (tp[1] != am_ac)) ||
+	    !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) {
+	    goto badarg;
+	}
+	type = tp[1];
+	bin = ((ProcBin *) binary_val(tp[2]))->val;
+	if (type == am_bm &&
+	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) {
+	    goto badarg;
+	}
+	if (type == am_ac &&
+	    ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) {
+	    goto badarg;
+	}
+	bin_term = tp[2];
+    } else if (do_binary_match_compile(arg2,&type,&bin)) {
+	goto badarg;
+    }
+    runres = do_binary_matches(p,arg1,hsstart,hsend,type,bin,
+			       NIL,&result);
+    if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) {
+	Eterm *hp = HAlloc(p, PROC_BIN_SIZE);
+	bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), bin);
+    } else if (bin_term == NIL) {
+	erts_bin_free(bin);
+    }
+    switch (runres) {
+    case DO_BIN_MATCH_OK:
+	BIF_RET(result);
+    case DO_BIN_MATCH_RESTART:
+	BUMP_ALL_REDS(p);
+	BIF_TRAP3(&binary_matches_trap_export, p, arg1, result,
+		  bin_term);
+    default:
+	goto badarg;
+    }
+ badarg:
+    BIF_ERROR(p,BADARG);
+}
+
+
+BIF_RETTYPE binary_match_2(BIF_ALIST_2)
+{
+    return binary_match(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0));
+}
+
+
+BIF_RETTYPE binary_matches_2(BIF_ALIST_2)
+{
+    return binary_matches(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0));
+}
+
+
+BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen)
+{
+    Uint pos;
+    Sint len;
+    size_t orig_size;
+    Eterm orig;
+    Uint offset;
+    Uint bit_offset;
+    Uint bit_size;
+    Eterm* hp;
+    ErlSubBin* sb;
+
+    if (is_not_binary(binary)) {
+	goto badarg;
+    }
+    if (!term_to_Uint(epos, &pos)) {
+	goto badarg;
+    }
+    if (!term_to_Sint(elen, &len)) {
+	goto badarg;
+    }
+    if (len < 0) {
+	Sint lentmp = -len;
+	/* overflow */
+	if (lentmp == len || lentmp < 0 || -lentmp != len) {
+	    goto badarg;
+	}
+	len = lentmp;
+	if (len > pos) {
+	    goto badarg;
+	}
+	pos -= len;
+    }
+    /* overflow */
+    if ((pos + len) < pos || (len > 0 && (pos + len) == pos)){
+	goto badarg;
+    }
+    if ((orig_size = binary_size(binary)) < pos ||
+	orig_size < (pos + len)) {
+	goto badarg;
+    }
+
+
+
+    hp = HAlloc(p, ERL_SUB_BIN_SIZE);
+
+    ERTS_GET_REAL_BIN(binary, orig, offset, bit_offset, bit_size);
+    sb = (ErlSubBin *) hp;
+    sb->thing_word = HEADER_SUB_BIN;
+    sb->size = len;
+    sb->offs = offset + pos;
+    sb->orig = orig;
+    sb->bitoffs = bit_offset;
+    sb->bitsize = 0;
+    sb->is_writable = 0;
+
+    BIF_RET(make_binary(sb));
+
+ badarg:
+    BIF_ERROR(p, BADARG);
+}
+
+#define ERTS_NEED_GC(p, need) ((HEAP_LIMIT((p)) - HEAP_TOP((p))) <= (need))
+
+BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is_tuple)
+{
+    Uint pos;
+    Sint len;
+    size_t orig_size;
+    Eterm orig;
+    Uint offset;
+    Uint bit_offset;
+    Uint bit_size;
+    Eterm* hp;
+    ErlSubBin* sb;
+    Eterm binary;
+    Eterm *tp;
+    Eterm epos, elen;
+    int extra_args;
+
+
+    if (range_is_tuple) {
+	Eterm tpl = reg[live];
+	extra_args = 1;
+	if (is_not_tuple(tpl)) {
+	    goto badarg;
+	}
+	tp = tuple_val(tpl);
+	if (arityval(*tp) != 2) {
+	    goto badarg;
+	}
+
+	epos = tp[1];
+	elen = tp[2];
+    } else {
+	extra_args = 2;
+	epos = reg[live-1];
+	elen = reg[live];
+    }
+    binary = reg[live-extra_args];
+
+    if (is_not_binary(binary)) {
+	goto badarg;
+    }
+    if (!term_to_Uint(epos, &pos)) {
+	goto badarg;
+    }
+    if (!term_to_Sint(elen, &len)) {
+	goto badarg;
+    }
+    if (len < 0) {
+	Sint lentmp = -len;
+	/* overflow */
+	if (lentmp == len || lentmp < 0 || -lentmp != len) {
+	    goto badarg;
+	}
+	len = lentmp;
+	if (len > pos) {
+	    goto badarg;
+	}
+	pos -= len;
+    }
+    /* overflow */
+    if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) {
+	goto badarg;
+    }
+    if ((orig_size = binary_size(binary)) < pos ||
+	orig_size < (pos + len)) {
+	goto badarg;
+    }
+
+    if (ERTS_NEED_GC(p, ERL_SUB_BIN_SIZE)) {
+	erts_garbage_collect(p, ERL_SUB_BIN_SIZE, reg, live+1-extra_args); /* I don't need the tuple
+									      or indices any more */
+	binary = reg[live-extra_args];
+    }
+
+    hp = p->htop;
+    p->htop += ERL_SUB_BIN_SIZE;
+
+    ERTS_GET_REAL_BIN(binary, orig, offset, bit_offset, bit_size);
+
+    sb = (ErlSubBin *) hp;
+    sb->thing_word = HEADER_SUB_BIN;
+    sb->size = len;
+    sb->offs = offset + pos;
+    sb->orig = orig;
+    sb->bitoffs = bit_offset;
+    sb->bitsize = 0;
+    sb->is_writable = 0;
+
+    BIF_RET(make_binary(sb));
+
+ badarg:
+    BIF_ERROR(p, BADARG);
+}
+/*************************************************************
+ * The actual guard BIFs are in erl_bif_guard.c
+ * but the implementation of both the non-gc and the gc
+ * variants are here. Note that the functions are named so that they do
+ * not clash with the guard bif's erlang:binary_part/2,3
+ *************************************************************/
+
+BIF_RETTYPE binary_binary_part_3(BIF_ALIST_3)
+{
+    return erts_binary_part(BIF_P,BIF_ARG_1,BIF_ARG_2, BIF_ARG_3);
+}
+
+BIF_RETTYPE binary_binary_part_2(BIF_ALIST_2)
+{
+    Eterm *tp;
+    if (is_not_tuple(BIF_ARG_2)) {
+	goto badarg;
+    }
+    tp = tuple_val(BIF_ARG_2);
+    if (arityval(*tp) != 2) {
+	goto badarg;
+    }
+    return erts_binary_part(BIF_P,BIF_ARG_1,tp[1], tp[2]);
+ badarg:
+   BIF_ERROR(BIF_P,BADARG);
+}
+
+typedef struct {
+    int type;            /* CL_TYPE_XXX */
+    byte *temp_alloc;    /* Used for erts_get/free_aligned, i.e. CL_TYPE_ALIGNED */
+    unsigned char *buff; /* Used for all types, malloced if CL_TYPE_HEAP */
+    Uint bufflen;        /* The length (in bytes) of buffer */
+} CommonData;
+
+#define COMMON_LOOP_FACTOR 10
+
+#define DIRECTION_PREFIX 0
+#define DIRECTION_SUFFIX 1
+
+#define CL_OK 0
+#define CL_RESTART 1
+
+/* The type field in the above structure */
+#define CL_TYPE_EMPTY 0 /* End of array */
+#define CL_TYPE_HEAP 1
+#define CL_TYPE_ALIGNED 2
+#define CL_TYPE_COMMON 3 /* emacsulated */
+#define CL_TYPE_HEAP_NOALLOC 4 /* Will need allocating when trapping */
+
+
+static int do_search_forward(CommonData *cd, Uint *posp, Uint *redsp)
+{
+    Uint pos = *posp;
+    Sint reds = (Sint) *redsp;
+    int i;
+    unsigned char current = 0;
+
+    for(;;) {
+	for(i = 0; cd[i].type != CL_TYPE_EMPTY; ++i) {
+	    if (pos >= cd[i].bufflen) {
+		*posp = pos;
+		if (reds > 0) {
+		    *redsp = (Uint) reds;
+		} else {
+		    *redsp = 0;
+		}
+		return CL_OK;
+	    }
+	    if (i == 0) {
+		current = cd[i].buff[pos];
+	    } else {
+		if (cd[i].buff[pos] != current) {
+		    *posp = pos;
+		    if (reds > 0) {
+			*redsp = (Uint) reds;
+		    } else {
+			*redsp = 0;
+		    }
+		    return CL_OK;
+		}
+	    }
+	    --reds;
+	}
+	++pos;
+	if (reds <= 0) {
+	    *posp = pos;
+	    *redsp = 0;
+	    return CL_RESTART;
+	}
+    }
+}
+static int do_search_backward(CommonData *cd, Uint *posp, Uint *redsp)
+{
+    Uint pos = *posp;
+    Sint reds = (Sint) *redsp;
+    int i;
+    unsigned char current = 0;
+
+    for(;;) {
+	for(i = 0; cd[i].type != CL_TYPE_EMPTY; ++i) {
+	    if (pos >= cd[i].bufflen) {
+		*posp = pos;
+		if (reds > 0) {
+		    *redsp = (Uint) reds;
+		} else {
+		    *redsp = 0;
+		}
+		return CL_OK;
+	    }
+	    if (i == 0) {
+		current = cd[i].buff[cd[i].bufflen - 1 - pos];
+	    } else {
+		if (cd[i].buff[cd[i].bufflen - 1 - pos] != current) {
+		    *posp = pos;
+		    if (reds > 0) {
+			*redsp = (Uint) reds;
+		    } else {
+			*redsp = 0;
+		    }
+		    return CL_OK;
+		}
+	    }
+	    --reds;
+	}
+	++pos;
+	if (reds <= 0) {
+	    *posp = pos;
+	    *redsp = 0;
+	    return CL_RESTART;
+	}
+    }
+}
+
+static void cleanup_common_data(Binary *bp)
+{
+    int i;
+    CommonData *cd;
+    cd = (CommonData *) ERTS_MAGIC_BIN_DATA(bp);
+    for (i=0;cd[i].type != CL_TYPE_EMPTY;++i) {
+	switch (cd[i].type) {
+	case CL_TYPE_HEAP:
+	    erts_free(ERTS_ALC_T_BINARY_BUFFER,cd[i].buff);
+	    break;
+	case CL_TYPE_ALIGNED:
+	    erts_free_aligned_binary_bytes_extra(cd[i].temp_alloc, ERTS_ALC_T_BINARY_BUFFER);
+	    break;
+	default:
+	    break;
+	}
+    }
+    return;
+}
+
+static BIF_RETTYPE do_longest_common(Process *p, Eterm list, int direction)
+{
+    Eterm l = list;
+    int n = 0;
+    Binary *mb;
+    CommonData *cd;
+    int i = 0;
+    Uint reds = get_reds(p, COMMON_LOOP_FACTOR);
+    Uint save_reds = reds;
+    int res;
+    Export *trapper;
+    Uint pos;
+    Eterm epos;
+    Eterm *hp;
+    Eterm bin_term;
+    Eterm b;
+
+    /* First just count the number of binaries */
+    while (is_list(l)) {
+	b = CAR(list_val(l));
+	if (!is_binary(b)) {
+	    goto badarg;
+	}
+	++n;
+	l = CDR(list_val(l));
+    }
+    if (l != NIL || n == 0) {
+	goto badarg;
+    }
+
+    /* OK, now create a buffer of the right size, we can do a magic binary right away,
+       that's not too costly. */
+    mb = erts_create_magic_binary((n+1)*sizeof(CommonData),cleanup_common_data);
+    cd = (CommonData *) ERTS_MAGIC_BIN_DATA(mb);
+    l = list;
+    while (is_list(l)) {
+	ERTS_DECLARE_DUMMY(Uint bitoffs);
+	Uint bitsize;
+	ERTS_DECLARE_DUMMY(Uint offset);
+	Eterm real_bin;
+	ProcBin* pb;
+
+	cd[i].type = CL_TYPE_EMPTY;
+	b = CAR(list_val(l));
+	ERTS_GET_REAL_BIN(b, real_bin, offset, bitoffs, bitsize);
+	if (bitsize != 0) {
+	    erts_bin_free(mb);
+	    goto badarg;
+	}
+	cd[i].bufflen = binary_size(b);
+	cd[i].temp_alloc = NULL;
+	if (*(binary_val(real_bin)) == HEADER_PROC_BIN) {
+	    pb = (ProcBin *) binary_val(real_bin);
+	    if (pb->flags) {
+		erts_emasculate_writable_binary(pb);
+	    }
+	    cd[i].buff = erts_get_aligned_binary_bytes_extra(b, &(cd[i].temp_alloc),
+							     ERTS_ALC_T_BINARY_BUFFER,0);
+	    cd[i].type = (cd[i].temp_alloc != NULL) ? CL_TYPE_ALIGNED : CL_TYPE_COMMON;
+	} else { /* Heap binary */
+	    cd[i].buff = erts_get_aligned_binary_bytes_extra(b, &(cd[i].temp_alloc),
+							     ERTS_ALC_T_BINARY_BUFFER,0);
+	    /* CL_TYPE_HEAP_NOALLOC means you have to copy if trapping */
+	    cd[i].type = (cd[i].temp_alloc != NULL) ? CL_TYPE_ALIGNED : CL_TYPE_HEAP_NOALLOC;
+	}
+	++i;
+	l = CDR(list_val(l));
+    }
+    cd[i].type = CL_TYPE_EMPTY;
+#if defined(DEBUG) || defined(VALGRIND)
+    cd[i].temp_alloc = NULL;
+    cd[i].buff = NULL;
+    cd[i].bufflen = 0;
+#endif
+
+    pos = 0;
+    if (direction == DIRECTION_PREFIX) {
+	trapper = &binary_longest_prefix_trap_export;
+	res = do_search_forward(cd,&pos,&reds);
+    } else {
+	ASSERT(direction == DIRECTION_SUFFIX);
+	trapper = &binary_longest_suffix_trap_export;
+	res = do_search_backward(cd,&pos,&reds);
+    }
+    epos = erts_make_integer(pos,p);
+    if (res == CL_OK) {
+	erts_bin_free(mb);
+	BUMP_REDS(p, (save_reds - reds) / COMMON_LOOP_FACTOR);
+	BIF_RET(epos);
+    } else {
+	ASSERT(res == CL_RESTART);
+	/* Copy all heap binaries that are not already copied (aligned) */
+	for(i = 0; i < n; ++i) {
+	    if (cd[i].type == CL_TYPE_HEAP_NOALLOC) {
+		unsigned char *tmp = cd[i].buff;
+		cd[i].buff = erts_alloc(ERTS_ALC_T_BINARY_BUFFER, cd[i].bufflen);
+		memcpy(cd[i].buff,tmp,cd[i].bufflen);
+		cd[i].type = CL_TYPE_HEAP;
+	    }
+	}
+	hp = HAlloc(p, PROC_BIN_SIZE);
+	bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), mb);
+	BUMP_ALL_REDS(p);
+	BIF_TRAP3(trapper, p, bin_term, epos,list);
+    }
+ badarg:
+    BIF_ERROR(p,BADARG);
+}
+
+static BIF_RETTYPE do_longest_common_trap(Process *p, Eterm bin_term, Eterm current_pos,
+					  Eterm orig_list, int direction)
+{
+    Uint reds = get_reds(p, COMMON_LOOP_FACTOR);
+    Uint save_reds = reds;
+    Uint pos;
+    Binary *bin;
+    CommonData *cd;
+    int res;
+    Eterm epos;
+    Export *trapper;
+
+#ifdef DEBUG
+    int r;
+    r = term_to_Uint(current_pos, &pos);
+    ASSERT(r != 0);
+#else
+    term_to_Uint(current_pos, &pos);
+#endif
+    ASSERT(ERTS_TERM_IS_MAGIC_BINARY(bin_term));
+    bin = ((ProcBin *) binary_val(bin_term))->val;
+    cd = (CommonData *) ERTS_MAGIC_BIN_DATA(bin);
+    if (direction == DIRECTION_PREFIX) {
+	trapper = &binary_longest_prefix_trap_export;
+	res = do_search_forward(cd,&pos,&reds);
+    } else {
+	ASSERT(direction == DIRECTION_SUFFIX);
+	trapper = &binary_longest_suffix_trap_export;
+	res = do_search_backward(cd,&pos,&reds);
+    }
+    epos = erts_make_integer(pos,p);
+    if (res == CL_OK) {
+	BUMP_REDS(p, (save_reds - reds) / COMMON_LOOP_FACTOR);
+	BIF_RET(epos);
+    } else {
+	ASSERT(res == CL_RESTART);
+	/* Copy all heap binaries that are not already copied (aligned) */
+	BUMP_ALL_REDS(p);
+	BIF_TRAP3(trapper, p, bin_term, epos, orig_list);
+    }
+}
+
+static BIF_RETTYPE binary_longest_prefix_trap(BIF_ALIST_3)
+{
+    return do_longest_common_trap(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3,DIRECTION_PREFIX);
+}
+
+static BIF_RETTYPE binary_longest_suffix_trap(BIF_ALIST_3)
+{
+    return do_longest_common_trap(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3,DIRECTION_SUFFIX);
+}
+
+BIF_RETTYPE binary_longest_common_prefix_1(BIF_ALIST_1)
+{
+    return do_longest_common(BIF_P,BIF_ARG_1,DIRECTION_PREFIX);
+}
+
+BIF_RETTYPE binary_longest_common_suffix_1(BIF_ALIST_1)
+{
+    return do_longest_common(BIF_P,BIF_ARG_1,DIRECTION_SUFFIX);
+}
+
+BIF_RETTYPE binary_first_1(BIF_ALIST_1)
+{
+    byte* bytes;
+    Uint byte_size;
+    Uint bit_offs;
+    Uint bit_size;
+    Uint res;
+
+    if (is_not_binary(BIF_ARG_1)) {
+	goto badarg;
+    }
+    byte_size = binary_size(BIF_ARG_1);
+    if (!byte_size) {
+	goto badarg;
+    }
+    ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size);
+    if (bit_size) {
+	goto badarg;
+    }
+    if (bit_offs) {
+	res = ((((Uint) bytes[0]) << bit_offs) | (((Uint) bytes[1]) >> (8-bit_offs))) & 0xFF;
+    } else {
+	res = bytes[0];
+    }
+    BIF_RET(make_small(res));
+ badarg:
+    BIF_ERROR(BIF_P,BADARG);
+}
+
+BIF_RETTYPE binary_last_1(BIF_ALIST_1)
+{
+    byte* bytes;
+    Uint byte_size;
+    Uint bit_offs;
+    Uint bit_size;
+    Uint res;
+
+    if (is_not_binary(BIF_ARG_1)) {
+	goto badarg;
+    }
+    byte_size = binary_size(BIF_ARG_1);
+    if (!byte_size) {
+	goto badarg;
+    }
+    ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size);
+    if (bit_size) {
+	goto badarg;
+    }
+    if (bit_offs) {
+	res = ((((Uint) bytes[byte_size-1]) << bit_offs) |
+	       (((Uint) bytes[byte_size]) >> (8-bit_offs))) & 0xFF;
+    } else {
+	res = bytes[byte_size-1];
+    }
+    BIF_RET(make_small(res));
+ badarg:
+    BIF_ERROR(BIF_P,BADARG);
+}
+
+BIF_RETTYPE binary_at_2(BIF_ALIST_2)
+{
+    byte* bytes;
+    Uint byte_size;
+    Uint bit_offs;
+    Uint bit_size;
+    Uint res;
+    Uint index;
+
+    if (is_not_binary(BIF_ARG_1)) {
+	goto badarg;
+    }
+    byte_size = binary_size(BIF_ARG_1);
+    if (!byte_size) {
+	goto badarg;
+    }
+    if (!term_to_Uint(BIF_ARG_2, &index)) {
+	goto badarg;
+    }
+    if (index >= byte_size) {
+	goto badarg;
+    }
+    ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size);
+    if (bit_size) {
+	goto badarg;
+    }
+    if (bit_offs) {
+	res = ((((Uint) bytes[index]) << bit_offs) |
+	       (((Uint) bytes[index+1]) >> (8-bit_offs))) & 0xFF;
+    } else {
+	res = bytes[index];
+    }
+    BIF_RET(make_small(res));
+ badarg:
+    BIF_ERROR(BIF_P,BADARG);
+}
+
+#define BIN_TO_LIST_OK 0
+#define BIN_TO_LIST_TRAP 1
+/* No badarg, checked before call */
+
+#define BIN_TO_LIST_LOOP_FACTOR 10
+
+static int do_bin_to_list(Process *p, byte *bytes, Uint bit_offs,
+			  Uint start, Sint *lenp, Eterm *termp)
+{
+    Uint reds = get_reds(p, BIN_TO_LIST_LOOP_FACTOR); /* reds can never be 0 */
+    Uint len = *lenp;
+    Uint loops;
+    Eterm *hp;
+    Eterm term = *termp;
+    Uint n;
+
+    ASSERT(reds > 0);
+
+    loops = MIN(reds,len);
+
+    BUMP_REDS(p, loops / BIN_TO_LIST_LOOP_FACTOR);
+
+    hp = HAlloc(p,2*loops);
+    while (loops--) {
+	--len;
+	if (bit_offs) {
+	    n = ((((Uint) bytes[start+len]) << bit_offs) |
+		 (((Uint) bytes[start+len+1]) >> (8-bit_offs))) & 0xFF;
+	} else {
+	    n = bytes[start+len];
+	}
+
+	term = CONS(hp,make_small(n),term);
+	hp +=2;
+    }
+    *termp = term;
+    *lenp = len;
+    if (len) {
+	BUMP_ALL_REDS(p);
+	return BIN_TO_LIST_TRAP;
+    }
+    return BIN_TO_LIST_OK;
+}
+
+
+static BIF_RETTYPE do_trap_bin_to_list(Process *p, Eterm binary,
+				       Uint start, Sint len, Eterm sofar)
+{
+    Eterm *hp;
+    Eterm blob;
+
+    hp = HAlloc(p,3);
+    hp[0] = make_pos_bignum_header(2);
+    hp[1] = start;
+    hp[2] = (Uint) len;
+    blob = make_big(hp);
+    BIF_TRAP3(&binary_bin_to_list_trap_export, p, binary, blob, sofar);
+}
+
+static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3)
+{
+    Eterm *ptr;
+    Uint start;
+    Sint len;
+    byte *bytes;
+    Uint bit_offs;
+    Uint bit_size;
+    Eterm res = BIF_ARG_3;
+
+    ptr  = big_val(BIF_ARG_2);
+    start = ptr[1];
+    len = (Sint) ptr[2];
+
+    ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size);
+    if (do_bin_to_list(BIF_P, bytes, bit_offs, start, &len, &res) ==
+	BIN_TO_LIST_OK) {
+	BIF_RET(res);
+    }
+    return do_trap_bin_to_list(BIF_P,BIF_ARG_1,start,len,res);
+}
+
+static BIF_RETTYPE binary_bin_to_list_common(Process *p,
+					     Eterm bin,
+					     Eterm epos,
+					     Eterm elen)
+{
+    Uint pos;
+    Sint len;
+    size_t sz;
+    byte *bytes;
+    Uint bit_offs;
+    Uint bit_size;
+    Eterm res = NIL;
+
+    if (is_not_binary(bin)) {
+	goto badarg;
+    }
+    if (!term_to_Uint(epos, &pos)) {
+	goto badarg;
+    }
+    if (!term_to_Sint(elen, &len)) {
+	goto badarg;
+    }
+    if (len < 0) {
+	Sint lentmp = -len;
+	/* overflow */
+	if (lentmp == len || lentmp < 0 || -lentmp != len) {
+	    goto badarg;
+	}
+	len = lentmp;
+	if (len > pos) {
+	    goto badarg;
+	}
+	pos -= len;
+    }
+    /* overflow */
+    if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) {
+	goto badarg;
+    }
+    sz = binary_size(bin);
+
+    if (pos+len > sz) {
+	goto badarg;
+    }
+    ERTS_GET_BINARY_BYTES(bin,bytes,bit_offs,bit_size);
+    if (bit_size != 0) {
+	goto badarg;
+    }
+    if(do_bin_to_list(p, bytes, bit_offs, pos, &len, &res) ==
+       BIN_TO_LIST_OK) {
+	BIF_RET(res);
+    }
+    return do_trap_bin_to_list(p,bin,pos,len,res);
+
+ badarg:
+    BIF_ERROR(p,BADARG);
+}
+
+BIF_RETTYPE binary_bin_to_list_3(BIF_ALIST_3)
+{
+    return binary_bin_to_list_common(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3);
+}
+
+BIF_RETTYPE binary_bin_to_list_2(BIF_ALIST_2)
+{
+    Eterm *tp;
+
+    if (is_not_tuple(BIF_ARG_2)) {
+	goto badarg;
+    }
+    tp = tuple_val(BIF_ARG_2);
+    if (arityval(*tp) != 2) {
+	goto badarg;
+    }
+    return binary_bin_to_list_common(BIF_P,BIF_ARG_1,tp[1],tp[2]);
+ badarg:
+    BIF_ERROR(BIF_P,BADARG);
+}
+
+BIF_RETTYPE binary_bin_to_list_1(BIF_ALIST_1)
+{
+    Uint pos = 0;
+    Sint len;
+    byte *bytes;
+    Uint bit_offs;
+    Uint bit_size;
+    Eterm res = NIL;
+
+    if (is_not_binary(BIF_ARG_1)) {
+	goto badarg;
+    }
+    len = binary_size(BIF_ARG_1);
+    ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size);
+    if (bit_size != 0) {
+	goto badarg;
+    }
+    if(do_bin_to_list(BIF_P, bytes, bit_offs, pos, &len, &res) ==
+       BIN_TO_LIST_OK) {
+	BIF_RET(res);
+    }
+    return do_trap_bin_to_list(BIF_P,BIF_ARG_1,pos,len,res);
+ badarg:
+    BIF_ERROR(BIF_P,BADARG);
+}
+
+/*
+ * Ok, erlang:list_to_binary does not interrupt, and we really don't want
+ * an alternative implementation for the exact same thing, why we
+ * have descided to use the old non-restarting implementation for now.
+ * In reality, there are seldom many iterations involved in doing this, so the
+ * problem of long-running bifs is not really that big in this case.
+ * So, for now we use the old implementation also in the module binary.
+ */
+
+BIF_RETTYPE binary_list_to_bin_1(BIF_ALIST_1)
+{
+    return erts_list_to_binary_bif(BIF_P, BIF_ARG_1);
+}
+
+typedef struct {
+    Uint times_left;
+    Uint source_size;
+    int source_type;
+    byte *source;
+    byte *temp_alloc;
+    Uint result_pos;
+    Binary *result;
+} CopyBinState;
+
+#define BC_TYPE_EMPTY 0
+#define BC_TYPE_HEAP 1
+#define BC_TYPE_ALIGNED 2 /* May or may not point to (emasculated) binary, temp_alloc field is set
+			     so that erts_free_aligned_binary_bytes_extra can handle either */
+
+
+#define BINARY_COPY_LOOP_FACTOR 100
+
+static void cleanup_copy_bin_state(Binary *bp)
+{
+    CopyBinState *cbs = (CopyBinState *) ERTS_MAGIC_BIN_DATA(bp);
+    if (cbs->result != NULL) {
+	erts_bin_free(cbs->result);
+	cbs->result = NULL;
+    }
+    switch (cbs->source_type) {
+    case BC_TYPE_HEAP:
+	erts_free(ERTS_ALC_T_BINARY_BUFFER,cbs->source);
+	break;
+    case BC_TYPE_ALIGNED:
+	erts_free_aligned_binary_bytes_extra(cbs->temp_alloc,
+					     ERTS_ALC_T_BINARY_BUFFER);
+	break;
+    default:
+	/* otherwise do nothing */
+	break;
+    }
+    cbs->source_type =  BC_TYPE_EMPTY;
+}
+
+/*
+ * Binary *erts_bin_nrml_alloc(Uint size);
+ * Binary *erts_bin_realloc(Binary *bp, Uint size);
+ * void erts_bin_free(Binary *bp);
+ */
+static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en)
+{
+    Uint n;
+    byte *bytes;
+    ERTS_DECLARE_DUMMY(Uint bit_offs);
+    Uint bit_size;
+    size_t size;
+    Uint reds = get_reds(p, BINARY_COPY_LOOP_FACTOR);
+    Uint target_size;
+    byte *t;
+    Uint pos;
+
+
+    if (is_not_binary(bin)) {
+	goto badarg;
+    }
+    if (!term_to_Uint(en, &n)) {
+	goto badarg;
+    }
+    if (!n) {
+	Eterm res_term = erts_new_heap_binary(p,NULL,0,&bytes);
+	BIF_RET(res_term);
+    }
+    ERTS_GET_BINARY_BYTES(bin,bytes,bit_offs,bit_size);
+    if (bit_size != 0) {
+	goto badarg;
+    }
+
+    size = binary_size(bin);
+    target_size = size * n;
+
+    if ((target_size - size) >= reds) {
+	Eterm orig;
+	ERTS_DECLARE_DUMMY(Uint offset);
+	ERTS_DECLARE_DUMMY(Uint bit_offset);
+	ERTS_DECLARE_DUMMY(Uint bit_size);
+	CopyBinState *cbs;
+	Eterm *hp;
+	Eterm trap_term;
+	int i;
+
+	/* We will trap, set up the structure for trapping right away */
+	Binary *mb = erts_create_magic_binary(sizeof(CopyBinState),
+					      cleanup_copy_bin_state);
+	cbs = ERTS_MAGIC_BIN_DATA(mb);
+
+	cbs->temp_alloc = NULL;
+	cbs->source = NULL;
+
+	ERTS_GET_REAL_BIN(bin, orig, offset, bit_offset, bit_size);
+	if (*(binary_val(orig)) == HEADER_PROC_BIN) {
+	    ProcBin* pb = (ProcBin *) binary_val(orig);
+	    if (pb->flags) {
+		erts_emasculate_writable_binary(pb);
+	    }
+	    cbs->source =
+		erts_get_aligned_binary_bytes_extra(bin,
+						    &(cbs->temp_alloc),
+						    ERTS_ALC_T_BINARY_BUFFER,
+						    0);
+	    cbs->source_type = BC_TYPE_ALIGNED;
+	} else { /* Heap binary */
+	    cbs->source =
+		erts_get_aligned_binary_bytes_extra(bin,
+						    &(cbs->temp_alloc),
+						    ERTS_ALC_T_BINARY_BUFFER,
+						    0);
+	    if (!(cbs->temp_alloc)) { /* alignment not needed, need to copy */
+		byte *tmp = erts_alloc(ERTS_ALC_T_BINARY_BUFFER,size);
+		memcpy(tmp,cbs->source,size);
+		cbs->source = tmp;
+		cbs->source_type = BC_TYPE_HEAP;
+	    } else {
+		cbs->source_type = BC_TYPE_ALIGNED;
+	    }
+	}
+	cbs->result = erts_bin_nrml_alloc(target_size); /* Always offheap
+							   if trapping */
+	cbs->result->flags = 0;
+	cbs->result->orig_size = target_size;
+	erts_refc_init(&(cbs->result->refc), 1);
+	t = (byte *) cbs->result->orig_bytes; /* No offset or anything */
+	pos = 0;
+	i = 0;
+	while (pos < reds) {
+	    memcpy(t+pos,cbs->source, size);
+	    pos += size;
+	    ++i;
+	}
+	cbs->source_size = size;
+	cbs->result_pos = pos;
+	cbs->times_left = n-i;
+	hp = HAlloc(p,PROC_BIN_SIZE);
+	trap_term = erts_mk_magic_binary_term(&hp, &MSO(p), mb);
+	BUMP_ALL_REDS(p);
+	BIF_TRAP2(&binary_copy_trap_export, p, bin, trap_term);
+    } else {
+	Eterm res_term;
+	byte *temp_alloc = NULL;
+	byte *source =
+	    erts_get_aligned_binary_bytes(bin,
+					  &temp_alloc);
+	if (target_size <= ERL_ONHEAP_BIN_LIMIT) {
+	    res_term = erts_new_heap_binary(p,NULL,target_size,&t);
+	} else {
+	    res_term = erts_new_mso_binary(p,NULL,target_size);
+	    t = ((ProcBin *) binary_val(res_term))->bytes;
+	}
+	pos = 0;
+	while (pos < target_size) {
+	    memcpy(t+pos,source, size);
+	    pos += size;
+	}
+	erts_free_aligned_binary_bytes(temp_alloc);
+	BUMP_REDS(p,pos / BINARY_COPY_LOOP_FACTOR);
+	BIF_RET(res_term);
+    }
+ badarg:
+    BIF_ERROR(p,BADARG);
+}
+
+BIF_RETTYPE binary_copy_trap(BIF_ALIST_2)
+{
+    Uint n;
+    size_t size;
+    Uint reds = get_reds(BIF_P, BINARY_COPY_LOOP_FACTOR);
+    byte *t;
+    Uint pos;
+    Binary *mb = ((ProcBin *) binary_val(BIF_ARG_2))->val;
+    CopyBinState *cbs = (CopyBinState *) ERTS_MAGIC_BIN_DATA(mb);
+    Uint opos;
+
+    /* swapout... */
+    n = cbs->times_left;
+    size = cbs->source_size;
+    opos = pos = cbs->result_pos;
+    t = (byte *) cbs->result->orig_bytes; /* "well behaved" binary */
+    if ((n-1) * size >= reds) {
+	Uint i = 0;
+	while ((pos - opos) < reds) {
+	    memcpy(t+pos,cbs->source, size);
+	    pos += size;
+	    ++i;
+	}
+	cbs->result_pos = pos;
+	cbs->times_left -= i;
+	BUMP_ALL_REDS(BIF_P);
+	BIF_TRAP2(&binary_copy_trap_export, BIF_P, BIF_ARG_1, BIF_ARG_2);
+    } else {
+	Binary *save;
+	ProcBin* pb;
+	Uint target_size = cbs->result->orig_size;
+	while (pos < target_size) {
+	    memcpy(t+pos,cbs->source, size);
+	    pos += size;
+	}
+	save =  cbs->result;
+	cbs->result = NULL;
+	cleanup_copy_bin_state(mb); /* now cbs is dead */
+	pb = (ProcBin *) HAlloc(BIF_P, PROC_BIN_SIZE);
+	pb->thing_word = HEADER_PROC_BIN;
+	pb->size = target_size;
+	pb->next = MSO(BIF_P).first;
+	MSO(BIF_P).first = (struct erl_off_heap_header*) pb;
+	pb->val = save;
+	pb->bytes = t;
+	pb->flags = 0;
+
+	OH_OVERHEAD(&(MSO(BIF_P)), target_size / sizeof(Eterm));
+	BUMP_REDS(BIF_P,(pos - opos) / BINARY_COPY_LOOP_FACTOR);
+
+	BIF_RET(make_binary(pb));
+    }
+}
+
+
+BIF_RETTYPE binary_copy_1(BIF_ALIST_1)
+{
+    return do_binary_copy(BIF_P,BIF_ARG_1,make_small(1));
+}
+
+BIF_RETTYPE binary_copy_2(BIF_ALIST_2)
+{
+    return do_binary_copy(BIF_P,BIF_ARG_1,BIF_ARG_2);
+}
+
+BIF_RETTYPE binary_referenced_byte_size_1(BIF_ALIST_1)
+{
+    ErlSubBin *sb;
+    ProcBin *pb;
+    Eterm res;
+    Eterm bin = BIF_ARG_1;
+
+    if (is_not_binary(BIF_ARG_1)) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    sb = (ErlSubBin *) binary_val(bin);
+    if (sb->thing_word == HEADER_SUB_BIN) {
+	bin = sb->orig;
+    }
+    pb = (ProcBin *) binary_val(bin);
+    if (pb->thing_word == HEADER_PROC_BIN) {
+	/* XXX:PaN - Halfword - orig_size is a long, we should handle that */
+	res = erts_make_integer((Uint) pb->val->orig_size, BIF_P);
+    } else { /* heap binary */
+	res = erts_make_integer((Uint) ((ErlHeapBin *) pb)->size, BIF_P);
+    }
+    BIF_RET(res);
+}
+
+#define END_BIG 0
+#define END_SMALL 1
+
+#ifdef WORDS_BIGENDIAN
+#define END_NATIVE END_BIG
+#else
+#define END_NATIVE END_SMALL
+#endif
+
+static int get_need(Uint u) {
+#if defined(ARCH_64) && !HALFWORD_HEAP
+    if (u > 0xFFFFFFFFUL) {
+	if (u > 0xFFFFFFFFFFFFUL) {
+	    if (u > 0xFFFFFFFFFFFFFFUL) {
+		return 8;
+	    }
+	    return 7;
+	}
+	if (u > 0xFFFFFFFFFFUL) {
+	    return 6;
+	}
+	return 5;
+    }
+#endif
+    if (u > 0xFFFFUL) {
+	if (u > 0xFFFFFFUL) {
+	    return 4;
+	}
+	return 3;
+    }
+    if (u > 0xFFUL) {
+	return 2;
+    }
+    return 1;
+}
+
+static BIF_RETTYPE do_encode_unsigned(Process *p, Eterm uns, Eterm endianess)
+{
+    Eterm res;
+    if ((is_not_small(uns) && is_not_big(uns)) || is_not_atom(endianess) ||
+	(endianess != am_big && endianess != am_little)) {
+	goto badarg;
+    }
+    if (is_small(uns)) {
+	Sint x = signed_val(uns);
+	Uint u;
+	int n,i;
+	byte *b;
+
+	if (x < 0) {
+	    goto badarg;
+	}
+
+	u = (Uint) x;
+	n = get_need(u);
+	ASSERT(n <= ERL_ONHEAP_BIN_LIMIT);
+	res = erts_new_heap_binary(p, NULL, n, &b);
+	if (endianess == am_big) {
+	    for(i=n-1;i>=0;--i) {
+		b[i] = u & 0xFF;
+		u >>= 8;
+	    }
+	} else {
+	    for(i=0;i<n;++i) {
+		b[i] = u & 0xFF;
+		u >>= 8;
+	    }
+	}
+	BIF_RET(res);
+    } else {
+	/* Big */
+	Eterm *bigp = big_val(uns);
+	Uint n;
+	dsize_t num_parts = BIG_SIZE(bigp);
+	Eterm res;
+	byte *b;
+	ErtsDigit d;
+
+	if(BIG_SIGN(bigp)) {
+	    goto badarg;
+	}
+	n = (num_parts-1)*sizeof(ErtsDigit)+get_need(BIG_DIGIT(bigp,(num_parts-1)));
+	if (n <= ERL_ONHEAP_BIN_LIMIT) {
+	    res = erts_new_heap_binary(p,NULL,n,&b);
+	} else {
+	    res = erts_new_mso_binary(p,NULL,n);
+	    b = ((ProcBin *) binary_val(res))->bytes;
+	}
+
+	if (endianess == am_big) {
+	    Sint i,j;
+	    j = 0;
+	    d = BIG_DIGIT(bigp,0);
+	    for (i=n-1;i>=0;--i) {
+		b[i] = d & 0xFF;
+		if (!((++j) % sizeof(ErtsDigit))) {
+		    d = BIG_DIGIT(bigp,j / sizeof(ErtsDigit));
+		} else {
+		    d >>= 8;
+		}
+	    }
+	} else {
+	    Sint i,j;
+	    j = 0;
+	    d = BIG_DIGIT(bigp,0);
+	    for (i=0;i<n;++i) {
+		b[i] = d & 0xFF;
+		if (!((++j) % sizeof(ErtsDigit))) {
+		    d = BIG_DIGIT(bigp,j / sizeof(ErtsDigit));
+		} else {
+		    d >>= 8;
+		}
+	    }
+
+	}
+	BIF_RET(res);
+    }
+ badarg:
+    BIF_ERROR(p,BADARG);
+}
+
+static BIF_RETTYPE do_decode_unsigned(Process *p, Eterm uns, Eterm endianess)
+{
+    byte *bytes;
+    Uint bitoffs, bitsize;
+    Uint size;
+    Eterm res;
+
+    if (is_not_binary(uns) || is_not_atom(endianess) ||
+	(endianess != am_big && endianess != am_little)) {
+	goto badarg;
+    }
+    ERTS_GET_BINARY_BYTES(uns, bytes, bitoffs, bitsize);
+    if (bitsize != 0) {
+	goto badarg;
+    }
+    /* align while rolling */
+    size = binary_size(uns);
+    if (bitoffs) {
+	if (endianess == am_big) {
+	    while (size && (((((Uint) bytes[0]) << bitoffs) |
+			    (((Uint) bytes[1]) >> (8-bitoffs))) & 0xFF) == 0) {
+		++bytes;
+		--size;
+	    }
+	} else {
+	    while(size &&
+		  (((((Uint) bytes[size-1]) << bitoffs) |
+		    (((Uint) bytes[size]) >> (8-bitoffs))) & 0xFF) == 0) {
+		--size;
+	    }
+	}
+    } else {
+	if (endianess == am_big) {
+	    while (size && *bytes == 0) {
+		++bytes;
+		--size;
+	    }
+	} else {
+	    while(size && bytes[size-1] == 0) {
+		--size;
+	    }
+	}
+    }
+    if (!size) {
+	BIF_RET(make_small(0));
+    }
+
+    if (size <= sizeof(Uint)) {
+	Uint u = 0;
+	Sint i;
+
+	if (endianess == am_big) {
+		if (bitoffs) {
+		    for(i=0;i<size;++i) {
+			u <<=8;
+			u |= (((((Uint) bytes[i]) << bitoffs) |
+			       (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF);
+		    }
+		} else {
+		    for(i=0;i<size;++i) {
+			u <<=8;
+			u |= bytes[i];
+		    }
+		}
+	} else {
+
+		if (bitoffs) {
+		    for(i=size-1;i>=0;--i) {
+			u <<=8;
+			u |= (((((Uint) bytes[i]) << bitoffs) |
+			       (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF);
+		    }
+		} else {
+		    for(i=size-1;i>=0;--i) {
+			u <<=8;
+			u |= bytes[i];
+		    }
+		}
+	}
+	res = erts_make_integer(u,p);
+	BIF_RET(res);
+    } else {
+	/* Assume big, as we stripped away all zeroes from the MSB part of the binary */
+	dsize_t num_parts = size / sizeof(ErtsDigit) + !!(size % sizeof(ErtsDigit));
+	Eterm *bigp;
+
+	bigp = HAlloc(p, BIG_NEED_SIZE(num_parts));
+	*bigp = make_pos_bignum_header(num_parts);
+	res = make_big(bigp);
+
+	if (endianess == am_big) {
+	    Sint i,j;
+	    ErtsDigit *d;
+	    j = size;
+	    d = &(BIG_DIGIT(bigp,num_parts - 1));
+	    *d = 0;
+	    i = 0;
+	    if(bitoffs) {
+		for (;;){
+		    (*d) <<= 8;
+		    (*d) |= (((((Uint) bytes[i]) << bitoffs) |
+			      (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF);
+		    if (++i >= size) {
+			break;
+		    }
+		    if (!(--j % sizeof(ErtsDigit))) {
+			--d;
+			*d = 0;
+		    }
+		}
+	    } else {
+		for (;;){
+		    (*d) <<= 8;
+		    (*d) |= bytes[i];
+		    if (++i >= size) {
+			break;
+		    }
+		    if (!(--j % sizeof(ErtsDigit))) {
+			--d;
+			*d = 0;
+		    }
+		}
+	    }
+	} else {
+	    Sint i,j;
+	    ErtsDigit *d;
+	    j = size;
+	    d = &(BIG_DIGIT(bigp,num_parts - 1));
+	    *d = 0;
+	    i = size-1;
+	    if (bitoffs) {
+		for (;;){
+		    (*d) <<= 8;
+		    (*d) |= (((((Uint) bytes[i]) << bitoffs) |
+			      (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF);
+		    if (--i < 0) {
+			break;
+		    }
+		    if (!(--j % sizeof(ErtsDigit))) {
+			--d;
+			*d = 0;
+		    }
+		}
+	    } else {
+		for (;;){
+		    (*d) <<= 8;
+		    (*d) |= bytes[i];
+		    if (--i < 0) {
+			break;
+		    }
+		    if (!(--j % sizeof(ErtsDigit))) {
+			--d;
+			*d = 0;
+		    }
+		}
+	    }
+	}
+	BIF_RET(res);
+    }
+ badarg:
+    BIF_ERROR(p,BADARG);
+}
+
+BIF_RETTYPE binary_encode_unsigned_1(BIF_ALIST_1)
+{
+    return do_encode_unsigned(BIF_P,BIF_ARG_1,am_big);
+}
+
+BIF_RETTYPE binary_encode_unsigned_2(BIF_ALIST_2)
+{
+    return do_encode_unsigned(BIF_P,BIF_ARG_1,BIF_ARG_2);
+}
+
+BIF_RETTYPE binary_decode_unsigned_1(BIF_ALIST_1)
+{
+    return do_decode_unsigned(BIF_P,BIF_ARG_1,am_big);
+}
+
+BIF_RETTYPE binary_decode_unsigned_2(BIF_ALIST_2)
+{
+    return do_decode_unsigned(BIF_P,BIF_ARG_1,BIF_ARG_2);
+}
+
+/*
+ * Hard debug functions (dump) for the search structures
+ */
+
+#ifdef HARDDEBUG
+static void dump_bm_data(BMData *bm)
+{
+    int i,j;
+    erts_printf("Dumping Boyer-Moore structure.\n");
+    erts_printf("=============================\n");
+    erts_printf("Searchstring [%ld]:\n", bm->len);
+    erts_printf("<<");
+    for (i = 0; i < bm->len; ++i) {
+	if (i > 0) {
+	    erts_printf(", ");
+	}
+	erts_printf("%d", (int) bm->x[i]);
+	if (bm->x[i] >= 'A') {
+	    erts_printf(" ($%c)",(char) bm->x[i]);
+	}
+    }
+    erts_printf(">>\n");
+    erts_printf("GoodShift array:\n");
+    for (i = 0; i < bm->len; ++i) {
+	erts_printf("GoodShift[%d]: %ld\n", i, bm->goodshift[i]);
+    }
+    erts_printf("BadShift array:\n");
+    j = 0;
+    for (i = 0; i < ALPHABET_SIZE; i += j) {
+	for (j = 0; i + j < ALPHABET_SIZE && j < 6; ++j) {
+	    erts_printf("BS[%03d]:%02ld, ", i+j, bm->badshift[i+j]);
+	}
+	erts_printf("\n");
+    }
+}
+
+static void dump_ac_node(ACNode *node, int indent, int ch) {
+    int i;
+    char *spaces = erts_alloc(ERTS_ALC_T_TMP, 10 * indent + 1);
+    memset(spaces,' ',10*indent);
+    spaces[10*indent] = '\0';
+    erts_printf("%s-> %c\n",spaces,ch);
+    erts_printf("%sId: %u\n",spaces,(unsigned) node->id);
+    erts_printf("%sD: %u\n",spaces,(unsigned)node->d);
+    erts_printf("%sFinal: %d\n",spaces,(int)node->final);
+    erts_printf("%sFail: %u\n",spaces,(unsigned)node->h->id);
+    erts_free(ERTS_ALC_T_TMP,spaces);
+    for(i=0;i<ALPHABET_SIZE;++i) {
+	if (node->g[i] != NULL && node->g[i] != node) {
+	    dump_ac_node(node->g[i],indent+1,i);
+	}
+    }
+}
+
+
+static void dump_ac_trie(ACTrie *act)
+{
+    erts_printf("Aho Corasick Trie dump.\n");
+    erts_printf("=======================\n");
+    erts_printf("Node counter: %u\n", (unsigned) act->idc);
+    erts_printf("Searchstring counter: %u\n", (unsigned) act->counter);
+    erts_printf("Trie:\n");
+    dump_ac_node(act->root, 0, '0');
+    return;
+}
+#endif
diff --git a/erts/emulator/beam/erl_bif_chksum.c b/erts/emulator/beam/erl_bif_chksum.c
index 445ba00ca7..06b7ffdf32 100644
--- a/erts/emulator/beam/erl_bif_chksum.c
+++ b/erts/emulator/beam/erl_bif_chksum.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2008-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -49,9 +49,9 @@ void erts_init_bif_chksum(void)
     chksum_md5_2_exp.code[1] = am_atom_put("md5_trap",8);
     chksum_md5_2_exp.code[2] = 2;
     chksum_md5_2_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     chksum_md5_2_exp.code[4] = 
-	(Eterm) &md5_2;
+	(BeamInstr) &md5_2;
 }
     
 
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index 9d5f0d9c02..c338ee1c4b 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2012. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -45,6 +45,7 @@
 #include "big.h"
 #include "dist.h"
 #include "erl_version.h"
+#include "dtrace-wrapper.h"
 
 #ifdef ERTS_SMP
 #define DDLL_SMP 1
@@ -142,11 +143,13 @@ static void ddll_no_more_references(void *vdh);
  * really load and add as LOADED {ok,loaded} {ok,pending_driver}
  * {error, permanent} {error,load_error()}
  */
-BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term, 
-				Eterm name_term, Eterm options)
+BIF_RETTYPE erl_ddll_try_load_3(BIF_ALIST_3)
 {
+    Eterm path_term = BIF_ARG_1;
+    Eterm name_term = BIF_ARG_2;
+    Eterm options = BIF_ARG_3;
     char *path = NULL;
-    int path_len;
+    Uint path_len;
     char *name = NULL;
     DE_Handle *dh;
     erts_driver_t *drv;
@@ -221,9 +224,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
 	goto error;
     }
 
-    path_len = io_list_len(path_term);
-
-    if (path_len <= 0) {
+    if (erts_iolist_size(path_term, &path_len)) {
 	goto error;
     }
     path = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, path_len + 1 /* might need path separator */ + sys_strlen(name) + 1);
@@ -238,7 +239,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
     sys_strcpy(path+path_len,name);
 
 #if DDLL_SMP
-    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
+    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
     lock_drv_list();
 #endif
     if ((drv = lookup_driver(name)) != NULL) {
@@ -249,7 +250,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
 	} else {
 	    dh = drv->handle;
 	    if (dh->status == ERL_DE_OK) {
-		int is_last = is_last_user(dh,p);
+		int is_last = is_last_user(dh, BIF_P);
 		if (reload == 1 && !is_last) {
 		    /*Want reload if no other users, 
 		      but there are others...*/
@@ -263,7 +264,8 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
 			soft_error_term = am_inconsistent;
 			goto soft_error;
 		    }
-		    if ((old = find_proc_entry(dh, p, ERL_DE_PROC_LOADED)) == 
+		    if ((old = find_proc_entry(dh, BIF_P,
+					       ERL_DE_PROC_LOADED)) ==
 			NULL) {
 			soft_error_term = am_not_loaded_by_this_process;
 			goto soft_error;
@@ -274,7 +276,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
 		    }
 		    /* Reload requested and granted */
 		    dereference_all_processes(dh);
-		    set_driver_reloading(dh, p, path, name, flags);
+		    set_driver_reloading(dh, BIF_P, path, name, flags);
 		    if (dh->flags & ERL_DE_FL_KILL_PORTS) {
 			kill_ports = 1;
 		    }
@@ -288,7 +290,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
 			soft_error_term = am_inconsistent;
 			goto soft_error;
 		    }
-		    add_proc_loaded(dh,p);
+		    add_proc_loaded(dh, BIF_P);
 		    erts_ddll_reference_driver(dh);
 		    monitor = 0;
 		    ok_term = mkatom("already_loaded");
@@ -310,7 +312,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
 		notify_all(dh, drv->name, 
 			   ERL_DE_PROC_AWAIT_UNLOAD, am_UP, 
 			   am_unload_cancelled);
-		add_proc_loaded(dh,p);
+		add_proc_loaded(dh, BIF_P);
 		erts_ddll_reference_driver(dh);
 		monitor = 0;
 		ok_term = mkatom("already_loaded");
@@ -327,7 +329,8 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
 		    goto soft_error;
 		}
 		/* Load of granted unload... */
-		add_proc_loaded_deref(dh,p); /* Dont reference, will happen after reload */
+		/* Don't reference, will happen after reload */
+		add_proc_loaded_deref(dh, BIF_P);
 		++monitor;
 		ok_term = am_pending_driver;
 	    } else { /* ERL_DE_PERMANENT */
@@ -347,7 +350,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
 	    goto soft_error;
 	} else {
 	    dh->flags = flags;
-	    add_proc_loaded(dh,p);
+	    add_proc_loaded(dh, BIF_P);
 	    first_ddll_reference(dh);
 	    monitor = 0;
 	    ok_term = mkatom("loaded");
@@ -371,7 +374,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
 	    if (!(prt->status & FREE_PORT_FLAGS) &&
 		prt->drv_ptr->handle == dh) {
 #if DDLL_SMP
-		erts_smp_atomic_inc(&prt->refc);
+		erts_smp_atomic_inc_nob(&prt->refc);
 		/* Extremely rare spinlock */
 		while(prt->status & ERTS_PORT_SFLG_INITIALIZING) {
 		       erts_smp_port_state_unlock(prt);
@@ -399,18 +402,18 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
 #if DDLL_SMP
     erts_ddll_reference_driver(dh);
     unlock_drv_list();
-    erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
+    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
     lock_drv_list();
     erts_ddll_dereference_driver(dh);
 #endif
 
-    p->flags |= F_USING_DDLL;
+    BIF_P->flags |= F_USING_DDLL;
     if (monitor) {
-	Eterm mref = add_monitor(p, dh, ERL_DE_PROC_AWAIT_LOAD);
-	hp = HAlloc(p,4);
+	Eterm mref = add_monitor(BIF_P, dh, ERL_DE_PROC_AWAIT_LOAD);
+	hp = HAlloc(BIF_P, 4);
 	t = TUPLE3(hp, am_ok, ok_term, mref);
     } else {
-	hp = HAlloc(p,3);
+	hp = HAlloc(BIF_P, 3);
 	t = TUPLE2(hp, am_ok, ok_term);
     }
 #if DDLL_SMP
@@ -418,33 +421,33 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
 #endif
     erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) path);
     erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name);
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));
+    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P));
     BIF_RET(t);
  soft_error:
 #if DDLL_SMP
     unlock_drv_list();
-    erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
+    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 #endif
     if (do_build_load_error) {
-	soft_error_term = build_load_error(p, build_this_load_error);
+	soft_error_term = build_load_error(BIF_P, build_this_load_error);
     }
 
-    hp = HAlloc(p,3);
+    hp = HAlloc(BIF_P, 3);
     t = TUPLE2(hp, am_error, soft_error_term);
     erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) path);
     erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name);
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));
+    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P));
     BIF_RET(t);
  error:
     assert_drv_list_not_locked();
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));
+    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(BIF_P));
     if (path != NULL) {
 	erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) path);
     }
     if (name != NULL) {
 	erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name);
     }
-    BIF_ERROR(p,BADARG);
+    BIF_ERROR(BIF_P, BADARG);
 }
 
 /*
@@ -483,8 +486,10 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
    any AWAIT_LOAD-waiters with {'DOWN', ref(), driver, name(), load_cancelled}
    If the driver made itself permanent, {'UP', ref(), driver, name(), permanent}
 */
-Eterm erl_ddll_try_unload_2(Process *p, Eterm name_term, Eterm options)
+Eterm erl_ddll_try_unload_2(BIF_ALIST_2)
 {
+    Eterm name_term = BIF_ARG_1;
+    Eterm options = BIF_ARG_2;
     char *name = NULL;
     Eterm ok_term = NIL;
     Eterm soft_error_term = NIL;
@@ -497,7 +502,7 @@ Eterm erl_ddll_try_unload_2(Process *p, Eterm name_term, Eterm options)
     Eterm l;
     int kill_ports = 0;
 
-    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
+    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
     for(l = options; is_list(l); l =  CDR(list_val(l))) {
 	Eterm opt = CAR(list_val(l));
@@ -550,7 +555,7 @@ Eterm erl_ddll_try_unload_2(Process *p, Eterm name_term, Eterm options)
     if (dh->flags & ERL_DE_FL_KILL_PORTS) {
 	kill_ports = 1;
     }
-    if ((pe = find_proc_entry(dh, p, ERL_DE_PROC_LOADED)) == NULL) {
+    if ((pe = find_proc_entry(dh, BIF_P, ERL_DE_PROC_LOADED)) == NULL) {
 	if (num_procs(dh, ERL_DE_PROC_LOADED) > 0) {
 	    soft_error_term = am_not_loaded_by_this_process;
 	    goto soft_error;
@@ -599,7 +604,7 @@ done:
 	    if (!(prt->status &  FREE_PORT_FLAGS) 
 		&& prt->drv_ptr->handle == dh) {
 #if DDLL_SMP
-		erts_smp_atomic_inc(&prt->refc);
+		erts_smp_atomic_inc_nob(&prt->refc);
 		/* Extremely rare spinlock */
 		while(prt->status & ERTS_PORT_SFLG_INITIALIZING) {
 		       erts_smp_port_state_unlock(prt);
@@ -626,22 +631,22 @@ done:
 #if DDLL_SMP
     erts_ddll_reference_driver(dh);
     unlock_drv_list();
-    erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
+    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
     lock_drv_list();
     erts_ddll_dereference_driver(dh);
 #endif
     erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name);
-    p->flags |= F_USING_DDLL;
+    BIF_P->flags |= F_USING_DDLL;
     if (monitor > 0) {
-	Eterm mref = add_monitor(p, dh, ERL_DE_PROC_AWAIT_UNLOAD);
-	hp = HAlloc(p,4);
+	Eterm mref = add_monitor(BIF_P, dh, ERL_DE_PROC_AWAIT_UNLOAD);
+	hp = HAlloc(BIF_P, 4);
 	t = TUPLE3(hp, am_ok, ok_term, mref);
     } else {
-	hp = HAlloc(p,3);
+	hp = HAlloc(BIF_P, 3);
 	t = TUPLE2(hp, am_ok, ok_term);
     }
     if (kill_ports > 1) {
-	ERTS_BIF_CHK_EXITED(p); /* May be exited by port killing */
+	ERTS_BIF_CHK_EXITED(BIF_P); /* May be exited by port killing */
     }
 #if DDLL_SMP
     unlock_drv_list();
@@ -653,8 +658,8 @@ soft_error:
     unlock_drv_list();
 #endif
     erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name);
-    erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
-    hp = HAlloc(p,3);
+    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    hp = HAlloc(BIF_P, 3);
     t = TUPLE2(hp, am_error, soft_error_term);
     BIF_RET(t);
  
@@ -663,21 +668,21 @@ soft_error:
     if (name != NULL) {
 	erts_free(ERTS_ALC_T_DDLL_TMP_BUF, (void *) name);
     }
-    erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
-    BIF_ERROR(p,BADARG);
+    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    BIF_ERROR(BIF_P, BADARG);
 }
 
 
 /* 
  * A shadow of the "real" demonitor BIF
  */
-BIF_RETTYPE erl_ddll_demonitor_1(Process *p, Eterm ref)
+BIF_RETTYPE erl_ddll_demonitor_1(BIF_ALIST_1)
 {
-   if (is_not_internal_ref(ref)) {
-       BIF_ERROR(p, BADARG);
+   if (is_not_internal_ref(BIF_ARG_1)) {
+       BIF_ERROR(BIF_P, BADARG);
    }
-   if (p->flags & F_USING_DDLL) {
-       erts_ddll_remove_monitor(p, ref, ERTS_PROC_LOCK_MAIN);
+   if (BIF_P->flags & F_USING_DDLL) {
+       erts_ddll_remove_monitor(BIF_P, BIF_ARG_1, ERTS_PROC_LOCK_MAIN);
    }
    BIF_RET(am_true);
 }
@@ -685,18 +690,18 @@ BIF_RETTYPE erl_ddll_demonitor_1(Process *p, Eterm ref)
 /* 
  * A shadow of the "real" monitor BIF
  */
-BIF_RETTYPE erl_ddll_monitor_2(Process *p, Eterm dr, Eterm what)
+BIF_RETTYPE erl_ddll_monitor_2(BIF_ALIST_2)
 {
-    if (dr != am_driver) {
-	BIF_ERROR(p,BADARG);
+    if (BIF_ARG_1 != am_driver) {
+	BIF_ERROR(BIF_P, BADARG);
     }
-    return erts_ddll_monitor_driver(p, what, ERTS_PROC_LOCK_MAIN);
+    return erts_ddll_monitor_driver(BIF_P, BIF_ARG_2, ERTS_PROC_LOCK_MAIN);
 }
 
 /* 
  * Return list of loaded drivers {ok,[string()]} 
  */
-Eterm erl_ddll_loaded_drivers_0(Process *p)
+BIF_RETTYPE erl_ddll_loaded_drivers_0(BIF_ALIST_0)
 {
     Eterm *hp;
     int need = 3;
@@ -708,7 +713,7 @@ Eterm erl_ddll_loaded_drivers_0(Process *p)
     for (drv = driver_list; drv; drv = drv->next) {
 	need += sys_strlen(drv->name)*2+2;
     }
-    hp = HAlloc(p,need);
+    hp = HAlloc(BIF_P, need);
     for (drv = driver_list; drv; drv = drv->next) {
 	Eterm l;
 	l = buf_to_intlist(&hp, drv->name, sys_strlen(drv->name), NIL);
@@ -728,8 +733,11 @@ Eterm erl_ddll_loaded_drivers_0(Process *p)
  * item is processes, driver_options, port_count, linked_in_driver, 
  * permanent, awaiting_load, awaiting_unload 
  */
-Eterm erl_ddll_info_2(Process *p, Eterm name_term, Eterm item) 
+BIF_RETTYPE erl_ddll_info_2(BIF_ALIST_2)
 {
+    Process *p = BIF_P;
+    Eterm name_term = BIF_ARG_1;
+    Eterm item = BIF_ARG_2;
     char *name = NULL;
     Eterm res = NIL;
     erts_driver_t *drv;
@@ -852,8 +860,10 @@ Eterm erl_ddll_info_2(Process *p, Eterm name_term, Eterm item)
  * Backend for erl_ddll:format_error, handles all "soft" errors returned by builtins,
  * possibly by calling the system specific error handler
  */
-Eterm erl_ddll_format_error_int_1(Process *p, Eterm code_term)
+BIF_RETTYPE erl_ddll_format_error_int_1(BIF_ALIST_1)
 {
+    Process *p = BIF_P;
+    Eterm code_term = BIF_ARG_1;
     char *errstring = NULL;
     int errint;
     int len;
@@ -1056,7 +1066,7 @@ void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks)
 			if (!(prt->status & FREE_PORT_FLAGS) &&
 			    prt->drv_ptr->handle == dh) {
 #if DDLL_SMP
-			    erts_smp_atomic_inc(&prt->refc);
+			    erts_smp_atomic_inc_nob(&prt->refc);
 			    while(prt->status & ERTS_PORT_SFLG_INITIALIZING) {
 				erts_smp_port_state_unlock(prt);
 				erts_smp_port_state_lock(prt);
@@ -1193,7 +1203,7 @@ int erts_ddll_driver_ok(DE_Handle *dh)
 static void ddll_no_more_references(void *vdh)
 {
     DE_Handle *dh = (DE_Handle *) vdh;
-    int x;
+    erts_aint_t x;
 
     lock_drv_list();
 
@@ -1560,51 +1570,36 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name)
     
     if ((res = erts_sys_ddll_load_driver_init(dh->handle, 
 					      &init_handle)) != ERL_DE_NO_ERROR) {
-	erts_sys_ddll_close(dh->handle);
-	return ERL_DE_LOAD_ERROR_NO_INIT;
+	res = ERL_DE_LOAD_ERROR_NO_INIT;
+	goto error;
     }
     
     dp = erts_sys_ddll_call_init(init_handle);
     if (dp == NULL) {
-	erts_sys_ddll_close(dh->handle);
-	return ERL_DE_LOAD_ERROR_FAILED_INIT;
+	res = ERL_DE_LOAD_ERROR_FAILED_INIT;
+	goto error;
     }
 
     switch (dp->extended_marker) {
-    case 0:
-	/*
-	 * This may be an old driver that has been recompiled. If so,
-	 * at least the fields that existed in extended driver version
-	 * 1.0 should be zero. If not, a it is a bad driver. We cannot
-	 * be completely certain that this is a valid driver but this is
-	 * the best we can do with old drivers...
-	 */
-	if (dp->major_version != 0
-	    || dp->minor_version != 0
-	    || dp->driver_flags != 0
-	    || dp->handle2 != NULL
-	    || dp->process_exit != NULL) {
-	    /* Old driver; needs to be recompiled... */
-	    return ERL_DE_LOAD_ERROR_INCORRECT_VERSION;
-	}
-	break;
     case ERL_DRV_EXTENDED_MARKER:
 	if (ERL_DRV_EXTENDED_MAJOR_VERSION != dp->major_version
 	    || ERL_DRV_EXTENDED_MINOR_VERSION < dp->minor_version) {
 	    /* Incompatible driver version */
-	    return ERL_DE_LOAD_ERROR_INCORRECT_VERSION;
+	    res = ERL_DE_LOAD_ERROR_INCORRECT_VERSION;
+	    goto error;
 	}
 	break;
     default:
 	/* Old driver; needs to be recompiled... */
-	return ERL_DE_LOAD_ERROR_INCORRECT_VERSION;
+	res = ERL_DE_LOAD_ERROR_INCORRECT_VERSION;
+	goto error;
     }
 
     if (strcmp(name, dp->driver_name) != 0) {
-	erts_sys_ddll_close(dh->handle);
-	return ERL_DE_LOAD_ERROR_BAD_NAME;
+	res = ERL_DE_LOAD_ERROR_BAD_NAME;
+	goto error;
     }
-    erts_smp_atomic_init(&(dh->refc), (long) 0);
+    erts_smp_atomic_init_nob(&(dh->refc), (erts_aint_t) 0);
     dh->port_count = 0;
     dh->full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1);
     sys_strcpy(dh->full_path, path);
@@ -1617,11 +1612,14 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name)
 	 */
 	erts_free(ERTS_ALC_T_DDLL_HANDLE, dh->full_path);
 	dh->full_path = NULL;
-	erts_sys_ddll_close(dh->handle);
-	return ERL_DE_LOAD_ERROR_FAILED_INIT;
+	res = ERL_DE_LOAD_ERROR_FAILED_INIT;
+	goto error;
     }
-
     return ERL_DE_NO_ERROR;
+
+error:
+    erts_sys_ddll_close(dh->handle);
+    return res;
 }
 
 static int do_unload_driver_entry(DE_Handle *dh, Eterm *save_name)
@@ -1646,9 +1644,11 @@ static int do_unload_driver_entry(DE_Handle *dh, Eterm *save_name)
 	    if (save_name != NULL) {
 		*save_name = mkatom(q->name);
 	    } 
-	    /* XXX:PaN Future locking problems? Don't dare to let go of the diver_list lock here!*/
+	    /* Future locking problems? Don't dare to let go of the
+	       diver_list lock here!*/
 	    if (q->finish) {
 		int fpe_was_unmasked = erts_block_fpe();
+		DTRACE1(driver_finish, q->name);
 		(*(q->finish))();
 		erts_unblock_fpe(fpe_was_unmasked);
 	    }
@@ -1671,7 +1671,7 @@ static int load_driver_entry(DE_Handle **dhp, char *path, char *name)
     dh->handle = NULL;
     dh->procs = NULL;
     dh->port_count = 0;
-    erts_refc_init(&(dh->refc), (long) 0);
+    erts_refc_init(&(dh->refc), (erts_aint_t) 0);
     dh->status = -1;
     dh->reload_full_path = NULL;
     dh->reload_driver_name = NULL;
@@ -1762,7 +1762,11 @@ static void notify_proc(Process *proc, Eterm ref, Eterm driver_name, Eterm type,
 	hp += REF_THING_SIZE;
 	mess = TUPLE5(hp,type,r,am_driver,driver_name,tag);
     }
-    erts_queue_message(proc, &rp_locks, bp, mess, am_undefined);
+    erts_queue_message(proc, &rp_locks, bp, mess, am_undefined
+#ifdef USE_VM_PROBES
+		       , NIL
+#endif
+		       );
     erts_smp_proc_unlock(proc, rp_locks);
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 }
@@ -1877,7 +1881,7 @@ static Eterm mkatom(char *str)
 static char *pick_list_or_atom(Eterm name_term)
 { 
     char *name = NULL;
-    int name_len;
+    Uint name_len;
     if (is_atom(name_term)) {
 	Atom *ap = atom_tab(atom_val(name_term));
 	if (ap->len == 0) {
@@ -1889,8 +1893,7 @@ static char *pick_list_or_atom(Eterm name_term)
 	memcpy(name,ap->name,ap->len);
 	name[ap->len] = '\0';
     } else {
-	name_len = io_list_len(name_term);
-	if (name_len <= 0) {
+	if (erts_iolist_size(name_term, &name_len)) {
 	    goto error;
 	}
 	name = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, name_len + 1);
diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c
index 440b0b4f14..a715756c15 100644
--- a/erts/emulator/beam/erl_bif_guard.c
+++ b/erts/emulator/beam/erl_bif_guard.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2006-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -52,7 +52,7 @@ BIF_RETTYPE abs_1(BIF_ALIST_1)
     /* integer arguments */
     if (is_small(BIF_ARG_1)) {
 	i0 = signed_val(BIF_ARG_1);
-	i = labs(i0);
+	i = ERTS_SMALL_ABS(i0);
 	if (i0 == MIN_SMALL) {
 	    hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE);
 	    BIF_RET(uint_to_big(i, hp));
@@ -314,6 +314,30 @@ double_to_integer(Process* p, double x)
     return res;
 }
 
+/********************************************************************************
+ * binary_part guards. The actual implementation is in erl_bif_binary.c
+ ********************************************************************************/
+BIF_RETTYPE binary_part_3(BIF_ALIST_3)
+{
+    return erts_binary_part(BIF_P,BIF_ARG_1,BIF_ARG_2, BIF_ARG_3);
+}
+
+BIF_RETTYPE binary_part_2(BIF_ALIST_2)
+{
+    Eterm *tp;
+    if (is_not_tuple(BIF_ARG_2)) {
+	goto badarg;
+    }
+    tp = tuple_val(BIF_ARG_2);
+    if (arityval(*tp) != 2) {
+	goto badarg;
+    }
+    return erts_binary_part(BIF_P,BIF_ARG_1,tp[1], tp[2]);
+ badarg:
+   BIF_ERROR(BIF_P,BADARG);
+}
+
+
 /*
  * The following code is used when a guard that may build on the
  * heap is called directly. They must not use HAlloc(), but must
@@ -443,7 +467,7 @@ Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live)
     /* integer arguments */
     if (is_small(arg)) {
 	i0 = signed_val(arg);
-	i = labs(i0);
+	i = ERTS_SMALL_ABS(i0);
 	if (i0 == MIN_SMALL) {
 	    if (ERTS_NEED_GC(p, BIG_UINT_HEAP_SIZE)) {
 		erts_garbage_collect(p, BIG_UINT_HEAP_SIZE, reg, live+1);
@@ -630,3 +654,16 @@ gc_double_to_integer(Process* p, double x, Eterm* reg, Uint live)
     }
     return res;
 }
+
+/********************************************************************************
+ * binary_part guards. The actual implementation is in erl_bif_binary.c
+ ********************************************************************************/
+Eterm erts_gc_binary_part_3(Process* p, Eterm* reg, Uint live)
+{
+    return erts_gc_binary_part(p,reg,live,0);
+}
+
+Eterm erts_gc_binary_part_2(Process* p, Eterm* reg, Uint live)
+{
+    return erts_gc_binary_part(p,reg,live,1);
+}
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index a34d400ed8..f56b00287f 100644..100755
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -25,7 +25,6 @@
 #include "erl_vm.h"
 #include "global.h"
 #include "erl_process.h"
-#include "erl_nmgc.h"
 #include "error.h"
 #include "erl_driver.h"
 #include "bif.h"
@@ -38,9 +37,9 @@
 #include "erl_instrument.h"
 #include "dist.h"
 #include "erl_gc.h"
-#ifdef ELIB_ALLOC_IS_CLIB
-#include "elib_stat.h"
-#endif
+#include "erl_cpu_topology.h"
+#include "erl_async.h"
+#include "erl_thr_progress.h"
 #ifdef HIPE
 #include "hipe_arch.h"
 #endif
@@ -54,21 +53,32 @@
 #include <valgrind/memcheck.h>
 #endif
 
+static Export* alloc_info_trap = NULL;
+static Export* alloc_sizes_trap = NULL;
+
+static Export *gather_sched_wall_time_res_trap;
+
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 
 /* Keep erts_system_version as a global variable for easy access from a core */
 static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE
 				     " (erts-" ERLANG_VERSION ")"
+#if !HEAP_ON_C_STACK && !HALFWORD_HEAP
+				     " [no-c-stack-objects]"
+#endif
 #ifndef OTP_RELEASE
 				     " [source]"
 #endif	
 #ifdef ARCH_64
+#if HALFWORD_HEAP
+				     " [64-bit halfword]"
+#else
 				     " [64-bit]"
 #endif
+#endif
 #ifdef ERTS_SMP
-				     " [smp:%bpu:%bpu]"
+				     " [smp:%beu:%beu]"
 #endif
-				     " [rq:%bpu]"
 #ifdef USE_THREADS
 				     " [async-threads:%d]"
 #endif
@@ -78,12 +88,6 @@ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE
 #ifdef ERTS_ENABLE_KERNEL_POLL
 				     " [kernel-poll:%s]"
 #endif	
-#ifdef HYBRID
-				     " [hybrid heap]"
-#endif
-#ifdef INCREMENTAL
-				     " [incremental GC]"
-#endif
 #ifdef ET_DEBUG
 #if ET_DEBUG
 				     " [type-assertions]"
@@ -104,6 +108,12 @@ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE
 #ifdef VALGRIND
 				     " [valgrind-compiled]"
 #endif
+#ifdef USE_DTRACE
+				     " [dtrace]"
+#endif
+#ifdef USE_SYSTEMTAP
+				     " [systemtap]"
+#endif
 				     "\n");
 
 #define ASIZE(a) (sizeof(a)/sizeof(a[0]))
@@ -114,23 +124,37 @@ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE
 # define PERFMON_GETPCR			_IOR('P', 2, unsigned long long)
 #endif
 
+/* Cached, pre-built {OsType,OsFlavor} and {Major,Minor,Build} tuples */
+static Eterm os_type_tuple;
+static Eterm os_version_tuple;
+
+static BIF_RETTYPE port_info(Process* p, Eterm portid, Eterm item);
+
+static Eterm
+current_function(Process* p, Process* rp, Eterm** hpp, int full_info);
+static Eterm current_stacktrace(Process* p, Process* rp, Eterm** hpp);
+
 static Eterm
-bld_bin_list(Uint **hpp, Uint *szp, ProcBin* pb)
+bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh)
 {
+    struct erl_off_heap_header* ohh;
     Eterm res = NIL;
     Eterm tuple;
 
-    for (; pb; pb = pb->next) {
-	Eterm val = erts_bld_uint(hpp, szp, (Uint) pb->val);
-	Eterm orig_size = erts_bld_uint(hpp, szp, pb->val->orig_size);
-
-	if (szp)
-	    *szp += 4+2;
-	if (hpp) {
-	    Uint refc = (Uint) erts_smp_atomic_read(&pb->val->refc);
-	    tuple = TUPLE3(*hpp, val, orig_size, make_small(refc));
-	    res = CONS(*hpp + 4, tuple, res);
-	    *hpp += 4+2;
+    for (ohh = oh->first; ohh; ohh = ohh->next) {
+	if (ohh->thing_word == HEADER_PROC_BIN) {
+	    ProcBin* pb = (ProcBin*) ohh;
+	    Eterm val = erts_bld_uword(hpp, szp, (UWord) pb->val);
+	    Eterm orig_size = erts_bld_uint(hpp, szp, pb->val->orig_size);
+    
+	    if (szp)
+		*szp += 4+2;
+	    if (hpp) {
+		Uint refc = (Uint) erts_smp_atomic_read_nob(&pb->val->refc);
+		tuple = TUPLE3(*hpp, val, orig_size, make_small(refc));
+		res = CONS(*hpp + 4, tuple, res);
+		*hpp += 4+2;
+	    }
 	}
     }
     return res;
@@ -169,10 +193,10 @@ static void do_make_one_mon_element(ErtsMonitor *mon, void * vpmlc)
     Eterm tup;
     Eterm r = (IS_CONST(mon->ref)
 	       ? mon->ref
-	       : STORE_NC(&(pmlc->hp), &MSO(pmlc->p).externals, mon->ref));
+	       : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->ref));
     Eterm p = (IS_CONST(mon->pid)
 	       ? mon->pid
-	       : STORE_NC(&(pmlc->hp), &MSO(pmlc->p).externals, mon->pid));
+	       : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->pid));
     tup = TUPLE5(pmlc->hp, pmlc->tag, make_small(mon->type), r, p, mon->name);
     pmlc->hp += 6;
     pmlc->res = CONS(pmlc->hp, tup, pmlc->res);
@@ -233,7 +257,7 @@ static void do_make_one_lnk_element(ErtsLink *lnk, void * vpllc)
     Eterm old_res, targets = NIL;
     Eterm p = (IS_CONST(lnk->pid)
 	       ? lnk->pid
-	       : STORE_NC(&(pllc->hp), &MSO(pllc->p).externals, lnk->pid));
+	       : STORE_NC(&(pllc->hp), &MSO(pllc->p), lnk->pid));
     if (lnk->type == LINK_NODE) {
 	targets = make_small(ERTS_LINK_REFC(lnk));
     } else if (ERTS_LINK_ROOT(lnk) != NULL) {
@@ -277,9 +301,7 @@ erts_print_system_version(int to, void *arg, Process *c_p)
 #endif
     return erts_print(to, arg, erts_system_version
 #ifdef ERTS_SMP
-		      , total, online, erts_no_run_queues
-#else
-		      , 1
+		      , total, online
 #endif
 #ifdef USE_THREADS
 		      , erts_async_max_threads
@@ -545,9 +567,8 @@ static Eterm pi_args[] = {
     am_suspending,
     am_min_heap_size,
     am_min_bin_vheap_size,
-#ifdef HYBRID
-    am_message_binary
-#endif
+    am_current_location,
+    am_current_stacktrace,
 };    
 
 #define ERTS_PI_ARGS ((int) (sizeof(pi_args)/sizeof(Eterm)))
@@ -593,9 +614,8 @@ pi_arg2ix(Eterm arg)
     case am_suspending:				return 26;
     case am_min_heap_size:			return 27;
     case am_min_bin_vheap_size:			return 28;
-#ifdef HYBRID
-    case am_message_binary:			return 29;
-#endif
+    case am_current_location:			return 29;
+    case am_current_stacktrace:			return 30;
     default:					return -1;
     }
 }
@@ -624,12 +644,18 @@ static Eterm pi_1_keys[] = {
 #define ERTS_PI_1_NO_OF_KEYS (sizeof(pi_1_keys)/sizeof(Eterm))
 
 static Eterm pi_1_keys_list;
-static Uint pi_1_keys_list_heap[2*ERTS_PI_1_NO_OF_KEYS];
+#if HEAP_ON_C_STACK
+static Eterm pi_1_keys_list_heap[2*ERTS_PI_1_NO_OF_KEYS];
+#endif
 
 static void
 process_info_init(void)
 {
+#if HEAP_ON_C_STACK
     Eterm *hp = &pi_1_keys_list_heap[0];
+#else
+    Eterm *hp = erts_alloc(ERTS_ALC_T_LL_TEMP_TERM,sizeof(Eterm)*2*ERTS_PI_1_NO_OF_KEYS);
+#endif
     int i;
 
     pi_1_keys_list = NIL;
@@ -991,35 +1017,15 @@ process_info_aux(Process *BIF_P,
 	break;
 
     case am_current_function:
-	if (rp->current == NULL) {
-	    rp->current = find_function_from_pc(rp->i);
-	}
-	if (rp->current == NULL) {
-	    hp = HAlloc(BIF_P, 3);
-	    res = am_undefined;
-	} else {
-	    Eterm* current;
-
-	    if (rp->current[0] == am_erlang &&
-		rp->current[1] == am_process_info &&
-		(rp->current[2] == 1 || rp->current[2] == 2) &&
-		(current = find_function_from_pc(rp->cp)) != NULL) {
-
-		/*
-		 * The current function is erlang:process_info/2,
-		 * which is not the answer that the application want.
-		 * We will use the function pointed into by rp->cp
-		 * instead.
-		 */
+	res = current_function(BIF_P, rp, &hp, 0);
+	break;
 
-		rp->current = current;
-	    }
+    case am_current_location:
+	res = current_function(BIF_P, rp, &hp, 1);
+	break;
 
-	    hp = HAlloc(BIF_P, 3+4);
-	    res = TUPLE3(hp, rp->current[0],
-			 rp->current[1], make_small(rp->current[2]));
-	    hp += 4;
-	}
+    case am_current_stacktrace:
+	res = current_stacktrace(BIF_P, rp, &hp);
 	break;
 
     case am_initial_call:
@@ -1062,12 +1068,8 @@ process_info_aux(Process *BIF_P,
 		if (rp != BIF_P) {
 		    Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp);
 		    if (is_value(msg)) {
-			mq[i].copy_struct_size = (is_immed(msg)
-#ifdef HYBRID
-						  || NO_COPY(msg)
-#endif
-						  ? 0
-						  : size_object(msg));
+			mq[i].copy_struct_size = (is_immed(msg)? 0 :
+						  size_object(msg));
 		    }
 		    else if (mq[i].msgp->data.attached) {
 			mq[i].copy_struct_size
@@ -1127,9 +1129,9 @@ process_info_aux(Process *BIF_P,
 			}
 			else {
 			    /* Make our copy of the message */
-			    ASSERT(size_object(msg) == hfp->size);
+			    ASSERT(size_object(msg) == hfp->used_size);
 			    msg = copy_struct(msg,
-					      hfp->size,
+					      hfp->used_size,
 					      &hp,
 					      &MSO(BIF_P));
 			}
@@ -1212,7 +1214,7 @@ process_info_aux(Process *BIF_P,
 	hp = HAlloc(BIF_P, 3 + mic.sz);
 	res = NIL;
 	for (i = 0; i < mic.mi_i; i++) {
-	    item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); 
+	    item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); 
 	    res = CONS(hp, item, res);
 	    hp += 2;
 	}
@@ -1245,9 +1247,7 @@ process_info_aux(Process *BIF_P,
 	    else {
 		/* Monitor by pid. Build {process, Pid} and cons it. */
 		Eterm t;
-		Eterm pid = STORE_NC(&hp,
-				     &MSO(BIF_P).externals,
-				     mic.mi[i].entity);
+		Eterm pid = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity);
 		t = TUPLE2(hp, am_process, pid);
 		hp += 3;
 		res = CONS(hp, t, res);
@@ -1269,7 +1269,7 @@ process_info_aux(Process *BIF_P,
 
 	res = NIL;
 	for (i = 0; i < mic.mi_i; ++i) {
-	    item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); 
+	    item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); 
 	    res = CONS(hp, item, res);
 	    hp += 2;
 	}
@@ -1478,7 +1478,7 @@ process_info_aux(Process *BIF_P,
     case am_group_leader: {
 	int sz = NC_HEAP_SIZE(rp->group_leader);
 	hp = HAlloc(BIF_P, 3 + sz);
-	res = STORE_NC(&hp, &MSO(BIF_P).externals, rp->group_leader);
+	res = STORE_NC(&hp, &MSO(BIF_P), rp->group_leader);
 	break;
     }
 
@@ -1503,22 +1503,12 @@ process_info_aux(Process *BIF_P,
 
     case am_binary: {
 	Uint sz = 3;
-	(void) bld_bin_list(NULL, &sz, MSO(rp).mso);
+	(void) bld_bin_list(NULL, &sz, &MSO(rp));
 	hp = HAlloc(BIF_P, sz);
-	res = bld_bin_list(&hp, NULL, MSO(rp).mso);
+	res = bld_bin_list(&hp, NULL, &MSO(rp));
 	break;
     }
 
-#ifdef HYBRID
-    case am_message_binary: {
-	Uint sz = 3;
-	(void) bld_bin_list(NULL, &sz, erts_global_offheap.mso);
-	hp = HAlloc(BIF_P, sz);
-	res = bld_bin_list(&hp, NULL, erts_global_offheap.mso);
-	break;
-    }
-#endif
-
     case am_sequential_trace_token:
 	res = copy_object(rp->seq_trace_token, BIF_P);
 	hp = HAlloc(BIF_P, 3);
@@ -1532,7 +1522,7 @@ process_info_aux(Process *BIF_P,
     case am_backtrace: {
 	erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0);
 	erts_stack_dump(ERTS_PRINT_DSBUF, (void *) dsbufp, rp);
-	res = new_binary(BIF_P, (byte *) dsbufp->str, (int) dsbufp->str_len);
+	res = new_binary(BIF_P, (byte *) dsbufp->str, dsbufp->str_len);
 	erts_destroy_tmp_dsbuf(dsbufp);
 	hp = HAlloc(BIF_P, 3);
 	break;
@@ -1595,6 +1585,113 @@ process_info_aux(Process *BIF_P,
 }
 #undef MI_INC
 
+static Eterm
+current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info)
+{
+    Eterm* hp;
+    Eterm res;
+    FunctionInfo fi;
+
+    if (rp->current == NULL) {
+	erts_lookup_function_info(&fi, rp->i, full_info);
+	rp->current = fi.current;
+    } else if (full_info) {
+	erts_lookup_function_info(&fi, rp->i, full_info);
+	if (fi.current == NULL) {
+	    /* Use the current function without location info */
+	    erts_set_current_function(&fi, rp->current);
+	}
+    }
+
+    if (BIF_P->id == rp->id) {
+	FunctionInfo fi2;
+
+	/*
+	 * The current function is erlang:process_info/{1,2},
+	 * which is not the answer that the application want.
+	 * We will use the function pointed into by rp->cp
+	 * instead if it can be looked up.
+	 */
+	erts_lookup_function_info(&fi2, rp->cp, full_info);
+	if (fi2.current) {
+	    fi = fi2;
+	    rp->current = fi2.current;
+	}
+    }
+
+    /*
+     * Return the result.
+     */
+    if (rp->current == NULL) {
+	hp = HAlloc(BIF_P, 3);
+	res = am_undefined;
+    } else if (full_info) {
+	hp = HAlloc(BIF_P, 3+fi.needed);
+	hp = erts_build_mfa_item(&fi, hp, am_true, &res);
+    } else {
+	hp = HAlloc(BIF_P, 3+4);
+	res = TUPLE3(hp, rp->current[0],
+		     rp->current[1], make_small(rp->current[2]));
+	hp += 4;
+    }
+    *hpp = hp;
+    return res;
+}
+
+static Eterm
+current_stacktrace(Process* p, Process* rp, Eterm** hpp)
+{
+    Uint sz;
+    struct StackTrace* s;
+    int depth;
+    FunctionInfo* stk;
+    FunctionInfo* stkp;
+    Uint heap_size;
+    int i;
+    Eterm* hp = *hpp;
+    Eterm mfa;
+    Eterm res = NIL;
+
+    depth = 8;
+    sz = offsetof(struct StackTrace, trace) + sizeof(BeamInstr *)*depth;
+    s = (struct StackTrace *) erts_alloc(ERTS_ALC_T_TMP, sz);
+    s->depth = 0;
+    if (rp->i) {
+	s->trace[s->depth++] = rp->i;
+	depth--;
+    }
+    if (depth > 0 && rp->cp != 0) {
+	s->trace[s->depth++] = rp->cp - 1;
+	depth--;
+    }
+    erts_save_stacktrace(rp, s, depth);
+
+    depth = s->depth;
+    stk = stkp = (FunctionInfo *) erts_alloc(ERTS_ALC_T_TMP,
+					     depth*sizeof(FunctionInfo));
+    heap_size = 3;
+    for (i = 0; i < depth; i++) {
+	erts_lookup_function_info(stkp, s->trace[i], 1);
+	if (stkp->current) {
+	    heap_size += stkp->needed + 2;
+	    stkp++;
+	}
+    }
+
+    hp = HAlloc(p, heap_size);
+    while (stkp > stk) {
+	stkp--;
+	hp = erts_build_mfa_item(stkp, hp, am_true, &mfa);
+	res = CONS(hp, mfa, res);
+	hp += 2;
+    }
+
+    erts_free(ERTS_ALC_T_TMP, stk);
+    erts_free(ERTS_ALC_T_TMP, s);
+    *hpp = hp;
+    return res;
+}
+
 #if defined(VALGRIND)
 static int check_if_xml(void)
 {
@@ -1620,8 +1717,26 @@ info_1_tuple(Process* BIF_P,	/* Pointer to current process. */
 
     sel = *tp++;
 
-    if (sel == am_allocator_sizes && arity == 2) {
-	return erts_allocator_info_term(BIF_P, *tp, 1);
+    if (sel == am_allocator_sizes) {
+	switch (arity) {
+	case 2:
+	    ERTS_BIF_PREP_TRAP1(ret, alloc_sizes_trap, BIF_P, *tp);
+	    return ret;
+	case 3:
+	    if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 1))
+		return am_true;
+	default:
+	    goto badarg;
+	}
+    }
+    else if (sel == am_wordsize && arity == 2) {
+	if (tp[0] == am_internal) {
+	    return make_small(sizeof(Eterm));
+	}
+	if (tp[0] == am_external) {
+	    return make_small(sizeof(UWord));
+	}
+	goto badarg;
     } else if (sel == am_allocated) {
 	if (arity == 2) {
 	    Eterm res = THE_NON_VALUE;
@@ -1661,12 +1776,23 @@ info_1_tuple(Process* BIF_P,	/* Pointer to current process. */
 	}
 	else
 	    goto badarg;
-    } else if (sel == am_allocator && arity == 2) {
-	return erts_allocator_info_term(BIF_P, *tp, 0);
+    } else if (sel == am_allocator) {
+	switch (arity) {
+	case 2:
+	    ERTS_BIF_PREP_TRAP1(ret, alloc_info_trap, BIF_P, *tp);
+	    return ret;
+	case 3:
+	    if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 0))
+		return am_true;
+	default:
+	    goto badarg;
+	}
     } else if (ERTS_IS_ATOM_STR("internal_cpu_topology", sel) && arity == 2) {
 	return erts_get_cpu_topology_term(BIF_P, *tp);
     } else if (ERTS_IS_ATOM_STR("cpu_topology", sel) && arity == 2) {
 	Eterm res = erts_get_cpu_topology_term(BIF_P, *tp);
+	if (res == THE_NON_VALUE)
+	    goto badarg;
 	ERTS_BIF_PREP_TRAP1(ret, erts_format_cpu_topology_trap, BIF_P, res);
 	return ret;
 #if defined(PURIFY) || defined(VALGRIND)
@@ -1700,17 +1826,23 @@ info_1_tuple(Process* BIF_P,	/* Pointer to current process. */
 	} else if (is_list(*tp)) {
 #if defined(PURIFY)
 #define ERTS_ERROR_CHECKER_PRINTF purify_printf
+#define ERTS_ERROR_CHECKER_PRINTF_XML purify_printf
 #elif defined(VALGRIND)
 #define ERTS_ERROR_CHECKER_PRINTF VALGRIND_PRINTF
+#  ifndef HAVE_VALGRIND_PRINTF_XML
+#    define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF
+#  else
+#    define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF_XML
+#  endif
 #endif
-	    int buf_size = 8*1024; /* Try with 8KB first */
+	    Uint buf_size = 8*1024; /* Try with 8KB first */
 	    char *buf = erts_alloc(ERTS_ALC_T_TMP, buf_size);
 	    int r = io_list_to_buf(*tp, (char*) buf, buf_size - 1);
 	    if (r < 0) {
 		erts_free(ERTS_ALC_T_TMP, (void *) buf);
-		buf_size = io_list_len(*tp);
-		if (buf_size < 0)
+		if (erts_iolist_size(*tp, &buf_size)) {
 		    goto badarg;
+		}
 		buf_size++;
 		buf = erts_alloc(ERTS_ALC_T_TMP, buf_size);
 		r = io_list_to_buf(*tp, (char*) buf, buf_size - 1);
@@ -1718,8 +1850,8 @@ info_1_tuple(Process* BIF_P,	/* Pointer to current process. */
 	    }
 	    buf[buf_size - 1 - r] = '\0';
 	    if (check_if_xml()) {
-		ERTS_ERROR_CHECKER_PRINTF("<erlang_info_log>"
-					  "%s</erlang_info_log>\n", buf);
+		ERTS_ERROR_CHECKER_PRINTF_XML("<erlang_info_log>"
+					      "%s</erlang_info_log>\n", buf);
 	    } else {
 		ERTS_ERROR_CHECKER_PRINTF("%s\n", buf);
 	    }
@@ -1871,6 +2003,37 @@ c_compiler_used(Eterm **hpp, Uint *szp)
 
 }
 
+static int is_snif_term(Eterm module_atom) {
+    int i;
+    Atom *a = atom_tab(atom_val(module_atom));
+    char *aname = (char *) a->name;
+
+    /* if a->name has a '.' then the bif (snif) is bogus i.e a package */
+    for (i = 0; i < a->len; i++) {
+	if (aname[i] == '.')
+	    return 0;
+    }
+
+    return 1;
+}
+
+static Eterm build_snif_term(Eterm **hpp, Uint *szp, int ix, Eterm res) {
+    Eterm tup;
+    tup = erts_bld_tuple(hpp, szp, 3, bif_table[ix].module, bif_table[ix].name, make_small(bif_table[ix].arity));
+    res = erts_bld_cons( hpp, szp, tup, res);
+    return res;
+}
+
+static Eterm build_snifs_term(Eterm **hpp, Uint *szp, Eterm res) {
+    int i;
+    for (i = 0; i < BIF_SIZE; i++) {
+	if (is_snif_term(bif_table[i].module)) {
+	    res = build_snif_term(hpp, szp, i, res);
+	}
+    }
+    return res;
+}
+
 BIF_RETTYPE system_info_1(BIF_ALIST_1)
 {
     Eterm res;
@@ -1904,6 +2067,35 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 		    : am_enabled);
 	}
 #endif
+    } else if (BIF_ARG_1 == am_build_type) {
+#if defined(DEBUG)
+	ERTS_DECL_AM(debug);
+	BIF_RET(AM_debug);
+#elif defined(PURIFY)
+	ERTS_DECL_AM(purify);
+	BIF_RET(AM_purify);
+#elif defined(QUANTIFY)
+	ERTS_DECL_AM(quantify);
+	BIF_RET(AM_quantify);
+#elif defined(PURECOV)
+	ERTS_DECL_AM(purecov);
+	BIF_RET(AM_purecov);
+#elif defined(ERTS_GCOV)
+	ERTS_DECL_AM(gcov);
+	BIF_RET(AM_gcov);
+#elif defined(VALGRIND)
+	ERTS_DECL_AM(valgrind);
+	BIF_RET(AM_valgrind);
+#elif defined(GPROF)
+	ERTS_DECL_AM(gprof);
+	BIF_RET(AM_gprof);
+#elif defined(ERTS_ENABLE_LOCK_COUNT)
+	ERTS_DECL_AM(lcnt);
+	BIF_RET(AM_lcnt);
+#else
+	BIF_RET(am_opt);
+#endif
+	BIF_RET(res);
     } else if (BIF_ARG_1 == am_allocated_areas) {
 	res = erts_allocated_areas(NULL, NULL, BIF_P);
 	BIF_RET(res);
@@ -1916,9 +2108,20 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(am_undefined);
 #endif
     } else if (BIF_ARG_1 == am_trace_control_word) {
-	BIF_RET(db_get_trace_control_word_0(BIF_P));
+	BIF_RET(db_get_trace_control_word(BIF_P));
     } else if (ERTS_IS_ATOM_STR("ets_realloc_moves", BIF_ARG_1)) {
  	BIF_RET((erts_ets_realloc_always_moves) ? am_true : am_false);
+    } else if (ERTS_IS_ATOM_STR("ets_always_compress", BIF_ARG_1)) {
+	BIF_RET((erts_ets_always_compress) ? am_true : am_false);
+    } else if (ERTS_IS_ATOM_STR("snifs", BIF_ARG_1)) {
+	Uint size = 0;
+	Uint *szp;
+
+	szp = &size;
+	build_snifs_term(NULL, szp, NIL);
+	hp = HAlloc(BIF_P, size);
+	res = build_snifs_term(&hp, NULL, NIL);
+	BIF_RET(res);
     } else if (BIF_ARG_1 == am_sequential_tracer) {
 	val = erts_get_system_seq_tracer();
 	ASSERT(is_internal_pid(val) || is_internal_port(val) || val==am_false)
@@ -1926,7 +2129,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	res = TUPLE2(hp, am_sequential_tracer, val);
 	BIF_RET(res);
     } else if (BIF_ARG_1 == am_garbage_collection){
-	Uint val = (Uint) erts_smp_atomic_read(&erts_max_gen_gcs);
+	Uint val = (Uint) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
 	Eterm tup;
 	hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2);
 
@@ -1941,7 +2144,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 
 	BIF_RET(res);
     } else if (BIF_ARG_1 == am_fullsweep_after){
-	Uint val = (Uint) erts_smp_atomic_read(&erts_max_gen_gcs);
+	Uint val = (Uint) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
 	hp = HAlloc(BIF_P, 3);
 	res = TUPLE2(hp, am_fullsweep_after, make_small(val));
 	BIF_RET(res);
@@ -1965,7 +2168,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 
 	/* Need to be the only thread running... */
 	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erts_smp_block_system(0);
+	erts_smp_thr_progress_block();
 
 	if (BIF_ARG_1 == am_info)
 	    info(ERTS_PRINT_DSBUF, (void *) dsbufp);
@@ -1976,11 +2179,11 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	else
 	    distribution_info(ERTS_PRINT_DSBUF, (void *) dsbufp);
 
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
 	ASSERT(dsbufp && dsbufp->str);
-	res = new_binary(BIF_P, (byte *) dsbufp->str, (int) dsbufp->str_len);
+	res = new_binary(BIF_P, (byte *) dsbufp->str, dsbufp->str_len);
 	erts_destroy_info_dsbuf(dsbufp);
 	BIF_RET(res);
     } else if (ERTS_IS_ATOM_STR("dist_ctrl", BIF_ARG_1)) {
@@ -1988,7 +2191,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	i = 0;
 	/* Need to be the only thread running... */
 	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erts_smp_block_system(0);
+	erts_smp_thr_progress_block();
 	for (dep = erts_visible_dist_entries; dep; dep = dep->next) 
 	    ++i;
 	for (dep = erts_hidden_dist_entries; dep; dep = dep->next)
@@ -2011,7 +2214,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	    res = CONS(hp, tpl, res);
 	    hp += 2;
 	}
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	BIF_RET(res);
     } else if (BIF_ARG_1 == am_system_version) {
@@ -2032,16 +2235,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	return erts_instr_get_type_info(BIF_P);
     }
     else if (BIF_ARG_1 == am_os_type) {
-       Eterm type = am_atom_put(os_type, strlen(os_type));
-       Eterm flav, tup;
-       char *buf = erts_alloc(ERTS_ALC_T_TMP, 1024); /* More than enough */
-
-       os_flavor(buf, 1024);
-       flav = am_atom_put(buf, strlen(buf));
-       hp = HAlloc(BIF_P, 3);
-       tup = TUPLE2(hp, type, flav);
-       erts_free(ERTS_ALC_T_TMP, (void *) buf);
-       BIF_RET(tup);
+	BIF_RET(os_type_tuple);
     }
     else if (BIF_ARG_1 == am_allocator) {
 	BIF_RET(erts_allocator_options((void *) BIF_P));
@@ -2063,98 +2257,11 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(erts_alloc_util_allocators((void *) BIF_P));
     }
     else if (BIF_ARG_1 == am_elib_malloc) {
-#ifdef ELIB_ALLOC_IS_CLIB
-	struct elib_stat stat;
-	DECL_AM(heap_size);
-	DECL_AM(max_alloced_size);
-	DECL_AM(alloced_size);
-	DECL_AM(free_size);
-	DECL_AM(no_alloced_blocks);
-	DECL_AM(no_free_blocks);
-	DECL_AM(smallest_alloced_block);
-	DECL_AM(largest_free_block);
-	Eterm atoms[8];
-	Eterm ints[8];
-	Uint **hpp;
-	Uint sz;
-	Uint *szp;
-	int length;
-#ifdef DEBUG
-	Uint *endp;
-#endif
-
-	elib_stat(&stat);
-
-	/* First find out the heap size needed ... */
-	hpp = NULL;
-	szp = &sz;
-	sz = 0;
-
-    build_elib_malloc_term:
-	length = 0;
-	atoms[length] = AM_heap_size;
-	ints[length++] = erts_bld_uint(hpp, szp,
-				       (Uint) stat.mem_total*sizeof(Uint));
-	atoms[length] = AM_max_alloced_size;
-	ints[length++] = erts_bld_uint(hpp, szp,
-				       (Uint) stat.mem_max_alloc*sizeof(Uint));
-	atoms[length] = AM_alloced_size;
-	ints[length++] = erts_bld_uint(hpp, szp,
-				       (Uint) stat.mem_alloc*sizeof(Uint));
-	atoms[length] = AM_free_size;
-	ints[length++] = erts_bld_uint(hpp, szp,
-				       (Uint) stat.mem_free*sizeof(Uint));
-	atoms[length] = AM_no_alloced_blocks;
-	ints[length++] = erts_bld_uint(hpp, szp, (Uint) stat.mem_blocks);
-	atoms[length] = AM_no_free_blocks;
-	ints[length++] = erts_bld_uint(hpp, szp, (Uint) stat.free_blocks);
-	atoms[length] = AM_smallest_alloced_block;
-	ints[length++] = erts_bld_uint(hpp, szp,
-				       (Uint) stat.min_used*sizeof(Uint));
-	atoms[length] = AM_largest_free_block;
-	ints[length++] = erts_bld_uint(hpp, szp,
-				       (Uint) stat.max_free*sizeof(Uint));
-
-
-
-	ASSERT(length <= sizeof(atoms)/sizeof(Eterm));
-	ASSERT(length <= sizeof(ints)/sizeof(Eterm));
-
-	res = erts_bld_2tup_list(hpp, szp, length, atoms, ints);
-
-	if (szp) {
-	    /* ... and then build the term */
-	    hp = HAlloc(BIF_P, sz);
-#ifdef DEBUG
-	    endp = hp + sz;
-#endif
-
-	    szp = NULL;
-	    hpp = &hp;
-	    goto build_elib_malloc_term;
-	}
-
-#ifdef DEBUG
-	ASSERT(endp == hp);
-#endif
-
-#else /* #ifdef ELIB_ALLOC_IS_CLIB */
-	res = am_false;
-#endif /* #ifdef ELIB_ALLOC_IS_CLIB */
-
-	BIF_RET(res);
+	/* To be removed in R15 */
+        BIF_RET(am_false);
     }
     else if (BIF_ARG_1 == am_os_version) {
-       int major, minor, build;
-       Eterm tup;
-
-       os_version(&major, &minor, &build);
-       hp = HAlloc(BIF_P, 4);
-       tup = TUPLE3(hp,
-		    make_small(major),
-		    make_small(minor),
-		    make_small(build));
-       BIF_RET(tup);
+	BIF_RET(os_version_tuple);
     }
     else if (BIF_ARG_1 == am_version) {
 	int n = strlen(ERLANG_VERSION);
@@ -2220,39 +2327,20 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 #endif
     } else if (BIF_ARG_1 == am_heap_sizes) {
 	return erts_heap_sizes(BIF_P);
-    } else if (BIF_ARG_1 == am_global_heaps_size) {
-#ifdef HYBRID
-	Uint hsz = 0;
-	Uint sz = 0;
-
-	sz += global_heap_sz;
-#ifdef INCREMENTAL
-        /* The size of the old generation is a bit hard to define here...
-         * The amount of live data in the last collection perhaps..? */
-        sz = 0;
-#else
-	if (global_old_hend && global_old_heap)
-	    sz += global_old_hend - global_old_heap;
-#endif
-
-	sz *= sizeof(Eterm);
-
-	(void) erts_bld_uint(NULL, &hsz, sz);
-	hp = hsz ? HAlloc(BIF_P, hsz) : NULL;
-	res = erts_bld_uint(&hp, NULL, sz);
-#else
-	res = make_small(0);
-#endif
-	return res;
     } else if (BIF_ARG_1 == am_heap_type) {
-#if defined(HYBRID)
-        return am_hybrid;
-#else
 	return am_private;
-#endif
     } else if (ERTS_IS_ATOM_STR("cpu_topology", BIF_ARG_1)) {
 	res = erts_get_cpu_topology_term(BIF_P, am_used);
 	BIF_TRAP1(erts_format_cpu_topology_trap, BIF_P, res);
+    } else if (ERTS_IS_ATOM_STR("update_cpu_info", BIF_ARG_1)) {
+	if (erts_update_cpu_info()) {
+	    ERTS_DECL_AM(changed);
+	    BIF_RET(AM_changed);
+	}
+	else {
+	    ERTS_DECL_AM(unchanged);
+	    BIF_RET(AM_unchanged);
+	}
 #if defined(__GNUC__) && defined(HAVE_SOLARIS_SPARC_PERFMON)
     } else if (ERTS_IS_ATOM_STR("ultrasparc_read_tick1", BIF_ARG_1)) {
 	register unsigned high asm("%l0");
@@ -2324,7 +2412,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
     }
     /* Arguments that are unusual follow ... */
     else if (ERTS_IS_ATOM_STR("logical_processors", BIF_ARG_1)) {
-	int no = erts_get_cpu_configured(erts_cpuinfo);
+	int no;
+	erts_get_logical_processors(&no, NULL, NULL);
 	if (no > 0)
 	    BIF_RET(make_small((Uint) no));
 	else {
@@ -2333,7 +2422,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	}
     }
     else if (ERTS_IS_ATOM_STR("logical_processors_online", BIF_ARG_1)) {
-	int no = erts_get_cpu_online(erts_cpuinfo);
+	int no;
+	erts_get_logical_processors(NULL, &no, NULL);
 	if (no > 0)
 	    BIF_RET(make_small((Uint) no));
 	else {
@@ -2342,7 +2432,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	}
     }
     else if (ERTS_IS_ATOM_STR("logical_processors_available", BIF_ARG_1)) {
-	int no = erts_get_cpu_available(erts_cpuinfo);
+	int no;
+	erts_get_logical_processors(NULL, NULL, &no);
 	if (no > 0)
 	    BIF_RET(make_small((Uint) no));
 	else {
@@ -2502,21 +2593,125 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	BIF_RET(erts_sched_stat_term(BIF_P, 1));
     } else if (ERTS_IS_ATOM_STR("taints", BIF_ARG_1)) {
 	BIF_RET(erts_nif_taints(BIF_P));
+    } else if (ERTS_IS_ATOM_STR("reader_groups_map", BIF_ARG_1)) {
+	BIF_RET(erts_get_reader_groups_map(BIF_P));
+    } else if (ERTS_IS_ATOM_STR("dist_buf_busy_limit", BIF_ARG_1)) {
+	Uint hsz = 0;
+
+ 	(void) erts_bld_uint(NULL, &hsz, erts_dist_buf_busy_limit);
+	hp = hsz ? HAlloc(BIF_P, hsz) : NULL;
+	res = erts_bld_uint(&hp, NULL, erts_dist_buf_busy_limit);
+	BIF_RET(res);
+    } else if (ERTS_IS_ATOM_STR("print_ethread_info", BIF_ARG_1)) {
+#if defined(ETHR_NATIVE_ATOMIC32_IMPL) \
+    || defined(ETHR_NATIVE_ATOMIC64_IMPL) \
+    || defined(ETHR_NATIVE_DW_ATOMIC_IMPL)
+	int i;
+	char **str;
+#endif
+#ifdef ETHR_NATIVE_ATOMIC32_IMPL
+	erts_printf("32-bit native atomics: %s\n",
+		    ETHR_NATIVE_ATOMIC32_IMPL);
+	str = ethr_native_atomic32_ops();
+	for (i = 0; str[i]; i++)
+	    erts_printf("ethr_native_atomic32_%s()\n", str[i]);
+#endif
+#ifdef ETHR_NATIVE_ATOMIC64_IMPL
+	erts_printf("64-bit native atomics: %s\n",
+		    ETHR_NATIVE_ATOMIC64_IMPL);
+	str = ethr_native_atomic64_ops();
+	for (i = 0; str[i]; i++)
+	    erts_printf("ethr_native_atomic64_%s()\n", str[i]);
+#endif
+#ifdef ETHR_NATIVE_DW_ATOMIC_IMPL
+	if (ethr_have_native_dw_atomic()) {
+	    erts_printf("Double word native atomics: %s\n",
+			ETHR_NATIVE_DW_ATOMIC_IMPL);
+	    str = ethr_native_dw_atomic_ops();
+	    for (i = 0; str[i]; i++)
+		erts_printf("ethr_native_dw_atomic_%s()\n", str[i]);
+	    str = ethr_native_su_dw_atomic_ops();
+	    for (i = 0; str[i]; i++)
+		erts_printf("ethr_native_su_dw_atomic_%s()\n", str[i]);
+	}
+#endif
+#ifdef ETHR_NATIVE_SPINLOCK_IMPL
+	erts_printf("Native spin-locks: %s\n", ETHR_NATIVE_SPINLOCK_IMPL);
+#endif
+#ifdef ETHR_NATIVE_RWSPINLOCK_IMPL
+	erts_printf("Native rwspin-locks: %s\n", ETHR_NATIVE_RWSPINLOCK_IMPL);
+#endif
+#ifdef ETHR_X86_RUNTIME_CONF_HAVE_SSE2__
+	erts_printf("SSE2 support: %s\n", (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__
+					   ? "yes" : "no"));
+#endif
+#ifdef ETHR_X86_OUT_OF_ORDER
+	erts_printf("x86"
+#ifdef ARCH_64
+		    "_64"
+#endif
+		    " out of order\n");
+#endif
+#ifdef ETHR_SPARC_TSO
+	erts_printf("Sparc TSO\n");
+#endif
+#ifdef ETHR_SPARC_PSO
+	erts_printf("Sparc PSO\n");
+#endif
+#ifdef ETHR_SPARC_RMO
+	erts_printf("Sparc RMO\n");
+#endif
+#if defined(ETHR_PPC_HAVE_LWSYNC)
+	erts_printf("Have lwsync instruction: yes\n");
+#elif defined(ETHR_PPC_HAVE_NO_LWSYNC)
+	erts_printf("Have lwsync instruction: no\n");
+#elif defined(ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__)
+	erts_printf("Have lwsync instruction: %s (runtime test)\n",
+		    ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ ? "yes" : "no");
+#endif
+	BIF_RET(am_true);
+    }
+    else if (ERTS_IS_ATOM_STR("dynamic_trace", BIF_ARG_1)) {
+#if defined(USE_DTRACE)
+	DECL_AM(dtrace);
+	BIF_RET(AM_dtrace);
+#elif defined(USE_SYSTEMTAP)
+	DECL_AM(systemtap);
+	BIF_RET(AM_systemtap);
+#else
+	BIF_RET(am_none);
+#endif
+    }	    
+    else if (ERTS_IS_ATOM_STR("dynamic_trace_probes", BIF_ARG_1)) {
+#if defined(USE_VM_PROBES)
+	BIF_RET(am_true);
+#else
+	BIF_RET(am_false);
+#endif	
     }
+#ifdef ERTS_SMP
+    else if (ERTS_IS_ATOM_STR("thread_progress", BIF_ARG_1)) {
+	erts_thr_progress_dbg_print_state();
+	BIF_RET(am_true);
+    }
+#endif
 
     BIF_ERROR(BIF_P, BADARG);
 }
 
-Eterm
-port_info_1(Process* p, Eterm pid)
+BIF_RETTYPE
+port_info_1(BIF_ALIST_1)
 {
+    Process* p = BIF_P;
+    Eterm pid = BIF_ARG_1;
     static Eterm keys[] = {
 	am_name,
 	am_links,
 	am_id,
 	am_connected,
 	am_input,
-	am_output
+	am_output,
+	am_os_pid
     };
     Eterm items[ASIZE(keys)];
     Eterm result = NIL;
@@ -2532,7 +2727,7 @@ port_info_1(Process* p, Eterm pid)
     for (i = 0; i < ASIZE(keys); i++) {
 	Eterm item;
 
-	item = port_info_2(p, pid, keys[i]);
+	item = port_info(p, pid, keys[i]);
 	if (is_non_value(item)) {
 	    return THE_NON_VALUE;
 	}
@@ -2541,7 +2736,7 @@ port_info_1(Process* p, Eterm pid)
 	}
 	items[i] = item;
     }
-    reg_name = port_info_2(p, pid, am_registered_name);
+    reg_name = port_info(p, pid, am_registered_name);
 
     /*
      * Build the resulting list.
@@ -2573,28 +2768,32 @@ port_info_1(Process* p, Eterm pid)
 **    name        String
 **    input       Number of bytes input from port program
 **    output      Number of bytes output to the port program
+**    os_pid      The child's process ID
 */
 
 BIF_RETTYPE port_info_2(BIF_ALIST_2)
 {
+    return port_info(BIF_P, BIF_ARG_1, BIF_ARG_2);
+}
+
+static BIF_RETTYPE port_info(Process* p, Eterm portid, Eterm item)
+{
     BIF_RETTYPE ret;
-    Eterm portid = BIF_ARG_1;
     Port *prt;
-    Eterm item = BIF_ARG_2;
     Eterm res;
     Eterm* hp;
     int count;
 
     if (is_internal_port(portid))
-	prt = erts_id2port(portid, BIF_P, ERTS_PROC_LOCK_MAIN);
+	prt = erts_id2port(portid, p, ERTS_PROC_LOCK_MAIN);
     else if (is_atom(portid))
-	erts_whereis_name(BIF_P, ERTS_PROC_LOCK_MAIN,
+	erts_whereis_name(p, ERTS_PROC_LOCK_MAIN,
 			  portid, NULL, 0, 0, &prt);
     else if (is_external_port(portid)
 	     && external_port_dist_entry(portid) == erts_this_dist_entry)
 	BIF_RET(am_undefined);
     else {
-	BIF_ERROR(BIF_P, BADARG);
+	BIF_ERROR(p, BADARG);
     }
 
     if (!prt) {
@@ -2602,7 +2801,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2)
     }
 
     if (item == am_id) {
-	hp = HAlloc(BIF_P, 3);
+	hp = HAlloc(p, 3);
 	res = make_small(internal_port_number(portid));
     }
     else if (item == am_links) {
@@ -2614,10 +2813,10 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2)
 
 	erts_doforall_links(prt->nlinks, &collect_one_link, &mic);
 
-	hp = HAlloc(BIF_P, 3 + mic.sz);
+	hp = HAlloc(p, 3 + mic.sz);
 	res = NIL;
 	for (i = 0; i < mic.mi_i; i++) {
-	    item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); 
+	    item = STORE_NC(&hp, &MSO(p), mic.mi[i].entity);
 	    res = CONS(hp, item, res);
 	    hp += 2;
 	}
@@ -2633,11 +2832,11 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2)
 
 	erts_doforall_monitors(prt->monitors, &collect_one_origin_monitor, &mic);
 
-	hp = HAlloc(BIF_P, 3 + mic.sz);
+	hp = HAlloc(p, 3 + mic.sz);
 	res = NIL;
 	for (i = 0; i < mic.mi_i; i++) {
 	    Eterm t;
-	    item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); 
+	    item = STORE_NC(&hp, &MSO(p), mic.mi[i].entity);
 	    t = TUPLE2(hp, am_process, item);
 	    hp += 3;
 	    res = CONS(hp, t, res);
@@ -2649,27 +2848,39 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2)
     else if (item == am_name) {
 	count = sys_strlen(prt->name);
 
-	hp = HAlloc(BIF_P, 3 + 2*count);
+	hp = HAlloc(p, 3 + 2*count);
 	res = buf_to_intlist(&hp, prt->name, count, NIL);
     }
     else if (item == am_connected) {
-	hp = HAlloc(BIF_P, 3);
+	hp = HAlloc(p, 3);
 	res = prt->connected; /* internal pid */
     }
     else if (item == am_input) {
 	Uint hsz = 3;
 	Uint n = prt->bytes_in;
 	(void) erts_bld_uint(NULL, &hsz, n);
-	hp = HAlloc(BIF_P, hsz);
+	hp = HAlloc(p, hsz);
 	res = erts_bld_uint(&hp, NULL, n);
     }
     else if (item == am_output) {
 	Uint hsz = 3;
 	Uint n = prt->bytes_out;
 	(void) erts_bld_uint(NULL, &hsz, n);
-	hp = HAlloc(BIF_P, hsz);
+	hp = HAlloc(p, hsz);
 	res = erts_bld_uint(&hp, NULL, n);
     }
+    else if (item == am_os_pid) {
+        if (prt->os_pid >= 0) {
+            Uint hsz = 3;
+            UWord n = prt->os_pid;
+            (void) erts_bld_uword(NULL, &hsz, n);
+            hp = HAlloc(p, hsz);
+            res = erts_bld_uword(&hp, NULL, n);
+        } else {
+	    hp = HAlloc(p, 3);
+	    res = am_undefined;
+        }
+    }
     else if (item == am_registered_name) {
 	RegProc *reg;
 	reg = prt->reg;
@@ -2677,7 +2888,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2)
 	    ERTS_BIF_PREP_RET(ret, NIL);
 	    goto done;
 	} else {
-	    hp = HAlloc(BIF_P, 3);
+	    hp = HAlloc(p, 3);
 	    res = reg->name;
 	}
     }
@@ -2689,12 +2900,12 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2)
 	Uint size = 0;
 	ErlHeapFragment* bp;
 
-	hp = HAlloc(BIF_P, 3);
+	hp = HAlloc(p, 3);
 
 	erts_doforall_links(prt->nlinks, &one_link_size, &size);
 
 	for (bp = prt->bp; bp; bp = bp->next)
-	    size += sizeof(ErlHeapFragment) + (bp->size - 1)*sizeof(Eterm);
+	    size += sizeof(ErlHeapFragment) + (bp->alloc_size - 1)*sizeof(Eterm);
 
 	if (prt->linebuf)
 	    size += sizeof(LineBuf) + prt->linebuf->ovsiz;
@@ -2706,18 +2917,18 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2)
 	   hard to retrieve... */
 	
 	(void) erts_bld_uint(NULL, &hsz, size);
-	hp = HAlloc(BIF_P, hsz);
+	hp = HAlloc(p, hsz);
 	res = erts_bld_uint(&hp, NULL, size);
     }
     else if (item == am_queue_size) {
 	Uint ioq_size = erts_port_ioq_size(prt);
 	Uint hsz = 3;
 	(void) erts_bld_uint(NULL, &hsz, ioq_size);
-	hp = HAlloc(BIF_P, hsz);
+	hp = HAlloc(p, hsz);
 	res = erts_bld_uint(&hp, NULL, ioq_size);
     }
     else if (ERTS_IS_ATOM_STR("locking", item)) {
-	hp = HAlloc(BIF_P, 3);
+	hp = HAlloc(p, 3);
 #ifndef ERTS_SMP
 	res = am_false;
 #else
@@ -2736,7 +2947,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2)
 #endif
     }
     else {
-	ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG);
+	ERTS_BIF_PREP_ERROR(ret, p, BADARG);
 	goto done;
     }
 
@@ -2750,9 +2961,12 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2)
 }
 
 
-Eterm
-fun_info_2(Process* p, Eterm fun, Eterm what)
+BIF_RETTYPE
+fun_info_2(BIF_ALIST_2)
 {
+    Process* p = BIF_P;
+    Eterm fun = BIF_ARG_1;
+    Eterm what = BIF_ARG_2;
     Eterm* hp;
     Eterm val;
 
@@ -2802,7 +3016,7 @@ fun_info_2(Process* p, Eterm fun, Eterm what)
 	    }
 	    break;
 	case am_refc:
-	    val = erts_make_integer(erts_smp_atomic_read(&funp->fe->refc), p);
+	    val = erts_make_integer(erts_smp_atomic_read_nob(&funp->fe->refc), p);
 	    hp = HAlloc(p, 3);
 	    break;
 	case am_arity:
@@ -2817,7 +3031,7 @@ fun_info_2(Process* p, Eterm fun, Eterm what)
 	    goto error;
 	}
     } else if (is_export(fun)) {
-	Export* exp = (Export *) (export_val(fun))[1];
+	Export* exp = (Export *) ((UWord) (export_val(fun))[1]);
 	switch (what) {
 	case am_type:
 	    hp = HAlloc(p, 3);
@@ -2951,7 +3165,12 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
     Eterm res;
     Eterm* hp;
 
-    if (BIF_ARG_1 == am_context_switches) {
+    if (BIF_ARG_1 == am_scheduler_wall_time) {
+	res = erts_sched_wall_time_request(BIF_P, 0, 0);
+	if (is_non_value(res))
+	    BIF_RET(am_undefined);
+	BIF_TRAP1(gather_sched_wall_time_res_trap, BIF_P, res);
+    } else if (BIF_ARG_1 == am_context_switches) {
 	Eterm cs = erts_make_integer(erts_get_total_context_switches(), BIF_P);
 	hp = HAlloc(BIF_P, 3);
 	res = TUPLE2(hp, cs, SMALL_ZERO);
@@ -2998,7 +3217,7 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
 	res = TUPLE2(hp, b1, b2); 
 	BIF_RET(res);
     } else if (BIF_ARG_1 == am_runtime) {
-	unsigned long u1, u2, dummy;
+	UWord u1, u2, dummy;
 	Eterm b1, b2;
 	elapsed_time_both(&u1,&dummy,&u2,&dummy);
 	b1 = erts_make_integer(u1,BIF_P);
@@ -3010,11 +3229,11 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
 	res = erts_run_queues_len(NULL);
 	BIF_RET(make_small(res));
     } else if (BIF_ARG_1 == am_wall_clock) {
-	Uint w1, w2;
+	UWord w1, w2;
 	Eterm b1, b2;
 	wall_clock_elapsed_time_both(&w1, &w2);
-	b1 = erts_make_integer(w1,BIF_P);
-	b2 = erts_make_integer(w2,BIF_P);
+	b1 = erts_make_integer((Uint) w1,BIF_P);
+	b2 = erts_make_integer((Uint) w2,BIF_P);
 	hp = HAlloc(BIF_P,3);
 	res = TUPLE2(hp, b1, b2);
 	BIF_RET(res);
@@ -3022,8 +3241,8 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
 	Eterm r1, r2;
 	Eterm in, out;
 	Uint hsz = 9;
-	Uint bytes_in = (Uint) erts_smp_atomic_read(&erts_bytes_in);
-	Uint bytes_out = (Uint) erts_smp_atomic_read(&erts_bytes_out);
+	Uint bytes_in = (Uint) erts_smp_atomic_read_nob(&erts_bytes_in);
+	Uint bytes_out = (Uint) erts_smp_atomic_read_nob(&erts_bytes_out);
 
 	(void) erts_bld_uint(NULL, &hsz, bytes_in);
 	(void) erts_bld_uint(NULL, &hsz, bytes_out);
@@ -3063,26 +3282,6 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
     BIF_ERROR(BIF_P, BADARG);
 }
 
-BIF_RETTYPE memory_0(BIF_ALIST_0)
-{
-    BIF_RETTYPE res = erts_memory(NULL, NULL, BIF_P, THE_NON_VALUE);
-    switch (res) {
-    case am_badarg: BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); /* never... */
-    case am_notsup: BIF_ERROR(BIF_P, EXC_NOTSUP);
-    default: BIF_RET(res);
-    }
-}
-
-BIF_RETTYPE memory_1(BIF_ALIST_1)
-{
-    BIF_RETTYPE res = erts_memory(NULL, NULL, BIF_P, BIF_ARG_1);
-    switch (res) {
-    case am_badarg: BIF_ERROR(BIF_P, BADARG);
-    case am_notsup: BIF_ERROR(BIF_P, EXC_NOTSUP);
-    default: BIF_RET(res);
-    }
-}
-
 BIF_RETTYPE error_logger_warning_map_0(BIF_ALIST_0)
 {
     BIF_RET(erts_error_logger_warnings);
@@ -3096,7 +3295,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
      * NOTE: Only supposed to be used for testing, and debugging.
      */
 
-    if (!erts_smp_atomic_read(&available_internal_state)) {
+    if (!erts_smp_atomic_read_nob(&available_internal_state)) {
 	BIF_ERROR(BIF_P, EXC_UNDEF);
     }
 
@@ -3184,6 +3383,15 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	    BIF_RET(am_false);
 #endif
 	}
+	else if (ERTS_IS_ATOM_STR("memory", BIF_ARG_1)) {
+	    Eterm res;
+	    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	    erts_smp_thr_progress_block();
+	    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+	    res = erts_memory(NULL, NULL, BIF_P, THE_NON_VALUE);
+	    erts_smp_thr_progress_unblock();
+	    BIF_RET(res);
+	}
     }
     else if (is_tuple(BIF_ARG_1)) {
 	Eterm* tp = tuple_val(BIF_ARG_1);
@@ -3365,6 +3573,16 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 	    else if (ERTS_IS_ATOM_STR("fake_scheduler_bindings", tp[1])) {
 		return erts_fake_scheduler_bindings(BIF_P, tp[2]);
 	    }
+	    else if (ERTS_IS_ATOM_STR("reader_groups_map", tp[1])) {
+		Sint groups;
+		if (is_not_small(tp[2]))
+		    BIF_ERROR(BIF_P, BADARG);
+		groups = signed_val(tp[2]);
+		if (groups < (Sint) 1 || groups > (Sint) INT_MAX)
+		    BIF_ERROR(BIF_P, BADARG);
+
+		BIF_RET(erts_debug_reader_groups_map(BIF_P, (int) groups));
+	    }
 	    break;
 	}
 	default:
@@ -3376,6 +3594,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
 
 static erts_smp_atomic_t hipe_test_reschedule_flag;
 
+
 BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 {
     /*
@@ -3383,8 +3602,8 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
      */
     if (ERTS_IS_ATOM_STR("available_internal_state", BIF_ARG_1)
 	&& (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false)) {
-	long on = (long) (BIF_ARG_2 == am_true);
-	long prev_on = erts_smp_atomic_xchg(&available_internal_state, on);
+	erts_aint_t on = (erts_aint_t) (BIF_ARG_2 == am_true);
+	erts_aint_t prev_on = erts_smp_atomic_xchg_nob(&available_internal_state, on);
 	if (on) {
 	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
 	    erts_dsprintf(dsbufp, "Process %T ", BIF_P->id);
@@ -3400,7 +3619,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 	BIF_RET(prev_on ? am_true : am_false);
     }
 
-    if (!erts_smp_atomic_read(&available_internal_state)) {
+    if (!erts_smp_atomic_read_nob(&available_internal_state)) {
 	BIF_ERROR(BIF_P, EXC_UNDEF);
     }
 
@@ -3426,10 +3645,10 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 		if (ms > 0) {
 		    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 		    if (block)
-			erts_smp_block_system(0);
+			erts_smp_thr_progress_block();
 		    while (erts_milli_sleep((long) ms) != 0);
 		    if (block)
-			erts_smp_release_system();
+			erts_smp_thr_progress_unblock();
 		    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 		}
 		BIF_RET(am_true);
@@ -3546,6 +3765,17 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
                 }
 	    }
 	}
+	else if (ERTS_IS_ATOM_STR("binary_loop_limit", BIF_ARG_1)) {
+	    /* Used by binary_module_SUITE (stdlib) */
+	    Uint max_loops;
+	    if (is_atom(BIF_ARG_2) && ERTS_IS_ATOM_STR("default", BIF_ARG_2)) {
+		max_loops = erts_binary_set_loop_limit(-1);
+		BIF_RET(make_small(max_loops));
+	    } else if (term_to_Uint(BIF_ARG_2, &max_loops) != 0) {
+		max_loops = erts_binary_set_loop_limit(max_loops);
+		BIF_RET(make_small(max_loops));
+	    }
+	}
 	else if (ERTS_IS_ATOM_STR("re_loop_limit", BIF_ARG_1)) {
 	    /* Used by re_SUITE (stdlib) */
 	    Uint max_loops;
@@ -3570,14 +3800,14 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 	}
 	else if (ERTS_IS_ATOM_STR("hipe_test_reschedule_suspend", BIF_ARG_1)) {
 	    /* Used by hipe test suites */
-	    long flag = erts_smp_atomic_read(&hipe_test_reschedule_flag);
+	    erts_aint_t flag = erts_smp_atomic_read_nob(&hipe_test_reschedule_flag);
 	    if (!flag && BIF_ARG_2 != am_false) {
-		erts_smp_atomic_set(&hipe_test_reschedule_flag, 1);
+		erts_smp_atomic_set_nob(&hipe_test_reschedule_flag, 1);
 		erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL);
 		ERTS_BIF_YIELD2(bif_export[BIF_erts_debug_set_internal_state_2],
 				BIF_P, BIF_ARG_1, BIF_ARG_2);
 	    }
-	    erts_smp_atomic_set(&hipe_test_reschedule_flag, !flag);
+	    erts_smp_atomic_set_nob(&hipe_test_reschedule_flag, !flag);
 	    BIF_RET(NIL);
 	}
 	else if (ERTS_IS_ATOM_STR("hipe_test_reschedule_resume", BIF_ARG_1)) {
@@ -3628,16 +3858,23 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 	    }
 
 	    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	    erts_smp_block_system(0);
+	    erts_smp_thr_progress_block();
 	    old_use_opt = !erts_disable_proc_not_running_opt;
 	    erts_disable_proc_not_running_opt = !use_opt;
-	    erts_smp_release_system();
+	    erts_smp_thr_progress_unblock();
 	    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	    BIF_RET(old_use_opt ? am_true : am_false);
 #else
 	    BIF_ERROR(BIF_P,  EXC_NOTSUP);
 #endif
 	}
+	else if (ERTS_IS_ATOM_STR("wait", BIF_ARG_1)) {
+	    if (ERTS_IS_ATOM_STR("deallocations", BIF_ARG_2)) {
+		if (erts_debug_wait_deallocations(BIF_P)) {
+		    ERTS_BIF_YIELD_RETURN(BIF_P, am_ok);
+		}
+	    }
+	}
     }
 
     BIF_ERROR(BIF_P, BADARG);
@@ -3645,7 +3882,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
 static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_stats_t *stats, Eterm res) {
-    unsigned long tries = 0, colls = 0;
+    Uint tries = 0, colls = 0;
     unsigned long timer_s = 0, timer_ns = 0, timer_n = 0;
     unsigned int  line = 0;
     
@@ -3658,8 +3895,8 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s
      * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}}}]
      */
     
-    ethr_atomic_read(&stats->tries, (long *)&tries);
-    ethr_atomic_read(&stats->colls, (long *)&colls);
+    tries = (Uint) ethr_atomic_read(&stats->tries);
+    colls = (Uint) ethr_atomic_read(&stats->colls);
    
     line     = stats->line; 
     timer_s  = stats->timer.s;
@@ -3796,7 +4033,7 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1)
     	Eterm* hp;
 
 	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erts_smp_block_system(0);
+	erts_smp_thr_progress_block();
 
 	erts_lcnt_set_rt_opt(ERTS_LCNT_OPT_SUSPEND);
 	data = erts_lcnt_get_data();
@@ -3814,17 +4051,17 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1)
 	
 	erts_lcnt_clear_rt_opt(ERTS_LCNT_OPT_SUSPEND);
 
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	
 	BIF_RET(res);
     } else if (BIF_ARG_1 == am_clear) {
 	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erts_smp_block_system(0);
+	erts_smp_thr_progress_block();
 
 	erts_lcnt_clear_counters();
 
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
 	BIF_RET(am_ok);
@@ -3832,48 +4069,52 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1)
 	Eterm* tp = tuple_val(BIF_ARG_1);
 
 	switch (arityval(tp[0])) {
-	    case 2:
+	    case 2: {
+		int opt = 0;
+		int val = 0;
 		if (ERTS_IS_ATOM_STR("copy_save", tp[1])) {
-		    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-		    erts_smp_block_system(0);
-		    if (tp[2] == am_true) {
-
-			res = erts_lcnt_set_rt_opt(ERTS_LCNT_OPT_COPYSAVE) ? am_true : am_false;
-
-		    } else if (tp[2] == am_false) {
-
-			res = erts_lcnt_clear_rt_opt(ERTS_LCNT_OPT_COPYSAVE) ? am_true : am_false;
-
-		    } else {
-			erts_smp_release_system();
-			erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
-			BIF_ERROR(BIF_P, BADARG);
-		    }
-		    erts_smp_release_system();
-		    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
-		    BIF_RET(res);
-
+		    opt = ERTS_LCNT_OPT_COPYSAVE;
 		} else if (ERTS_IS_ATOM_STR("process_locks", tp[1])) {
-		    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-		    erts_smp_block_system(0);
-		    if (tp[2] == am_true) {
-
-			res = erts_lcnt_set_rt_opt(ERTS_LCNT_OPT_PROCLOCK) ? am_true : am_false;
-
-		    } else if (tp[2] == am_false) {
-
-			res = erts_lcnt_set_rt_opt(ERTS_LCNT_OPT_PROCLOCK) ? am_true : am_false;
+		    opt = ERTS_LCNT_OPT_PROCLOCK;
+		} else if (ERTS_IS_ATOM_STR("port_locks", tp[1])) {
+		    opt = ERTS_LCNT_OPT_PORTLOCK;
+		} else if (ERTS_IS_ATOM_STR("suspend", tp[1])) {
+		    opt = ERTS_LCNT_OPT_SUSPEND;
+		} else if (ERTS_IS_ATOM_STR("location", tp[1])) {
+		    opt = ERTS_LCNT_OPT_LOCATION;
+		} else {
+		    BIF_ERROR(BIF_P, BADARG);
+		}
+		if (tp[2] == am_true) {
+		    val = 1;
+		} else if (tp[2] == am_false) {
+		    val = 0;
+		} else {
+		    BIF_ERROR(BIF_P, BADARG);
+		}
 
-		    } else {
-			erts_smp_release_system();
-			erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
-			BIF_ERROR(BIF_P, BADARG);
+		erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+		erts_smp_thr_progress_block();
+		
+		if (val) {
+		    res = erts_lcnt_set_rt_opt(opt) ? am_true : am_false;
+		} else {
+		    res = erts_lcnt_clear_rt_opt(opt) ? am_true : am_false;
+		}
+#ifdef ERTS_SMP
+		if (res != tp[2]) {
+		    if (opt == ERTS_LCNT_OPT_PORTLOCK) {
+			erts_lcnt_enable_io_lock_count(val);
+		    } else if (opt == ERTS_LCNT_OPT_PROCLOCK) {
+			erts_lcnt_enable_proc_lock_count(val);
 		    }
-		    erts_smp_release_system();
-		    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
-		    BIF_RET(res);
-		 }
-	    break;
+		}
+#endif
+		erts_smp_thr_progress_unblock();
+		erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+		BIF_RET(res);
+		break;
+	    }
      
 	    default:
 	    break;
@@ -3884,11 +4125,37 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1)
     BIF_ERROR(BIF_P, BADARG);
 }
 
+static void os_info_init(void)
+{
+    Eterm type = am_atom_put(os_type, strlen(os_type));
+    Eterm flav;
+    int major, minor, build;
+    char* buf = erts_alloc(ERTS_ALC_T_TMP, 1024); /* More than enough */
+    Eterm* hp;
+
+    os_flavor(buf, 1024);
+    flav = am_atom_put(buf, strlen(buf));
+    erts_free(ERTS_ALC_T_TMP, (void *) buf);
+    hp = erts_alloc(ERTS_ALC_T_LL_TEMP_TERM, (3+4)*sizeof(Eterm));
+    os_type_tuple = TUPLE2(hp, type, flav);
+    hp += 3;
+    os_version(&major, &minor, &build);
+    os_version_tuple = TUPLE3(hp,
+			      make_small(major),
+			      make_small(minor),
+			      make_small(build));
+}
+
 void
 erts_bif_info_init(void)
 {
-    erts_smp_atomic_init(&available_internal_state, 0);
-    erts_smp_atomic_init(&hipe_test_reschedule_flag, 0);
+    erts_smp_atomic_init_nob(&available_internal_state, 0);
+    erts_smp_atomic_init_nob(&hipe_test_reschedule_flag, 0);
 
+    alloc_info_trap = erts_export_put(am_erlang, am_alloc_info, 1);
+    alloc_sizes_trap = erts_export_put(am_erlang, am_alloc_sizes, 1);
+    gather_sched_wall_time_res_trap
+	= erts_export_put(am_erlang, am_gather_sched_wall_time_result, 1);
     process_info_init();
+    os_info_init();
 }
diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c
index a9e8dd86f7..1805366cfe 100644
--- a/erts/emulator/beam/erl_bif_lists.c
+++ b/erts/emulator/beam/erl_bif_lists.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1999-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1999-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -34,27 +34,7 @@
 
 static Eterm keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List);
 
-/*
- * erlang:'++'/2
- */
-
-Eterm
-ebif_plusplus_2(Process* p, Eterm A, Eterm B)
-{
-    return append_2(p, A, B);
-}
-
-/*
- * erlang:'--'/2
- */
-
-Eterm
-ebif_minusminus_2(Process* p, Eterm A, Eterm B)
-{
-    return subtract_2(p, A, B);
-}
-
-BIF_RETTYPE append_2(BIF_ALIST_2)
+static BIF_RETTYPE append(Process* p, Eterm A, Eterm B)
 {
     Eterm list;
     Eterm copy;
@@ -63,18 +43,18 @@ BIF_RETTYPE append_2(BIF_ALIST_2)
     Eterm* hp;
     int i;
 
-    if ((i = list_length(BIF_ARG_1)) < 0) {
-	BIF_ERROR(BIF_P, BADARG);
+    if ((i = list_length(A)) < 0) {
+	BIF_ERROR(p, BADARG);
     }
     if (i == 0) {
-	BIF_RET(BIF_ARG_2);
-    } else if (is_nil(BIF_ARG_2)) {
-	BIF_RET(BIF_ARG_1);
+	BIF_RET(B);
+    } else if (is_nil(B)) {
+	BIF_RET(A);
     }
 
     need = 2*i;
-    hp = HAlloc(BIF_P, need); 
-    list = BIF_ARG_1;
+    hp = HAlloc(p, need);
+    list = A;
     copy = last = CONS(hp, CAR(list_val(list)), make_list(hp+2));
     list = CDR(list_val(list));
     hp += 2;
@@ -85,44 +65,64 @@ BIF_RETTYPE append_2(BIF_ALIST_2)
 	list = CDR(listp);
 	hp += 2;
     }
-    CDR(list_val(last)) = BIF_ARG_2;
+    CDR(list_val(last)) = B;
     BIF_RET(copy);
 }
 
-BIF_RETTYPE subtract_2(BIF_ALIST_2)
+/*
+ * erlang:'++'/2
+ */
+
+Eterm
+ebif_plusplus_2(BIF_ALIST_2)
+{
+    return append(BIF_P, BIF_ARG_1, BIF_ARG_2);
+}
+
+BIF_RETTYPE append_2(BIF_ALIST_2)
+{
+    return append(BIF_P, BIF_ARG_1, BIF_ARG_2);
+}
+
+/*
+ * erlang:'--'/2
+ */
+
+#define SMALL_VEC_SIZE 10
+static Eterm subtract(Process* p, Eterm A, Eterm B)
 {
     Eterm  list;
     Eterm* hp;
     Uint  need;
     Eterm  res;
-    Eterm  small_vec[10];	/* Preallocated memory for small lists */
+    Eterm small_vec[SMALL_VEC_SIZE];	/* Preallocated memory for small lists */
     Eterm* vec_p;
     Eterm* vp;
     int     i;
     int     n;
     int     m;
     
-    if ((n = list_length(BIF_ARG_1)) < 0) {
-	BIF_ERROR(BIF_P, BADARG);
+    if ((n = list_length(A)) < 0) {
+	BIF_ERROR(p, BADARG);
     }
-    if ((m = list_length(BIF_ARG_2)) < 0) {
-	BIF_ERROR(BIF_P, BADARG);
+    if ((m = list_length(B)) < 0) {
+	BIF_ERROR(p, BADARG);
     }
     
     if (n == 0)
 	BIF_RET(NIL);
     if (m == 0)
-	BIF_RET(BIF_ARG_1);
+	BIF_RET(A);
     
     /* allocate element vector */
-    if (n <= sizeof(small_vec)/sizeof(small_vec[0]))
+    if (n <= SMALL_VEC_SIZE)
 	vec_p = small_vec;
     else
 	vec_p = (Eterm*) erts_alloc(ERTS_ALC_T_TMP, n * sizeof(Eterm));
     
     /* PUT ALL ELEMENTS IN VP */
     vp = vec_p;
-    list = BIF_ARG_1;
+    list = A;
     i = n;
     while(i--) {
 	Eterm* listp = list_val(list);
@@ -131,7 +131,7 @@ BIF_RETTYPE subtract_2(BIF_ALIST_2)
     }
     
     /* UNMARK ALL DELETED CELLS */
-    list = BIF_ARG_2;
+    list = B;
     m = 0;  /* number of deleted elements */
     while(is_list(list)) {
 	Eterm* listp = list_val(list);
@@ -152,11 +152,11 @@ BIF_RETTYPE subtract_2(BIF_ALIST_2)
     if (m == n)      /* All deleted ? */
 	res = NIL;
     else if (m == 0)  /* None deleted ? */
-	res = BIF_ARG_1;
+	res = A;
     else {			/* REBUILD LIST */
 	res = NIL;
 	need = 2*(n - m);
-	hp = HAlloc(BIF_P, need);
+	hp = HAlloc(p, need);
 	vp = vec_p + n - 1;
 	while(vp >= vec_p) {
 	    if (is_value(*vp)) {
@@ -171,6 +171,16 @@ BIF_RETTYPE subtract_2(BIF_ALIST_2)
     BIF_RET(res);
 }
 
+BIF_RETTYPE ebif_minusminus_2(BIF_ALIST_2)
+{
+    return subtract(BIF_P, BIF_ARG_1, BIF_ARG_2);
+}
+
+BIF_RETTYPE subtract_2(BIF_ALIST_2)
+{
+    return subtract(BIF_P, BIF_ARG_1, BIF_ARG_2);
+}
+
 BIF_RETTYPE lists_member_2(BIF_ALIST_2)
 {
     Eterm term;
@@ -277,11 +287,12 @@ BIF_RETTYPE lists_reverse_2(BIF_ALIST_2)
 }
 
 BIF_RETTYPE
-lists_keymember_3(Process* p, Eterm Key, Eterm Pos, Eterm List)
+lists_keymember_3(BIF_ALIST_3)
 {
     Eterm res;
 
-    res = keyfind(BIF_lists_keymember_3, p, Key, Pos, List);
+    res = keyfind(BIF_lists_keymember_3, BIF_P,
+		  BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
     if (is_value(res) && is_tuple(res)) {
 	return am_true;
     } else {
@@ -290,23 +301,25 @@ lists_keymember_3(Process* p, Eterm Key, Eterm Pos, Eterm List)
 }
 
 BIF_RETTYPE
-lists_keysearch_3(Process* p, Eterm Key, Eterm Pos, Eterm List)
+lists_keysearch_3(BIF_ALIST_3)
 {
     Eterm res;
     
-    res = keyfind(BIF_lists_keysearch_3, p, Key, Pos, List);
+    res = keyfind(BIF_lists_keysearch_3, BIF_P,
+		  BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
     if (is_non_value(res) || is_not_tuple(res)) {
 	return res;
     } else {			/* Tuple */
-	Eterm* hp = HAlloc(p, 3);
+	Eterm* hp = HAlloc(BIF_P, 3);
 	return TUPLE2(hp, am_value, res);
     }
 }
 
 BIF_RETTYPE
-lists_keyfind_3(Process* p, Eterm Key, Eterm Pos, Eterm List)
+lists_keyfind_3(BIF_ALIST_3)
 {
-    return keyfind(BIF_lists_keyfind_3, p, Key, Pos, List);
+    return keyfind(BIF_lists_keyfind_3, BIF_P,
+		   BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
 }
 
 static Eterm
@@ -377,7 +390,7 @@ keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List)
 		Eterm *tuple_ptr = tuple_val(term);
 		if (pos <= arityval(*tuple_ptr)) {
 		    Eterm element = tuple_ptr[pos];
-		    if (cmp(Key, element) == 0) {
+		    if (CMP(Key, element) == 0) {
 			return term;
 		    }
 		}
diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c
index 6da72dcef9..13f8b1f63c 100644
--- a/erts/emulator/beam/erl_bif_op.c
+++ b/erts/emulator/beam/erl_bif_op.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1999-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -225,18 +225,23 @@ BIF_RETTYPE is_function_1(BIF_ALIST_1)
 
 BIF_RETTYPE is_function_2(BIF_ALIST_2)
 {
+    BIF_RET(erl_is_function(BIF_P, BIF_ARG_1, BIF_ARG_2));
+}
+
+Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2)
+{
     Sint arity;
 
     /*
      * Verify argument 2 (arity); arity must be >= 0.
      */ 
-    if (is_small(BIF_ARG_2)) {
-	arity = signed_val(BIF_ARG_2);
+    if (is_small(arg2)) {
+	arity = signed_val(arg2);
 	if (arity < 0) {
 	error:
-	    BIF_ERROR(BIF_P, BADARG);
+	    BIF_ERROR(p, BADARG);
 	}
-    } else if (is_big(BIF_ARG_2) && !bignum_header_is_neg(*big_val(BIF_ARG_2))) {
+    } else if (is_big(arg2) && !bignum_header_is_neg(*big_val(arg2))) {
 	/* A positive bignum is OK, but can't possibly match. */
 	arity = -1;
     } else {
@@ -244,20 +249,20 @@ BIF_RETTYPE is_function_2(BIF_ALIST_2)
 	goto error;
     }
 
-    if (is_fun(BIF_ARG_1)) {
-	ErlFunThing* funp = (ErlFunThing *) fun_val(BIF_ARG_1);
+    if (is_fun(arg1)) {
+	ErlFunThing* funp = (ErlFunThing *) fun_val(arg1);
 
 	if (funp->arity == (Uint) arity) {
 	    BIF_RET(am_true);
 	}
-    } else if (is_export(BIF_ARG_1)) {
-	Export* exp = (Export *) (export_val(BIF_ARG_1))[1];
+    } else if (is_export(arg1)) {
+	Export* exp = (Export *) EXPAND_POINTER((export_val(arg1))[1]);
 
 	if (exp->code[2] == (Uint) arity) {
 	    BIF_RET(am_true);
 	}
-    } else if (is_tuple(BIF_ARG_1)) {
-	Eterm* tp = tuple_val(BIF_ARG_1);
+    } else if (is_tuple(arg1)) {
+	Eterm* tp = tuple_val(arg1);
 	if (tp[0] == make_arityval(2) && is_atom(tp[1]) && is_atom(tp[2])) {
 	    BIF_RET(am_true);
 	}
diff --git a/erts/emulator/beam/erl_bif_os.c b/erts/emulator/beam/erl_bif_os.c
index 954b1f9729..58d48199fa 100644
--- a/erts/emulator/beam/erl_bif_os.c
+++ b/erts/emulator/beam/erl_bif_os.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1999-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2010. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -53,20 +53,18 @@ BIF_RETTYPE os_timestamp_0(BIF_ALIST_0)
 }
 
 
-Eterm
-os_getpid_0(Process* p)
+BIF_RETTYPE os_getpid_0(BIF_ALIST_0)
 {
      char pid_string[21]; /* enough for a 64 bit number */
      int n;
      Eterm* hp;
      sys_get_pid(pid_string); /* In sys.c */
      n = sys_strlen(pid_string);
-     hp = HAlloc(p, n*2);
+     hp = HAlloc(BIF_P, n*2);
      BIF_RET(buf_to_intlist(&hp, pid_string, n, NIL));
 }
 
-Eterm
-os_getenv_0(Process* p)
+BIF_RETTYPE os_getenv_0(BIF_ALIST_0)
 {
     GETENV_STATE state;
     char *cp;
@@ -80,7 +78,7 @@ os_getenv_0(Process* p)
     ret = NIL;
     while ((cp = getenv_string(&state)) != NULL) {
 	len = strlen(cp);
-	hp = HAlloc(p, len*2+2);
+	hp = HAlloc(BIF_P, len*2+2);
 	str = buf_to_intlist(&hp, cp, len, NIL);
 	ret = CONS(hp, str, ret);
     }
@@ -90,9 +88,11 @@ os_getenv_0(Process* p)
     return ret;
 }
 
-Eterm
-os_getenv_1(Process* p, Eterm key)
+
+BIF_RETTYPE os_getenv_1(BIF_ALIST_1)
 {
+    Process* p = BIF_P;
+    Eterm key = BIF_ARG_1;
     Eterm str;
     int len, res;
     char *key_str, *val;
@@ -145,9 +145,11 @@ os_getenv_1(Process* p, Eterm key)
     BIF_RET(str);
 }
 
-Eterm
-os_putenv_2(Process* p, Eterm key, Eterm value)
+BIF_RETTYPE os_putenv_2(BIF_ALIST_2)
 {
+    Process* p = BIF_P;
+    Eterm key = BIF_ARG_1;
+    Eterm value = BIF_ARG_2;
     char def_buf[1024];
     char *buf = NULL;
     int sep_ix, i, key_len, value_len, tot_len;
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index f454f2e12d..2a1b01b107 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -21,10 +21,6 @@
 #  include "config.h"
 #endif
 
-#ifdef _OSE_
-#  include "ose.h"
-#endif
-
 #include <ctype.h>
 
 #define ERTS_WANT_EXTERNAL_TAGS
@@ -44,6 +40,7 @@
 #include "external.h"
 #include "packet_parser.h"
 #include "erl_bits.h"
+#include "dtrace-wrapper.h"
 
 static int open_port(Process* p, Eterm name, Eterm settings, int *err_nump);
 static byte* convert_environment(Process* p, Eterm env);
@@ -52,6 +49,9 @@ static void free_args(char **);
 
 char *erts_default_arg0 = "default";
 
+static BIF_RETTYPE
+port_call(Process* p, Eterm arg1, Eterm arg2, Eterm arg3);
+
 BIF_RETTYPE open_port_2(BIF_ALIST_2)
 {
     int port_num;
@@ -121,11 +121,9 @@ id_or_name2port(Process *c_p, Eterm id)
 #define ERTS_PORT_COMMAND_FLAG_FORCE		(((Uint32) 1) << 0)
 #define ERTS_PORT_COMMAND_FLAG_NOSUSPEND	(((Uint32) 1) << 1)
 
-static BIF_RETTYPE do_port_command(Process *BIF_P,
-				   Eterm BIF_ARG_1,
-				   Eterm BIF_ARG_2,
-				   Eterm BIF_ARG_3,
-				   Uint32 flags)
+static BIF_RETTYPE
+do_port_command(Process *BIF_P, Eterm arg1, Eterm arg2, Eterm arg3,
+		Uint32 flags)
 {
     BIF_RETTYPE res;
     Port *p;
@@ -139,7 +137,7 @@ static BIF_RETTYPE do_port_command(Process *BIF_P,
     	profile_runnable_proc(BIF_P, am_inactive);
     }
 
-    p = id_or_name2port(BIF_P, BIF_ARG_1);
+    p = id_or_name2port(BIF_P, arg1);
     if (!p) {
     	if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) {
 	    trace_virtual_sched(BIF_P, am_in);
@@ -176,13 +174,13 @@ static BIF_RETTYPE do_port_command(Process *BIF_P,
 		monitor_generic(BIF_P, am_busy_port, p->id);
 	    }
 	    ERTS_BIF_PREP_YIELD3(res, bif_export[BIF_port_command_3], BIF_P,
-				 BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);    
+				 arg1, arg2, arg3);
 	}
     } else {
 	int wres;
 	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	ERTS_SMP_CHK_NO_PROC_LOCKS;
-	wres = erts_write_to_port(BIF_P->id, p, BIF_ARG_2);
+	wres = erts_write_to_port(BIF_P->id, p, arg2);
 	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	if (wres != 0) {
 	    ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
@@ -241,27 +239,33 @@ BIF_RETTYPE port_command_3(BIF_ALIST_3)
 
 BIF_RETTYPE port_call_2(BIF_ALIST_2)
 {
-    return port_call_3(BIF_P,BIF_ARG_1,make_small(0),BIF_ARG_2);
+    return port_call(BIF_P,BIF_ARG_1, make_small(0), BIF_ARG_2);
 }
 
 BIF_RETTYPE port_call_3(BIF_ALIST_3)
 {
+    return port_call(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+}
+
+static BIF_RETTYPE
+port_call(Process* c_p, Eterm arg1, Eterm arg2, Eterm arg3)
+{
     Uint op;
     Port *p;
     Uint size;
     byte *bytes;
     byte *endp;
-    size_t real_size;
+    ErlDrvSizeT real_size;
     erts_driver_t *drv;
     byte port_input[256];	/* Default input buffer to encode in */
     byte port_result[256];	/* Buffer for result from port. */
     byte* port_resp;		/* Pointer to result buffer. */
     char *prc;
-    int ret;
+    ErlDrvSSizeT ret;
     Eterm res;
     Sint result_size;
     Eterm *hp;
-    Eterm *hp_end;              /* To satisfy hybrid heap architecture */
+    Eterm *hp_end;
     unsigned ret_flags = 0U;
     int fpe_was_unmasked;
 
@@ -270,15 +274,15 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3)
     /* trace of port scheduling with virtual process descheduling
      * lock wait 
      */
-    if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) {
-    	trace_virtual_sched(BIF_P, am_out);
+    if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) {
+	trace_virtual_sched(c_p, am_out);
     }
 
     if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
-    	profile_runnable_proc(BIF_P, am_inactive);
+	profile_runnable_proc(c_p, am_inactive);
     }
 
-    p = id_or_name2port(BIF_P, BIF_ARG_1);
+    p = id_or_name2port(c_p, arg1);
     if (!p) {
     error:
 	if (port_resp != port_result && 
@@ -290,22 +294,22 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3)
 	/* Need to virtual schedule in the process if there
 	 * was an error.
 	 */
-    	if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) {
-    	    trace_virtual_sched(BIF_P, am_in);
+	if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) {
+	    trace_virtual_sched(c_p, am_in);
     	}
 
 	if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
-    	    profile_runnable_proc(BIF_P, am_active);
+	    profile_runnable_proc(c_p, am_active);
     	}
 
 	if (p)
 	    erts_port_release(p);
 #ifdef ERTS_SMP
-	ERTS_SMP_BIF_CHK_PENDING_EXIT(BIF_P, ERTS_PROC_LOCK_MAIN);
+	ERTS_SMP_BIF_CHK_PENDING_EXIT(c_p, ERTS_PROC_LOCK_MAIN);
 #else
-	ERTS_BIF_CHK_EXITED(BIF_P);
+	ERTS_BIF_CHK_EXITED(c_p);
 #endif
-	BIF_ERROR(BIF_P, BADARG);
+	BIF_ERROR(c_p, BADARG);
     }
 
     if ((drv = p->drv_ptr) == NULL) {
@@ -314,10 +318,10 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3)
     if (drv->call == NULL) {
 	goto error;
     }
-    if (!term_to_Uint(BIF_ARG_2, &op)) {
+    if (!term_to_Uint(arg2, &op)) {
 	goto error;
     }
-    p->caller = BIF_P->id;
+    p->caller = c_p->id;
     
     /* Lock taken, virtual schedule of port */
     if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) {
@@ -327,19 +331,29 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3)
     if (erts_system_profile_flags.runnable_ports && !erts_port_is_scheduled(p)) {
     	profile_runnable_port(p, am_active);
     }
-    size = erts_encode_ext_size(BIF_ARG_3);
+    size = erts_encode_ext_size(arg3);
     if (size > sizeof(port_input))
 	bytes = erts_alloc(ERTS_ALC_T_PORT_CALL_BUF, size);
 
     endp = bytes;
-    erts_encode_ext(BIF_ARG_3, &endp);
+    erts_encode_ext(arg3, &endp);
 
     real_size = endp - bytes;
     if (real_size > size) {
 	erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n",
 		 __FILE__, __LINE__, endp - (bytes + size));
     }
-    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+#ifdef USE_VM_PROBES
+    if (DTRACE_ENABLED(driver_call)) {
+        DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE);
+        DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE);
+
+        dtrace_pid_str(p->connected, process_str);
+        dtrace_port_str(p, port_str);
+        DTRACE5(driver_call, process_str, port_str, p->name, op, real_size);
+    }
+#endif
     prc  = (char *) port_resp;
     fpe_was_unmasked = erts_block_fpe();
     ret = drv->call((ErlDrvData)p->drv_data, 
@@ -360,12 +374,12 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3)
    
     port_resp = (byte *) prc;
     p->caller = NIL;
-    erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
 #ifdef HARDDEBUG
     { 
-	int z;
-	printf("real_size = %ld,%d, ret = %d\r\n",real_size, 
-	       (int) real_size, ret);
+	ErlDrvSizeT z;
+	printf("real_size = %ld,%d, ret = %ld,%d\r\n", (unsigned long) real_size,
+	       (int) real_size, (unsigned long)ret, (int) ret);
 	printf("[");
 	for(z = 0; z < real_size; ++z) {
 	    printf("%d, ",(int) bytes[z]);
@@ -382,18 +396,18 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3)
 	/* Error or a binary without magic/ with wrong magic */
 	goto error;
     }
-    result_size = erts_decode_ext_size(port_resp, ret, 0);
+    result_size = erts_decode_ext_size(port_resp, ret);
     if (result_size < 0) {
 	goto error;
     }
-    hp = HAlloc(BIF_P, result_size);
+    hp = HAlloc(c_p, result_size);
     hp_end = hp + result_size;
     endp = port_resp;
-    res = erts_decode_ext(&hp, &MSO(BIF_P), &endp);
+    res = erts_decode_ext(&hp, &MSO(c_p), &endp);
     if (res == THE_NON_VALUE) {
 	goto error;
     }
-    HRelease(BIF_P, hp_end, hp);
+    HRelease(c_p, hp_end, hp);
     if (port_resp != port_result && !(ret_flags & DRIVER_CALL_KEEP_BUFFER)) {
 	driver_free(port_resp);
     }
@@ -402,16 +416,16 @@ BIF_RETTYPE port_call_3(BIF_ALIST_3)
     if (p)
 	erts_port_release(p);
 #ifdef ERTS_SMP
-    ERTS_SMP_BIF_CHK_PENDING_EXIT(BIF_P, ERTS_PROC_LOCK_MAIN);
+    ERTS_SMP_BIF_CHK_PENDING_EXIT(c_p, ERTS_PROC_LOCK_MAIN);
 #else
-    ERTS_BIF_CHK_EXITED(BIF_P);
+    ERTS_BIF_CHK_EXITED(c_p);
 #endif
-    if (IS_TRACED_FL(BIF_P, F_TRACE_SCHED_PROCS)) {
-    	trace_virtual_sched(BIF_P, am_in);
+    if (IS_TRACED_FL(c_p, F_TRACE_SCHED_PROCS)) {
+	trace_virtual_sched(c_p, am_in);
     }
 
     if (erts_system_profile_flags.runnable_procs && erts_system_profile_flags.exclusive) {
-    	profile_runnable_proc(BIF_P, am_active);
+	profile_runnable_proc(c_p, am_active);
     }
   
     return res;
@@ -536,6 +550,18 @@ BIF_RETTYPE port_connect_2(BIF_ALIST_2)
 
     prt->connected = pid; /* internal pid */
     erts_smp_port_unlock(prt);
+#ifdef USE_VM_PROBES
+    if (DTRACE_ENABLED(port_connect)) {
+        DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE);
+        DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE);
+        DTRACE_CHARBUF(newprocess_str, DTRACE_TERM_BUF_SIZE);
+
+        dtrace_pid_str(prt->connected, process_str);
+        erts_snprintf(port_str, sizeof(port_str), "%T", prt->id);
+        dtrace_proc_str(rp, newprocess_str);
+        DTRACE4(port_connect, process_str, port_str, prt->name, newprocess_str);
+    }
+#endif
     BIF_RET(am_true);
 }
 
@@ -583,8 +609,8 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1)
     if (prt->bp == NULL) {	/* MUST be CONST! */
 	res = prt->data;
     } else {
-	Eterm* hp = HAlloc(BIF_P, prt->bp->size);
-	res = copy_struct(prt->data, prt->bp->size, &hp, &MSO(BIF_P));
+	Eterm* hp = HAlloc(BIF_P, prt->bp->used_size);
+	res = copy_struct(prt->data, prt->bp->used_size, &hp, &MSO(BIF_P));
     }
     erts_smp_port_unlock(prt);
     BIF_RET(res);
@@ -614,6 +640,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
     int binary_io;
     int soft_eof;
     Sint linebuf;
+    Eterm edir = NIL;
     byte dir[MAXPATHLEN];
 
     /* These are the defaults */
@@ -690,19 +717,10 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
 
 		} else if (option == am_arg0) {
 		    char *a0;
-		    int n;
-		    if (is_nil(*tp)) {
-			n = 0;
-		    } else if( (n = is_string(*tp)) == 0) {
+
+		    if ((a0 = erts_convert_filename_to_native(*tp, ERTS_ALC_T_TMP, 1)) == NULL) {
 			goto badarg;
 		    }
-		    a0 = (char *) erts_alloc(ERTS_ALC_T_TMP, 
-					    (n + 1) * sizeof(byte));
-		    if (intlist_to_buf(*tp, a0, n) != n) {
-			erl_exit(1, "%s:%d: Internal error\n",
-				 __FILE__, __LINE__);
-		    }
-		    a0[n] = '\0';		    
 		    if (opts.argv == NULL) {
 			opts.argv = erts_alloc(ERTS_ALC_T_TMP, 
 					       2 * sizeof(char **));
@@ -715,20 +733,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
 			opts.argv[0] = a0;
 		    }
 		} else if (option == am_cd) {
-		    Eterm iolist;
-		    Eterm heap[4];
-		    int r;
-
-		    heap[0] = *tp;
-		    heap[1] = make_list(heap+2);
-		    heap[2] = make_small(0);
-		    heap[3] = NIL;
-		    iolist = make_list(heap);
-		    r = io_list_to_buf(iolist, (char*) dir, MAXPATHLEN);
-		    if (r < 0) {
-			goto badarg;
-		    }
-		    opts.wd = (char *) dir;
+		    edir = *tp;
 		} else {
 		    goto badarg;
 		}
@@ -840,19 +845,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
 		goto badarg;
 	    }
 	    name = tp[1];
-	    if (is_atom(name)) {
-		name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP,
-					       atom_tab(atom_val(name))->len+1);
-		sys_memcpy((void *) name_buf,
-			   (void *) atom_tab(atom_val(name))->name, 
-			   atom_tab(atom_val(name))->len);
-		name_buf[atom_tab(atom_val(name))->len] = '\0';
-	    } else if ((i = is_string(name))) {
-		name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1);
-		if (intlist_to_buf(name, name_buf, i) != i)
-		    erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
-		name_buf[i] = '\0';
-	    } else {
+	    if ((name_buf = erts_convert_filename_to_native(name,ERTS_ALC_T_TMP,0)) == NULL) {
 		goto badarg;
 	    }
 	    opts.spawn_type = ERTS_SPAWN_EXECUTABLE;
@@ -894,7 +887,33 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
 	/* Argument vector only if explicit spawn_executable */
 	goto badarg;
     }
-	
+
+    if (edir != NIL) {
+	/* A working directory is expressed differently if spawn_executable, i.e. Unicode is handles 
+	   for spawn_executable... */
+	if (opts.spawn_type != ERTS_SPAWN_EXECUTABLE) {
+	    Eterm iolist;
+	    DeclareTmpHeap(heap,4,p);
+	    int r;
+	    
+	    UseTmpHeap(4,p);
+	    heap[0] = edir;
+	    heap[1] = make_list(heap+2);
+	    heap[2] = make_small(0);
+	    heap[3] = NIL;
+	    iolist = make_list(heap);
+	    r = io_list_to_buf(iolist, (char*) dir, MAXPATHLEN);
+	    UnUseTmpHeap(4,p);
+	    if (r < 0) {
+		goto badarg;
+	    }
+	    opts.wd = (char *) dir;
+	} else {
+	    if ((opts.wd = erts_convert_filename_to_native(edir,ERTS_ALC_T_TMP,0)) == NULL) {
+		goto badarg;
+	    }
+	}
+    }
 
     if (driver != &spawn_driver && opts.exit_status) {
 	goto badarg;
@@ -908,7 +927,16 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
     erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
 
     port_num = erts_open_driver(driver, p->id, name_buf, &opts, err_nump);
+#ifdef USE_VM_PROBES
+    if (port_num >= 0 && DTRACE_ENABLED(port_open)) {
+        DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE);
+        DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE);
 
+        dtrace_proc_str(p, process_str);
+        erts_snprintf(port_str, sizeof(port_str), "%T", erts_port[port_num].id);
+        DTRACE3(port_open, process_str, name_buf, port_str);
+    }
+#endif
     erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
 
     if (port_num < 0) {
@@ -943,6 +971,9 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
     if (opts.argv) {
 	free_args(opts.argv);
     }
+    if (opts.wd && opts.wd != ((char *)dir)) {
+	erts_free(ERTS_ALC_T_TMP, (void *) opts.wd);
+    }
     return port_num;
     
  badarg:
@@ -952,6 +983,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
 #undef OPEN_PORT_ERROR
 }
 
+/* Arguments can be given i unicode and as raw binaries, convert filename is used to convert */
 static char **convert_args(Eterm l)
 {
     char **pp;
@@ -968,22 +1000,14 @@ static char **convert_args(Eterm l)
     pp[i++] = erts_default_arg0;
     while (is_list(l)) {
 	str = CAR(list_val(l));
-
-	if (is_nil(str)) {
-	    n = 0;
-	} else if( (n = is_string(str)) == 0) {
-	    /* Not a string... */
+	if ((b = erts_convert_filename_to_native(str,ERTS_ALC_T_TMP,1)) == NULL) {
 	    int j;
 	    for (j = 1; j < i; ++j)
 		erts_free(ERTS_ALC_T_TMP, pp[j]);
 	    erts_free(ERTS_ALC_T_TMP, pp);
 	    return NULL;
-	}
-	b = (char *) erts_alloc(ERTS_ALC_T_TMP, (n + 1) * sizeof(byte));
-	pp[i++] = (char *) b;
-	if (intlist_to_buf(str, b, n) != n)
-	    erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
-	b[n] = '\0';
+	}	    
+	pp[i++] = b;
 	l = CDR(list_val(l));
     }
     pp[i] = NULL;
@@ -1011,6 +1035,7 @@ static byte* convert_environment(Process* p, Eterm env)
     Eterm* hp;
     Uint heap_size;
     int n;
+    Uint size;
     byte* bytes;
 
     if ((n = list_length(env)) < 0) {
@@ -1054,15 +1079,15 @@ static byte* convert_environment(Process* p, Eterm env)
     if (is_not_nil(env)) {
 	goto done;
     }
-    if ((n = io_list_len(all)) < 0) {
+    if (erts_iolist_size(all, &size)) {
 	goto done;
     }
 
     /*
      * Put the result in a binary (no risk for a memory leak that way).
      */
-    (void) erts_new_heap_binary(p, NULL, n, &bytes);
-    io_list_to_buf(all, (char*)bytes, n);
+    (void) erts_new_heap_binary(p, NULL, size, &bytes);
+    io_list_to_buf(all, (char*)bytes, size);
 
  done:
     erts_free(ERTS_ALC_T_TMP, temp_heap);
@@ -1077,27 +1102,33 @@ struct packet_callback_args
     Eterm res;   /* Out */
     int string_as_bin; /* return strings as binaries (http_bin): */
     byte* aligned_ptr;
+    Uint bin_sz;
     Eterm orig;
     Uint bin_offs;
     byte bin_bitoffs;
 };
 
+#define in_area(ptr,start,nbytes) \
+    ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
+
 static Eterm
 http_bld_string(struct packet_callback_args* pca, Uint **hpp, Uint *szp,
 		const char *str, Sint len)
 {
     Eterm res = THE_NON_VALUE;
     Uint size;
+    int make_subbin;
 
     if (pca->string_as_bin) {
 	size = heap_bin_size(len);
-    
+	make_subbin = (size > ERL_SUB_BIN_SIZE
+		       && in_area(str, pca->aligned_ptr, pca->bin_sz));
 	if (szp) {
-	    *szp += (size > ERL_SUB_BIN_SIZE) ? ERL_SUB_BIN_SIZE : size;	
+	    *szp += make_subbin ? ERL_SUB_BIN_SIZE : size;
 	}
 	if (hpp) {
 	    res = make_binary(*hpp);
-	    if (size > ERL_SUB_BIN_SIZE) {
+	    if (make_subbin) {
 		ErlSubBin* bin = (ErlSubBin*) *hpp;
 		bin->thing_word = HEADER_SUB_BIN;
 		bin->size = len;
@@ -1328,7 +1359,7 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3)
     int packet_sz;           /*-------Binaries involved: ------------------*/
     byte* bin_ptr;           /*| orig: original binary                     */
     byte bin_bitsz;          /*| bin: BIF_ARG_2, may be sub-binary of orig */
-    Uint bin_sz;             /*| packet: prefix of bin                     */
+	                     /*| packet: prefix of bin                     */
     char* body_ptr;          /*| body: part of packet to return            */
     int body_sz;             /*| rest: bin without packet                  */
     struct packet_callback_args pca;
@@ -1389,18 +1420,18 @@ BIF_RETTYPE decode_packet_3(BIF_ALIST_3)
     }
 
 
-    bin_sz = binary_size(BIF_ARG_2);
+    pca.bin_sz = binary_size(BIF_ARG_2);
     ERTS_GET_BINARY_BYTES(BIF_ARG_2, bin_ptr, pca.bin_bitoffs, bin_bitsz);  
     if (pca.bin_bitoffs != 0) {
-        pca.aligned_ptr = erts_alloc(ERTS_ALC_T_TMP, bin_sz);
-        erts_copy_bits(bin_ptr, pca.bin_bitoffs, 1, pca.aligned_ptr, 0, 1, bin_sz*8);
+        pca.aligned_ptr = erts_alloc(ERTS_ALC_T_TMP, pca.bin_sz);
+        erts_copy_bits(bin_ptr, pca.bin_bitoffs, 1, pca.aligned_ptr, 0, 1, pca.bin_sz*8);
     }
     else {
         pca.aligned_ptr = bin_ptr;
     }
-    packet_sz = packet_get_length(type, (char*)pca.aligned_ptr, bin_sz,
+    packet_sz = packet_get_length(type, (char*)pca.aligned_ptr, pca.bin_sz,
                                   max_plen, trunc_len, &http_state);
-    if (!(packet_sz > 0 && packet_sz <= bin_sz)) {
+    if (!(packet_sz > 0 && packet_sz <= pca.bin_sz)) {
         if (packet_sz < 0) {
 	    goto error;
         }
@@ -1456,7 +1487,7 @@ error:
 
     rest = (ErlSubBin *) hp;
     rest->thing_word = HEADER_SUB_BIN;
-    rest->size = bin_sz - packet_sz;
+    rest->size = pca.bin_sz - packet_sz;
     rest->offs = pca.bin_offs + packet_sz;
     rest->orig = pca.orig;
     rest->bitoffs = pca.bin_bitoffs;
diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index c027cd5984..6b843d2e08 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -45,6 +45,7 @@ static Export *urun_trap_exportp = NULL;
 static Export *ucompile_trap_exportp = NULL;
 
 static BIF_RETTYPE re_exec_trap(BIF_ALIST_3);
+static BIF_RETTYPE re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3);
 
 static void *erts_erts_pcre_malloc(size_t size) {
     return erts_alloc(ERTS_ALC_T_RE_HEAP,size);
@@ -76,8 +77,8 @@ void erts_init_bif_re(void)
     re_exec_trap_export.code[0] = am_erlang;
     re_exec_trap_export.code[1] = am_re_run_trap;
     re_exec_trap_export.code[2] = 3;
-    re_exec_trap_export.code[3] = (Eterm) em_apply_bif;
-    re_exec_trap_export.code[4] = (Eterm) &re_exec_trap;
+    re_exec_trap_export.code[3] = (BeamInstr) em_apply_bif;
+    re_exec_trap_export.code[4] = (BeamInstr) &re_exec_trap;
 
     grun_trap_exportp =  erts_export_put(am_re,am_grun,3);
     urun_trap_exportp =  erts_export_put(am_re,am_urun,3);
@@ -103,7 +104,7 @@ Sint erts_re_set_loop_limit(Sint limit)
 
 static int term_to_int(Eterm term, int *sp)
 {
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 
     if (is_small(term)) {
 	Uint x = signed_val(term);
@@ -154,7 +155,7 @@ static int term_to_int(Eterm term, int *sp)
 
 static Eterm make_signed_integer(int x, Process *p)
 {
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
     return make_small(x);
 #else
     Eterm* hp;
@@ -414,10 +415,10 @@ build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, con
  * Compile BIFs
  */
 
-BIF_RETTYPE
-re_compile_2(BIF_ALIST_2)
+static BIF_RETTYPE
+re_compile(Process* p, Eterm arg1, Eterm arg2)
 {
-    int slen;
+    Uint slen;
     char *expr;
     pcre *result;
     int errcode = 0;
@@ -429,43 +430,49 @@ re_compile_2(BIF_ALIST_2)
     int unicode = 0;
 
 
-    if (parse_options(BIF_ARG_2,&options,NULL,&pflags,NULL,NULL)
+    if (parse_options(arg2,&options,NULL,&pflags,NULL,NULL)
 	< 0) {
-	BIF_ERROR(BIF_P,BADARG);
+	BIF_ERROR(p,BADARG);
     }
 
     if (pflags & PARSE_FLAG_UNIQUE_EXEC_OPT) {
-	BIF_ERROR(BIF_P,BADARG);
+	BIF_ERROR(p,BADARG);
     }
 
     unicode = (pflags & PARSE_FLAG_UNICODE) ? 1 : 0;
 
-    if (pflags & PARSE_FLAG_UNICODE && !is_binary(BIF_ARG_1)) { 
-	BIF_TRAP2(ucompile_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2);
+    if (pflags & PARSE_FLAG_UNICODE && !is_binary(arg1)) {
+	BIF_TRAP2(ucompile_trap_exportp, p, arg1, arg2);
     }
 
-    if ((slen = io_list_len(BIF_ARG_1)) < 0) {
-        BIF_ERROR(BIF_P,BADARG);
+    if (erts_iolist_size(arg1, &slen)) {
+        BIF_ERROR(p,BADARG);
     }
     expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1);
-    if (io_list_to_buf(BIF_ARG_1, expr, slen) != 0) {
+    if (io_list_to_buf(arg1, expr, slen) != 0) {
 	erts_free(ERTS_ALC_T_RE_TMP_BUF, expr);
-	BIF_ERROR(BIF_P,BADARG);
+	BIF_ERROR(p,BADARG);
     }
     expr[slen]='\0';
     result = erts_pcre_compile2(expr, options, &errcode, 
 			   &errstr, &errofset, default_table);
 
-    ret = build_compile_result(BIF_P, am_error, result, errcode, 
+    ret = build_compile_result(p, am_error, result, errcode,
 			       errstr, errofset, unicode, 1);
     erts_free(ERTS_ALC_T_RE_TMP_BUF, expr);
     BIF_RET(ret);
 }
 
 BIF_RETTYPE
+re_compile_2(BIF_ALIST_2)
+{
+    return re_compile(BIF_P, BIF_ARG_1, BIF_ARG_2);
+}
+
+BIF_RETTYPE
 re_compile_1(BIF_ALIST_1)
 {
-    return re_compile_2(BIF_P,BIF_ARG_1,NIL);
+    return re_compile(BIF_P, BIF_ARG_1, NIL);
 }
 
 /*
@@ -795,8 +802,8 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
 			memcpy(tmpb,ap->name,ap->len);
 			tmpb[ap->len] = '\0';
 		    } else {
-			int slen = io_list_len(val);
-			if (slen < 0) {
+			Uint slen;
+			if (erts_iolist_size(val, &slen)) {
 			    goto error;
 			}
 			if ((slen + 1) > tmpbsiz) {
@@ -845,13 +852,13 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
 /*
  * The actual re:run/2,3 BIFs
  */
-BIF_RETTYPE
-re_run_3(BIF_ALIST_3)
+static BIF_RETTYPE
+re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3)
 {
     const pcre *code_tmp;
     RestartContext restart;
     byte *temp_alloc = NULL;
-    int slength;
+    Uint slength;
     int startoffset = 0;
     int options = 0, comp_options = 0;
     int ovsize;
@@ -865,17 +872,17 @@ re_run_3(BIF_ALIST_3)
     Eterm capture[CAPSPEC_SIZE];
     int is_list_cap;
 
-    if (parse_options(BIF_ARG_3,&comp_options,&options,&pflags,&startoffset,capture)
+    if (parse_options(arg3,&comp_options,&options,&pflags,&startoffset,capture)
 	< 0) {
-	BIF_ERROR(BIF_P,BADARG);
+	BIF_ERROR(p,BADARG);
     }
     is_list_cap = ((pflags & PARSE_FLAG_CAPTURE_OPT) && 
 		   (capture[CAPSPEC_TYPE] == am_list));
 
-    if (is_not_tuple(BIF_ARG_2) || (arityval(*tuple_val(BIF_ARG_2)) != 4)) {
-	if (is_binary(BIF_ARG_2) || is_list(BIF_ARG_2) || is_nil(BIF_ARG_2)) { 
+    if (is_not_tuple(arg2) || (arityval(*tuple_val(arg2)) != 4)) {
+	if (is_binary(arg2) || is_list(arg2) || is_nil(arg2)) {
 	    /* Compile from textual RE */
-	    int slen;
+	    Uint slen;
 	    char *expr;
 	    pcre *result;
 	    int errcode = 0;
@@ -884,19 +891,19 @@ re_run_3(BIF_ALIST_3)
 	    int capture_count;
 
 	    if (pflags & PARSE_FLAG_UNICODE && 
-		(!is_binary(BIF_ARG_2) || !is_binary(BIF_ARG_1) ||
+		(!is_binary(arg2) || !is_binary(arg1) ||
 		 (is_list_cap && !(pflags & PARSE_FLAG_GLOBAL)))) { 
-		BIF_TRAP3(urun_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+		BIF_TRAP3(urun_trap_exportp, p, arg1, arg2, arg3);
 	    }
 	    
-	    if ((slen = io_list_len(BIF_ARG_2)) < 0) {
-		BIF_ERROR(BIF_P,BADARG);
+	    if (erts_iolist_size(arg2, &slen)) {
+		BIF_ERROR(p,BADARG);
 	    }
 	    
 	    expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1);
-	    if (io_list_to_buf(BIF_ARG_2, expr, slen) != 0) {
+	    if (io_list_to_buf(arg2, expr, slen) != 0) {
 		erts_free(ERTS_ALC_T_RE_TMP_BUF, expr);
-		BIF_ERROR(BIF_P,BADARG);
+		BIF_ERROR(p,BADARG);
 	    }
 	    expr[slen]='\0';
 	    result = erts_pcre_compile2(expr, comp_options, &errcode, 
@@ -905,11 +912,11 @@ re_run_3(BIF_ALIST_3)
 		erts_free(ERTS_ALC_T_RE_TMP_BUF, expr);
 		/* Compilation error gives badarg except in the compile 
 		   function */
-		BIF_ERROR(BIF_P,BADARG);
+		BIF_ERROR(p,BADARG);
 	    }
 	    if (pflags & PARSE_FLAG_GLOBAL) {
 		Eterm precompiled = 
-		    build_compile_result(BIF_P, am_error, 
+		    build_compile_result(p, am_error,
 					 result, errcode, 
 					 errstr, errofset, 
 					 (pflags & 
@@ -917,13 +924,13 @@ re_run_3(BIF_ALIST_3)
 					 0);
 		Eterm *hp,r;
 		erts_free(ERTS_ALC_T_RE_TMP_BUF, expr);
-		hp = HAlloc(BIF_P,4);
-		/* BIF_ARG_2 is in the tuple just to make exceptions right */
-		r = TUPLE3(hp,BIF_ARG_3,
+		hp = HAlloc(p,4);
+		/* arg2 is in the tuple just to make exceptions right */
+		r = TUPLE3(hp,arg3,
 			   ((pflags & PARSE_FLAG_UNIQUE_COMPILE_OPT) ? 
 			    am_true : 
-			    am_false), BIF_ARG_2);
-		BIF_TRAP3(grun_trap_exportp, BIF_P, BIF_ARG_1, precompiled, r);
+			    am_false), arg2);
+		BIF_TRAP3(grun_trap_exportp, p, arg1, precompiled, r);
 	    }
 
 	    erts_pcre_fullinfo(result, NULL, PCRE_INFO_SIZE, &code_size);
@@ -935,31 +942,31 @@ re_run_3(BIF_ALIST_3)
 	    erts_free(ERTS_ALC_T_RE_TMP_BUF, expr);
 	    /*unicode = (pflags & PARSE_FLAG_UNICODE) ? 1 : 0;*/
 	} else {  
-	    BIF_ERROR(BIF_P,BADARG);
+	    BIF_ERROR(p,BADARG);
 	}
     } else {
 	if (pflags & PARSE_FLAG_UNIQUE_COMPILE_OPT) {
-	    BIF_ERROR(BIF_P,BADARG);
+	    BIF_ERROR(p,BADARG);
 	}
 
-	tp = tuple_val(BIF_ARG_2);
+	tp = tuple_val(arg2);
 	if (tp[1] != am_re_pattern || is_not_small(tp[2]) || 
 	    is_not_small(tp[3]) || is_not_binary(tp[4])) {
-	    BIF_ERROR(BIF_P,BADARG);
+	    BIF_ERROR(p,BADARG);
 	}
 
 	if (unsigned_val(tp[3]) && 
-	    (!is_binary(BIF_ARG_1) || 
+	    (!is_binary(arg1) ||
 	     (is_list_cap && !(pflags & PARSE_FLAG_GLOBAL)))) { /* unicode */
-	    BIF_TRAP3(urun_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2, 
-		      BIF_ARG_3);
+	    BIF_TRAP3(urun_trap_exportp, p, arg1, arg2,
+		      arg3);
 	}
 
 	if (pflags & PARSE_FLAG_GLOBAL) {
 	    Eterm *hp,r;
-	    hp = HAlloc(BIF_P,3);
-	    r = TUPLE2(hp,BIF_ARG_3,am_false);
-	    BIF_TRAP3(grun_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2, 
+	    hp = HAlloc(p,3);
+	    r = TUPLE2(hp,arg3,am_false);
+	    BIF_TRAP3(grun_trap_exportp, p, arg1, arg2,
 		      r);
 	}
 
@@ -968,7 +975,7 @@ re_run_3(BIF_ALIST_3)
 	if ((code_tmp = (const pcre *) 
 	     erts_get_aligned_binary_bytes(tp[4], &temp_alloc)) == NULL) {
 	    erts_free_aligned_binary_bytes(temp_alloc);
-	    BIF_ERROR(BIF_P, BADARG);
+	    BIF_ERROR(p, BADARG);
 	}
 	restart.code = erts_alloc(ERTS_ALC_T_RE_SUBJECT, code_size);
 	memcpy(restart.code, code_tmp, code_size);
@@ -980,7 +987,7 @@ re_run_3(BIF_ALIST_3)
     restart.ovector =  erts_alloc(ERTS_ALC_T_RE_SUBJECT, ovsize * sizeof(int));
     restart.extra.flags = PCRE_EXTRA_TABLES | PCRE_EXTRA_LOOP_LIMIT;
     restart.extra.tables = default_table;
-    restart.extra.loop_limit = ERTS_BIF_REDS_LEFT(BIF_P) * LOOP_FACTOR;
+    restart.extra.loop_limit = ERTS_BIF_REDS_LEFT(p) * LOOP_FACTOR;
     loop_limit_tmp = max_loop_limit; /* To lesser probability of race in debug
 					situation (erts_debug) */
     if (restart.extra.loop_limit > loop_limit_tmp) {
@@ -996,7 +1003,7 @@ re_run_3(BIF_ALIST_3)
 	if ((restart.ret_info = build_capture(capture,restart.code)) == NULL) {
 	    erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector);
 	    erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code);
-	    BIF_ERROR(BIF_P,BADARG);
+	    BIF_ERROR(p,BADARG);
 	}
     }
 	    
@@ -1004,7 +1011,7 @@ re_run_3(BIF_ALIST_3)
        copying, also binary returns can be sub binaries in that case */
 
     restart.flags = 0;
-    if (is_binary(BIF_ARG_1)) {
+    if (is_binary(arg1)) {
 	Eterm real_bin;
 	Uint offset;
 	Eterm* bptr;
@@ -1012,9 +1019,9 @@ re_run_3(BIF_ALIST_3)
 	int bitsize;
 	ProcBin* pb;
 
-	ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize);
+	ERTS_GET_REAL_BIN(arg1, real_bin, offset, bitoffs, bitsize);
 
-	slength = binary_size(BIF_ARG_1);
+	slength = binary_size(arg1);
 	bptr = binary_val(real_bin);
 	if (bitsize != 0 || bitoffs != 0 ||  (*bptr != HEADER_PROC_BIN)) {
 	    goto handle_iolist;
@@ -1027,24 +1034,24 @@ re_run_3(BIF_ALIST_3)
 	restart.flags |= RESTART_FLAG_SUBJECT_IN_BINARY;
     } else {
 handle_iolist:
-	if ((slength = io_list_len(BIF_ARG_1)) < 0) {
+	if (erts_iolist_size(arg1, &slength)) {
 	    erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector);
 	    erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code);
 	    if (restart.ret_info != NULL) {
 		erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ret_info);
 	    }
-	    BIF_ERROR(BIF_P,BADARG);
+	    BIF_ERROR(p,BADARG);
 	}
 	restart.subject = erts_alloc(ERTS_ALC_T_RE_SUBJECT, slength);
 
-	if (io_list_to_buf(BIF_ARG_1, restart.subject, slength) != 0) {
+	if (io_list_to_buf(arg1, restart.subject, slength) != 0) {
 	    erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector);
 	    erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code);
 	    erts_free(ERTS_ALC_T_RE_SUBJECT, restart.subject);
 	    if (restart.ret_info != NULL) {
 		erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ret_info);
 	    }
-	    BIF_ERROR(BIF_P,BADARG);
+	    BIF_ERROR(p,BADARG);
 	}
     }
 
@@ -1056,7 +1063,7 @@ handle_iolist:
     rc = erts_pcre_exec(restart.code, &(restart.extra), restart.subject, slength, startoffset, 
 		   options, restart.ovector, ovsize);
     ASSERT(loop_count != 0xFFFFFFFF);
-    BUMP_REDS(BIF_P, loop_count / LOOP_FACTOR);
+    BUMP_REDS(p, loop_count / LOOP_FACTOR);
     if (rc == PCRE_ERROR_LOOP_LIMIT) {
 	/* Trap */
 	Binary *mbp = erts_create_magic_binary(sizeof(RestartContext),
@@ -1065,17 +1072,17 @@ handle_iolist:
 	Eterm magic_bin;
 	Eterm *hp;
 	memcpy(restartp,&restart,sizeof(RestartContext));
-	BUMP_ALL_REDS(BIF_P);
-	hp = HAlloc(BIF_P, PROC_BIN_SIZE);
-	magic_bin = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), mbp);
+	BUMP_ALL_REDS(p);
+	hp = HAlloc(p, PROC_BIN_SIZE);
+	magic_bin = erts_mk_magic_binary_term(&hp, &MSO(p), mbp);
 	BIF_TRAP3(&re_exec_trap_export, 
-		  BIF_P, 
-		  BIF_ARG_1, 
-		  BIF_ARG_2 /* To avoid GC of precompiled code, XXX: not utilized yet */, 
+		  p,
+		  arg1,
+		  arg2 /* To avoid GC of precompiled code, XXX: not utilized yet */,
 		  magic_bin);
     }
     
-    res = build_exec_return(BIF_P, rc, &restart, BIF_ARG_1);
+    res = build_exec_return(p, rc, &restart, arg1);
  
     cleanup_restart_context(&restart);
 
@@ -1083,9 +1090,15 @@ handle_iolist:
 }
 
 BIF_RETTYPE
+re_run_3(BIF_ALIST_3)
+{
+    return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+}
+
+BIF_RETTYPE
 re_run_2(BIF_ALIST_2) 
 {
-    return re_run_3(BIF_P,BIF_ARG_1, BIF_ARG_2, NIL);
+    return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, NIL);
 }
 
 /*
diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c
index 172bb37952..d806be0704 100644
--- a/erts/emulator/beam/erl_bif_timer.c
+++ b/erts/emulator/beam/erl_bif_timer.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -26,6 +26,7 @@
 #include "bif.h"
 #include "error.h"
 #include "big.h"
+#include "erl_thr_progress.h"
 
 /****************************************************************************
 ** BIF Timer support
@@ -155,7 +156,7 @@ create_ref(Uint *hp, Uint32 *ref_numbers, Uint32 len)
 	erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
     }
 
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
     hp[0] = make_ref_thing_header(len/2 + 1);
     datap = (Uint32 *) &hp[1];
     *(datap++) = len;
@@ -173,13 +174,13 @@ create_ref(Uint *hp, Uint32 *ref_numbers, Uint32 len)
 static int
 eq_non_standard_ref_numbers(Uint32 *rn1, Uint32 len1, Uint32 *rn2, Uint32 len2)
 {
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 #define MAX_REF_HEAP_SZ (1+(ERTS_MAX_REF_NUMBERS/2+1))
 #else
 #define MAX_REF_HEAP_SZ (1+ERTS_MAX_REF_NUMBERS)
 #endif
-    Uint r1_hp[MAX_REF_HEAP_SZ];
-    Uint r2_hp[MAX_REF_HEAP_SZ];
+    DeclareTmpHeapNoproc(r1_hp,(MAX_REF_HEAP_SZ * 2));
+    Eterm *r2_hp = r1_hp +MAX_REF_HEAP_SZ;
 
     return eq(create_ref(r1_hp, rn1, len1), create_ref(r2_hp, rn2, len2));
 #undef MAX_REF_HEAP_SZ
@@ -357,7 +358,7 @@ bif_timer_timeout(ErtsBifTimer* btm)
 						 rp,
 						 &rp_locks);
 		} else {
-		    Eterm old_size = bp->size;
+		    Eterm old_size = bp->used_size;
 		    bp = erts_resize_message_buffer(bp, old_size + wrap_size,
 						    &message, 1);
 		    hp = &bp->mem[0] + old_size;
@@ -372,7 +373,11 @@ bif_timer_timeout(ErtsBifTimer* btm)
 		message = TUPLE3(hp, am_timeout, ref, message);
 	    }
 
-	    erts_queue_message(rp, &rp_locks, bp, message, NIL);
+	    erts_queue_message(rp, &rp_locks, bp, message, NIL
+#ifdef USE_VM_PROBES
+			       , NIL
+#endif
+			       );
 	    erts_smp_proc_unlock(rp, rp_locks);
 	    erts_smp_proc_dec_refc(rp);
 	}
@@ -398,7 +403,7 @@ setup_bif_timer(Uint32 xflags,
     
     if (!term_to_Uint(time, &timeout))
 	return THE_NON_VALUE;
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
     if ((timeout >> 32) != 0)
 	return THE_NON_VALUE;
 #endif
@@ -478,7 +483,7 @@ setup_bif_timer(Uint32 xflags,
     tab_insert(btm);
     ASSERT(btm == tab_find(ref));
     btm->tm.active = 0; /* MUST be initalized */
-    erl_set_timer(&btm->tm,
+    erts_set_timer(&btm->tm,
 		  (ErlTimeoutProc) bif_timer_timeout,
 		  (ErlCancelProc) bif_timer_cleanup,
 		  (void *) btm,
@@ -550,7 +555,7 @@ BIF_RETTYPE cancel_timer_1(BIF_ALIST_1)
 	res = am_false;
     }
     else {
-	Uint left = time_left(&btm->tm);
+	Uint left = erts_time_left(&btm->tm);
 	if (!(btm->flags & BTM_FLG_BYNAME)) {
 	    erts_smp_proc_lock(btm->receiver.proc.ess, ERTS_PROC_LOCK_MSGQ);
 	    unlink_proc(btm);
@@ -558,7 +563,7 @@ BIF_RETTYPE cancel_timer_1(BIF_ALIST_1)
 	}
 	tab_remove(btm);
 	ASSERT(!tab_find(BIF_ARG_1));
-	erl_cancel_timer(&btm->tm);
+	erts_cancel_timer(&btm->tm);
 	erts_smp_btm_rwunlock();
 	res = erts_make_integer(left, BIF_P);
     }
@@ -587,7 +592,7 @@ BIF_RETTYPE read_timer_1(BIF_ALIST_1)
 	res = am_false;
     }
     else {
-	Uint left = time_left(&btm->tm);
+	Uint left = erts_time_left(&btm->tm);
 	res = erts_make_integer(left, BIF_P);
     }
 
@@ -613,7 +618,8 @@ erts_print_bif_timer_info(int to, void *to_arg)
 			      : btm->receiver.proc.ess->id);
 	    erts_print(to, to_arg, "=timer:%T\n", receiver);
 	    erts_print(to, to_arg, "Message: %T\n", btm->message);
-	    erts_print(to, to_arg, "Time left: %d ms\n", time_left(&btm->tm));
+	    erts_print(to, to_arg, "Time left: %u ms\n",
+		       erts_time_left(&btm->tm));
 	}
     }
 
@@ -640,7 +646,7 @@ erts_cancel_bif_timers(Process *p, ErtsProcLocks plocks)
 	tab_remove(btm);
 	tmp_btm = btm;
 	btm = btm->receiver.proc.next;
-	erl_cancel_timer(&tmp_btm->tm);
+	erts_cancel_timer(&tmp_btm->tm);
     }
 
     p->bif_timers = NULL;
@@ -685,7 +691,7 @@ erts_bif_timer_foreach(void (*func)(Eterm, Eterm, ErlHeapFragment *, void *),
 {
     int i;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0));
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
 
     for (i = 0; i < TIMER_HASH_VEC_SZ; i++) {
 	ErtsBifTimer *btm;
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 7dff5e0eeb..80f774523c 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1999-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1999-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -37,17 +37,22 @@
 #include "erl_version.h"
 #include "beam_bp.h"
 #include "erl_binary.h"
+#include "erl_thr_progress.h"
 
 #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
 
-static erts_smp_mtx_t              trace_pattern_mutex;
-const struct trace_pattern_flags   erts_trace_pattern_flags_off = {0, 0, 0, 0};
+const struct trace_pattern_flags   erts_trace_pattern_flags_off = {0, 0, 0, 0, 0};
 static int                         erts_default_trace_pattern_is_on;
 static Binary                     *erts_default_match_spec;
 static Binary                     *erts_default_meta_match_spec;
 static struct trace_pattern_flags  erts_default_trace_pattern_flags;
 static Eterm                       erts_default_meta_tracer_pid;
 
+static Eterm
+trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist);
+static BIF_RETTYPE
+system_monitor(Process *p, Eterm monitor_pid, Eterm list);
+
 static void new_seq_trace_token(Process* p); /* help func for seq_trace_2*/
 static int already_traced(Process *p, Process *tracee_p, Eterm tracer);
 static int port_already_traced(Process *p, Port *tracee_port, Eterm tracer);
@@ -65,7 +70,6 @@ static void clear_trace_bif(int bif_index);
 void
 erts_bif_trace_init(void)
 {
-    erts_smp_mtx_init(&trace_pattern_mutex, "trace_pattern");
     erts_default_trace_pattern_is_on = 0;
     erts_default_match_spec = NULL;
     erts_default_meta_match_spec = NULL;
@@ -78,15 +82,21 @@ erts_bif_trace_init(void)
  */
   
 Eterm
-trace_pattern_2(Process* p, Eterm MFA, Eterm Pattern)
+trace_pattern_2(BIF_ALIST_2)
 {
-    return trace_pattern_3(p,MFA,Pattern,NIL);
+    return trace_pattern(BIF_P, BIF_ARG_1, BIF_ARG_2, NIL);
 }
 
 Eterm
-trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)       
+trace_pattern_3(BIF_ALIST_3)
+{
+    return trace_pattern(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+}
+
+static Eterm
+trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 {
-    Eterm mfa[3];
+    DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */
     int i;
     int matches = 0;
     int specified = 0;
@@ -99,8 +109,9 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
     Eterm meta_tracer_pid = p->id;
 
     erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
-    erts_smp_block_system(0);
+    erts_smp_thr_progress_block();
 
+    UseTmpHeap(3,p);
     /*
      * Check and compile the match specification.
      */
@@ -185,6 +196,14 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 		flags.breakpoint = 1;
 		flags.call_count = 1;
 		break;
+	    case am_call_time:
+		if (is_global) {
+		    goto error;
+		}
+		flags.breakpoint = 1;
+		flags.call_time = 1;
+		break;
+
 	    default:
 		goto error;
 	    }
@@ -194,8 +213,8 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 	goto error;
     }
     
-    if (match_prog_set && !flags.local && !flags.meta && flags.call_count) {
-	/* A match prog is not allowed with just call_count */
+    if (match_prog_set && !flags.local && !flags.meta && (flags.call_count || flags.call_time)) {
+	/* A match prog is not allowed with just call_count or call_time*/
 	goto error;
     }
 
@@ -234,6 +253,8 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 		    |= flags.meta;
 		erts_default_trace_pattern_flags.call_count
 		    |= (on == 1) ? flags.call_count : 0;
+		erts_default_trace_pattern_flags.call_time
+		    |= (on == 1) ? flags.call_time : 0;
 	    } else {
 		erts_default_trace_pattern_flags.local
 		    &= ~flags.local;
@@ -241,10 +262,13 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 		    &= ~flags.meta;
 		erts_default_trace_pattern_flags.call_count
 		    &= ~flags.call_count;
+		erts_default_trace_pattern_flags.call_time
+		    &= ~flags.call_time;
 		if (! (erts_default_trace_pattern_flags.breakpoint =
 		       erts_default_trace_pattern_flags.local |
 		       erts_default_trace_pattern_flags.meta |
-		       erts_default_trace_pattern_flags.call_count)) {
+		       erts_default_trace_pattern_flags.call_count |
+		       erts_default_trace_pattern_flags.call_time)) {
 		    erts_default_trace_pattern_is_on = !!on; /* i.e off */
 		}
 	    }
@@ -266,8 +290,9 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 	    if (on) {
 		if (on != 1) {
 		    flags.call_count = 0;
+		    flags.call_time  = 0;
 		}
-		flags.breakpoint = flags.local | flags.meta | flags.call_count;
+		flags.breakpoint = flags.local | flags.meta | flags.call_count | flags.call_time;
 		erts_default_trace_pattern_flags = flags; /* Struct copy */
 		erts_default_trace_pattern_is_on = !!flags.breakpoint;
 	    }
@@ -312,8 +337,8 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
     MatchSetUnref(match_prog_set);
 
  done:
-
-    erts_smp_release_system();
+    UnUseTmpHeap(3,p);
+    erts_smp_thr_progress_unblock();
     erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
 
     return make_small(matches);
@@ -322,7 +347,8 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist)
 
     MatchSetUnref(match_prog_set);
 
-    erts_smp_release_system();
+    UnUseTmpHeap(3,p);
+    erts_smp_thr_progress_unblock();
     erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
     BIF_ERROR(p, BADARG);
 }
@@ -334,7 +360,6 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on,
 			       struct trace_pattern_flags *trace_pattern_flags,
 			       Eterm *meta_tracer_pid)
 {
-    erts_smp_mtx_lock(&trace_pattern_mutex);
     if (trace_pattern_is_on)
 	*trace_pattern_is_on = erts_default_trace_pattern_is_on;
     if (match_spec)
@@ -345,12 +370,10 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on,
 	*trace_pattern_flags = erts_default_trace_pattern_flags;
     if (meta_tracer_pid)
 	*meta_tracer_pid = erts_default_meta_tracer_pid;
-    erts_smp_mtx_unlock(&trace_pattern_mutex);
 }
 
 
 
-
 Uint 
 erts_trace_flag2bit(Eterm flag) 
 {
@@ -378,7 +401,7 @@ erts_trace_flag2bit(Eterm flag)
     default: return 0;
     }
 }
-
+
 /* Scan the argument list and sort out the trace flags.
 **
 ** Returns !0 on success, 0 on failure.
@@ -424,9 +447,12 @@ erts_trace_flags(Eterm List,
     return 0;
 }
 
-Eterm
-trace_3(Process* p, Eterm pid_spec, Eterm how, Eterm list)
+Eterm trace_3(BIF_ALIST_3)
 {
+    Process* p = BIF_P;
+    Eterm pid_spec = BIF_ARG_1;
+    Eterm how = BIF_ARG_2;
+    Eterm list = BIF_ARG_3;
     int on;
     Eterm tracer = NIL;
     int matches = 0;
@@ -619,7 +645,7 @@ trace_3(Process* p, Eterm pid_spec, Eterm how, Eterm list)
 
 #ifdef ERTS_SMP
 	    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
-	    erts_smp_block_system(0);
+	    erts_smp_thr_progress_block();
 	    system_blocked = 1;
 #endif
 
@@ -668,7 +694,7 @@ trace_3(Process* p, Eterm pid_spec, Eterm how, Eterm list)
 		    } else if (tracer != NIL) {
 			tracee_port->tracer_proc = tracer;
 		    }
-		    /* matches are not counted for ports since it would violate compability */	
+		    /* matches are not counted for ports since it would violate compatibility */
 		    /* This could be a reason to modify this function or make a new one. */
 		}
 	    }
@@ -700,7 +726,7 @@ trace_3(Process* p, Eterm pid_spec, Eterm how, Eterm list)
 
 #ifdef ERTS_SMP
     if (system_blocked) {
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
     }
 #endif
@@ -715,7 +741,7 @@ trace_3(Process* p, Eterm pid_spec, Eterm how, Eterm list)
 
 #ifdef ERTS_SMP
     if (system_blocked) {
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
     }
 #endif
@@ -809,9 +835,11 @@ static int already_traced(Process *c_p, Process *tracee_p, Eterm tracer)
  * Return information about a process or an external function being traced.
  */
 
-Eterm
-trace_info_2(Process* p, Eterm What, Eterm Key)
+Eterm trace_info_2(BIF_ALIST_2)
 {
+    Process* p = BIF_P;
+    Eterm What = BIF_ARG_1;
+    Eterm Key = BIF_ARG_2;
     Eterm res;
     if (What == am_on_load) {
 	res = trace_info_on_load(p, Key);
@@ -929,6 +957,7 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
 #define FUNC_TRACE_LOCAL_TRACE  (1<<2)
 #define FUNC_TRACE_META_TRACE   (1<<3)
 #define FUNC_TRACE_COUNT_TRACE  (1<<4)
+#define FUNC_TRACE_TIME_TRACE   (1<<5)
 /*
  * Returns either FUNC_TRACE_NOEXIST, FUNC_TRACE_UNTRACED,
  * FUNC_TRACE_GLOBAL_TRACE, or,
@@ -943,16 +972,18 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key)
  *
  * If the return value contains FUNC_TRACE_COUNT_TRACE, *count is set.
  */
-static int function_is_traced(Eterm mfa[3], 
-			      Binary **ms, /* out */
-			      Binary **ms_meta,  /* out */
+static int function_is_traced(Process *p,
+			      Eterm mfa[3],
+			      Binary **ms,              /* out */
+			      Binary **ms_meta,         /* out */
 			      Eterm   *tracer_pid_meta, /* out */
-			      Sint    *count)    /* out */
+			      Sint    *count,           /* out */
+			      Eterm   *call_time)       /* out */
 {
     Export e;
     Export* ep;
     int i;
-    Uint *code;
+    BeamInstr *code;
 
     /* First look for an export entry */
     e.code[0] = mfa[0];
@@ -960,12 +991,12 @@ static int function_is_traced(Eterm mfa[3],
     e.code[2] = mfa[2];
     if ((ep = export_get(&e)) != NULL) {
 	if (ep->address == ep->code+3 &&
-	    ep->code[3] != (Uint) em_call_error_handler) {
-	    if (ep->code[3] == (Uint) em_call_traced_function) {
+	    ep->code[3] != (BeamInstr) em_call_error_handler) {
+	    if (ep->code[3] == (BeamInstr) em_call_traced_function) {
 		*ms = ep->match_prog_set;
 		return FUNC_TRACE_GLOBAL_TRACE;
 	    }
-	    if (ep->code[3] == (Uint) em_apply_bif) {
+	    if (ep->code[3] == (BeamInstr) em_apply_bif) {
 		for (i = 0; i < BIF_SIZE; ++i) {
 		    if (bif_export[i] == ep) {
 			int r = 0;
@@ -978,10 +1009,13 @@ static int function_is_traced(Eterm mfa[3],
 				r |= FUNC_TRACE_LOCAL_TRACE;
 				*ms = ep->match_prog_set;
 			    }
-			    if (erts_is_mtrace_bif(ep->code+3, ms_meta, 
+			    if (erts_is_mtrace_break(ep->code+3, ms_meta,
 						   tracer_pid_meta)) {
 				r |= FUNC_TRACE_META_TRACE;
 			    }
+			    if (erts_is_time_break(p, ep->code+3, call_time)) {
+				r |= FUNC_TRACE_TIME_TRACE;
+			    }
 			}
 			return r ? r : FUNC_TRACE_UNTRACED;
 		    }
@@ -999,7 +1033,9 @@ static int function_is_traced(Eterm mfa[3],
 	    | (erts_is_mtrace_break(code, ms_meta, tracer_pid_meta)
 	       ? FUNC_TRACE_META_TRACE : 0)
 	    | (erts_is_count_break(code, count)
-	       ? FUNC_TRACE_COUNT_TRACE : 0);
+	       ? FUNC_TRACE_COUNT_TRACE : 0)
+	    | (erts_is_time_break(p, code, call_time)
+	       ? FUNC_TRACE_TIME_TRACE : 0);
 	
 	return r ? r : FUNC_TRACE_UNTRACED;
     } 
@@ -1011,15 +1047,19 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
 {
     Eterm* tp;
     Eterm* hp;
-    Eterm mfa[3];
+    DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */
     Binary *ms = NULL, *ms_meta = NULL;
     Sint count = 0;
     Eterm traced = am_false;
     Eterm match_spec = am_false;
     Eterm retval = am_false;
     Eterm meta = am_false;
+    Eterm call_time = NIL;
     int r;
 
+
+    UseTmpHeap(3,p);
+
     if (!is_tuple(func_spec)) {
 	goto error;
     }
@@ -1034,12 +1074,29 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
     mfa[1] = tp[2];
     mfa[2] = signed_val(tp[3]);
 
-    r = function_is_traced(mfa, &ms, &ms_meta, &meta, &count);
+#ifdef ERTS_SMP
+    if ( (key == am_call_time) || (key == am_all)) {
+	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
+	erts_smp_thr_progress_block();
+    }
+#endif
+
+    r = function_is_traced(p, mfa, &ms, &ms_meta, &meta, &count, &call_time);
+
+#ifdef ERTS_SMP
+    if ( (key == am_call_time) || (key == am_all)) {
+	erts_smp_thr_progress_unblock();
+	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
+    }
+#endif
+
     switch (r) {
     case FUNC_TRACE_NOEXIST:
+	UnUseTmpHeap(3,p);
 	hp = HAlloc(p, 3);
 	return TUPLE2(hp, key, am_undefined);
     case FUNC_TRACE_UNTRACED:
+	UnUseTmpHeap(3,p);
 	hp = HAlloc(p, 3);
 	return TUPLE2(hp, key, am_false);
     case FUNC_TRACE_GLOBAL_TRACE:
@@ -1085,8 +1142,13 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
 		erts_make_integer(count, p);
 	}
 	break;
+    case am_call_time:
+	if (r & FUNC_TRACE_TIME_TRACE) {
+	    retval = call_time;
+	}
+	break;
     case am_all: {
-	Eterm match_spec_meta = am_false, c = am_false, t;
+	Eterm match_spec_meta = am_false, c = am_false, t, ct = am_false;
 	
 	if (ms) {
 	    match_spec = MatchSetGetSource(ms);
@@ -1104,10 +1166,15 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
 		erts_make_integer(-count-1, p) : 
 		erts_make_integer(count, p);
 	}
-	hp = HAlloc(p, (3+2)*5);
+	if (r & FUNC_TRACE_TIME_TRACE) {
+	    ct = call_time;
+	}
+	hp = HAlloc(p, (3+2)*6);
 	retval = NIL;
 	t = TUPLE2(hp, am_call_count, c); hp += 3;
 	retval = CONS(hp, t, retval); hp += 2;
+	t = TUPLE2(hp, am_call_time, ct); hp += 3;
+	retval = CONS(hp, t, retval); hp += 2;
 	t = TUPLE2(hp, am_meta_match_spec, match_spec_meta); hp += 3;
 	retval = CONS(hp, t, retval); hp += 2;
 	t = TUPLE2(hp, am_meta, meta); hp += 3;
@@ -1120,10 +1187,12 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
     default:
 	goto error;
     }
+    UnUseTmpHeap(3,p);
     hp = HAlloc(p, 3);
     return TUPLE2(hp, key, retval);
 
  error:
+    UnUseTmpHeap(3,p);
     BIF_ERROR(p, BADARG);
 }
 
@@ -1201,6 +1270,13 @@ trace_info_on_load(Process* p, Eterm key)
 	} else {
 	    return TUPLE2(hp, key, am_false);
 	}
+    case am_call_time:
+	hp = HAlloc(p, 3);
+	if (erts_default_trace_pattern_flags.call_time) {
+	    return TUPLE2(hp, key, am_true);
+	} else {
+	    return TUPLE2(hp, key, am_false);
+	}
     case am_all:
 	{
 	    Eterm match_spec = am_false, meta_match_spec = am_false, r = NIL, t;
@@ -1275,6 +1351,7 @@ erts_set_trace_pattern(Eterm* mfa, int specified,
 	for (j = 0; j < specified && mfa[j] == ep->code[j]; j++) {
 	    /* Empty loop body */
 	}
+
 	if (j == specified) {
 	    if (on) {
 		if (! flags.breakpoint)
@@ -1312,7 +1389,7 @@ erts_set_trace_pattern(Eterm* mfa, int specified,
 		    if (erts_bif_trace_flags[i] & BIF_TRACE_AS_META) {
 			ASSERT(ExportIsBuiltIn(bif_export[i]));
 			erts_clear_mtrace_bif
-			    ((Uint *)bif_export[i]->code + 3);
+			    ((BeamInstr *)bif_export[i]->code + 3);
 			erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_META;
 		    }
 		    set_trace_bif(i, match_prog_set);
@@ -1341,12 +1418,18 @@ erts_set_trace_pattern(Eterm* mfa, int specified,
 		    }
 		    if (flags.meta) {
 			erts_set_mtrace_bif
-			    ((Uint *)bif_export[i]->code + 3,
+			    ((BeamInstr *)bif_export[i]->code + 3,
 			     meta_match_prog_set, meta_tracer_pid);
 			erts_bif_trace_flags[i] |= BIF_TRACE_AS_META;
 			erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_GLOBAL;
 			m = 1;
 		    }
+		    if (flags.call_time) {
+			erts_set_time_trace_bif(bif_export[i]->code + 3, on);
+			/* I don't want to remove any other tracers */
+			erts_bif_trace_flags[i] |= BIF_TRACE_AS_CALL_TIME;
+			m = 1;
+		    }
 		    if (erts_bif_trace_flags[i]) {
 			setup_bif_trace(i);
 		    }
@@ -1361,11 +1444,16 @@ erts_set_trace_pattern(Eterm* mfa, int specified,
 		    if (flags.meta) {
 			if (erts_bif_trace_flags[i] & BIF_TRACE_AS_META) {
 			    erts_clear_mtrace_bif
-				((Uint *)bif_export[i]->code + 3);
+				((BeamInstr *)bif_export[i]->code + 3);
 			    erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_META;
 			}
 			m = 1;
 		    }
+		    if (flags.call_time) {
+			erts_clear_time_trace_bif(bif_export[i]->code + 3);
+			erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_CALL_TIME;
+			m = 1;
+		    }
 		    if (! erts_bif_trace_flags[i]) {
 			reset_bif_trace(i);
 		    }
@@ -1383,6 +1471,7 @@ erts_set_trace_pattern(Eterm* mfa, int specified,
 	    erts_clear_trace_break(mfa, specified);
 	    erts_clear_mtrace_break(mfa, specified);
 	    erts_clear_count_break(mfa, specified);
+	    erts_clear_time_break(mfa, specified);
 	} else {
 	    int m = 0;
 	    if (flags.local) {
@@ -1396,6 +1485,9 @@ erts_set_trace_pattern(Eterm* mfa, int specified,
 	    if (flags.call_count) {
 		m = erts_set_count_break(mfa, specified, on);
 	    }
+	    if (flags.call_time) {
+		m = erts_set_time_break(mfa, specified, on);
+	    }
 	    /* All assignments to 'm' above should give the same value,
 	     * so just use the last */
 	    matches += m;
@@ -1411,6 +1503,9 @@ erts_set_trace_pattern(Eterm* mfa, int specified,
 	if (flags.call_count) {
 	    m = erts_clear_count_break(mfa, specified);
 	}
+	if (flags.call_time) {
+	    m = erts_clear_time_break(mfa, specified);
+	}
 	/* All assignments to 'm' above should give the same value,
 	 * so just use the last */
 	matches += m;
@@ -1430,9 +1525,9 @@ static int
 setup_func_trace(Export* ep, void* match_prog)
 {
     if (ep->address == ep->code+3) {
-	if (ep->code[3] == (Uint) em_call_error_handler) {
+	if (ep->code[3] == (BeamInstr) em_call_error_handler) {
 	    return 0;
-	} else if (ep->code[3] == (Uint) em_call_traced_function) {
+	} else if (ep->code[3] == (BeamInstr) em_call_traced_function) {
 	    MatchSetUnref(ep->match_prog_set);
 	    ep->match_prog_set = match_prog;
 	    MatchSetRef(ep->match_prog_set);
@@ -1452,8 +1547,8 @@ setup_func_trace(Export* ep, void* match_prog)
 	return 0;
     }
     
-    ep->code[3] = (Uint) em_call_traced_function;
-    ep->code[4] = (Uint) ep->address;
+    ep->code[3] = (BeamInstr) em_call_traced_function;
+    ep->code[4] = (BeamInstr) ep->address;
     ep->address = ep->code+3;
     ep->match_prog_set = match_prog;
     MatchSetRef(ep->match_prog_set);
@@ -1465,7 +1560,7 @@ static void setup_bif_trace(int bif_index) {
     
     ASSERT(ExportIsBuiltIn(ep));
     ASSERT(ep->code[4]);
-    ep->code[4] = (Uint) bif_table[bif_index].traced;
+    ep->code[4] = (BeamInstr) bif_table[bif_index].traced;
 }
 
 static void set_trace_bif(int bif_index, void* match_prog) {
@@ -1492,9 +1587,9 @@ static int
 reset_func_trace(Export* ep)
 {
     if (ep->address == ep->code+3) {
-	if (ep->code[3] == (Uint) em_call_error_handler) {
+	if (ep->code[3] == (BeamInstr) em_call_error_handler) {
 	    return 0;
-	} else if (ep->code[3] == (Uint) em_call_traced_function) {
+	} else if (ep->code[3] == (BeamInstr) em_call_traced_function) {
 	    ep->address = (Uint *) ep->code[4];
 	    MatchSetUnref(ep->match_prog_set);
 	    ep->match_prog_set = NULL;
@@ -1527,8 +1622,8 @@ static void reset_bif_trace(int bif_index) {
     ASSERT(ExportIsBuiltIn(ep));
     ASSERT(ep->code[4]);
     ASSERT(! ep->match_prog_set);
-    ASSERT(! erts_is_mtrace_bif((Uint *)ep->code+3, NULL, NULL));
-    ep->code[4] = (Uint) bif_table[bif_index].f;
+    ASSERT(! erts_is_mtrace_break((BeamInstr *)ep->code+3, NULL, NULL));
+    ep->code[4] = (BeamInstr) bif_table[bif_index].f;
 }
 
 static void clear_trace_bif(int bif_index) {
@@ -1649,9 +1744,17 @@ Eterm erts_seq_trace(Process *p, Eterm arg1, Eterm arg2,
 	    return THE_NON_VALUE;
         }
 	if (build_result) {
+#ifdef USE_VM_PROBES
+	    old_value = (SEQ_TRACE_TOKEN(p) == am_have_dt_utag) ? NIL : SEQ_TRACE_TOKEN(p);
+#else
 	    old_value = SEQ_TRACE_TOKEN(p);
+#endif
 	}
+#ifdef USE_VM_PROBES
+        SEQ_TRACE_TOKEN(p) = (DT_UTAG(p) != NIL) ? am_have_dt_utag : NIL;
+#else
         SEQ_TRACE_TOKEN(p) = NIL;
+#endif
         return old_value;
     }
     else {
@@ -1664,7 +1767,11 @@ new_seq_trace_token(Process* p)
 {
     Eterm* hp;
 
-    if (SEQ_TRACE_TOKEN(p) == NIL) {
+    if (SEQ_TRACE_TOKEN(p) == NIL
+#ifdef USE_VM_PROBES
+	|| SEQ_TRACE_TOKEN(p) == am_have_dt_utag
+#endif
+	) {
 	hp = HAlloc(p, 6);
 	SEQ_TRACE_TOKEN(p) = TUPLE5(hp, make_small(0),		/* Flags  */ 
 				    make_small(0),		/* Label  */
@@ -1674,23 +1781,24 @@ new_seq_trace_token(Process* p)
     }
 }
 
-BIF_RETTYPE seq_trace_info_1(BIF_ALIST_1)
+BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item)
 {
-    Eterm item;
     Eterm res;
     Eterm* hp;
     Uint current_flag;
 
-    if (is_not_atom(BIF_ARG_1)) {
-	BIF_ERROR(BIF_P, BADARG);
+    if (is_not_atom(item)) {
+	BIF_ERROR(p, BADARG);
     }
 
-    item = BIF_ARG_1;
-
-    if (SEQ_TRACE_TOKEN(BIF_P) == NIL) {
+    if (SEQ_TRACE_TOKEN(p) == NIL
+#ifdef USE_VM_PROBES
+	|| SEQ_TRACE_TOKEN(p) == am_have_dt_utag
+#endif
+	) {
 	if ((item == am_send) || (item == am_receive) || 
 	    (item == am_print) || (item == am_timestamp)) {
-	    hp = HAlloc(BIF_P,3);
+	    hp = HAlloc(p,3);
 	    res = TUPLE2(hp, item, am_false);
 	    BIF_RET(res);
 	} else if ((item == am_label) || (item == am_serial)) {
@@ -1700,35 +1808,40 @@ BIF_RETTYPE seq_trace_info_1(BIF_ALIST_1)
 	}
     }
 
-    if (BIF_ARG_1 == am_send) {
+    if (item == am_send) {
 	current_flag = SEQ_TRACE_SEND;
-    } else if (BIF_ARG_1 == am_receive) {
+    } else if (item == am_receive) {
 	current_flag = SEQ_TRACE_RECEIVE; 
-    } else if (BIF_ARG_1 == am_print) {
+    } else if (item == am_print) {
 	current_flag = SEQ_TRACE_PRINT; 
-    } else if (BIF_ARG_1 == am_timestamp) {
+    } else if (item == am_timestamp) {
 	current_flag = SEQ_TRACE_TIMESTAMP; 
     } else {
 	current_flag = 0;
     }
 
     if (current_flag) {
-	res = unsigned_val(SEQ_TRACE_TOKEN_FLAGS(BIF_P)) & current_flag ? 
+	res = unsigned_val(SEQ_TRACE_TOKEN_FLAGS(p)) & current_flag ?
 	    am_true : am_false;
     } else if (item == am_label) {
-	res = SEQ_TRACE_TOKEN_LABEL(BIF_P);
+	res = SEQ_TRACE_TOKEN_LABEL(p);
     } else if (item  == am_serial) {
-	hp = HAlloc(BIF_P, 3);
-	res = TUPLE2(hp, SEQ_TRACE_TOKEN_LASTCNT(BIF_P), SEQ_TRACE_TOKEN_SERIAL(BIF_P));
+	hp = HAlloc(p, 3);
+	res = TUPLE2(hp, SEQ_TRACE_TOKEN_LASTCNT(p), SEQ_TRACE_TOKEN_SERIAL(p));
     } else {
     error:
-	BIF_ERROR(BIF_P, BADARG);
+	BIF_ERROR(p, BADARG);
     }
-    hp = HAlloc(BIF_P, 3);
+    hp = HAlloc(p, 3);
     res = TUPLE2(hp, item, res);
     BIF_RET(res);
 }
 
+BIF_RETTYPE seq_trace_info_1(BIF_ALIST_1)
+{
+    BIF_RET(erl_seq_trace_info(BIF_P, BIF_ARG_1));
+}
+
 /*
    seq_trace_print(Message) -> true | false
    This function passes Message to the system_tracer
@@ -1739,8 +1852,13 @@ BIF_RETTYPE seq_trace_info_1(BIF_ALIST_1)
  */
 BIF_RETTYPE seq_trace_print_1(BIF_ALIST_1)    
 {
-    if (SEQ_TRACE_TOKEN(BIF_P) == NIL) 
+    if (SEQ_TRACE_TOKEN(BIF_P) == NIL 
+#ifdef USE_VM_PROBES
+	|| SEQ_TRACE_TOKEN(BIF_P) == am_have_dt_utag
+#endif
+	) {
 	BIF_RET(am_false);
+    }
     seq_trace_update_send(BIF_P);
     seq_trace_output(SEQ_TRACE_TOKEN(BIF_P), BIF_ARG_1, 
 		     SEQ_TRACE_PRINT, NIL, BIF_P);
@@ -1757,8 +1875,13 @@ BIF_RETTYPE seq_trace_print_1(BIF_ALIST_1)
  */
 BIF_RETTYPE seq_trace_print_2(BIF_ALIST_2)    
 {
-    if (SEQ_TRACE_TOKEN(BIF_P) == NIL) 
+    if (SEQ_TRACE_TOKEN(BIF_P) == NIL
+#ifdef USE_VM_PROBES
+	|| SEQ_TRACE_TOKEN(BIF_P) == am_have_dt_utag
+#endif
+	) {
 	BIF_RET(am_false);
+    }
     if (!(is_atom(BIF_ARG_1) || is_small(BIF_ARG_1))) {
 	BIF_ERROR(BIF_P, BADARG);
     }
@@ -1774,7 +1897,7 @@ void erts_system_monitor_clear(Process *c_p) {
 #ifdef ERTS_SMP
     if (c_p) {
 	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
-	erts_smp_block_system(0);
+	erts_smp_thr_progress_block();
     }
 #endif
     erts_set_system_monitor(NIL);
@@ -1784,7 +1907,7 @@ void erts_system_monitor_clear(Process *c_p) {
     erts_system_monitor_flags.busy_dist_port = 0;
 #ifdef ERTS_SMP
     if (c_p) {
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
     }
 #endif
@@ -1841,23 +1964,35 @@ static Eterm system_monitor_get(Process *p)
 }
 
 
-BIF_RETTYPE system_monitor_0(Process *p) {
-    BIF_RET(system_monitor_get(p));
+BIF_RETTYPE system_monitor_0(BIF_ALIST_0)
+{
+    BIF_RET(system_monitor_get(BIF_P));
 }
 
-BIF_RETTYPE system_monitor_1(Process *p, Eterm spec) {
+BIF_RETTYPE system_monitor_1(BIF_ALIST_1)
+{
+    Process* p = BIF_P;
+    Eterm spec = BIF_ARG_1;
+
     if (spec == am_undefined) {
-	BIF_RET(system_monitor_2(p, spec, NIL));
+	BIF_RET(system_monitor(p, spec, NIL));
     } else if (is_tuple(spec)) {
 	Eterm *tp = tuple_val(spec);
 	if (tp[0] != make_arityval(2)) goto error;
-	BIF_RET(system_monitor_2(p, tp[1], tp[2]));
+	BIF_RET(system_monitor(p, tp[1], tp[2]));
     }
  error:
     BIF_ERROR(p, BADARG);
 }
 
-BIF_RETTYPE system_monitor_2(Process *p, Eterm monitor_pid, Eterm list) {
+BIF_RETTYPE system_monitor_2(BIF_ALIST_2)
+{
+    return system_monitor(BIF_P, BIF_ARG_1, BIF_ARG_2);
+}
+
+static BIF_RETTYPE
+system_monitor(Process *p, Eterm monitor_pid, Eterm list)
+{
     Eterm prev;
     int system_blocked = 0;
 
@@ -1873,7 +2008,7 @@ BIF_RETTYPE system_monitor_2(Process *p, Eterm monitor_pid, Eterm list) {
 
 	system_blocked = 1;
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
-	erts_smp_block_system(0);
+	erts_smp_thr_progress_block();
 
 	if (!erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, monitor_pid, 0))
 	    goto error;
@@ -1907,7 +2042,7 @@ BIF_RETTYPE system_monitor_2(Process *p, Eterm monitor_pid, Eterm list) {
 	erts_system_monitor_flags.busy_port = !!busy_port;
 	erts_system_monitor_flags.busy_dist_port = !!busy_dist_port;
 
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
 	BIF_RET(prev);
     }
@@ -1915,7 +2050,7 @@ BIF_RETTYPE system_monitor_2(Process *p, Eterm monitor_pid, Eterm list) {
  error:
 
     if (system_blocked) {
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
     }
 
@@ -1928,7 +2063,7 @@ void erts_system_profile_clear(Process *c_p) {
 #ifdef ERTS_SMP
     if (c_p) {
 	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
-	erts_smp_block_system(0);
+	erts_smp_thr_progress_block();
     }
 #endif
     erts_set_system_profile(NIL);
@@ -1938,7 +2073,7 @@ void erts_system_profile_clear(Process *c_p) {
     erts_system_profile_flags.exclusive = 0;
 #ifdef ERTS_SMP
     if (c_p) {
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
     }
 #endif
@@ -1975,11 +2110,16 @@ static Eterm system_profile_get(Process *p) {
     }
 }
 
-BIF_RETTYPE system_profile_0(Process *p) {
-    BIF_RET(system_profile_get(p));
+BIF_RETTYPE system_profile_0(BIF_ALIST_0)
+{
+    BIF_RET(system_profile_get(BIF_P));
 }
 
-BIF_RETTYPE system_profile_2(Process *p, Eterm profiler, Eterm list) {
+BIF_RETTYPE system_profile_2(BIF_ALIST_2)
+{
+    Process *p = BIF_P;
+    Eterm profiler = BIF_ARG_1;
+    Eterm list = BIF_ARG_2;
     Eterm prev;
     int system_blocked = 0;
     Process *profiler_p = NULL;
@@ -1997,7 +2137,7 @@ BIF_RETTYPE system_profile_2(Process *p, Eterm profiler, Eterm list) {
 	system_blocked = 1;
 	
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
-	erts_smp_block_system(0);
+	erts_smp_thr_progress_block();
 
 	/* Check if valid process, no locks are taken */
 
@@ -2039,7 +2179,7 @@ BIF_RETTYPE system_profile_2(Process *p, Eterm profiler, Eterm list) {
 	erts_system_profile_flags.runnable_procs = !!runnable_procs;
 	erts_system_profile_flags.exclusive = !!exclusive;
 
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
 	
 	BIF_RET(prev);
@@ -2048,7 +2188,7 @@ BIF_RETTYPE system_profile_2(Process *p, Eterm profiler, Eterm list) {
 
     error:
 	if (system_blocked) {
-	    erts_smp_release_system();
+	    erts_smp_thr_progress_unblock();
 	    erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
     	}
 
@@ -2083,7 +2223,7 @@ trace_delivered_1(BIF_ALIST_1)
 #ifdef ERTS_SMP
     bp = new_message_buffer(REF_THING_SIZE + 4);
     hp = &bp->mem[0];
-    msg_ref = STORE_NC(&hp, &bp->off_heap.externals, ref);
+    msg_ref = STORE_NC(&hp, &bp->off_heap, ref);
 #else
     hp = HAlloc(BIF_P, 4);
     msg_ref = ref;
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 1f948a9684..506c4813fa 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -21,6 +21,7 @@
 #define __ERL_BINARY_H
 
 #include "erl_threads.h"
+#include "bif.h"
 
 /*
  * Maximum number of bytes to place in a heap binary.
@@ -70,6 +71,7 @@ typedef struct erl_heap_bin {
  */
 
 #define binary_size(Bin) (binary_val(Bin)[1])
+#define binary_size_rel(Bin,BasePtr) (binary_val_rel(Bin,BasePtr)[1])
 
 #define binary_bitsize(Bin)			\
   ((*binary_val(Bin) == HEADER_SUB_BIN) ?	\
@@ -92,9 +94,12 @@ typedef struct erl_heap_bin {
  * Bitsize: output variable (Uint)
  */
 
-#define ERTS_GET_BINARY_BYTES(Bin,Bytep,Bitoffs,Bitsize)		\
+#define ERTS_GET_BINARY_BYTES(Bin,Bytep,Bitoffs,Bitsize) \
+     ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,NULL)
+
+#define ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,BasePtr)    \
 do {									\
-    Eterm* _real_bin = binary_val(Bin);					\
+    Eterm* _real_bin = binary_val_rel(Bin,BasePtr);			\
     Uint _offs = 0;							\
     Bitoffs = Bitsize = 0;						\
     if (*_real_bin == HEADER_SUB_BIN) {					\
@@ -102,7 +107,7 @@ do {									\
 	_offs = _sb->offs;						\
         Bitoffs = _sb->bitoffs;						\
         Bitsize = _sb->bitsize;						\
-	_real_bin = binary_val(_sb->orig);				\
+	_real_bin = binary_val_rel(_sb->orig,BasePtr);			\
     }									\
     if (*_real_bin == HEADER_PROC_BIN) {				\
 	Bytep = ((ProcBin *) _real_bin)->bytes + _offs;			\
@@ -124,9 +129,12 @@ do {									\
  * BitSize: Extra bit size (Uint)
  */
 
-#define ERTS_GET_REAL_BIN(Bin, RealBin, ByteOffset, BitOffset, BitSize)	\
+#define ERTS_GET_REAL_BIN(Bin, RealBin, ByteOffset, BitOffset, BitSize) \
+     ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, NULL)
+
+#define ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, BasePtr) \
   do {									\
-    ErlSubBin* _sb = (ErlSubBin *) binary_val(Bin);			\
+    ErlSubBin* _sb = (ErlSubBin *) binary_val_rel(Bin,BasePtr);	        \
     if (_sb->thing_word == HEADER_SUB_BIN) {				\
       RealBin = _sb->orig;						\
       ByteOffset = _sb->offs;						\
@@ -150,7 +158,18 @@ do {									\
 
 void erts_init_binary(void);
 
-byte* erts_get_aligned_binary_bytes_extra(Eterm, byte**, unsigned extra);
+byte* erts_get_aligned_binary_bytes_extra(Eterm, byte**, ErtsAlcType_t, unsigned extra);
+/* Used by unicode module */
+Eterm erts_bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs);
+
+/*
+ * Common implementation for erlang:list_to_binary/1 and binary:list_to_bin/1
+ */
+
+BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg);
+BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is_tuple);
+BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen);
+
 
 #if defined(__i386__) || !defined(__GNUC__)
 /*
@@ -164,10 +183,11 @@ byte* erts_get_aligned_binary_bytes_extra(Eterm, byte**, unsigned extra);
 #endif
 
 #define ERTS_CHK_BIN_ALIGNMENT(B) \
-  do { ASSERT(!(B) || (((Uint) &((Binary *)(B))->orig_bytes[0]) & ERTS_BIN_ALIGNMENT_MASK) == ((Uint) 0)) } while(0)
+  do { ASSERT(!(B) || (((UWord) &((Binary *)(B))->orig_bytes[0]) & ERTS_BIN_ALIGNMENT_MASK) == ((UWord) 0)) } while(0)
 
 ERTS_GLB_INLINE byte* erts_get_aligned_binary_bytes(Eterm bin, byte** base_ptr);
 ERTS_GLB_INLINE void erts_free_aligned_binary_bytes(byte* buf);
+ERTS_GLB_INLINE void erts_free_aligned_binary_bytes_extra(byte* buf, ErtsAlcType_t);
 ERTS_GLB_INLINE Binary *erts_bin_drv_alloc_fnf(Uint size);
 ERTS_GLB_INLINE Binary *erts_bin_drv_alloc(Uint size);
 ERTS_GLB_INLINE Binary *erts_bin_nrml_alloc(Uint size);
@@ -184,17 +204,23 @@ ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size,
 ERTS_GLB_INLINE byte*
 erts_get_aligned_binary_bytes(Eterm bin, byte** base_ptr)
 {
-    return erts_get_aligned_binary_bytes_extra(bin, base_ptr, 0);
+    return erts_get_aligned_binary_bytes_extra(bin, base_ptr, ERTS_ALC_T_TMP, 0);
 }
 
 ERTS_GLB_INLINE void
-erts_free_aligned_binary_bytes(byte* buf)
+erts_free_aligned_binary_bytes_extra(byte* buf, ErtsAlcType_t allocator)
 {
     if (buf) {
-	erts_free(ERTS_ALC_T_TMP, (void *) buf);
+	erts_free(allocator, (void *) buf);
     }
 }
 
+ERTS_GLB_INLINE void
+erts_free_aligned_binary_bytes(byte* buf)
+{
+    erts_free_aligned_binary_bytes_extra(buf,ERTS_ALC_T_TMP);
+}
+
 /* Explicit extra bytes allocated to counter buggy drivers.
 ** These extra bytes where earlier (< R13B04) added by an alignment-bug
 ** in this code. Do we dare remove this in some major release (R14?) maybe?
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index e4f5d50ddf..6f7309f493 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1999-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1999-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -76,14 +76,12 @@ struct erl_bits_state ErlBitsState;
 #define byte_buf	(ErlBitsState.byte_buf_)
 #define byte_buf_len	(ErlBitsState.byte_buf_len_)
 
-#ifdef ERTS_SMP
 static erts_smp_atomic_t bits_bufs_size;
-#endif
 
 Uint
 erts_bits_bufs_size(void)
 {
-    return 0;
+    return (Uint) erts_smp_atomic_read_nob(&bits_bufs_size);
 }
 
 #if !defined(ERTS_SMP)
@@ -109,8 +107,8 @@ erts_bits_destroy_state(ERL_BITS_PROTO_0)
 void
 erts_init_bits(void)
 {
+    erts_smp_atomic_init_nob(&bits_bufs_size, 0);
 #if defined(ERTS_SMP)
-    erts_smp_atomic_init(&bits_bufs_size, 0);
     /* erl_process.c calls erts_bits_init_state() on all state instances */
 #else
     ERL_BITS_DECLARE_STATEP;
@@ -177,7 +175,6 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff
     byte* LSB;
     byte* MSB;
     Uint* hp;
-    Uint* hp_end;
     Uint words_needed;
     Uint actual;
     Uint v32;
@@ -255,7 +252,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff
 	 * Simply shift whole bytes into the result.
 	 */
 	switch (BYTE_OFFSET(n)) {
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 	case 7: w = (w << 8) | *bp++;
 	case 6: w = (w << 8) | *bp++;
 	case 5: w = (w << 8) | *bp++;
@@ -360,7 +357,7 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff
     case 3: 
 	v32 = LSB[0] + (LSB[1]<<8) + (LSB[2]<<16); 
 	goto big_small;
-#if !defined(ARCH_64)
+#if !defined(ARCH_64) || HALFWORD_HEAP
     case 4:
 	v32 = (LSB[0] + (LSB[1]<<8) + (LSB[2]<<16) + (LSB[3]<<24));
 	if (!IS_USMALL(sgn, v32)) {
@@ -405,7 +402,6 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff
     default:
 	words_needed = 1+WSIZE(bytes);
 	hp = HeapOnlyAlloc(p, words_needed);
-	hp_end = hp + words_needed;
 	res = bytes_to_big(LSB, bytes, sgn, hp); 
 	if (is_small(res)) {
 	    p->htop = hp;
@@ -425,7 +421,6 @@ Eterm
 erts_bs_get_binary_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb)
 {
     ErlSubBin* sb;
-    size_t num_bytes;		/* Number of bytes in binary. */
 
     if (mb->size - mb->offset < num_bits) {	/* Asked for too many bits.  */
 	return THE_NON_VALUE;
@@ -435,7 +430,6 @@ erts_bs_get_binary_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffe
      * From now on, we can't fail.
      */
 
-    num_bytes = NBYTES(num_bits);
     sb = (ErlSubBin *) HeapOnlyAlloc(p, ERL_SUB_BIN_SIZE);
     
     sb->thing_word = HEADER_SUB_BIN;
@@ -555,10 +549,11 @@ fmt_int(byte *buf, Uint sz, Eterm val, Uint size, Uint flags)
 {
     unsigned long offs;
 
-    ASSERT(size != 0);
     offs = BIT_OFFSET(size);
     if (is_small(val)) {
 	Sint v = signed_val(val);
+
+	ASSERT(size != 0);	  /* Tested by caller */
 	if (flags & BSF_LITTLE) { /* Little endian */
 	    sz--;
 	    COPY_VAL(buf,1,v,sz);
@@ -578,6 +573,9 @@ fmt_int(byte *buf, Uint sz, Eterm val, Uint size, Uint flags)
 	ErtsDigit* dp = big_v(val);
 	int n = MIN(sz,ds);
 
+	if (size == 0) {
+	    return 0;
+	}
 	if (flags & BSF_LITTLE) {
 	    sz -= n;                       /* pad with this amount */
 	    if (sign) {
@@ -713,9 +711,7 @@ static void
 ERTS_INLINE need_byte_buf(ERL_BITS_PROTO_1(int need))
 {
     if (byte_buf_len < need) {
-#ifdef ERTS_SMP
-	erts_smp_atomic_add(&bits_bufs_size, need - byte_buf_len);
-#endif
+	erts_smp_atomic_add_nob(&bits_bufs_size, need - byte_buf_len);
 	byte_buf_len = need;
 	byte_buf = erts_realloc(ERTS_ALC_T_BITS_BUF, byte_buf, byte_buf_len);
     }
@@ -729,15 +725,13 @@ erts_new_bs_put_integer(ERL_BITS_PROTO_3(Eterm arg, Uint num_bits, unsigned flag
     Uint b;
     byte *iptr;
 
-    if (num_bits == 0) {
-	return 1;
-    }
-
     bit_offset = BIT_OFFSET(bin_offset);
     if (is_small(arg)) {
 	Uint rbits = 8 - bit_offset;
 
-	if (bit_offset + num_bits <= 8) {
+	if (num_bits == 0) {
+	    return 1;
+	} else if (bit_offset + num_bits <= 8) {
 	    /*
 	     * All bits are in the same byte.
 	     */ 
@@ -851,8 +845,7 @@ erts_bs_put_utf8(ERL_BITS_PROTO_1(Eterm arg))
 	dst[1] = 0x80 | (val & 0x3F);
 	num_bits = 16;
     } else if (val < 0x10000UL) {
-	if ((0xD800 <= val && val <= 0xDFFF) ||
-	    val == 0xFFFE || val == 0xFFFF) {
+	if (0xD800 <= val && val <= 0xDFFF) {
 	    return 0;
 	}
 	dst[0] = 0xE0 | (val >> 12);
@@ -892,8 +885,7 @@ erts_bs_put_utf16(ERL_BITS_PROTO_2(Eterm arg, Uint flags))
 	return 0;
     }
     val = unsigned_val(arg);
-    if (val > 0x10FFFF || (0xD800 <= val && val <= 0xDFFF) ||
-	val == 0xFFFE || val == 0xFFFF) {
+    if (val > 0x10FFFF || (0xD800 <= val && val <= 0xDFFF)) {
 	return 0;
     }
 
@@ -1335,12 +1327,12 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term,
 	hp += PROC_BIN_SIZE;
 	pb->thing_word = HEADER_PROC_BIN;
 	pb->size = used_size_in_bytes;
-	pb->next = MSO(c_p).mso;
-	MSO(c_p).mso = pb;
+	pb->next = MSO(c_p).first;
+	MSO(c_p).first = (struct erl_off_heap_header*)pb;
 	pb->val = bptr;
 	pb->bytes = (byte*) bptr->orig_bytes;
 	pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER;
-	MSO(c_p).overhead += pb->size / sizeof(Eterm);
+	OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm));
 
 	/*
 	 * Now allocate the sub binary and set its size to include the
@@ -1506,12 +1498,12 @@ erts_bs_init_writable(Process* p, Eterm sz)
     hp += PROC_BIN_SIZE;
     pb->thing_word = HEADER_PROC_BIN;
     pb->size = 0;
-    pb->next = MSO(p).mso;
-    MSO(p).mso = pb;
+    pb->next = MSO(p).first;
+    MSO(p).first = (struct erl_off_heap_header*) pb;
     pb->val = bptr;
     pb->bytes = (byte*) bptr->orig_bytes;
     pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER;
-    MSO(p).overhead += pb->size / sizeof(Eterm);
+    OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
     
     /*
      * Now allocate the sub binary.
@@ -1555,7 +1547,6 @@ Uint32
 erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb)
 {
     Uint bytes;
-    Uint bits;
     Uint offs;
     byte bigbuf[4];
     byte* LSB;
@@ -1565,7 +1556,6 @@ erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb)
     ASSERT(mb->size - mb->offset >= 32);
 
     bytes = 4;
-    bits = 8;
     offs = 0;
 
     LSB = bigbuf;
@@ -1660,8 +1650,7 @@ erts_bs_get_utf8(ErlBinMatchBuffer* mb)
 	    return THE_NON_VALUE;
 	}
 	result = (((result << 6) + a) << 6) + b - (Eterm) 0x000E2080UL;
-	if ((0xD800 <= result && result <= 0xDFFF) ||
-	    result == 0xFFFE || result == 0xFFFF) {
+	if (0xD800 <= result && result <= 0xDFFF) {
 	    return THE_NON_VALUE;
 	}
 	mb->offset += 24;
@@ -1731,9 +1720,6 @@ erts_bs_get_utf16(ErlBinMatchBuffer* mb, Uint flags)
 	w1 = (src[0] << 8) | src[1];
     }
     if (w1 < 0xD800 || w1 > 0xDFFF) {
-	if (w1 == 0xFFFE || w1 == 0xFFFF) {
-	    return THE_NON_VALUE;
-	}
 	mb->offset += 16;
 	return make_small(w1);
     } else if (w1 > 0xDBFF) {
diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h
index e3f8e0b679..388d943755 100644
--- a/erts/emulator/beam/erl_bits.h
+++ b/erts/emulator/beam/erl_bits.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1999-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1999-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -63,7 +63,7 @@ typedef struct erl_bin_match_struct{
 #define HEADER_NUM_SLOTS(hdr) (header_arity(hdr)-sizeof(ErlBinMatchState)/sizeof(Eterm)+1)
 
 #define make_matchstate(_Ms) make_boxed((Eterm*)(_Ms))  
-#define ms_matchbuffer(_Ms) &(((ErlBinMatchState*)(_Ms - TAG_PRIMARY_BOXED))->mb)
+#define ms_matchbuffer(_Ms) &(((ErlBinMatchState*) boxed_val(_Ms))->mb)
 
 
 #if defined(ERTS_SMP)
@@ -150,7 +150,7 @@ void erts_bits_destroy_state(ERL_BITS_PROTO_0);
  * NBYTES(x) returns the number of bytes needed to store x bits.
  */
 
-#define NBYTES(x)  (((x) + 7) >> 3) 
+#define NBYTES(x)  (((Uint64)(x) + (Uint64) 7) >> 3) 
 #define BYTE_OFFSET(ofs) ((Uint) (ofs) >> 3)
 #define BIT_OFFSET(ofs) ((ofs) & 7)
 
diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c
new file mode 100644
index 0000000000..fe3693d0ca
--- /dev/null
+++ b/erts/emulator/beam/erl_cpu_topology.c
@@ -0,0 +1,2346 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description:	CPU topology and related functionality
+ *
+ * Author: 	Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include <ctype.h>
+
+#include "global.h"
+#include "error.h"
+#include "bif.h"
+#include "erl_cpu_topology.h"
+
+#define ERTS_MAX_READER_GROUPS 8
+
+/*
+ * Cpu topology hierarchy.
+ */
+#define ERTS_TOPOLOGY_NODE		0
+#define ERTS_TOPOLOGY_PROCESSOR		1
+#define ERTS_TOPOLOGY_PROCESSOR_NODE	2
+#define ERTS_TOPOLOGY_CORE		3
+#define ERTS_TOPOLOGY_THREAD		4
+#define ERTS_TOPOLOGY_LOGICAL		5
+
+#define ERTS_TOPOLOGY_MAX_DEPTH		6
+
+typedef struct {
+    int bind_id;
+    int bound_id;
+} ErtsCpuBindData;
+
+static erts_cpu_info_t *cpuinfo;
+
+static int max_main_threads;
+static int reader_groups;
+
+static ErtsCpuBindData *scheduler2cpu_map;
+static erts_smp_rwmtx_t cpuinfo_rwmtx;
+
+typedef enum {
+    ERTS_CPU_BIND_UNDEFINED,
+    ERTS_CPU_BIND_SPREAD,
+    ERTS_CPU_BIND_PROCESSOR_SPREAD,
+    ERTS_CPU_BIND_THREAD_SPREAD,
+    ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD,
+    ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD,
+    ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD,
+    ERTS_CPU_BIND_NO_SPREAD,
+    ERTS_CPU_BIND_NONE
+} ErtsCpuBindOrder;
+
+#define ERTS_CPU_BIND_DEFAULT_BIND \
+  ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD
+
+static int no_cpu_groups_callbacks;
+static ErtsCpuBindOrder cpu_bind_order;
+
+static erts_cpu_topology_t *user_cpudata;
+static int user_cpudata_size;
+static erts_cpu_topology_t *system_cpudata;
+static int system_cpudata_size;
+
+typedef struct {
+    int level[ERTS_TOPOLOGY_MAX_DEPTH+1];
+} erts_avail_cput;
+
+typedef struct {
+    int id;
+    int sub_levels;
+    int cpu_groups;
+} erts_cpu_groups_count_t;
+
+typedef struct {
+    int logical;
+    int cpu_group;
+} erts_cpu_groups_map_array_t;
+
+typedef struct erts_cpu_groups_callback_list_t_ erts_cpu_groups_callback_list_t;
+struct erts_cpu_groups_callback_list_t_ {
+    erts_cpu_groups_callback_list_t *next;
+    erts_cpu_groups_callback_t callback;
+    void *arg;
+};
+
+typedef struct erts_cpu_groups_map_t_ erts_cpu_groups_map_t;
+struct erts_cpu_groups_map_t_ {
+    erts_cpu_groups_map_t *next;
+    int groups;
+    erts_cpu_groups_map_array_t *array;
+    int size;
+    int logical_processors;
+    erts_cpu_groups_callback_list_t *callback_list;
+};
+
+typedef struct {
+    erts_cpu_groups_callback_t callback;
+    int ix;
+    void *arg;
+} erts_cpu_groups_callback_call_t;
+
+static erts_cpu_groups_map_t *cpu_groups_maps;
+
+static erts_cpu_groups_map_t *reader_groups_map;
+
+#define ERTS_TOPOLOGY_CG ERTS_TOPOLOGY_MAX_DEPTH
+
+#define ERTS_MAX_CPU_TOPOLOGY_ID ((int) 0xffff)
+
+#ifdef ERTS_SMP
+static void cpu_bind_order_sort(erts_cpu_topology_t *cpudata,
+				int size,
+				ErtsCpuBindOrder bind_order,
+				int mk_seq);
+static void write_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size);
+#endif
+
+static void reader_groups_callback(int, ErtsSchedulerData *, int, void *);
+static erts_cpu_groups_map_t *add_cpu_groups(int groups,
+					     erts_cpu_groups_callback_t callback,
+					     void *arg);
+static void update_cpu_groups_maps(void);
+static void make_cpu_groups_map(erts_cpu_groups_map_t *map, int test);
+static int cpu_groups_lookup(erts_cpu_groups_map_t *map,
+			     ErtsSchedulerData *esdp);
+
+static void create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata,
+					 int *cpudata_size);
+static void destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata);
+
+static int
+int_cmp(const void *vx, const void *vy)
+{
+    return *((int *) vx) - *((int *) vy);
+}
+
+static int
+cpu_spread_order_cmp(const void *vx, const void *vy)
+{
+    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
+    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
+
+    if (x->thread != y->thread)
+	return x->thread - y->thread;
+    if (x->core != y->core)
+	return x->core - y->core;
+    if (x->processor_node != y->processor_node)
+	return x->processor_node - y->processor_node;
+    if (x->processor != y->processor)
+	return x->processor - y->processor;
+    if (x->node != y->node)
+	return x->node - y->node;
+    return 0;
+}
+
+static int
+cpu_processor_spread_order_cmp(const void *vx, const void *vy)
+{
+    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
+    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
+
+    if (x->thread != y->thread)
+	return x->thread - y->thread;
+    if (x->processor_node != y->processor_node)
+	return x->processor_node - y->processor_node;
+    if (x->core != y->core)
+	return x->core - y->core;
+    if (x->node != y->node)
+	return x->node - y->node;
+    if (x->processor != y->processor)
+	return x->processor - y->processor;
+    return 0;
+}
+
+static int
+cpu_thread_spread_order_cmp(const void *vx, const void *vy)
+{
+    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
+    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
+
+    if (x->thread != y->thread)
+	return x->thread - y->thread;
+    if (x->node != y->node)
+	return x->node - y->node;
+    if (x->processor != y->processor)
+	return x->processor - y->processor;
+    if (x->processor_node != y->processor_node)
+	return x->processor_node - y->processor_node;
+    if (x->core != y->core)
+	return x->core - y->core;
+    return 0;
+}
+
+static int
+cpu_thread_no_node_processor_spread_order_cmp(const void *vx, const void *vy)
+{
+    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
+    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
+
+    if (x->thread != y->thread)
+	return x->thread - y->thread;
+    if (x->node != y->node)
+	return x->node - y->node;
+    if (x->core != y->core)
+	return x->core - y->core;
+    if (x->processor != y->processor)
+	return x->processor - y->processor;
+    return 0;
+}
+
+static int
+cpu_no_node_processor_spread_order_cmp(const void *vx, const void *vy)
+{
+    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
+    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
+
+    if (x->node != y->node)
+	return x->node - y->node;
+    if (x->thread != y->thread)
+	return x->thread - y->thread;
+    if (x->core != y->core)
+	return x->core - y->core;
+    if (x->processor != y->processor)
+	return x->processor - y->processor;
+    return 0;
+}
+
+static int
+cpu_no_node_thread_spread_order_cmp(const void *vx, const void *vy)
+{
+    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
+    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
+
+    if (x->node != y->node)
+	return x->node - y->node;
+    if (x->thread != y->thread)
+	return x->thread - y->thread;
+    if (x->processor != y->processor)
+	return x->processor - y->processor;
+    if (x->core != y->core)
+	return x->core - y->core;
+    return 0;
+}
+
+static int
+cpu_no_spread_order_cmp(const void *vx, const void *vy)
+{
+    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
+    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
+
+    if (x->node != y->node)
+	return x->node - y->node;
+    if (x->processor != y->processor)
+	return x->processor - y->processor;
+    if (x->processor_node != y->processor_node)
+	return x->processor_node - y->processor_node;
+    if (x->core != y->core)
+	return x->core - y->core;
+    if (x->thread != y->thread)
+	return x->thread - y->thread;
+    return 0;
+}
+
+static ERTS_INLINE void
+make_cpudata_id_seq(erts_cpu_topology_t *cpudata, int size, int no_node)
+{
+    int ix;
+    int node = -1;
+    int processor = -1;
+    int processor_node = -1;
+    int processor_node_node = -1;
+    int core = -1;
+    int thread = -1;
+    int old_node = -1;
+    int old_processor = -1;
+    int old_processor_node = -1;
+    int old_core = -1;
+    int old_thread = -1;
+
+    for (ix = 0; ix < size; ix++) {
+	if (!no_node || cpudata[ix].node >= 0) {
+	    if (old_node == cpudata[ix].node)
+		cpudata[ix].node = node;
+	    else {
+		old_node = cpudata[ix].node;
+		old_processor = processor = -1;
+		if (!no_node)
+		    old_processor_node = processor_node = -1;
+		old_core = core = -1;
+		old_thread = thread = -1;
+		if (no_node || cpudata[ix].node >= 0)
+		    cpudata[ix].node = ++node;
+	    }
+	}
+	if (old_processor == cpudata[ix].processor)
+	    cpudata[ix].processor = processor;
+	else {
+	    old_processor = cpudata[ix].processor;
+	    if (!no_node)
+		processor_node_node = old_processor_node = processor_node = -1;
+	    old_core = core = -1;
+	    old_thread = thread = -1;
+	    cpudata[ix].processor = ++processor;
+	}
+	if (no_node && cpudata[ix].processor_node < 0)
+	    old_processor_node = -1;
+	else {
+	    if (old_processor_node == cpudata[ix].processor_node) {
+		if (no_node)
+		    cpudata[ix].node = cpudata[ix].processor_node = node;
+		else {
+		    if (processor_node_node >= 0)
+			cpudata[ix].node = processor_node_node;
+		    cpudata[ix].processor_node = processor_node;
+		}
+	    }
+	    else {
+		old_processor_node = cpudata[ix].processor_node;
+		old_core = core = -1;
+		old_thread = thread = -1;
+		if (no_node)
+		    cpudata[ix].node = cpudata[ix].processor_node = ++node;
+		else {
+		    cpudata[ix].node = processor_node_node = ++node;
+		    cpudata[ix].processor_node = ++processor_node;
+		}
+	    }
+	}
+	if (!no_node && cpudata[ix].processor_node < 0)
+	    cpudata[ix].processor_node = 0;
+	if (old_core == cpudata[ix].core)
+	    cpudata[ix].core = core;
+	else {
+	    old_core = cpudata[ix].core;
+	    old_thread = thread = -1;
+	    cpudata[ix].core = ++core;
+	}
+	if (old_thread == cpudata[ix].thread)
+	    cpudata[ix].thread = thread;
+	else
+	    old_thread = cpudata[ix].thread = ++thread;
+    }
+}
+
+static void
+cpu_bind_order_sort(erts_cpu_topology_t *cpudata,
+		    int size,
+		    ErtsCpuBindOrder bind_order,
+		    int mk_seq)
+{
+    if (size > 1) {
+	int no_node = 0;
+	int (*cmp_func)(const void *, const void *);
+	switch (bind_order) {
+	case ERTS_CPU_BIND_SPREAD:
+	    cmp_func = cpu_spread_order_cmp;
+	    break;
+	case ERTS_CPU_BIND_PROCESSOR_SPREAD:
+	    cmp_func = cpu_processor_spread_order_cmp;
+	    break;
+	case ERTS_CPU_BIND_THREAD_SPREAD:
+	    cmp_func = cpu_thread_spread_order_cmp;
+	    break;
+	case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD:
+	    no_node = 1;
+	    cmp_func = cpu_thread_no_node_processor_spread_order_cmp;
+	    break;
+	case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD:
+	    no_node = 1;
+	    cmp_func = cpu_no_node_processor_spread_order_cmp;
+	    break;
+	case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD:
+	    no_node = 1;
+	    cmp_func = cpu_no_node_thread_spread_order_cmp;
+	    break;
+	case ERTS_CPU_BIND_NO_SPREAD:
+	    cmp_func = cpu_no_spread_order_cmp;
+	    break;
+	default:
+	    cmp_func = NULL;
+	    erl_exit(ERTS_ABORT_EXIT,
+		     "Bad cpu bind type: %d\n",
+		     (int) cpu_bind_order);
+	    break;
+	}
+
+	if (mk_seq)
+	    make_cpudata_id_seq(cpudata, size, no_node);
+
+	qsort(cpudata, size, sizeof(erts_cpu_topology_t), cmp_func);
+    }
+}
+
+static int
+processor_order_cmp(const void *vx, const void *vy)
+{
+    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
+    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
+
+    if (x->processor != y->processor)
+	return x->processor - y->processor;
+    if (x->node != y->node)
+	return x->node - y->node;
+    if (x->processor_node != y->processor_node)
+	return x->processor_node - y->processor_node;
+    if (x->core != y->core)
+	return x->core - y->core;
+    if (x->thread != y->thread)
+	return x->thread - y->thread;
+    return 0;
+}
+
+#ifdef ERTS_SMP
+void
+erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp)
+{
+    erts_cpu_groups_map_t *cgm;
+    erts_cpu_groups_callback_list_t *cgcl;
+    erts_cpu_groups_callback_call_t *cgcc;
+    int cgcc_ix;
+
+    /* Unbind from cpu */
+    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+    if (scheduler2cpu_map[esdp->no].bound_id >= 0
+	&& erts_unbind_from_cpu(cpuinfo) == 0) {
+	esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1;
+    }
+
+    cgcc = erts_alloc(ERTS_ALC_T_TMP,
+		      (no_cpu_groups_callbacks
+		       * sizeof(erts_cpu_groups_callback_call_t)));
+    cgcc_ix = 0;
+    for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) {
+	for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) {
+	    cgcc[cgcc_ix].callback = cgcl->callback;
+	    cgcc[cgcc_ix].ix = cpu_groups_lookup(cgm, esdp);
+	    cgcc[cgcc_ix].arg = cgcl->arg;
+	    cgcc_ix++;
+	}
+    }
+    ASSERT(no_cpu_groups_callbacks == cgcc_ix);
+    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+
+    for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++)
+	cgcc[cgcc_ix].callback(1,
+			       esdp,
+			       cgcc[cgcc_ix].ix,
+			       cgcc[cgcc_ix].arg);
+
+    erts_free(ERTS_ALC_T_TMP, cgcc);
+
+    if (esdp->no <= max_main_threads)
+	erts_thr_set_main_status(0, 0);
+
+}
+
+void
+erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp)
+{
+    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(esdp->run_queue));
+
+    if (esdp->no <= max_main_threads)
+	erts_thr_set_main_status(1, (int) esdp->no);
+
+    /* Make sure we check if we should bind to a cpu or not... */
+    esdp->run_queue->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND;
+}
+
+#endif
+
+void
+erts_sched_check_cpu_bind(ErtsSchedulerData *esdp)
+{
+    int res, cpu_id, cgcc_ix;
+    erts_cpu_groups_map_t *cgm;
+    erts_cpu_groups_callback_list_t *cgcl;
+    erts_cpu_groups_callback_call_t *cgcc;
+#ifdef ERTS_SMP
+    esdp->run_queue->flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND;
+#endif
+    erts_smp_runq_unlock(esdp->run_queue);
+    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+    cpu_id = scheduler2cpu_map[esdp->no].bind_id;
+    if (cpu_id >= 0 && cpu_id != scheduler2cpu_map[esdp->no].bound_id) {
+	res = erts_bind_to_cpu(cpuinfo, cpu_id);
+	if (res == 0)
+	    esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = cpu_id;
+	else {
+	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
+	    erts_dsprintf(dsbufp, "Scheduler %d failed to bind to cpu %d: %s\n",
+			  (int) esdp->no, cpu_id, erl_errno_id(-res));
+	    erts_send_error_to_logger_nogl(dsbufp);
+	    if (scheduler2cpu_map[esdp->no].bound_id >= 0)
+		goto unbind;
+	}
+    }
+    else if (cpu_id < 0) {
+    unbind:
+	/* Get rid of old binding */
+	res = erts_unbind_from_cpu(cpuinfo);
+	if (res == 0)
+	    esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1;
+	else if (res != -ENOTSUP) {
+	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
+	    erts_dsprintf(dsbufp, "Scheduler %d failed to unbind from cpu %d: %s\n",
+			  (int) esdp->no, cpu_id, erl_errno_id(-res));
+	    erts_send_error_to_logger_nogl(dsbufp);
+	}
+    }
+
+    cgcc = erts_alloc(ERTS_ALC_T_TMP,
+		      (no_cpu_groups_callbacks
+		       * sizeof(erts_cpu_groups_callback_call_t)));
+    cgcc_ix = 0;
+    for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) {
+	for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) {
+	    cgcc[cgcc_ix].callback = cgcl->callback;
+	    cgcc[cgcc_ix].ix = cpu_groups_lookup(cgm, esdp);
+	    cgcc[cgcc_ix].arg = cgcl->arg;
+	    cgcc_ix++;
+	}
+    }
+
+    ASSERT(no_cpu_groups_callbacks == cgcc_ix);
+    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+
+    for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++)
+	cgcc[cgcc_ix].callback(0,
+			       esdp,
+			       cgcc[cgcc_ix].ix,
+			       cgcc[cgcc_ix].arg);
+
+    erts_free(ERTS_ALC_T_TMP, cgcc);
+
+    erts_smp_runq_lock(esdp->run_queue);
+}
+
+#ifdef ERTS_SMP
+void
+erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp)
+{
+    int cgcc_ix;
+    erts_cpu_groups_map_t *cgm;
+    erts_cpu_groups_callback_list_t *cgcl;
+    erts_cpu_groups_callback_call_t *cgcc;
+
+    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+
+    cgcc = erts_alloc(ERTS_ALC_T_TMP,
+		      (no_cpu_groups_callbacks
+		       * sizeof(erts_cpu_groups_callback_call_t)));
+    cgcc_ix = 0;
+    for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) {
+	for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) {
+	    cgcc[cgcc_ix].callback = cgcl->callback;
+	    cgcc[cgcc_ix].ix = cpu_groups_lookup(cgm, esdp);
+	    cgcc[cgcc_ix].arg = cgcl->arg;
+	    cgcc_ix++;
+	}
+    }
+
+    ASSERT(no_cpu_groups_callbacks == cgcc_ix);
+    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+
+    for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++)
+	cgcc[cgcc_ix].callback(0,
+			       esdp,
+			       cgcc[cgcc_ix].ix,
+			       cgcc[cgcc_ix].arg);
+
+    erts_free(ERTS_ALC_T_TMP, cgcc);
+
+    if (esdp->no <= max_main_threads)
+	erts_thr_set_main_status(1, (int) esdp->no);
+}
+#endif
+
+static void
+write_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size)
+{
+    int s_ix = 1;
+    int cpu_ix;
+
+    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
+
+    if (cpu_bind_order != ERTS_CPU_BIND_NONE && size) {
+
+	cpu_bind_order_sort(cpudata, size, cpu_bind_order, 1);
+
+	for (cpu_ix = 0; cpu_ix < size && cpu_ix < erts_no_schedulers; cpu_ix++)
+	    if (erts_is_cpu_available(cpuinfo, cpudata[cpu_ix].logical))
+		scheduler2cpu_map[s_ix++].bind_id = cpudata[cpu_ix].logical;
+    }
+
+    if (s_ix <= erts_no_schedulers)
+	for (; s_ix <= erts_no_schedulers; s_ix++)
+	    scheduler2cpu_map[s_ix].bind_id = -1;
+}
+
+int
+erts_init_scheduler_bind_type_string(char *how)
+{
+    if (sys_strcmp(how, "u") == 0)
+	cpu_bind_order = ERTS_CPU_BIND_NONE;
+    else if (erts_bind_to_cpu(cpuinfo, -1) == -ENOTSUP)
+	return ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED;
+    else if (!system_cpudata && !user_cpudata)
+	return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY;
+    else if (sys_strcmp(how, "db") == 0)
+	cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND;
+    else if (sys_strcmp(how, "s") == 0)
+	cpu_bind_order = ERTS_CPU_BIND_SPREAD;
+    else if (sys_strcmp(how, "ps") == 0)
+	cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD;
+    else if (sys_strcmp(how, "ts") == 0)
+	cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD;
+    else if (sys_strcmp(how, "tnnps") == 0)
+	cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD;
+    else if (sys_strcmp(how, "nnps") == 0)
+	cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD;
+    else if (sys_strcmp(how, "nnts") == 0)
+	cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD;
+    else if (sys_strcmp(how, "ns") == 0)
+	cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD;
+    else
+	return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE;
+    return ERTS_INIT_SCHED_BIND_TYPE_SUCCESS;
+}
+
+static Eterm
+bound_schedulers_term(ErtsCpuBindOrder order)
+{
+    switch (order) {
+    case ERTS_CPU_BIND_SPREAD: {
+	ERTS_DECL_AM(spread);
+	return AM_spread;
+    }
+    case ERTS_CPU_BIND_PROCESSOR_SPREAD: {
+	ERTS_DECL_AM(processor_spread);
+	return AM_processor_spread;
+    }
+    case ERTS_CPU_BIND_THREAD_SPREAD: {
+	ERTS_DECL_AM(thread_spread);
+	return AM_thread_spread;
+    }
+    case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD: {
+	ERTS_DECL_AM(thread_no_node_processor_spread);
+	return AM_thread_no_node_processor_spread;
+    }
+    case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD: {
+	ERTS_DECL_AM(no_node_processor_spread);
+	return AM_no_node_processor_spread;
+    }
+    case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD: {
+	ERTS_DECL_AM(no_node_thread_spread);
+	return AM_no_node_thread_spread;
+    }
+    case ERTS_CPU_BIND_NO_SPREAD: {
+	ERTS_DECL_AM(no_spread);
+	return AM_no_spread;
+    }
+    case ERTS_CPU_BIND_NONE: {
+	ERTS_DECL_AM(unbound);
+	return AM_unbound;
+    }
+    default:
+	ASSERT(0);
+	return THE_NON_VALUE;
+    }
+}
+
+Eterm
+erts_bound_schedulers_term(Process *c_p)
+{
+    ErtsCpuBindOrder order;
+    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    order = cpu_bind_order;
+    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+    return bound_schedulers_term(order);
+}
+
+Eterm
+erts_bind_schedulers(Process *c_p, Eterm how)
+{
+    int notify = 0;
+    Eterm res;
+    erts_cpu_topology_t *cpudata;
+    int cpudata_size;
+    ErtsCpuBindOrder old_cpu_bind_order;
+
+    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+
+    if (erts_bind_to_cpu(cpuinfo, -1) == -ENOTSUP) {
+	if (cpu_bind_order == ERTS_CPU_BIND_NONE
+	    && ERTS_IS_ATOM_STR("unbound", how)) {
+	    res = bound_schedulers_term(ERTS_CPU_BIND_NONE);
+	    goto done;
+	}
+	ERTS_BIF_PREP_ERROR(res, c_p, EXC_NOTSUP);
+    }
+    else {
+
+	old_cpu_bind_order = cpu_bind_order;
+
+	if (ERTS_IS_ATOM_STR("default_bind", how))
+	    cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND;
+	else if (ERTS_IS_ATOM_STR("spread", how))
+	    cpu_bind_order = ERTS_CPU_BIND_SPREAD;
+	else if (ERTS_IS_ATOM_STR("processor_spread", how))
+	    cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD;
+	else if (ERTS_IS_ATOM_STR("thread_spread", how))
+	    cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD;
+	else if (ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how))
+	    cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD;
+	else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how))
+	    cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD;
+	else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how))
+	    cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD;
+	else if (ERTS_IS_ATOM_STR("no_spread", how))
+	    cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD;
+	else if (ERTS_IS_ATOM_STR("unbound", how))
+	    cpu_bind_order = ERTS_CPU_BIND_NONE;
+	else {
+	    cpu_bind_order = old_cpu_bind_order;
+	    ERTS_BIF_PREP_ERROR(res, c_p, BADARG);
+	    goto done;
+	}
+
+	create_tmp_cpu_topology_copy(&cpudata, &cpudata_size);
+
+	if (!cpudata) {
+	    cpu_bind_order = old_cpu_bind_order;
+	    ERTS_BIF_PREP_ERROR(res, c_p, BADARG);
+	    goto done;
+	}
+
+	write_schedulers_bind_change(cpudata, cpudata_size);
+	notify = 1;
+
+	destroy_tmp_cpu_topology_copy(cpudata);
+    
+	res = bound_schedulers_term(old_cpu_bind_order);
+    }
+
+ done:
+
+    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+
+    if (notify)
+	erts_sched_notify_check_cpu_bind();
+
+    return res;
+}
+
+int
+erts_sched_bind_atthrcreate_prepare(void)
+{
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    return esdp != NULL && erts_is_scheduler_bound(esdp);
+}
+
+int
+erts_sched_bind_atthrcreate_child(int unbind)
+{
+    int res = 0;
+    if (unbind) {
+	erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+	res = erts_unbind_from_cpu(cpuinfo);
+	erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+    }
+    return res;
+}
+
+void
+erts_sched_bind_atthrcreate_parent(int unbind)
+{
+
+}
+
+int
+erts_sched_bind_atfork_prepare(void)
+{
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    int unbind = esdp != NULL && erts_is_scheduler_bound(esdp);
+    if (unbind)
+	erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    return unbind;
+}
+
+int
+erts_sched_bind_atfork_child(int unbind)
+{
+    if (unbind) {
+	ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx)
+			   || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
+	return erts_unbind_from_cpu(cpuinfo);
+    }
+    return 0;
+}
+
+char *
+erts_sched_bind_atvfork_child(int unbind)
+{
+    if (unbind) {
+	ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx)
+			   || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
+	return erts_get_unbind_from_cpu_str(cpuinfo);
+    }
+    return "false";
+}
+
+void
+erts_sched_bind_atfork_parent(int unbind)
+{
+    if (unbind)
+	erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+}
+
+Eterm
+erts_fake_scheduler_bindings(Process *p, Eterm how)
+{
+    ErtsCpuBindOrder fake_cpu_bind_order;
+    erts_cpu_topology_t *cpudata;
+    int cpudata_size;
+    Eterm res;
+
+    if (ERTS_IS_ATOM_STR("default_bind", how))
+	fake_cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND;
+    else if (ERTS_IS_ATOM_STR("spread", how))
+	fake_cpu_bind_order = ERTS_CPU_BIND_SPREAD;
+    else if (ERTS_IS_ATOM_STR("processor_spread", how))
+	fake_cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD;
+    else if (ERTS_IS_ATOM_STR("thread_spread", how))
+	fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD;
+    else if (ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how))
+	fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD;
+    else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how))
+	fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD;
+    else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how))
+	fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD;
+    else if (ERTS_IS_ATOM_STR("no_spread", how))
+	fake_cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD;
+    else if (ERTS_IS_ATOM_STR("unbound", how))
+	fake_cpu_bind_order = ERTS_CPU_BIND_NONE;
+    else {
+	ERTS_BIF_PREP_ERROR(res, p, BADARG);
+	return res;
+    }
+
+    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    create_tmp_cpu_topology_copy(&cpudata, &cpudata_size);
+    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+
+    if (!cpudata || fake_cpu_bind_order == ERTS_CPU_BIND_NONE)
+	ERTS_BIF_PREP_RET(res, am_false);
+    else {
+	int i;
+	Eterm *hp;
+	
+	cpu_bind_order_sort(cpudata, cpudata_size, fake_cpu_bind_order, 1);
+
+#ifdef ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA
+
+	erts_fprintf(stderr, "node:          ");
+	for (i = 0; i < cpudata_size; i++)
+	    erts_fprintf(stderr, " %2d", cpudata[i].node);
+	erts_fprintf(stderr, "\n");
+	erts_fprintf(stderr, "processor:     ");
+	for (i = 0; i < cpudata_size; i++)
+	    erts_fprintf(stderr, " %2d", cpudata[i].processor);
+	erts_fprintf(stderr, "\n");
+	if (fake_cpu_bind_order != ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD
+	    && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD
+	    && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD) {
+	    erts_fprintf(stderr, "processor_node:");
+	    for (i = 0; i < cpudata_size; i++)
+		erts_fprintf(stderr, " %2d", cpudata[i].processor_node);
+	    erts_fprintf(stderr, "\n");
+	}
+	erts_fprintf(stderr, "core:          ");
+	for (i = 0; i < cpudata_size; i++)
+	    erts_fprintf(stderr, " %2d", cpudata[i].core);
+	erts_fprintf(stderr, "\n");
+	erts_fprintf(stderr, "thread:        ");
+	for (i = 0; i < cpudata_size; i++)
+	    erts_fprintf(stderr, " %2d", cpudata[i].thread);
+	erts_fprintf(stderr, "\n");
+	erts_fprintf(stderr, "logical:       ");
+	for (i = 0; i < cpudata_size; i++)
+	    erts_fprintf(stderr, " %2d", cpudata[i].logical);
+	erts_fprintf(stderr, "\n");
+#endif
+
+	hp = HAlloc(p, cpudata_size+1);
+	ERTS_BIF_PREP_RET(res, make_tuple(hp));
+	*hp++ = make_arityval((Uint) cpudata_size);
+	for (i = 0; i < cpudata_size; i++)
+	    *hp++ = make_small((Uint) cpudata[i].logical);
+    }
+
+    destroy_tmp_cpu_topology_copy(cpudata);
+
+    return res;
+}
+
+Eterm
+erts_get_schedulers_binds(Process *c_p)
+{
+    int ix;
+    ERTS_DECL_AM(unbound);
+    Eterm *hp = HAlloc(c_p, erts_no_schedulers+1);
+    Eterm res = make_tuple(hp);
+
+    *(hp++) = make_arityval(erts_no_schedulers);
+    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    for (ix = 1; ix <= erts_no_schedulers; ix++)
+	*(hp++) = (scheduler2cpu_map[ix].bound_id >= 0
+		   ? make_small(scheduler2cpu_map[ix].bound_id)
+		   : AM_unbound);
+    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+    return res;
+}
+
+/*
+ * CPU topology
+ */
+
+typedef struct {
+    int *id;
+    int used;
+    int size;
+} ErtsCpuTopIdSeq;
+
+typedef struct {
+    ErtsCpuTopIdSeq logical;
+    ErtsCpuTopIdSeq thread;
+    ErtsCpuTopIdSeq core;
+    ErtsCpuTopIdSeq processor_node;
+    ErtsCpuTopIdSeq processor;
+    ErtsCpuTopIdSeq node;
+} ErtsCpuTopEntry;
+
+static void
+init_cpu_top_entry(ErtsCpuTopEntry *cte)
+{
+    int size = 10;
+    cte->logical.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
+				 sizeof(int)*size);
+    cte->logical.size = size;
+    cte->thread.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
+				sizeof(int)*size);
+    cte->thread.size = size;
+    cte->core.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
+			      sizeof(int)*size);
+    cte->core.size = size;
+    cte->processor_node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
+					sizeof(int)*size);
+    cte->processor_node.size = size;
+    cte->processor.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
+				   sizeof(int)*size);
+    cte->processor.size = size;
+    cte->node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
+			      sizeof(int)*size);
+    cte->node.size = size;
+}
+
+static void
+destroy_cpu_top_entry(ErtsCpuTopEntry *cte)
+{
+    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->logical.id);
+    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->thread.id);
+    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->core.id);
+    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor_node.id);
+    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor.id);
+    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->node.id);
+}
+
+static int
+get_cput_value_or_range(int *v, int *vr, char **str)
+{
+    long l;
+    char *c = *str;
+    errno = 0;
+    if (!isdigit((unsigned char)*c))
+	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID;
+    l = strtol(c, &c, 10);
+    if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l)
+	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID;
+    *v = (int) l;
+    if (*c == '-') {
+	c++;
+	if (!isdigit((unsigned char)*c))
+	    return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
+	l = strtol(c, &c, 10);
+	if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l)
+	    return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
+	*vr = (int) l;
+    }
+    *str = c;
+    return ERTS_INIT_CPU_TOPOLOGY_OK;
+}
+
+static int
+get_cput_id_seq(ErtsCpuTopIdSeq *idseq, char **str)
+{
+    int ix = 0;
+    int need_size = 0;
+    char *c = *str;
+
+    while (1) {
+	int res;
+	int val;
+	int nids;
+	int val_range = -1;
+	res = get_cput_value_or_range(&val, &val_range, &c);
+	if (res != ERTS_INIT_CPU_TOPOLOGY_OK)
+	    return res;
+	if (val_range < 0 || val_range == val)
+	    nids = 1;
+	else {
+	    if (val_range > val)
+		nids = val_range - val + 1;
+	    else
+		nids = val - val_range + 1;
+	}
+	need_size += nids;
+	if (need_size > idseq->size) {
+	    idseq->size = need_size + 10;
+	    idseq->id = erts_realloc(ERTS_ALC_T_TMP_CPU_IDS,
+				      idseq->id,
+				      sizeof(int)*idseq->size);
+	}
+	if (nids == 1)
+	    idseq->id[ix++] = val;
+	else if (val_range > val) {
+	    for (; val <= val_range; val++)
+		idseq->id[ix++] = val;
+	}
+	else {
+	    for (; val >= val_range; val--)
+		idseq->id[ix++] = val;
+	}
+	if (*c != ',')
+	    break;
+	c++;
+    }
+    *str = c;
+    idseq->used = ix;
+    return ERTS_INIT_CPU_TOPOLOGY_OK;
+}
+
+static int
+get_cput_entry(ErtsCpuTopEntry *cput, char **str)
+{
+    int h;
+    char *c = *str;
+
+    cput->logical.used = 0;
+    cput->thread.id[0] = 0;
+    cput->thread.used = 1;
+    cput->core.id[0] = 0;
+    cput->core.used = 1;
+    cput->processor_node.id[0] = -1;
+    cput->processor_node.used = 1;
+    cput->processor.id[0] = 0;
+    cput->processor.used = 1;
+    cput->node.id[0] = -1;
+    cput->node.used = 1;
+
+    h = ERTS_TOPOLOGY_MAX_DEPTH;
+    while (*c != ':' && *c != '\0') {
+	int res;
+	ErtsCpuTopIdSeq *idseqp;
+	switch (*c++) {
+	case 'L':
+	    if (h <= ERTS_TOPOLOGY_LOGICAL)
+		return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
+	    idseqp = &cput->logical;
+	    h = ERTS_TOPOLOGY_LOGICAL;
+	    break;
+	case 't':
+	case 'T':
+	    if (h <= ERTS_TOPOLOGY_THREAD)
+		return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
+	    idseqp = &cput->thread;
+	    h = ERTS_TOPOLOGY_THREAD;
+	    break;
+	case 'c':
+	case 'C':
+	    if (h <= ERTS_TOPOLOGY_CORE)
+		return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
+	    idseqp = &cput->core;
+	    h = ERTS_TOPOLOGY_CORE;
+	    break;
+	case 'p':
+	case 'P':
+	    if (h <= ERTS_TOPOLOGY_PROCESSOR)
+		return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
+	    idseqp = &cput->processor;
+	    h = ERTS_TOPOLOGY_PROCESSOR;
+	    break;
+	case 'n':
+	case 'N':
+	    if (h <= ERTS_TOPOLOGY_PROCESSOR) {
+	    do_node:
+		if (h <= ERTS_TOPOLOGY_NODE)
+		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
+		idseqp = &cput->node;
+		h = ERTS_TOPOLOGY_NODE;
+	    }
+	    else {
+		int p_node = 0;
+		char *p_chk = c;
+		while (*p_chk != '\0' && *p_chk != ':') {
+		    if (*p_chk == 'p' || *p_chk == 'P') {
+			p_node = 1;
+			break;
+		    }
+		    p_chk++;
+		}
+		if (!p_node)
+		    goto do_node;
+		if (h <= ERTS_TOPOLOGY_PROCESSOR_NODE)
+		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
+		idseqp = &cput->processor_node;
+		h = ERTS_TOPOLOGY_PROCESSOR_NODE;
+	    }
+	    break;
+	default:
+	    return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE;
+	}
+	res = get_cput_id_seq(idseqp, &c);
+	if (res != ERTS_INIT_CPU_TOPOLOGY_OK)
+		return res;
+    }
+
+    if (cput->logical.used < 1)
+	return ERTS_INIT_CPU_TOPOLOGY_MISSING_LID;
+
+    if (*c == ':') {
+	c++;
+    }
+
+    if (cput->thread.used != 1
+	&& cput->thread.used != cput->logical.used)
+	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
+    if (cput->core.used != 1
+	&& cput->core.used != cput->logical.used)
+	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
+    if (cput->processor_node.used != 1
+	&& cput->processor_node.used != cput->logical.used)
+	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
+    if (cput->processor.used != 1
+	&& cput->processor.used != cput->logical.used)
+	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
+    if (cput->node.used != 1
+	&& cput->node.used != cput->logical.used)
+	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
+
+    *str = c;
+    return ERTS_INIT_CPU_TOPOLOGY_OK;
+}
+
+static int
+verify_topology(erts_cpu_topology_t *cpudata, int size)
+{
+    if (size > 0) {
+	int *logical;
+	int node, processor, no_nodes, i;
+
+	/* Verify logical ids */
+	logical = erts_alloc(ERTS_ALC_T_TMP, sizeof(int)*size);
+
+	for (i = 0; i < size; i++)
+	    logical[i] = cpudata[i].logical;
+
+	qsort(logical, size, sizeof(int), int_cmp);
+	for (i = 0; i < size-1; i++) {
+	    if (logical[i] == logical[i+1]) {
+		erts_free(ERTS_ALC_T_TMP, logical);
+		return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS;
+	    }
+	}
+
+	erts_free(ERTS_ALC_T_TMP, logical);
+
+	qsort(cpudata, size, sizeof(erts_cpu_topology_t), processor_order_cmp);
+
+	/* Verify unique entities */
+
+	for (i = 1; i < size; i++) {
+	    if (cpudata[i-1].processor == cpudata[i].processor
+		&& cpudata[i-1].node == cpudata[i].node
+		&& (cpudata[i-1].processor_node
+		    == cpudata[i].processor_node)
+		&& cpudata[i-1].core == cpudata[i].core
+		&& cpudata[i-1].thread == cpudata[i].thread) {
+		return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES;
+	    }
+	}
+
+	/* Verify numa nodes */
+	node = cpudata[0].node;
+	processor = cpudata[0].processor;
+	no_nodes = cpudata[0].node < 0 && cpudata[0].processor_node < 0;
+	for (i = 1; i < size; i++) {
+	    if (no_nodes) {
+		if (cpudata[i].node >= 0 || cpudata[i].processor_node >= 0)
+		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES;
+	    }
+	    else {
+		if (cpudata[i].processor == processor && cpudata[i].node != node)
+		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES;
+		node = cpudata[i].node;
+		processor = cpudata[i].processor;
+		if (node >= 0 && cpudata[i].processor_node >= 0)
+		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES;
+		if (node < 0 && cpudata[i].processor_node < 0)
+		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES;
+	    }
+	}
+    }
+
+    return ERTS_INIT_CPU_TOPOLOGY_OK;
+}
+
+int
+erts_init_cpu_topology_string(char *topology_str)
+{
+    ErtsCpuTopEntry cput;
+    int need_size;
+    char *c;
+    int ix;
+    int error = ERTS_INIT_CPU_TOPOLOGY_OK;
+
+    if (user_cpudata)
+	erts_free(ERTS_ALC_T_CPUDATA, user_cpudata);
+    user_cpudata_size = 10;
+
+    user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA,
+			      (sizeof(erts_cpu_topology_t)
+			       * user_cpudata_size));
+
+    init_cpu_top_entry(&cput);
+
+    ix = 0;
+    need_size = 0;
+
+    c = topology_str;
+    if (*c == '\0') {
+	error = ERTS_INIT_CPU_TOPOLOGY_MISSING;
+	goto fail;
+    }
+    do {
+	int r;
+	error = get_cput_entry(&cput, &c);
+	if (error != ERTS_INIT_CPU_TOPOLOGY_OK)
+	    goto fail;
+	need_size += cput.logical.used;
+	if (user_cpudata_size < need_size) {
+	    user_cpudata_size = need_size + 10;
+	    user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA,
+					user_cpudata,
+					(sizeof(erts_cpu_topology_t)
+					 * user_cpudata_size));
+	}
+
+	ASSERT(cput.thread.used == 1
+	       || cput.thread.used == cput.logical.used);
+	ASSERT(cput.core.used == 1
+	       || cput.core.used == cput.logical.used);
+	ASSERT(cput.processor_node.used == 1
+	       || cput.processor_node.used == cput.logical.used);
+	ASSERT(cput.processor.used == 1
+	       || cput.processor.used == cput.logical.used);
+	ASSERT(cput.node.used == 1
+	       || cput.node.used == cput.logical.used);
+
+	for (r = 0; r < cput.logical.used; r++) {
+	    user_cpudata[ix].logical = cput.logical.id[r];
+	    user_cpudata[ix].thread =
+		cput.thread.id[cput.thread.used == 1 ? 0 : r];
+	    user_cpudata[ix].core =
+		cput.core.id[cput.core.used == 1 ? 0 : r];
+	    user_cpudata[ix].processor_node =
+		cput.processor_node.id[cput.processor_node.used == 1 ? 0 : r];
+	    user_cpudata[ix].processor =
+		cput.processor.id[cput.processor.used == 1 ? 0 : r];
+	    user_cpudata[ix].node =
+		cput.node.id[cput.node.used == 1 ? 0 : r];
+	    ix++;
+	}
+    } while (*c != '\0');
+
+    if (user_cpudata_size != ix) {
+	user_cpudata_size = ix;
+	user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA,
+				    user_cpudata,
+				    (sizeof(erts_cpu_topology_t)
+				     * user_cpudata_size));
+    }
+
+    error = verify_topology(user_cpudata, user_cpudata_size);
+    if (error == ERTS_INIT_CPU_TOPOLOGY_OK) {
+	destroy_cpu_top_entry(&cput);
+	return ERTS_INIT_CPU_TOPOLOGY_OK;
+    }
+
+ fail:
+    if (user_cpudata)
+	erts_free(ERTS_ALC_T_CPUDATA, user_cpudata);
+    user_cpudata_size = 0;
+    destroy_cpu_top_entry(&cput);
+    return error;
+}
+
+#define ERTS_GET_CPU_TOPOLOGY_ERROR		-1
+#define ERTS_GET_USED_CPU_TOPOLOGY		0
+#define ERTS_GET_DETECTED_CPU_TOPOLOGY		1
+#define ERTS_GET_DEFINED_CPU_TOPOLOGY		2
+
+static Eterm get_cpu_topology_term(Process *c_p, int type);
+
+Eterm
+erts_set_cpu_topology(Process *c_p, Eterm term)
+{
+    erts_cpu_topology_t *cpudata = NULL;
+    int cpudata_size = 0;
+    Eterm res;
+
+    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+    res = get_cpu_topology_term(c_p, ERTS_GET_USED_CPU_TOPOLOGY);
+    if (term == am_undefined) {
+	if (user_cpudata)
+	    erts_free(ERTS_ALC_T_CPUDATA, user_cpudata);
+	user_cpudata = NULL;
+	user_cpudata_size = 0;
+
+	if (cpu_bind_order != ERTS_CPU_BIND_NONE && system_cpudata) {
+	    cpudata_size = system_cpudata_size;
+	    cpudata = erts_alloc(ERTS_ALC_T_TMP,
+				 (sizeof(erts_cpu_topology_t)
+				  * cpudata_size));
+
+	    sys_memcpy((void *) cpudata,
+		       (void *) system_cpudata,
+		       sizeof(erts_cpu_topology_t)*cpudata_size);
+	}
+    }
+    else if (is_not_list(term)) {
+    error:
+	erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+	res = THE_NON_VALUE;
+	goto done;
+    }
+    else {
+	Eterm list = term;
+	int ix = 0;
+
+	cpudata_size = 100;
+	cpudata = erts_alloc(ERTS_ALC_T_TMP,
+			     (sizeof(erts_cpu_topology_t)
+			      * cpudata_size));
+
+	while (is_list(list)) {
+	    Eterm *lp = list_val(list);
+	    Eterm cpu = CAR(lp);
+	    Eterm* tp;
+	    Sint id;
+		
+	    if (is_not_tuple(cpu))
+		goto error;
+
+	    tp = tuple_val(cpu);
+
+	    if (arityval(tp[0]) != 7 || tp[1] != am_cpu)
+		goto error;
+
+	    if (ix >= cpudata_size) {
+		cpudata_size += 100;
+		cpudata = erts_realloc(ERTS_ALC_T_TMP,
+				       cpudata,
+				       (sizeof(erts_cpu_topology_t)
+					* cpudata_size));
+	    }
+
+	    id = signed_val(tp[2]);
+	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
+		goto error;
+	    cpudata[ix].node = (int) id;
+
+	    id = signed_val(tp[3]);
+	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
+		goto error;
+	    cpudata[ix].processor = (int) id;
+
+	    id = signed_val(tp[4]);
+	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
+		goto error;
+	    cpudata[ix].processor_node = (int) id;
+
+	    id = signed_val(tp[5]);
+	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
+		goto error;
+	    cpudata[ix].core = (int) id;
+
+	    id = signed_val(tp[6]);
+	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
+		goto error;
+	    cpudata[ix].thread = (int) id;
+
+	    id = signed_val(tp[7]);
+	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
+		goto error;
+	    cpudata[ix].logical = (int) id;
+
+	    list = CDR(lp);
+	    ix++;
+	}
+
+	if (is_not_nil(list))
+	    goto error;
+	
+	cpudata_size = ix;
+
+	if (ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(cpudata, cpudata_size))
+	    goto error;
+
+	if (user_cpudata_size != cpudata_size) {
+	    if (user_cpudata)
+		erts_free(ERTS_ALC_T_CPUDATA, user_cpudata);
+	    user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA,
+				      sizeof(erts_cpu_topology_t)*cpudata_size);
+	    user_cpudata_size = cpudata_size;
+	}
+
+	sys_memcpy((void *) user_cpudata,
+		   (void *) cpudata,
+		   sizeof(erts_cpu_topology_t)*cpudata_size);
+    }
+
+    update_cpu_groups_maps();
+
+    write_schedulers_bind_change(cpudata, cpudata_size);
+
+    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+    erts_sched_notify_check_cpu_bind();
+
+ done:
+
+    if (cpudata)
+	erts_free(ERTS_ALC_T_TMP, cpudata);
+
+    return res;
+}
+
+static void
+create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata, int *cpudata_size)
+{
+    if (user_cpudata) {
+	*cpudata_size = user_cpudata_size;
+	*cpudata = erts_alloc(ERTS_ALC_T_TMP,
+			      (sizeof(erts_cpu_topology_t)
+			       * (*cpudata_size)));
+	sys_memcpy((void *) *cpudata,
+		   (void *) user_cpudata,
+		   sizeof(erts_cpu_topology_t)*(*cpudata_size));
+    }
+    else if (system_cpudata) {
+	*cpudata_size = system_cpudata_size;
+	*cpudata = erts_alloc(ERTS_ALC_T_TMP,
+			      (sizeof(erts_cpu_topology_t)
+			       * (*cpudata_size)));
+	sys_memcpy((void *) *cpudata,
+		   (void *) system_cpudata,
+		   sizeof(erts_cpu_topology_t)*(*cpudata_size));
+    }
+    else {
+	*cpudata = NULL;
+	*cpudata_size = 0;
+    }
+}
+
+static void
+destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata)
+{
+    if (cpudata)
+	erts_free(ERTS_ALC_T_TMP, cpudata);
+}
+
+
+static Eterm
+bld_topology_term(Eterm **hpp,
+		  Uint *hszp,
+		  erts_cpu_topology_t *cpudata,
+		  int size)
+{
+    Eterm res = NIL;
+    int i;
+
+    if (size == 0)
+	return am_undefined;
+
+    for (i = size-1; i >= 0; i--) {
+	res = erts_bld_cons(hpp,
+			    hszp,
+			    erts_bld_tuple(hpp,
+					   hszp,
+					   7,
+					   am_cpu,
+					   make_small(cpudata[i].node),
+					   make_small(cpudata[i].processor),
+					   make_small(cpudata[i].processor_node),
+					   make_small(cpudata[i].core),
+					   make_small(cpudata[i].thread),
+					   make_small(cpudata[i].logical)),
+			    res);
+    }
+    return res;
+}
+
+static Eterm
+get_cpu_topology_term(Process *c_p, int type)
+{
+#ifdef DEBUG
+    Eterm *hp_end;
+#endif
+    Eterm *hp;
+    Uint hsz;
+    Eterm res = THE_NON_VALUE;
+    erts_cpu_topology_t *cpudata = NULL;
+    int size = 0;
+
+    switch (type) {
+    case ERTS_GET_USED_CPU_TOPOLOGY:
+	if (user_cpudata)
+	    goto defined;
+	else
+	    goto detected;
+    case ERTS_GET_DETECTED_CPU_TOPOLOGY:
+    detected:
+	if (!system_cpudata)
+	    res = am_undefined;
+	else {
+	    size = system_cpudata_size;
+	    cpudata = erts_alloc(ERTS_ALC_T_TMP,
+				 (sizeof(erts_cpu_topology_t)
+				  * size));
+	    sys_memcpy((void *) cpudata,
+		       (void *) system_cpudata,
+		       sizeof(erts_cpu_topology_t)*size);
+	}
+	break;
+    case ERTS_GET_DEFINED_CPU_TOPOLOGY:
+    defined:
+	if (!user_cpudata)
+	    res = am_undefined;
+	else {
+	    size = user_cpudata_size;
+	    cpudata = user_cpudata;
+	}
+	break;
+    default:
+	erl_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type);
+	break;
+    }
+
+    if (res == am_undefined) {
+	ASSERT(!cpudata);
+	return res;
+    }
+
+    hsz = 0;
+
+    bld_topology_term(NULL, &hsz,
+		      cpudata, size);
+
+    hp = HAlloc(c_p, hsz);
+
+#ifdef DEBUG
+    hp_end = hp + hsz;
+#endif
+
+    res = bld_topology_term(&hp, NULL,
+			    cpudata, size);
+
+    ASSERT(hp_end == hp);
+
+    if (cpudata && cpudata != system_cpudata && cpudata != user_cpudata)
+	erts_free(ERTS_ALC_T_TMP, cpudata);
+
+    return res;
+}
+
+Eterm
+erts_get_cpu_topology_term(Process *c_p, Eterm which)
+{
+    Eterm res;
+    int type;
+    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    if (ERTS_IS_ATOM_STR("used", which))
+	type = ERTS_GET_USED_CPU_TOPOLOGY;
+    else if (ERTS_IS_ATOM_STR("detected", which))
+	type = ERTS_GET_DETECTED_CPU_TOPOLOGY;
+    else if (ERTS_IS_ATOM_STR("defined", which))
+	type = ERTS_GET_DEFINED_CPU_TOPOLOGY;
+    else
+	type = ERTS_GET_CPU_TOPOLOGY_ERROR;
+    if (type == ERTS_GET_CPU_TOPOLOGY_ERROR)
+	res = THE_NON_VALUE;
+    else
+	res = get_cpu_topology_term(c_p, type);
+    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+    return res;
+}
+
+static void
+get_logical_processors(int *conf, int *onln, int *avail)
+{
+    if (conf)
+	*conf = erts_get_cpu_configured(cpuinfo);
+    if (onln)
+	*onln = erts_get_cpu_online(cpuinfo);
+    if (avail)
+	*avail = erts_get_cpu_available(cpuinfo);
+}
+
+void
+erts_get_logical_processors(int *conf, int *onln, int *avail)
+{
+    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    get_logical_processors(conf, onln, avail);
+    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+}
+
+void
+erts_pre_early_init_cpu_topology(int *max_rg_p,
+				 int *conf_p,
+				 int *onln_p,
+				 int *avail_p)
+{
+    cpu_groups_maps = NULL;
+    no_cpu_groups_callbacks = 0;
+    *max_rg_p = ERTS_MAX_READER_GROUPS;
+    cpuinfo = erts_cpu_info_create();
+    get_logical_processors(conf_p, onln_p, avail_p);
+}
+
+void
+erts_early_init_cpu_topology(int no_schedulers,
+			     int *max_main_threads_p,
+			     int max_reader_groups,
+			     int *reader_groups_p)
+{
+    user_cpudata = NULL;
+    user_cpudata_size = 0;
+
+    system_cpudata_size = erts_get_cpu_topology_size(cpuinfo);
+    system_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA,
+				(sizeof(erts_cpu_topology_t)
+				 * system_cpudata_size));
+
+    cpu_bind_order = ERTS_CPU_BIND_UNDEFINED;
+
+    if (!erts_get_cpu_topology(cpuinfo, system_cpudata)
+	|| ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(system_cpudata,
+							system_cpudata_size)) {
+	erts_free(ERTS_ALC_T_CPUDATA, system_cpudata);
+	system_cpudata = NULL;
+	system_cpudata_size = 0;
+    }
+
+    max_main_threads = erts_get_cpu_configured(cpuinfo);
+    if (max_main_threads > no_schedulers)
+	max_main_threads = no_schedulers;
+    *max_main_threads_p = max_main_threads;
+
+    reader_groups = max_main_threads;
+    if (reader_groups <= 1 || max_reader_groups <= 1)
+	reader_groups = 0;
+    if (reader_groups > max_reader_groups)
+	reader_groups = max_reader_groups;
+    *reader_groups_p = reader_groups;
+}
+
+void
+erts_init_cpu_topology(void)
+{
+    int ix;
+
+    erts_smp_rwmtx_init(&cpuinfo_rwmtx, "cpu_info");
+    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+
+    scheduler2cpu_map = erts_alloc(ERTS_ALC_T_CPUDATA,
+				   (sizeof(ErtsCpuBindData)
+				    * (erts_no_schedulers+1)));
+    for (ix = 1; ix <= erts_no_schedulers; ix++) {
+	scheduler2cpu_map[ix].bind_id = -1;
+	scheduler2cpu_map[ix].bound_id = -1;
+    }
+
+    if (cpu_bind_order == ERTS_CPU_BIND_UNDEFINED)
+	cpu_bind_order = ERTS_CPU_BIND_NONE;
+
+    reader_groups_map = add_cpu_groups(reader_groups,
+				       reader_groups_callback,
+				       NULL);
+
+    if (cpu_bind_order == ERTS_CPU_BIND_NONE)
+	erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+    else {
+	erts_cpu_topology_t *cpudata;
+	int cpudata_size;
+	create_tmp_cpu_topology_copy(&cpudata, &cpudata_size);
+	write_schedulers_bind_change(cpudata, cpudata_size);
+	erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+	erts_sched_notify_check_cpu_bind();
+	destroy_tmp_cpu_topology_copy(cpudata);
+    }
+}
+
+int
+erts_update_cpu_info(void)
+{
+    int changed;
+    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+    changed = erts_cpu_info_update(cpuinfo);
+    if (changed) {
+	erts_cpu_topology_t *cpudata;
+	int cpudata_size;
+
+	if (system_cpudata)
+	    erts_free(ERTS_ALC_T_CPUDATA, system_cpudata);
+
+	system_cpudata_size = erts_get_cpu_topology_size(cpuinfo);
+	if (!system_cpudata_size)
+	    system_cpudata = NULL;
+	else {
+	    system_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA,
+					(sizeof(erts_cpu_topology_t)
+					 * system_cpudata_size));
+
+	    if (!erts_get_cpu_topology(cpuinfo, system_cpudata)
+		|| (ERTS_INIT_CPU_TOPOLOGY_OK
+		    != verify_topology(system_cpudata,
+				       system_cpudata_size))) {
+		erts_free(ERTS_ALC_T_CPUDATA, system_cpudata);
+		system_cpudata = NULL;
+		system_cpudata_size = 0;
+	    }
+	}
+
+	update_cpu_groups_maps();
+
+	create_tmp_cpu_topology_copy(&cpudata, &cpudata_size);
+	write_schedulers_bind_change(cpudata, cpudata_size);
+	destroy_tmp_cpu_topology_copy(cpudata);
+    }
+    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+    if (changed)
+	erts_sched_notify_check_cpu_bind();
+    return changed;
+}
+
+/*
+ * reader groups map
+ */
+
+void
+reader_groups_callback(int suspending,
+		       ErtsSchedulerData *esdp,
+		       int group,
+		       void *unused)
+{
+    if (reader_groups && esdp->no <= max_main_threads)
+	erts_smp_rwmtx_set_reader_group(suspending ? 0 : group+1);
+}
+
+static Eterm get_cpu_groups_map(Process *c_p,
+				erts_cpu_groups_map_t *map,
+				int offset);
+Eterm
+erts_debug_reader_groups_map(Process *c_p, int groups)
+{
+    Eterm res;
+    erts_cpu_groups_map_t test;
+
+    test.array = NULL;
+    test.groups = groups;
+    make_cpu_groups_map(&test, 1);
+    if (!test.array)
+	res = NIL;
+    else {
+	res = get_cpu_groups_map(c_p, &test, 1);
+	erts_free(ERTS_ALC_T_TMP, test.array);
+    }
+    return res;
+}
+
+
+Eterm
+erts_get_reader_groups_map(Process *c_p)
+{
+    Eterm res;
+    erts_smp_rwmtx_rlock(&cpuinfo_rwmtx);
+    res = get_cpu_groups_map(c_p, reader_groups_map, 1);
+    erts_smp_rwmtx_runlock(&cpuinfo_rwmtx);
+    return res;
+}
+
+/*
+ * CPU groups
+ */
+
+static Eterm
+get_cpu_groups_map(Process *c_p,
+		   erts_cpu_groups_map_t *map,
+		   int offset)
+{
+#ifdef DEBUG
+    Eterm *endp;
+#endif
+    Eterm res = NIL, tuple;
+    Eterm *hp;
+    int i;
+
+    hp = HAlloc(c_p, map->logical_processors*(2+3));
+#ifdef DEBUG
+    endp = hp + map->logical_processors*(2+3);
+#endif
+    for (i = map->size - 1; i >= 0; i--) {
+	if (map->array[i].logical >= 0) {
+	    tuple = TUPLE2(hp,
+			   make_small(map->array[i].logical),
+			   make_small(map->array[i].cpu_group + offset));
+	    hp += 3;
+	    res = CONS(hp, tuple, res);
+	    hp += 2;
+	}
+    }
+    ASSERT(hp == endp);
+    return res;
+}
+
+static void
+make_available_cpu_topology(erts_avail_cput *no,
+			    erts_avail_cput *avail,
+			    erts_cpu_topology_t *cpudata,
+			    int *size,
+			    int test)
+{
+    int len = *size;
+    erts_cpu_topology_t last;
+    int a, i, j;
+
+    no->level[ERTS_TOPOLOGY_NODE] = -1;
+    no->level[ERTS_TOPOLOGY_PROCESSOR] = -1;
+    no->level[ERTS_TOPOLOGY_PROCESSOR_NODE] = -1;
+    no->level[ERTS_TOPOLOGY_CORE] = -1;
+    no->level[ERTS_TOPOLOGY_THREAD] = -1;
+    no->level[ERTS_TOPOLOGY_LOGICAL] = -1;
+
+    last.node = INT_MIN;
+    last.processor = INT_MIN;
+    last.processor_node = INT_MIN;
+    last.core = INT_MIN;
+    last.thread = INT_MIN;
+    last.logical = INT_MIN;
+
+    a = 0;
+
+    for (i = 0; i < len; i++) {
+
+	if (!test && !erts_is_cpu_available(cpuinfo, cpudata[i].logical))
+	    continue;
+
+	if (last.node != cpudata[i].node)
+	    goto node;
+	if (last.processor != cpudata[i].processor)
+	    goto processor;
+	if (last.processor_node != cpudata[i].processor_node)
+	    goto processor_node;
+	if (last.core != cpudata[i].core)
+	    goto core;
+	ASSERT(last.thread != cpudata[i].thread);
+	goto thread;
+
+    node:
+	no->level[ERTS_TOPOLOGY_NODE]++;
+    processor:
+	no->level[ERTS_TOPOLOGY_PROCESSOR]++;
+    processor_node:
+	no->level[ERTS_TOPOLOGY_PROCESSOR_NODE]++;
+    core:
+	no->level[ERTS_TOPOLOGY_CORE]++;
+    thread:
+	no->level[ERTS_TOPOLOGY_THREAD]++;
+
+	no->level[ERTS_TOPOLOGY_LOGICAL]++;
+
+	for (j = 0; j < ERTS_TOPOLOGY_LOGICAL; j++)
+	    avail[a].level[j] = no->level[j];
+
+	avail[a].level[ERTS_TOPOLOGY_LOGICAL] = cpudata[i].logical;
+	avail[a].level[ERTS_TOPOLOGY_CG] = 0;
+
+	ASSERT(last.logical != cpudata[i].logical);
+
+	last = cpudata[i];
+	a++;
+    }
+
+    no->level[ERTS_TOPOLOGY_NODE]++;
+    no->level[ERTS_TOPOLOGY_PROCESSOR]++;
+    no->level[ERTS_TOPOLOGY_PROCESSOR_NODE]++;
+    no->level[ERTS_TOPOLOGY_CORE]++;
+    no->level[ERTS_TOPOLOGY_THREAD]++;
+    no->level[ERTS_TOPOLOGY_LOGICAL]++;
+
+    *size = a;
+}
+
+static void
+cpu_group_insert(erts_cpu_groups_map_t *map,
+		 int logical, int cpu_group)
+{
+    int start = logical % map->size;
+    int ix = start;
+
+    do {
+	if (map->array[ix].logical < 0) {
+	    map->array[ix].logical = logical;
+	    map->array[ix].cpu_group = cpu_group;
+	    return;
+	}
+	ix++;
+	if (ix == map->size)
+	    ix = 0;
+    } while (ix != start);
+
+    erl_exit(ERTS_ABORT_EXIT, "Reader groups map full\n");
+}
+
+
+static int
+sub_levels(erts_cpu_groups_count_t *cgc, int level, int aix,
+	   int avail_sz, erts_avail_cput *avail)
+{
+    int sub_level = level+1;
+    int last = -1;
+    cgc->sub_levels = 0;
+
+    do {
+	if (last != avail[aix].level[sub_level]) {
+	    cgc->sub_levels++;
+	    last = avail[aix].level[sub_level];
+	}
+	aix++;
+    }
+    while (aix < avail_sz && cgc->id == avail[aix].level[level]);
+    cgc->cpu_groups = 0;
+    return aix;
+}
+
+static int
+write_cpu_groups(int *cgp, erts_cpu_groups_count_t *cgcp,
+		    int level, int a,
+		    int avail_sz, erts_avail_cput *avail)
+{
+    int cg = *cgp;
+    int sub_level = level+1;
+    int sl_per_gr = cgcp->sub_levels / cgcp->cpu_groups;
+    int xsl = cgcp->sub_levels % cgcp->cpu_groups;
+    int sls = 0;
+    int last = -1;
+    int xsl_cg_lim = (cgcp->cpu_groups - xsl) + cg + 1;
+
+    ASSERT(level < 0 || avail[a].level[level] == cgcp->id);
+
+    do {
+	if (last != avail[a].level[sub_level]) {
+	    if (!sls) {
+		sls = sl_per_gr;
+		cg++;
+		if (cg >= xsl_cg_lim)
+		    sls++;
+	    }
+	    last = avail[a].level[sub_level];
+	    sls--;
+	}
+	avail[a].level[ERTS_TOPOLOGY_CG] = cg;
+	a++;
+    } while (a < avail_sz && (level < 0
+			      || avail[a].level[level] == cgcp->id));
+
+    ASSERT(cgcp->cpu_groups == cg - *cgp);
+
+    *cgp = cg;
+
+    return a;
+}
+
+static int
+cg_count_sub_levels_compare(const void *vx, const void *vy)
+{
+    erts_cpu_groups_count_t *x = (erts_cpu_groups_count_t *) vx;
+    erts_cpu_groups_count_t *y = (erts_cpu_groups_count_t *) vy;
+    if (x->sub_levels != y->sub_levels)
+	return y->sub_levels - x->sub_levels;
+    return x->id - y->id;
+}
+
+static int
+cg_count_id_compare(const void *vx, const void *vy)
+{
+    erts_cpu_groups_count_t *x = (erts_cpu_groups_count_t *) vx;
+    erts_cpu_groups_count_t *y = (erts_cpu_groups_count_t *) vy;
+    return x->id - y->id;
+}
+
+static void
+make_cpu_groups_map(erts_cpu_groups_map_t *map, int test)
+{
+    int i, spread_level, avail_sz;
+    erts_avail_cput no, *avail;
+    erts_cpu_topology_t *cpudata;
+    ErtsAlcType_t alc_type = (test
+			      ? ERTS_ALC_T_TMP
+			      : ERTS_ALC_T_CPU_GRPS_MAP);
+
+    if (map->array)
+	erts_free(alc_type, map->array);
+
+    map->array = NULL;
+    map->logical_processors = 0;
+    map->size = 0;
+
+    if (!map->groups)
+	return;
+
+    create_tmp_cpu_topology_copy(&cpudata, &avail_sz);
+
+    if (!cpudata)
+	return;
+
+    cpu_bind_order_sort(cpudata,
+			avail_sz,
+			ERTS_CPU_BIND_NO_SPREAD,
+			1);
+
+    avail = erts_alloc(ERTS_ALC_T_TMP,
+		       sizeof(erts_avail_cput)*avail_sz);
+
+    make_available_cpu_topology(&no, avail, cpudata,
+				&avail_sz, test);
+
+    destroy_tmp_cpu_topology_copy(cpudata);
+
+    map->size = avail_sz*2+1;
+
+    map->array = erts_alloc(alc_type,
+			    (sizeof(erts_cpu_groups_map_array_t)
+			     * map->size));;
+    map->logical_processors = avail_sz;
+
+    for (i = 0; i < map->size; i++) {
+	map->array[i].logical = -1;
+	map->array[i].cpu_group = -1;
+    }
+
+    spread_level = ERTS_TOPOLOGY_CORE;
+    for (i = ERTS_TOPOLOGY_NODE; i < ERTS_TOPOLOGY_THREAD; i++) {
+	if (no.level[i] > map->groups) {
+	    spread_level = i;
+	    break;
+	}
+    }
+
+    if (no.level[spread_level] <= map->groups) {
+	int a, cg, last = -1;
+	cg = -1;
+	ASSERT(spread_level == ERTS_TOPOLOGY_CORE);
+	for (a = 0; a < avail_sz; a++) {
+	    if (last != avail[a].level[spread_level]) {
+		cg++;
+		last = avail[a].level[spread_level];
+	    }
+	    cpu_group_insert(map,
+			     avail[a].level[ERTS_TOPOLOGY_LOGICAL],
+			     cg);
+	}
+    }
+    else { /* map->groups < no.level[spread_level] */
+	erts_cpu_groups_count_t *cg_count;
+	int a, cg, tl, toplevels;
+
+	tl = spread_level-1;
+
+	if (spread_level == ERTS_TOPOLOGY_NODE)
+	    toplevels = 1;
+	else
+	    toplevels = no.level[tl];
+
+	cg_count = erts_alloc(ERTS_ALC_T_TMP,
+			      toplevels*sizeof(erts_cpu_groups_count_t));
+
+	if (toplevels == 1) {
+	    cg_count[0].id = 0;
+	    cg_count[0].sub_levels = no.level[spread_level];
+	    cg_count[0].cpu_groups = map->groups;
+	}
+	else {
+	    int cgs_per_tl, cgs;
+	    cgs = map->groups;
+	    cgs_per_tl = cgs / toplevels;
+
+	    a = 0;
+	    for (i = 0; i < toplevels; i++) {
+		cg_count[i].id = avail[a].level[tl];
+		a = sub_levels(&cg_count[i], tl, a, avail_sz, avail);
+	    }
+
+	    qsort(cg_count,
+		  toplevels,
+		  sizeof(erts_cpu_groups_count_t),
+		  cg_count_sub_levels_compare);
+
+	    for (i = 0; i < toplevels; i++) {
+		if (cg_count[i].sub_levels < cgs_per_tl) {
+		    cg_count[i].cpu_groups = cg_count[i].sub_levels;
+		    cgs -= cg_count[i].sub_levels;
+		}
+		else {
+		    cg_count[i].cpu_groups = cgs_per_tl;
+		    cgs -= cgs_per_tl;
+		}
+	    }
+
+	    while (cgs > 0) {
+		for (i = 0; i < toplevels; i++) {
+		    if (cg_count[i].sub_levels == cg_count[i].cpu_groups)
+			break;
+		    else {
+			cg_count[i].cpu_groups++;
+			if (--cgs == 0)
+			    break;
+		    }
+		}
+	    }
+
+	    qsort(cg_count,
+		  toplevels,
+		  sizeof(erts_cpu_groups_count_t),
+		  cg_count_id_compare);
+	}
+
+	a = i = 0;
+	cg = -1;
+	while (a < avail_sz) {
+	    a = write_cpu_groups(&cg, &cg_count[i], tl,
+				 a, avail_sz, avail);
+	    i++;
+	}
+
+	ASSERT(map->groups == cg + 1);
+
+	for (a = 0; a < avail_sz; a++)
+	    cpu_group_insert(map,
+			     avail[a].level[ERTS_TOPOLOGY_LOGICAL],
+			     avail[a].level[ERTS_TOPOLOGY_CG]);
+
+	erts_free(ERTS_ALC_T_TMP, cg_count);
+    }
+
+    erts_free(ERTS_ALC_T_TMP, avail);
+}
+
+static erts_cpu_groups_map_t *
+add_cpu_groups(int groups,
+	       erts_cpu_groups_callback_t callback,
+	       void *arg)
+{
+    int use_groups = groups;
+    erts_cpu_groups_callback_list_t *cgcl;
+    erts_cpu_groups_map_t *cgm;
+
+    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
+
+    if (use_groups > max_main_threads)
+	use_groups = max_main_threads;
+
+    if (!use_groups)
+	return NULL;
+
+    no_cpu_groups_callbacks++;
+    cgcl = erts_alloc(ERTS_ALC_T_CPU_GRPS_MAP,
+		      sizeof(erts_cpu_groups_callback_list_t));
+    cgcl->callback = callback;
+    cgcl->arg = arg;
+
+    for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) {
+	if (cgm->groups == use_groups) {
+	    cgcl->next = cgm->callback_list;
+	    cgm->callback_list = cgcl;
+	    return cgm;
+	}
+    }
+
+
+    cgm = erts_alloc(ERTS_ALC_T_CPU_GRPS_MAP,
+		     sizeof(erts_cpu_groups_map_t));
+    cgm->next = cpu_groups_maps;
+    cgm->groups = use_groups;
+    cgm->array = NULL;
+    cgm->size = 0;
+    cgm->logical_processors = 0;
+    cgm->callback_list = cgcl;
+
+    cgcl->next = NULL;
+
+    make_cpu_groups_map(cgm, 0);
+
+    cpu_groups_maps = cgm;
+
+    return cgm;
+}
+
+static void
+remove_cpu_groups(erts_cpu_groups_callback_t callback, void *arg)
+{
+    erts_cpu_groups_map_t *prev_cgm, *cgm;
+    erts_cpu_groups_callback_list_t *prev_cgcl, *cgcl;
+
+    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
+
+    no_cpu_groups_callbacks--;
+
+    prev_cgm = NULL;
+    for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) {
+	prev_cgcl = NULL;
+	for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) {
+	    if (cgcl->callback == callback && cgcl->arg == arg) {
+		if (prev_cgcl)
+		    prev_cgcl->next = cgcl->next;
+		else
+		    cgm->callback_list = cgcl->next;
+		erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgcl);
+		if (!cgm->callback_list) {
+		    if (prev_cgm)
+			prev_cgm->next = cgm->next;
+		    else
+			cpu_groups_maps = cgm->next;
+		    if (cgm->array)
+			erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgm->array);
+		    erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgm);
+		}
+		return;
+	    }
+	    prev_cgcl = cgcl;
+	}
+	prev_cgm = cgm;
+    }
+
+    erl_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n");
+}
+
+static int
+cpu_groups_lookup(erts_cpu_groups_map_t *map,
+		  ErtsSchedulerData *esdp)
+{
+    int start, logical, ix;
+
+    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx)
+		       || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
+
+    if (esdp->cpu_id < 0)
+	return (((int) esdp->no) - 1) % map->groups;
+
+    logical = esdp->cpu_id;
+    start = logical % map->size;
+    ix = start;
+
+    do {
+	if (map->array[ix].logical == logical) {
+	    int group = map->array[ix].cpu_group;
+	    ASSERT(0 <= group && group < map->groups);
+	    return group;
+	}
+	ix++;
+	if (ix == map->size)
+	    ix = 0;
+    } while (ix != start);
+
+    erl_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical);
+}
+
+static void
+update_cpu_groups_maps(void)
+{
+    erts_cpu_groups_map_t *cgm;
+    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx));
+
+    for (cgm = cpu_groups_maps; cgm; cgm = cgm->next)
+	make_cpu_groups_map(cgm, 0);
+}
+
+void
+erts_add_cpu_groups(int groups,
+		    erts_cpu_groups_callback_t callback,
+		    void *arg)
+{
+    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+    add_cpu_groups(groups, callback, arg);
+    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+}
+
+void erts_remove_cpu_groups(erts_cpu_groups_callback_t callback,
+			    void *arg)
+{
+    erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx);
+    remove_cpu_groups(callback, arg);
+    erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx);
+}
diff --git a/erts/emulator/beam/erl_cpu_topology.h b/erts/emulator/beam/erl_cpu_topology.h
new file mode 100644
index 0000000000..c5a9520b61
--- /dev/null
+++ b/erts/emulator/beam/erl_cpu_topology.h
@@ -0,0 +1,105 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description:	CPU topology and related functionality
+ *
+ * Author: 	Rickard Green
+ */
+
+#ifndef ERL_CPU_TOPOLOGY_H__
+#define ERL_CPU_TOPOLOGY_H__
+
+void erts_pre_early_init_cpu_topology(int *max_rg_p,
+				      int *conf_p,
+				      int *onln_p,
+				      int *avail_p);
+void erts_early_init_cpu_topology(int no_schedulers,
+				  int *max_main_threads_p,
+				  int max_reader_groups,
+				  int *reader_groups_p);
+void erts_init_cpu_topology(void);
+
+
+#define ERTS_INIT_SCHED_BIND_TYPE_SUCCESS		0
+#define ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED		1
+#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY	2
+#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE	3
+
+int erts_init_scheduler_bind_type_string(char *how);
+
+
+#define ERTS_INIT_CPU_TOPOLOGY_OK			0
+#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID		1
+#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE		2
+#define ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY	3
+#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE		4
+#define ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES		5
+#define ERTS_INIT_CPU_TOPOLOGY_MISSING_LID		6
+#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS		7
+#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES	8
+#define ERTS_INIT_CPU_TOPOLOGY_MISSING			9
+
+int erts_init_cpu_topology_string(char *topology_str);
+
+void erts_sched_check_cpu_bind(ErtsSchedulerData *esdp);
+#ifdef ERTS_SMP
+void erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp);
+void erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp);
+void erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp);
+#endif
+
+int erts_update_cpu_info(void);
+
+Eterm erts_bind_schedulers(Process *c_p, Eterm how);
+Eterm erts_get_schedulers_binds(Process *c_p);
+
+Eterm erts_get_reader_groups_map(Process *c_p);
+
+Eterm erts_set_cpu_topology(Process *c_p, Eterm term);
+Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which);
+
+int erts_update_cpu_info(void);
+void erts_get_logical_processors(int *conf, int *onln, int *avail);
+
+int erts_sched_bind_atthrcreate_prepare(void);
+int erts_sched_bind_atthrcreate_child(int unbind);
+void erts_sched_bind_atthrcreate_parent(int unbind);
+
+int erts_sched_bind_atfork_prepare(void);
+int erts_sched_bind_atfork_child(int unbind);
+char *erts_sched_bind_atvfork_child(int unbind);
+void erts_sched_bind_atfork_parent(int unbind);
+
+Eterm erts_fake_scheduler_bindings(Process *p, Eterm how);
+Eterm erts_debug_cpu_groups_map(Process *c_p, int groups);
+
+
+typedef void (*erts_cpu_groups_callback_t)(int,
+					   ErtsSchedulerData *,
+					   int,
+					   void *);
+
+void erts_add_cpu_groups(int groups,
+			 erts_cpu_groups_callback_t callback,
+			 void *arg);
+void erts_remove_cpu_groups(erts_cpu_groups_callback_t callback,
+			    void *arg);
+
+#endif
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index 15b1c6bb56..51bdf53823 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -78,16 +78,24 @@ enum DbIterSafety {
 ** The main meta table, containing all ets tables.
 */
 #ifdef ERTS_SMP
-#  define META_MAIN_TAB_LOCK_CNT 16
-static union {
-    erts_smp_spinlock_t lck;
-    byte _cache_line_alignment[64];
-}meta_main_tab_locks[META_MAIN_TAB_LOCK_CNT];
+
+#define ERTS_META_MAIN_TAB_LOCK_TAB_BITS 8
+#define ERTS_META_MAIN_TAB_LOCK_TAB_SIZE (1 << ERTS_META_MAIN_TAB_LOCK_TAB_BITS)
+#define ERTS_META_MAIN_TAB_LOCK_TAB_MASK (ERTS_META_MAIN_TAB_LOCK_TAB_SIZE - 1)
+
+typedef union {
+    erts_smp_rwmtx_t rwmtx;
+    byte cache_line_align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(
+				sizeof(erts_smp_rwmtx_t))];
+} erts_meta_main_tab_lock_t;
+
+static erts_meta_main_tab_lock_t *meta_main_tab_locks;
+
 #endif
 static struct {
     union {
 	DbTable *tb;     /* Only directly readable if slot is ALIVE */
-	Uint next_free;  /* (index<<2)|1 if slot is FREE */
+	UWord next_free;  /* (index<<2)|1 if slot is FREE */
     }u;
 } *meta_main_tab;
 
@@ -104,17 +112,13 @@ static struct {
 #define MARK_SLOT_DEAD(i) (meta_main_tab[(i)].u.next_free |= 2)
 #define GET_ANY_SLOT_TAB(i) ((DbTable*)(meta_main_tab[(i)].u.next_free & ~(1|2))) /* dead or alive */
 
-static ERTS_INLINE void meta_main_tab_lock(unsigned slot)
+static ERTS_INLINE erts_smp_rwmtx_t *
+get_meta_main_tab_lock(unsigned slot)
 {
 #ifdef ERTS_SMP
-    erts_smp_spin_lock(&meta_main_tab_locks[slot % META_MAIN_TAB_LOCK_CNT].lck);
-#endif
-}
-
-static ERTS_INLINE void meta_main_tab_unlock(unsigned slot)
-{
-#ifdef ERTS_SMP
-    erts_smp_spin_unlock(&meta_main_tab_locks[slot % META_MAIN_TAB_LOCK_CNT].lck);
+    return &meta_main_tab_locks[slot & ERTS_META_MAIN_TAB_LOCK_TAB_MASK].rwmtx;
+#else
+    return NULL;
 #endif
 }
 
@@ -125,8 +129,6 @@ static Uint meta_main_tab_slot_mask;    /* The slot index part of an unnamed tab
 static Uint meta_main_tab_seq_incr;
 static Uint meta_main_tab_seq_cnt = 0;  /* To give unique(-ish) table identifiers */
 
-
-
 /* 
 ** The meta hash table of all NAMED ets tables
 */
@@ -166,7 +168,8 @@ struct meta_name_tab_entry* meta_name_tab_bucket(Eterm name,
 typedef enum {
     LCK_READ=1,     /* read only access */
     LCK_WRITE=2,    /* exclusive table write access */
-    LCK_WRITE_REC=3 /* record write access */
+    LCK_WRITE_REC=3, /* record write access */
+    LCK_NONE=4
 } db_lock_kind_t;
 
 extern DbTableMethod db_hash;
@@ -174,6 +177,7 @@ extern DbTableMethod db_tree;
 
 int user_requested_db_max_tabs;
 int erts_ets_realloc_always_moves;
+int erts_ets_always_compress;
 static int db_max_tabs;
 static DbTable *meta_pid_to_tab; /* Pid mapped to owned tables */
 static DbTable *meta_pid_to_fixed_tab; /* Pid mapped to fixed tables */
@@ -187,7 +191,7 @@ static Eterm ms_delete_all_buff[8]; /* To compare with for deletion
 
 static void fix_table_locked(Process* p, DbTable* tb);
 static void unfix_table_locked(Process* p,  DbTable* tb, db_lock_kind_t* kind);
-static void set_heir(Process* me, DbTable* tb, Eterm heir, Eterm heir_data);
+static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data);
 static void free_heir_data(DbTable*);
 static void free_fixations_locked(DbTable *tb);
 
@@ -196,12 +200,17 @@ static int free_table_cont(Process *p,
 			   int first,
 			   int clean_meta_tab);
 static void print_table(int to, void *to_arg, int show,  DbTable* tb);
-static BIF_RETTYPE ets_select_delete_1(Process *p, Eterm a1);
-static BIF_RETTYPE ets_select_count_1(Process *p, Eterm a1);
-static BIF_RETTYPE ets_select_trap_1(Process *p, Eterm a1);
-static BIF_RETTYPE ets_delete_trap(Process *p, Eterm a1);
+static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1);
+static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1);
+static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1);
+static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1);
 static Eterm table_info(Process* p, DbTable* tb, Eterm What);
 
+static BIF_RETTYPE ets_select1(Process* p, Eterm arg1);
+static BIF_RETTYPE ets_select2(Process* p, Eterm arg1, Eterm arg2);
+static BIF_RETTYPE ets_select3(Process* p, Eterm arg1, Eterm arg2, Eterm arg3);
+
+
 /* 
  * Exported global
  */
@@ -213,61 +222,87 @@ Export ets_select_continue_exp;
  * Static traps
  */
 static Export ets_delete_continue_exp;
-
-static ERTS_INLINE DbTable* db_ref(DbTable* tb)
-{
-    if (tb != NULL) {
-	erts_refc_inc(&tb->common.ref, 2);
-    }
-    return tb;
-}
-
-static ERTS_INLINE DbTable* db_unref(DbTable* tb)
+	
+static void
+free_dbtable(DbTable* tb)
 {
-    if (!erts_refc_dectest(&tb->common.ref, 0)) {
 #ifdef HARDDEBUG
-	if (erts_smp_atomic_read(&tb->common.memory_size) != sizeof(DbTable)) {
-	    erts_fprintf(stderr, "ets: db_unref memory remain=%ld fix=%x\n",
-			 erts_smp_atomic_read(&tb->common.memory_size)-sizeof(DbTable), 
+	if (erts_smp_atomic_read_nob(&tb->common.memory_size) != sizeof(DbTable)) {
+	    erts_fprintf(stderr, "ets: free_dbtable memory remain=%ld fix=%x\n",
+			 erts_smp_atomic_read_nob(&tb->common.memory_size)-sizeof(DbTable),
 			 tb->common.fixations);
 	}
-	erts_fprintf(stderr, "ets: db_unref(%T) deleted!!!\r\n", 
+	erts_fprintf(stderr, "ets: free_dbtable(%T) deleted!!!\r\n",
 		     tb->common.id);
 
-	erts_fprintf(stderr, "ets: db_unref: meta_pid_to_tab common.memory_size = %ld\n",
-		     erts_smp_atomic_read(&meta_pid_to_tab->common.memory_size));
+	erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_tab common.memory_size = %ld\n",
+		     erts_smp_atomic_read_nob(&meta_pid_to_tab->common.memory_size));
 	print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_tab);
 
 
-	erts_fprintf(stderr, "ets: db_unref: meta_pid_to_fixed_tab common.memory_size = %ld\n",
-		     erts_smp_atomic_read(&meta_pid_to_fixed_tab->common.memory_size));
+	erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_fixed_tab common.memory_size = %ld\n",
+		     erts_smp_atomic_read_nob(&meta_pid_to_fixed_tab->common.memory_size));
 	print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_fixed_tab);
-	
 #endif
 #ifdef ERTS_SMP
 	erts_smp_rwmtx_destroy(&tb->common.rwlock);
 	erts_smp_mtx_destroy(&tb->common.fixlock);
 #endif
 	ASSERT(is_immed(tb->common.heir_data));
-	erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable));		     
+	erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable));
 	ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable));
-	return NULL;
-    }
-    return tb;
+	ERTS_SMP_MEMORY_BARRIER;
 }
 
-static ERTS_INLINE void db_init_lock(DbTable* tb, char *rwname, char* fixname)
+#ifdef ERTS_SMP
+static void
+chk_free_dbtable(void *vtb)
+{
+    DbTable * tb = (DbTable *) vtb;
+    ERTS_THR_MEMORY_BARRIER;
+    if (erts_refc_dectest(&tb->common.ref, 0) == 0)
+	free_dbtable(tb);
+}
+#endif
+
+static void schedule_free_dbtable(DbTable* tb)
 {
-    erts_refc_init(&tb->common.ref, 1);
-    erts_refc_init(&tb->common.fixref, 0);
+    /*
+     * NON-SMP case: Caller is *not* allowed to access the *tb
+     *               structure after this function has returned!          
+     * SMP case:     Caller is allowed to access the *tb structure
+     *               until the bif has returned (we typically
+     *               need to unlock the table lock after this
+     *               function has returned).
+     */
+#ifdef ERTS_SMP
+    int scheds = erts_get_max_no_executing_schedulers();
+    ASSERT(scheds >= 1);
+    ASSERT(erts_refc_read(&tb->common.ref, 0) == 0);
+    erts_refc_init(&tb->common.ref, scheds);
+    erts_schedule_multi_misc_aux_work(0, scheds, chk_free_dbtable, tb);
+#else
+    free_dbtable(tb);
+#endif
+}
+
+static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock,
+				     char *rwname, char* fixname)
+{
+#ifdef ERTS_SMP
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+    if (use_frequent_read_lock)
+	rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+#endif
 #ifdef ERTS_SMP
-    erts_smp_rwmtx_init_x(&tb->common.rwlock, rwname, tb->common.the_name);
+    erts_smp_rwmtx_init_opt_x(&tb->common.rwlock, &rwmtx_opt,
+			      rwname, tb->common.the_name);
     erts_smp_mtx_init_x(&tb->common.fixlock, fixname, tb->common.the_name);
     tb->common.is_thread_safe = !(tb->common.status & DB_FINE_LOCKED);
 #endif
 }
 
-static ERTS_INLINE void db_lock_take_over_ref(DbTable* tb, db_lock_kind_t kind)
+static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind)
 {
 #ifdef ERTS_SMP
     ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab);
@@ -295,27 +330,24 @@ static ERTS_INLINE void db_lock_take_over_ref(DbTable* tb, db_lock_kind_t kind)
 #endif
 }
 
-static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind)
-{
-    (void) db_ref(tb);
-#ifdef ERTS_SMP
-    db_lock_take_over_ref(tb, kind);
-#endif
-}
-
 static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind)
 {
+    /*
+     * In NON-SMP case tb may refer to an already deallocated
+     * DbTable structure. That is, ONLY the SMP case is allowed
+     * to follow the tb pointer!
+     */
 #ifdef ERTS_SMP
     ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab);
 
     if (tb->common.type & DB_FINE_LOCKED) {
-	if (tb->common.is_thread_safe) {
-	    ASSERT(kind == LCK_WRITE);
+	if (kind == LCK_WRITE) {
+	    ASSERT(tb->common.is_thread_safe);
 	    tb->common.is_thread_safe = 0;
 	    erts_smp_rwmtx_rwunlock(&tb->common.rwlock);
 	}
 	else {
-	    ASSERT(kind != LCK_WRITE);
+	    ASSERT(!tb->common.is_thread_safe);
 	    erts_smp_rwmtx_runlock(&tb->common.rwlock);
 	}
     }
@@ -331,7 +363,6 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind)
 	}
     }
 #endif
-    (void) db_unref(tb); /* May delete table... */
 }
 
 
@@ -349,56 +380,85 @@ static ERTS_INLINE void db_meta_unlock(DbTable* tb, db_lock_kind_t kind)
 }
 
 static ERTS_INLINE
-DbTable* db_get_table(Process *p,
-		      Eterm id,
-		      int what,
-		      db_lock_kind_t kind)
+DbTable* db_get_table_aux(Process *p,
+			  Eterm id,
+			  int what,
+			  db_lock_kind_t kind,
+			  int meta_already_locked)
 {
     DbTable *tb = NULL;
+    erts_smp_rwmtx_t *mtl = NULL;
+
+    /*
+     * IMPORTANT: Only scheduler threads are allowed
+     *            to access tables. Memory management
+     *            depend on it.
+     */
+    ASSERT(erts_get_scheduler_data());
 
     if (is_small(id)) {
 	Uint slot = unsigned_val(id) & meta_main_tab_slot_mask;
-	meta_main_tab_lock(slot);
-	if (slot < db_max_tabs && IS_SLOT_ALIVE(slot)) {
-	    /* SMP: inc to prevent race, between unlock of meta_main_tab_lock
-	     * and the table locking outside the meta_main_tab_lock
-	     */
-	    tb = db_ref(meta_main_tab[slot].u.tb);
+	if (!meta_already_locked) {
+	    mtl = get_meta_main_tab_lock(slot);
+	    erts_smp_rwmtx_rlock(mtl);
+	}
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+	else {
+	    erts_smp_rwmtx_t *test_mtl = get_meta_main_tab_lock(slot);
+	    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(test_mtl)
+			       || erts_lc_rwmtx_is_rwlocked(test_mtl));
 	}
-	meta_main_tab_unlock(slot);
+#endif
+	if (slot < db_max_tabs && IS_SLOT_ALIVE(slot))
+	    tb = meta_main_tab[slot].u.tb;
     }
     else if (is_atom(id)) {
-	erts_smp_rwmtx_t* rwlock;
-	struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&rwlock);
-	erts_smp_rwmtx_rlock(rwlock);
+	struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&mtl);
+	if (!meta_already_locked)
+	    erts_smp_rwmtx_rlock(mtl);
+	else{
+	    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(mtl)
+			       || erts_lc_rwmtx_is_rwlocked(mtl));
+	    mtl = NULL;
+	}
+
 	if (bucket->pu.tb != NULL) {
 	    if (is_atom(bucket->u.name_atom)) { /* single */
-		if (bucket->u.name_atom == id) {
-		    tb = db_ref(bucket->pu.tb);
-		}
+		if (bucket->u.name_atom == id)
+		    tb = bucket->pu.tb;
 	    }
 	    else { /* multi */
 		Uint cnt = unsigned_val(bucket->u.mcnt);
 		Uint i;
 		for (i=0; i<cnt; i++) {
 		    if (bucket->pu.mvec[i].u.name_atom == id) {
-			tb = db_ref(bucket->pu.mvec[i].pu.tb);
+			tb = bucket->pu.mvec[i].pu.tb;
 			break;
 		    }
 		}
 	    }
 	}
-	erts_smp_rwmtx_runlock(rwlock);
     }
     if (tb) {
-	db_lock_take_over_ref(tb, kind);
-	if (tb->common.id == id && ((tb->common.status & what) != 0 || 
-				    p->id == tb->common.owner)) {
-	    return tb;
+	db_lock(tb, kind);
+	if (tb->common.id != id
+	    || ((tb->common.status & what) == 0 && p->id != tb->common.owner)) {
+	    db_unlock(tb, kind);
+	    tb = NULL;
 	}
-	db_unlock(tb, kind);
     }
-    return NULL;
+    if (mtl)
+	erts_smp_rwmtx_runlock(mtl);
+    return tb;
+}
+
+static ERTS_INLINE
+DbTable* db_get_table(Process *p,
+		      Eterm id,
+		      int what,
+		      db_lock_kind_t kind)
+{
+    return db_get_table_aux(p, id, what, kind, 0);
 }
 
 /* Requires meta_main_tab_locks[slot] locked.
@@ -413,15 +473,15 @@ static ERTS_INLINE void free_slot(int slot)
     erts_smp_spin_unlock(&meta_main_tab_main_lock);
 }
 
-static int insert_named_tab(Eterm name_atom, DbTable* tb)
+static int insert_named_tab(Eterm name_atom, DbTable* tb, int have_lock)
 {
     int ret = 0;
     erts_smp_rwmtx_t* rwlock;
     struct meta_name_tab_entry* new_entry;
     struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom,
 							      &rwlock);
-
-    erts_smp_rwmtx_rwlock(rwlock);
+    if (!have_lock)
+	erts_smp_rwmtx_rwlock(rwlock);
 
     if (bucket->pu.tb == NULL) { /* empty */
 	new_entry = bucket;
@@ -468,17 +528,32 @@ static int insert_named_tab(Eterm name_atom, DbTable* tb)
     ret = 1; /* Ok */
 
 done:
-    erts_smp_rwmtx_rwunlock(rwlock);
+    if (!have_lock)
+	erts_smp_rwmtx_rwunlock(rwlock);
     return ret;
 }
 
-static int remove_named_tab(Eterm name_atom)
+static int remove_named_tab(DbTable *tb, int have_lock)
 {
     int ret = 0;
     erts_smp_rwmtx_t* rwlock;
+    Eterm name_atom = tb->common.id;
     struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom,
 							      &rwlock);
-    erts_smp_rwmtx_rwlock(rwlock);
+#ifdef ERTS_SMP
+    if (!have_lock && erts_smp_rwmtx_tryrwlock(rwlock) == EBUSY) {
+	/*
+	 * We keep our increased refc over this op in order to
+	 * prevent the table from disapearing.
+	 */
+	db_unlock(tb, LCK_WRITE);
+	erts_smp_rwmtx_rwlock(rwlock);
+	db_lock(tb, LCK_WRITE);
+    }
+#endif
+
+    ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(rwlock));
+
     if (bucket->pu.tb == NULL) {
 	goto done;
     }
@@ -529,7 +604,8 @@ static int remove_named_tab(Eterm name_atom)
     ret = 1; /* Ok */
 
 done:
-    erts_smp_rwmtx_rwunlock(rwlock);
+    if (!have_lock)
+	erts_smp_rwmtx_rwunlock(rwlock);
     return ret;
 }
 
@@ -538,11 +614,11 @@ done:
 */
 static ERTS_INLINE void local_fix_table(DbTable* tb)
 {
-    erts_refc_inc(&tb->common.fixref, 1);
+    erts_refc_inc(&tb->common.ref, 1);
 }	    
 static ERTS_INLINE void local_unfix_table(DbTable* tb)
 {	
-    if (erts_refc_dectest(&tb->common.fixref, 0) == 0) {
+    if (erts_refc_dectest(&tb->common.ref, 0) == 0) {
 	ASSERT(IS_HASH_TABLE(tb->common.status));
 	db_unfix_table_hash(&(tb->hash));
     }
@@ -704,12 +780,13 @@ BIF_RETTYPE ets_update_element_3(BIF_ALIST_3)
     int cret = DB_ERROR_BADITEM;
     Eterm list;
     Eterm iter;
-    Eterm cell[2];
+    DeclareTmpHeap(cell,2,BIF_P);
     DbUpdateHandle handle;
 
     if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
 	BIF_ERROR(BIF_P, BADARG);
     }
+    UseTmpHeap(2,BIF_P);
     if (!(tb->common.status & (DB_SET | DB_ORDERED_SET))) {
 	goto bail_out;
     }
@@ -762,6 +839,7 @@ finalize:
     tb->common.meth->db_finalize_dbterm(&handle);
 
 bail_out:
+    UnUseTmpHeap(2,BIF_P);
     db_unlock(tb, LCK_WRITE_REC);
 
     switch (cret) {
@@ -794,8 +872,8 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
     Eterm* ret_list_currp = NULL;
     Eterm* ret_list_prevp = NULL;
     Eterm iter;
-    Eterm cell[2];
-    Eterm tuple[3];
+    DeclareTmpHeap(cell,5,BIF_P);
+    Eterm *tuple = cell+2;
     DbUpdateHandle handle;
     Uint halloc_size = 0; /* overestimated heap usage */
     Eterm* htop;          /* actual heap usage */
@@ -805,6 +883,9 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
     if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC)) == NULL) {
 	BIF_ERROR(BIF_P, BADARG);
     }
+
+    UseTmpHeap(5,BIF_P);
+
     if (!(tb->common.status & (DB_SET | DB_ORDERED_SET))) {
 	goto bail_out;
     }
@@ -832,7 +913,8 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
 	Eterm upop;
 	Eterm* tpl;
 	Sint position;
-	Eterm incr, warp, oldcnt;
+	Eterm incr, warp;
+	Wterm oldcnt;
 
 	if (is_not_list(iter)) {
 	    goto finalize;
@@ -871,7 +953,7 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
 		position > arityval(handle.dbterm->tpl[0])) {
 		goto finalize;
 	    }
-	    oldcnt = handle.dbterm->tpl[position];
+	    oldcnt = db_do_read_element(&handle, position);
 	    if (is_big(oldcnt)) {
 		halloc_size += BIG_NEED_SIZE(big_arity(oldcnt));
 	    }
@@ -907,7 +989,7 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
 	Eterm* tpl = tuple_val(CAR(list_val(iter)));
 	Sint position = signed_val(tpl[1]);
 	Eterm incr = tpl[2];
-	Eterm oldcnt = handle.dbterm->tpl[position];
+	Wterm oldcnt = db_do_read_element(&handle,position);
 	Eterm newcnt = db_add_counter(&htop, oldcnt, incr);
 
 	if (newcnt == NIL) {
@@ -920,9 +1002,9 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3)
 
 	if (arityval(*tpl) == 4) { /* Maybe warp it */
 	    Eterm threshold = tpl[3];
-	    if ((cmp(incr,make_small(0)) < 0) ? /* negative increment? */
-		(cmp(newcnt,threshold) < 0) :  /* if negative, check if below */
-		(cmp(newcnt,threshold) > 0)) { /* else check if above threshold */
+	    if ((CMP(incr,make_small(0)) < 0) ? /* negative increment? */
+		(CMP(newcnt,threshold) < 0) :  /* if negative, check if below */
+		(CMP(newcnt,threshold) > 0)) { /* else check if above threshold */
 
 		newcnt = tpl[4];
 	    }
@@ -951,6 +1033,7 @@ finalize:
     tb->common.meth->db_finalize_dbterm(&handle);
 
 bail_out:
+    UnUseTmpHeap(5,BIF_P);
     db_unlock(tb, LCK_WRITE_REC);
 
     switch (cret) {
@@ -1127,6 +1210,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
 {
     DbTable* tb;
     Eterm ret;
+    erts_smp_rwmtx_t *lck1, *lck2;
 
 #ifdef HARDDEBUG
     erts_fprintf(stderr,
@@ -1135,34 +1219,65 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2)
 		BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]);
 #endif
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL) {
+
+    if (is_not_atom(BIF_ARG_2)) {
 	BIF_ERROR(BIF_P, BADARG);
     }
 
-    if (is_not_atom(BIF_ARG_2)) {
-	goto badarg;
+    (void) meta_name_tab_bucket(BIF_ARG_2, &lck1);
+
+    if (is_small(BIF_ARG_1)) {
+	Uint slot = unsigned_val(BIF_ARG_1) & meta_main_tab_slot_mask;
+	lck2 = get_meta_main_tab_lock(slot);
     }
+    else if (is_atom(BIF_ARG_1)) {
+	(void) meta_name_tab_bucket(BIF_ARG_1, &lck2);
+	if (lck1 == lck2)
+	    lck2 = NULL;
+	else if (lck1 > lck2) {
+	    erts_smp_rwmtx_t *tmp = lck1;
+	    lck1 = lck2;
+	    lck2 = tmp;
+	}
+    }
+    else {
+	BIF_ERROR(BIF_P, BADARG);
+    }
+
+    erts_smp_rwmtx_rwlock(lck1);
+    if (lck2)
+	erts_smp_rwmtx_rwlock(lck2);
+
+    tb = db_get_table_aux(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE, 1);
+    if (!tb)
+	goto badarg;
 
     if (is_not_atom(tb->common.id)) { /* Not a named table */
 	tb->common.the_name = BIF_ARG_2;
 	goto done;
     }
 
-    if (!insert_named_tab(BIF_ARG_2,tb)) {
+    if (!insert_named_tab(BIF_ARG_2, tb, 1))
 	goto badarg;
-    }
-    if (!remove_named_tab(tb->common.id)) {
+
+    if (!remove_named_tab(tb, 1))
 	erl_exit(1,"Could not find named tab %s", tb->common.id);
-    }
 
     tb->common.id = tb->common.the_name = BIF_ARG_2;
 
  done:
     ret = tb->common.id;
     db_unlock(tb, LCK_WRITE);
+    erts_smp_rwmtx_rwunlock(lck1);
+    if (lck2)
+	erts_smp_rwmtx_rwunlock(lck2);
     BIF_RET(ret);
  badarg:
-    db_unlock(tb, LCK_WRITE);
+    if (tb)
+	db_unlock(tb, LCK_WRITE);
+    erts_smp_rwmtx_rwunlock(lck1);
+    if (lck2)
+	erts_smp_rwmtx_rwunlock(lck2);
     BIF_ERROR(BIF_P, BADARG);    
 }
 
@@ -1180,13 +1295,19 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     Eterm val;
     Eterm ret;
     Eterm heir;
-    Eterm heir_data;
+    UWord heir_data;
     Uint32 status;
     Sint keypos;
-    int is_named, is_fine_locked;
+    int is_named, is_compressed;
+#ifdef ERTS_SMP
+    int is_fine_locked, frequent_read;
+#endif
+#ifdef DEBUG
     int cret;
-    Eterm meta_tuple[3];
+#endif
+    DeclareTmpHeap(meta_tuple,3,BIF_P);
     DbTableMethod* meth;
+    erts_smp_rwmtx_t *mmtl;
 
     if (is_not_atom(BIF_ARG_1)) {
 	BIF_ERROR(BIF_P, BADARG);
@@ -1198,9 +1319,13 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     status = DB_NORMAL | DB_SET | DB_PROTECTED;
     keypos = 1;
     is_named = 0;
+#ifdef ERTS_SMP
     is_fine_locked = 0;
+    frequent_read = 0;
+#endif
     heir = am_none;
-    heir_data = am_undefined;
+    heir_data = (UWord) am_undefined;
+    is_compressed = erts_ets_always_compress;
 
     list = BIF_ARG_2;
     while(is_list(list)) {
@@ -1226,11 +1351,31 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 		    keypos = signed_val(tp[2]);
 		}		
 		else if (tp[1] == am_write_concurrency) {
+#ifdef ERTS_SMP
 		    if (tp[2] == am_true) {
 			is_fine_locked = 1;
 		    } else if (tp[2] == am_false) {
 			is_fine_locked = 0;
 		    } else break;
+#else
+		    if ((tp[2] != am_true) &&  (tp[2] != am_false)) {
+			break;
+		    }
+#endif
+		}
+		else if (tp[1] == am_read_concurrency) {
+#ifdef ERTS_SMP
+		    if (tp[2] == am_true) {
+			frequent_read = 1;
+		    } else if (tp[2] == am_false) {
+			frequent_read = 0;
+		    } else break;
+#else
+		    if ((tp[2] != am_true) &&  (tp[2] != am_false)) {
+			break;
+		    }
+#endif
+		    
 		}
 		else if (tp[1] == am_heir && tp[2] == am_none) {
 		    heir = am_none;
@@ -1256,6 +1401,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 	else if (val == am_named_table) {
 	    is_named = 1;
 	}
+	else if (val == am_compressed) {
+	    is_compressed = 1;
+	}
 	else if (val == am_set || val == am_protected)
 	    ;
 	else break;
@@ -1267,11 +1415,11 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     }
     if (IS_HASH_TABLE(status)) {
 	meth = &db_hash;
-	#ifdef ERTS_SMP
+#ifdef ERTS_SMP
 	if (is_fine_locked && !(status & DB_PRIVATE)) {
 	    status |= DB_FINE_LOCKED;
 	}
-	#endif
+#endif
     }
     else if (IS_TREE_TABLE(status)) {
 	meth = &db_tree;
@@ -1280,6 +1428,11 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 	BIF_ERROR(BIF_P, BADARG);
     }
 
+#ifdef ERTS_SMP
+    if (frequent_read && !(status & DB_PRIVATE))
+	status |= DB_FREQ_READ;
+#endif
+
     /* we create table outside any table lock
      * and take the unusal cost of destroy table if it
      * fails to find a slot 
@@ -1287,12 +1440,12 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     {
         DbTable init_tb;
 
-	erts_smp_atomic_init(&init_tb.common.memory_size, 0);
+	erts_smp_atomic_init_nob(&init_tb.common.memory_size, 0);
 	tb = (DbTable*) erts_db_alloc(ERTS_ALC_T_DB_TABLE,
 				      &init_tb, sizeof(DbTable));
 	ERTS_ETS_MISC_MEM_ADD(sizeof(DbTable));
-	erts_smp_atomic_init(&tb->common.memory_size,
-			     erts_smp_atomic_read(&init_tb.common.memory_size));
+	erts_smp_atomic_init_nob(&tb->common.memory_size,
+				 erts_smp_atomic_read_nob(&init_tb.common.memory_size));
     }
 
     tb->common.meth = meth;
@@ -1302,16 +1455,22 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     tb->common.type = status & ERTS_ETS_TABLE_TYPES;
     /* Note, 'type' is *read only* from now on... */
 #endif
-    db_init_lock(tb, "db_tab", "db_tab_fix");
+    erts_refc_init(&tb->common.ref, 0);
+    db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ),
+		 "db_tab", "db_tab_fix");
     tb->common.keypos = keypos;
     tb->common.owner = BIF_P->id;
     set_heir(BIF_P, tb, heir, heir_data);
 
-    erts_smp_atomic_init(&tb->common.nitems, 0);
+    erts_smp_atomic_init_nob(&tb->common.nitems, 0);
 
     tb->common.fixations = NULL;
+    tb->common.compress = is_compressed;
 
-    cret = meth->db_create(BIF_P, tb);
+#ifdef DEBUG
+    cret = 
+#endif
+	meth->db_create(BIF_P, tb);
     ASSERT(cret == DB_ERROR_NONE);
 
     erts_smp_spin_lock(&meta_main_tab_main_lock);
@@ -1322,8 +1481,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 				      "** Too many db tables **\n");
 	free_heir_data(tb);
 	tb->common.meth->db_free_table(tb);
-	erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable));
-	ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable));
+	free_dbtable(tb);
 	BIF_ERROR(BIF_P, SYSTEM_LIMIT);
     }
 
@@ -1345,19 +1503,22 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     tb->common.id = ret;
     tb->common.slot = slot;           /* store slot for erase */
 
-    meta_main_tab_lock(slot);
+    mmtl = get_meta_main_tab_lock(slot);
+    erts_smp_rwmtx_rwlock(mmtl);
     meta_main_tab[slot].u.tb = tb;
     ASSERT(IS_SLOT_ALIVE(slot));
-    meta_main_tab_unlock(slot);
+    erts_smp_rwmtx_rwunlock(mmtl);
 
-    if (is_named && !insert_named_tab(BIF_ARG_1, tb)) {
-	meta_main_tab_lock(slot);
+    if (is_named && !insert_named_tab(BIF_ARG_1, tb, 0)) {
+	mmtl = get_meta_main_tab_lock(slot);
+	erts_smp_rwmtx_rwlock(mmtl);
 	free_slot(slot);
-	meta_main_tab_unlock(slot);
+	erts_smp_rwmtx_rwunlock(mmtl);
 
-	db_lock_take_over_ref(tb,LCK_WRITE);
+	db_lock(tb,LCK_WRITE);
 	free_heir_data(tb);
 	tb->common.meth->db_free_table(tb);
+	schedule_free_dbtable(tb);
 	db_unlock(tb,LCK_WRITE);
 	BIF_ERROR(BIF_P, BADARG);
     }
@@ -1370,11 +1531,13 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
 		 BIF_ARG_1, BIF_ARG_2, ret, BIF_P->id,
 		 BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]);
 	erts_fprintf(stderr, "ets: new: meta_pid_to_tab common.memory_size = %ld\n",
-		     erts_smp_atomic_read(&meta_pid_to_tab->common.memory_size));
+		     erts_smp_atomic_read_nob(&meta_pid_to_tab->common.memory_size));
 	erts_fprintf(stderr, "ets: new: meta_pid_to_fixed_tab common.memory_size = %ld\n",
-		     erts_smp_atomic_read(&meta_pid_to_fixed_tab->common.memory_size));
+		     erts_smp_atomic_read_nob(&meta_pid_to_fixed_tab->common.memory_size));
 #endif
 
+    UseTmpHeap(3,BIF_P);
+
     db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
     if (db_put_hash(meta_pid_to_tab,
 		    TUPLE2(meta_tuple, BIF_P->id, make_small(slot)),
@@ -1383,6 +1546,8 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2)
     }
     db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
 
+    UnUseTmpHeap(3,BIF_P);
+
     BIF_RET(ret);
 }
 
@@ -1489,6 +1654,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 {
     int trap;
     DbTable* tb;
+    erts_smp_rwmtx_t *mmtl;
 
 #ifdef HARDDEBUG
     erts_fprintf(stderr,
@@ -1510,16 +1676,8 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
     tb->common.status &= ~(DB_PROTECTED|DB_PUBLIC|DB_PRIVATE);
     tb->common.status |= DB_DELETE;
 
-    meta_main_tab_lock(tb->common.slot);
-    /* We must keep the slot, to be found by db_proc_dead() if process dies */
-    MARK_SLOT_DEAD(tb->common.slot);
-    meta_main_tab_unlock(tb->common.slot);
-    if (is_atom(tb->common.id)) {
-	remove_named_tab(tb->common.id);
-    }
-    
     if (tb->common.owner != BIF_P->id) {
-	Eterm meta_tuple[3];
+	DeclareTmpHeap(meta_tuple,3,BIF_P);
 
 	/*
 	 * The table is being deleted by a process other than its owner.
@@ -1527,6 +1685,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 	 * current process will be killed (e.g. by an EXIT signal), we will
 	 * now transfer the ownership to the current process.
 	 */
+	UseTmpHeap(3,BIF_P);
 	db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
 	db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner,
 			    make_small(tb->common.slot));
@@ -1538,7 +1697,27 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 		    TUPLE2(meta_tuple,BIF_P->id,make_small(tb->common.slot)),
 		    0);
 	db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
+	UnUseTmpHeap(3,BIF_P);
     }    
+
+    mmtl = get_meta_main_tab_lock(tb->common.slot);
+#ifdef ERTS_SMP
+    if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) {
+	/*
+	 * We keep our increased refc over this op in order to
+	 * prevent the table from disapearing.
+	 */
+	db_unlock(tb, LCK_WRITE);
+	erts_smp_rwmtx_rwlock(mmtl);
+	db_lock(tb, LCK_WRITE);
+    }
+#endif
+    /* We must keep the slot, to be found by db_proc_dead() if process dies */
+    MARK_SLOT_DEAD(tb->common.slot);
+    erts_smp_rwmtx_rwunlock(mmtl);
+    if (is_atom(tb->common.id))
+	remove_named_tab(tb, 0);
+    
     /* disable inheritance */
     free_heir_data(tb);
     tb->common.heir = am_none;
@@ -1554,9 +1733,15 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1)
 	 * (it looks like an continuation pointer), but that is will crash the
 	 * emulator if this BIF is call traced.
 	 */
+#if HALFWORD_HEAP
+	Eterm *hp = HAlloc(BIF_P, 3);
+	hp[0] = make_pos_bignum_header(2);
+	*((UWord *) (UWord) (hp+1)) = (UWord) tb;
+#else
 	Eterm *hp = HAlloc(BIF_P, 2);
 	hp[0] = make_pos_bignum_header(1);
 	hp[1] = (Eterm) tb;
+#endif
 	BIF_TRAP1(&ets_delete_continue_exp, BIF_P, make_big(hp));
     }
     else {
@@ -1571,7 +1756,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
 {
     Process* to_proc = NULL;
     ErtsProcLocks to_locks = ERTS_PROC_LOCK_MAIN;
-    Eterm buf[5];
+    DeclareTmpHeap(buf,5,BIF_P);
     Eterm to_pid = BIF_ARG_2;
     Eterm from_pid;
     DbTable* tb = NULL;
@@ -1593,6 +1778,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
 	goto badarg;  /* or should we be idempotent? return false maybe */
     }
 
+    UseTmpHeap(5,BIF_P);
     db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
     db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner,
 			make_small(tb->common.slot));
@@ -1610,6 +1796,7 @@ BIF_RETTYPE ets_give_away_3(BIF_ALIST_3)
 		      TUPLE4(buf, am_ETS_TRANSFER, tb->common.id, from_pid, BIF_ARG_3), 
 		      0);
     erts_smp_proc_unlock(to_proc, to_locks);
+    UnUseTmpHeap(5,BIF_P);
     BIF_RET(am_true);
 
 badarg:
@@ -1624,11 +1811,12 @@ BIF_RETTYPE ets_setopts_2(BIF_ALIST_2)
     Eterm* tp;
     Eterm opt;
     Eterm heir = THE_NON_VALUE;
-    Eterm heir_data = THE_NON_VALUE;
+    UWord heir_data = (UWord) THE_NON_VALUE;
     Uint32 protection = 0;
-    Eterm fakelist[2];
+    DeclareTmpHeap(fakelist,2,BIF_P);
     Eterm tail;
 
+    UseTmpHeap(2,BIF_P);
     for (tail = is_tuple(BIF_ARG_2) ? CONS(fakelist, BIF_ARG_2, NIL) : BIF_ARG_2;	
 	  is_list(tail);
 	  tail = CDR(list_val(tail))) {
@@ -1681,9 +1869,11 @@ BIF_RETTYPE ets_setopts_2(BIF_ALIST_2)
     }
 
     db_unlock (tb,LCK_WRITE);
+    UnUseTmpHeap(2,BIF_P);
     BIF_RET(am_true);
 
 badarg:
+    UnUseTmpHeap(2,BIF_P);
     if (tb != NULL) {
 	db_unlock(tb,LCK_WRITE);
     }
@@ -1777,8 +1967,10 @@ BIF_RETTYPE ets_delete_object_2(BIF_ALIST_2)
 /*
 ** This is for trapping, cannot be called directly.
 */
-static BIF_RETTYPE ets_select_delete_1(Process *p, Eterm a1)
+static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1)
 {
+    Process *p = BIF_P;
+    Eterm a1 = BIF_ARG_1;
     BIF_RETTYPE result;
     DbTable* tb;
     int cret;
@@ -1832,7 +2024,7 @@ BIF_RETTYPE ets_select_delete_2(BIF_ALIST_2)
 	if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL) {
 	    BIF_ERROR(BIF_P, BADARG);
 	}
-	nitems = erts_smp_atomic_read(&tb->common.nitems);
+	nitems = erts_smp_atomic_read_nob(&tb->common.nitems);
 	tb->common.meth->db_delete_all_objects(BIF_P, tb);
 	db_unlock(tb, LCK_WRITE);
 	BIF_RET(erts_make_integer(nitems,BIF_P));
@@ -1896,14 +2088,15 @@ BIF_RETTYPE ets_all_0(BIF_ALIST_0)
     previous = NIL;
     j = 0;
     for(i = 0; (i < t_max_tabs && j < t_tabs_cnt); i++) {
-	meta_main_tab_lock(i);
+	erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(i);
+	erts_smp_rwmtx_rlock(mmtl);
 	if (IS_SLOT_ALIVE(i)) {
 	    j++;
 	    tb = meta_main_tab[i].u.tb;
 	    previous = CONS(hp, tb->common.id, previous);
 	    hp += 2;
 	}
-	meta_main_tab_unlock(i);
+	erts_smp_rwmtx_runlock(mmtl);
     }
     HRelease(BIF_P, hendp, hp);
     BIF_RET(previous);
@@ -1943,40 +2136,54 @@ BIF_RETTYPE ets_slot_2(BIF_ALIST_2)
 
 BIF_RETTYPE ets_match_1(BIF_ALIST_1)
 {
-    return ets_select_1(BIF_P, BIF_ARG_1);
+    return ets_select1(BIF_P, BIF_ARG_1);
 }
 
 BIF_RETTYPE ets_match_2(BIF_ALIST_2)
 {
     Eterm ms;
-    Eterm buff[8];
+    DeclareTmpHeap(buff,8,BIF_P);
     Eterm *hp = buff;
-    /*hp = HAlloc(BIF_P, 8);*/
+    Eterm res;
+
+    UseTmpHeap(8,BIF_P);
     ms = CONS(hp, am_DollarDollar, NIL);
     hp += 2;
     ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); 
     hp += 4;
     ms = CONS(hp, ms, NIL);
-    return ets_select_2(BIF_P, BIF_ARG_1, ms);
+    res = ets_select2(BIF_P, BIF_ARG_1, ms);
+    UnUseTmpHeap(8,BIF_P);
+    return res;
 }
 
 BIF_RETTYPE ets_match_3(BIF_ALIST_3)
 {
     Eterm ms;
-    Eterm buff[8];
+    DeclareTmpHeap(buff,8,BIF_P);
     Eterm *hp = buff;
-    /*hp = HAlloc(BIF_P, 8);*/
+    Eterm res;
+
+    UseTmpHeap(8,BIF_P);
     ms = CONS(hp, am_DollarDollar, NIL);
     hp += 2;
     ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); 
     hp += 4;
     ms = CONS(hp, ms, NIL);
-    return ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3);
+    res = ets_select3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3);
+    UnUseTmpHeap(8,BIF_P);
+    return res;
 }
 
 
 BIF_RETTYPE ets_select_3(BIF_ALIST_3)
 {
+    return ets_select3(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+}
+
+static BIF_RETTYPE
+ets_select3(Process* p, Eterm arg1, Eterm arg2, Eterm arg3)
+{
     BIF_RETTYPE result;
     DbTable* tb;
     int cret;
@@ -1987,22 +2194,22 @@ BIF_RETTYPE ets_select_3(BIF_ALIST_3)
     CHECK_TABLES();
 
     /* Chunk size strictly greater than 0 */
-    if (is_not_small(BIF_ARG_3) || (chunk_size = signed_val(BIF_ARG_3)) <= 0) {
-	BIF_ERROR(BIF_P, BADARG);
+    if (is_not_small(arg3) || (chunk_size = signed_val(arg3)) <= 0) {
+	BIF_ERROR(p, BADARG);
     }
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_READ, LCK_READ)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
+    if ((tb = db_get_table(p, arg1, DB_READ, LCK_READ)) == NULL) {
+	BIF_ERROR(p, BADARG);
     }
-    safety = ITERATION_SAFETY(BIF_P,tb);
+    safety = ITERATION_SAFETY(p,tb);
     if (safety == ITER_UNSAFE) {
 	local_fix_table(tb);
     }
-    cret = tb->common.meth->db_select_chunk(BIF_P, tb,
-					    BIF_ARG_2, chunk_size, 
+    cret = tb->common.meth->db_select_chunk(p, tb,
+					    arg2, chunk_size,
 					    0 /* not reversed */,
 					    &ret);
-    if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) {
-	fix_table_locked(BIF_P, tb);
+    if (DID_TRAP(p,ret) && safety != ITER_SAFE) {
+	fix_table_locked(p, tb);
     }
     if (safety == ITER_UNSAFE) {
 	local_unfix_table(tb);
@@ -2014,22 +2221,24 @@ BIF_RETTYPE ets_select_3(BIF_ALIST_3)
 	ERTS_BIF_PREP_RET(result, ret);
 	break;
     case DB_ERROR_SYSRES:
-	ERTS_BIF_PREP_ERROR(result, BIF_P, SYSTEM_LIMIT);
+	ERTS_BIF_PREP_ERROR(result, p, SYSTEM_LIMIT);
 	break;
     default:
-	ERTS_BIF_PREP_ERROR(result, BIF_P, BADARG);
+	ERTS_BIF_PREP_ERROR(result, p, BADARG);
 	break;
     }
 
-    erts_match_set_release_result(BIF_P);
+    erts_match_set_release_result(p);
 
     return result;
 }
 
 
 /* We get here instead of in the real BIF when trapping */
-static BIF_RETTYPE ets_select_trap_1(Process *p, Eterm a1)
+static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1)
 {
+    Process *p = BIF_P;
+    Eterm a1 = BIF_ARG_1;
     BIF_RETTYPE result;
     DbTable* tb;
     int cret;
@@ -2074,6 +2283,11 @@ static BIF_RETTYPE ets_select_trap_1(Process *p, Eterm a1)
 
 BIF_RETTYPE ets_select_1(BIF_ALIST_1)
 {
+    return ets_select1(BIF_P, BIF_ARG_1);
+}
+
+static BIF_RETTYPE ets_select1(Process *p, Eterm arg1)
+{
     BIF_RETTYPE result;
     DbTable* tb;
     int cret;
@@ -2087,28 +2301,27 @@ BIF_RETTYPE ets_select_1(BIF_ALIST_1)
      * Make sure that the table exists.
      */
 
-    if (!is_tuple(BIF_ARG_1)) {
-	if (BIF_ARG_1 == am_EOT) {
+    if (!is_tuple(arg1)) {
+	if (arg1 == am_EOT) {
 	    BIF_RET(am_EOT);
 	}
-	BIF_ERROR(BIF_P, BADARG);
+	BIF_ERROR(p, BADARG);
     }
-    tptr = tuple_val(BIF_ARG_1);
+    tptr = tuple_val(arg1);
     if (arityval(*tptr) < 1 ||
-	(tb = db_get_table(BIF_P, tptr[1], DB_READ, LCK_READ)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
+	(tb = db_get_table(p, tptr[1], DB_READ, LCK_READ)) == NULL) {
+	BIF_ERROR(p, BADARG);
     }
 
-    safety = ITERATION_SAFETY(BIF_P,tb);
+    safety = ITERATION_SAFETY(p,tb);
     if (safety == ITER_UNSAFE) {
 	local_fix_table(tb);
     }
 
-    cret = tb->common.meth->db_select_continue(BIF_P,tb,
-					       BIF_ARG_1, &ret);
+    cret = tb->common.meth->db_select_continue(p,tb, arg1, &ret);
 
-    if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) {
-	fix_table_locked(BIF_P, tb);
+    if (DID_TRAP(p,ret) && safety != ITER_SAFE) {
+	fix_table_locked(p, tb);
     }
     if (safety == ITER_UNSAFE) {
 	local_unfix_table(tb);
@@ -2120,20 +2333,26 @@ BIF_RETTYPE ets_select_1(BIF_ALIST_1)
 	ERTS_BIF_PREP_RET(result, ret);
 	break;
     case DB_ERROR_SYSRES:
-	ERTS_BIF_PREP_ERROR(result, BIF_P, SYSTEM_LIMIT);
+	ERTS_BIF_PREP_ERROR(result, p, SYSTEM_LIMIT);
 	break;
     default:
-	ERTS_BIF_PREP_ERROR(result, BIF_P, BADARG);
+	ERTS_BIF_PREP_ERROR(result, p, BADARG);
 	break;
     }
 
-    erts_match_set_release_result(BIF_P);
+    erts_match_set_release_result(p);
 
     return result;
 }
 
 BIF_RETTYPE ets_select_2(BIF_ALIST_2)
 {
+    return ets_select2(BIF_P, BIF_ARG_1, BIF_ARG_2);
+}
+
+static BIF_RETTYPE
+ets_select2(Process* p, Eterm arg1, Eterm arg2)
+{
     BIF_RETTYPE result;
     DbTable* tb;
     int cret;
@@ -2146,19 +2365,19 @@ BIF_RETTYPE ets_select_2(BIF_ALIST_2)
      * Make sure that the table exists.
      */
 
-    if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_READ, LCK_READ)) == NULL) {
-	BIF_ERROR(BIF_P, BADARG);
+    if ((tb = db_get_table(p, arg1, DB_READ, LCK_READ)) == NULL) {
+	BIF_ERROR(p, BADARG);
     }
-    safety = ITERATION_SAFETY(BIF_P,tb);
+    safety = ITERATION_SAFETY(p,tb);
     if (safety == ITER_UNSAFE) {
 	local_fix_table(tb);
     }
 
-    cret = tb->common.meth->db_select(BIF_P, tb, BIF_ARG_2,
+    cret = tb->common.meth->db_select(p, tb, arg2,
 				      0, &ret);
 
-    if (DID_TRAP(BIF_P,ret) && safety != ITER_SAFE) {
-	fix_table_locked(BIF_P, tb);
+    if (DID_TRAP(p,ret) && safety != ITER_SAFE) {
+	fix_table_locked(p, tb);
     }    
     if (safety == ITER_UNSAFE) {
 	local_unfix_table(tb);
@@ -2170,21 +2389,23 @@ BIF_RETTYPE ets_select_2(BIF_ALIST_2)
 	ERTS_BIF_PREP_RET(result, ret);
 	break;
     case DB_ERROR_SYSRES:
-	ERTS_BIF_PREP_ERROR(result, BIF_P, SYSTEM_LIMIT);
+	ERTS_BIF_PREP_ERROR(result, p, SYSTEM_LIMIT);
 	break;
     default:
-	ERTS_BIF_PREP_ERROR(result, BIF_P, BADARG);
+	ERTS_BIF_PREP_ERROR(result, p, BADARG);
 	break;
     }
 
-    erts_match_set_release_result(BIF_P);
+    erts_match_set_release_result(p);
 
     return result;
 }
 
 /* We get here instead of in the real BIF when trapping */
-static BIF_RETTYPE ets_select_count_1(Process *p, Eterm a1)
+static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1)
 {
+    Process *p = BIF_P;
+    Eterm a1 = BIF_ARG_1;
     BIF_RETTYPE result;
     DbTable* tb;
     int cret;
@@ -2325,7 +2546,7 @@ BIF_RETTYPE ets_select_reverse_3(BIF_ALIST_3)
 
 BIF_RETTYPE ets_select_reverse_1(BIF_ALIST_1)
 {
-    return ets_select_1(BIF_P, BIF_ARG_1);
+    return ets_select1(BIF_P, BIF_ARG_1);
 }
 
 BIF_RETTYPE ets_select_reverse_2(BIF_ALIST_2)
@@ -2379,35 +2600,43 @@ BIF_RETTYPE ets_select_reverse_2(BIF_ALIST_2)
 */
 BIF_RETTYPE ets_match_object_1(BIF_ALIST_1)
 {
-    return ets_select_1(BIF_P, BIF_ARG_1);
+    return ets_select1(BIF_P, BIF_ARG_1);
 }
 
 BIF_RETTYPE ets_match_object_2(BIF_ALIST_2)
 {
     Eterm ms;
-    Eterm buff[8];
+    DeclareTmpHeap(buff,8,BIF_P);
     Eterm *hp = buff;
-    /*hp = HAlloc(BIF_P, 8);*/
+    Eterm res;
+
+    UseTmpHeap(8,BIF_P);
     ms = CONS(hp, am_DollarUnderscore, NIL);
     hp += 2;
     ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); 
     hp += 4;
     ms = CONS(hp, ms, NIL);
-    return ets_select_2(BIF_P, BIF_ARG_1, ms);
+    res = ets_select2(BIF_P, BIF_ARG_1, ms);
+    UnUseTmpHeap(8,BIF_P);
+    return res;
 }
 
 BIF_RETTYPE ets_match_object_3(BIF_ALIST_3)
 {
     Eterm ms;
-    Eterm buff[8];
+    DeclareTmpHeap(buff,8,BIF_P);
     Eterm *hp = buff;
-    /*hp = HAlloc(BIF_P, 8);*/
+    Eterm res;
+
+    UseTmpHeap(8,BIF_P);
     ms = CONS(hp, am_DollarUnderscore, NIL);
     hp += 2;
     ms = TUPLE3(hp, BIF_ARG_2, NIL, ms); 
     hp += 4;
     ms = CONS(hp, ms, NIL);
-    return ets_select_3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3);
+    res = ets_select3(BIF_P, BIF_ARG_1, ms, BIF_ARG_3);
+    UnUseTmpHeap(8,BIF_P);
+    return res;
 }
 
 /* 
@@ -2417,14 +2646,16 @@ BIF_RETTYPE ets_match_object_3(BIF_ALIST_3)
 BIF_RETTYPE ets_info_1(BIF_ALIST_1)
 {
     static Eterm fields[] = {am_protection, am_keypos, am_type, am_named_table,
-	am_node, am_size, am_name, am_heir, am_owner, am_memory};
+	am_node, am_size, am_name, am_heir, am_owner, am_memory, am_compressed};
     Eterm results[sizeof(fields)/sizeof(Eterm)];
     DbTable* tb;
     Eterm res;
     int i;
     Eterm* hp;
     /*Process* rp = NULL;*/
+    /* If/when we implement lockless private tables:
     Eterm owner;
+    */
 
     if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_INFO, LCK_READ)) == NULL) {
 	if (is_atom(BIF_ARG_1) || is_small(BIF_ARG_1)) {
@@ -2433,7 +2664,9 @@ BIF_RETTYPE ets_info_1(BIF_ALIST_1)
 	BIF_ERROR(BIF_P, BADARG);
     }
 
+    /* If/when we implement lockless private tables:
     owner = tb->common.owner;
+    */
 
     /* If/when we implement lockless private tables:
     if ((tb->common.status & DB_PRIVATE) && owner != BIF_P->id) {
@@ -2533,7 +2766,6 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3)
     Binary *mp;
     Eterm res;
     Uint32 dummy;
-    Uint sz;
 
     if (!(is_list(BIF_ARG_1) || BIF_ARG_1 == NIL) || !is_binary(BIF_ARG_2)) {
     error:
@@ -2558,11 +2790,10 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3)
 	    BIF_TRAP3(bif_export[BIF_ets_match_spec_run_r_3],
 		      BIF_P,lst,BIF_ARG_2,ret);
 	}
-	res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), 0, &dummy);
+	res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), NULL, NULL, 0,
+			    ERTS_PAM_COPY_RESULT, &dummy);
 	if (is_value(res)) {
-	    sz = size_object(res);
-	    hp = HAlloc(BIF_P, sz + 2);
-	    res = copy_struct(res, sz, &hp, &MSO(BIF_P));
+	    hp = HAlloc(BIF_P, 2);
 	    ret = CONS(hp,res,ret);
 	    /*hp += 2;*/
 	} 
@@ -2585,22 +2816,33 @@ void init_db(void)
 {
     DbTable init_tb;
     int i;
-    extern Eterm* em_apply_bif;
+    extern BeamInstr* em_apply_bif;
     Eterm *hp;
     unsigned bits;
     size_t size;
 
 #ifdef ERTS_SMP
-    for (i=0; i<META_MAIN_TAB_LOCK_CNT; i++) {
-	erts_smp_spinlock_init_x(&meta_main_tab_locks[i].lck, "meta_main_tab_slot", make_small(i));
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+
+    meta_main_tab_locks =
+	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_DB_TABLES,
+					   sizeof(erts_meta_main_tab_lock_t)
+					   * ERTS_META_MAIN_TAB_LOCK_TAB_SIZE);
+
+    for (i = 0; i < ERTS_META_MAIN_TAB_LOCK_TAB_SIZE; i++) {
+	erts_smp_rwmtx_init_opt_x(&meta_main_tab_locks[i].rwmtx, &rwmtx_opt,
+				  "meta_main_tab_slot", make_small(i));
     }
     erts_smp_spinlock_init(&meta_main_tab_main_lock, "meta_main_tab_main");
     for (i=0; i<META_NAME_TAB_LOCK_CNT; i++) {
-	erts_smp_rwmtx_init_x(&meta_name_tab_rwlocks[i].lck, "meta_name_tab", make_small(i));
+	erts_smp_rwmtx_init_opt_x(&meta_name_tab_rwlocks[i].lck, &rwmtx_opt,
+				  "meta_name_tab", make_small(i));
     }
 #endif
 
-    erts_smp_atomic_init(&erts_ets_misc_mem_size, 0);
+    erts_smp_atomic_init_nob(&erts_ets_misc_mem_size, 0);
     db_initialize_util();
 
     if (user_requested_db_max_tabs < DB_DEF_MAX_TABS)
@@ -2611,10 +2853,10 @@ void init_db(void)
     bits = erts_fit_in_bits(db_max_tabs-1);
     if (bits > SMALL_BITS) {
 	erl_exit(1,"Max limit for ets tabled too high %u (max %u).",
-		 db_max_tabs, 1L<<SMALL_BITS);
+		 db_max_tabs, ((Uint)1)<<SMALL_BITS);
     }
-    meta_main_tab_slot_mask = (1L<<bits) - 1;
-    meta_main_tab_seq_incr = (1L<<bits);
+    meta_main_tab_slot_mask = (((Uint)1)<<bits) - 1;
+    meta_main_tab_seq_incr = (((Uint)1)<<bits);
 
     size = sizeof(*meta_main_tab)*db_max_tabs;
     meta_main_tab = erts_db_alloc_nt(ERTS_ALC_T_DB_TABLES, size);
@@ -2627,7 +2869,7 @@ void init_db(void)
     SET_NEXT_FREE_SLOT(db_max_tabs-1, (Uint)-1);
     meta_main_tab_first_free = 0;
 
-    meta_name_tab_mask = (1L<<(bits-1)) - 1; /* At least half the size of main tab */
+    meta_name_tab_mask = (((Uint) 1)<<(bits-1)) - 1; /* At least half the size of main tab */
     size = sizeof(struct meta_name_tab_entry)*(meta_name_tab_mask+1);
     meta_name_tab = erts_db_alloc_nt(ERTS_ALC_T_DB_TABLES, size);
     ERTS_ETS_MISC_MEM_ADD(size);
@@ -2642,13 +2884,13 @@ void init_db(void)
 
     /*TT*/
     /* Create meta table invertion. */
-    erts_smp_atomic_init(&init_tb.common.memory_size, 0);
+    erts_smp_atomic_init_nob(&init_tb.common.memory_size, 0);
     meta_pid_to_tab = (DbTable*) erts_db_alloc(ERTS_ALC_T_DB_TABLE,
 					       &init_tb,
 					       sizeof(DbTable));
     ERTS_ETS_MISC_MEM_ADD(sizeof(DbTable));
-    erts_smp_atomic_init(&meta_pid_to_tab->common.memory_size,
-			 erts_smp_atomic_read(&init_tb.common.memory_size));
+    erts_smp_atomic_init_nob(&meta_pid_to_tab->common.memory_size,
+			     erts_smp_atomic_read_nob(&init_tb.common.memory_size));
 
     meta_pid_to_tab->common.id = NIL;
     meta_pid_to_tab->common.the_name = am_true;
@@ -2661,12 +2903,12 @@ void init_db(void)
 #endif
     meta_pid_to_tab->common.keypos = 1;
     meta_pid_to_tab->common.owner  = NIL;
-    erts_smp_atomic_init(&meta_pid_to_tab->common.nitems, 0);
+    erts_smp_atomic_init_nob(&meta_pid_to_tab->common.nitems, 0);
     meta_pid_to_tab->common.slot   = -1;
     meta_pid_to_tab->common.meth   = &db_hash;
+    meta_pid_to_tab->common.compress = 0;
 
-    erts_refc_init(&meta_pid_to_tab->common.ref, 1);
-    erts_refc_init(&meta_pid_to_tab->common.fixref, 0);
+    erts_refc_init(&meta_pid_to_tab->common.ref, 0);
     /* Neither rwlock or fixlock used
     db_init_lock(meta_pid_to_tab, "meta_pid_to_tab", "meta_pid_to_tab_FIX");*/
 
@@ -2674,13 +2916,13 @@ void init_db(void)
 	erl_exit(1,"Unable to create ets metadata tables.");
     }
 
-    erts_smp_atomic_set(&init_tb.common.memory_size, 0);
+    erts_smp_atomic_set_nob(&init_tb.common.memory_size, 0);
     meta_pid_to_fixed_tab = (DbTable*) erts_db_alloc(ERTS_ALC_T_DB_TABLE,
 						     &init_tb,
 						     sizeof(DbTable));
     ERTS_ETS_MISC_MEM_ADD(sizeof(DbTable));
-    erts_smp_atomic_init(&meta_pid_to_fixed_tab->common.memory_size,
-			 erts_smp_atomic_read(&init_tb.common.memory_size));
+    erts_smp_atomic_init_nob(&meta_pid_to_fixed_tab->common.memory_size,
+			     erts_smp_atomic_read_nob(&init_tb.common.memory_size));
 
     meta_pid_to_fixed_tab->common.id = NIL;
     meta_pid_to_fixed_tab->common.the_name = am_true;
@@ -2693,12 +2935,12 @@ void init_db(void)
 #endif
     meta_pid_to_fixed_tab->common.keypos = 1;
     meta_pid_to_fixed_tab->common.owner  = NIL;
-    erts_smp_atomic_init(&meta_pid_to_fixed_tab->common.nitems, 0);
+    erts_smp_atomic_init_nob(&meta_pid_to_fixed_tab->common.nitems, 0);
     meta_pid_to_fixed_tab->common.slot   = -1;
     meta_pid_to_fixed_tab->common.meth   = &db_hash;
+    meta_pid_to_fixed_tab->common.compress = 0;
 
-    erts_refc_init(&meta_pid_to_fixed_tab->common.ref, 1);
-    erts_refc_init(&meta_pid_to_fixed_tab->common.fixref, 0);
+    erts_refc_init(&meta_pid_to_fixed_tab->common.ref, 0);
     /* Neither rwlock or fixlock used
     db_init_lock(meta_pid_to_fixed_tab, "meta_pid_to_fixed_tab", "meta_pid_to_fixed_tab_FIX");*/
 
@@ -2714,9 +2956,9 @@ void init_db(void)
     ets_select_delete_continue_exp.code[1] = am_atom_put("delete_trap",11);
     ets_select_delete_continue_exp.code[2] = 1;
     ets_select_delete_continue_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     ets_select_delete_continue_exp.code[4] = 
-	(Eterm) &ets_select_delete_1;
+	(BeamInstr) &ets_select_delete_1;
 
     /* Non visual BIF to trap to. */
     memset(&ets_select_count_continue_exp, 0, sizeof(Export));
@@ -2726,9 +2968,9 @@ void init_db(void)
     ets_select_count_continue_exp.code[1] = am_atom_put("count_trap",11);
     ets_select_count_continue_exp.code[2] = 1;
     ets_select_count_continue_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     ets_select_count_continue_exp.code[4] = 
-	(Eterm) &ets_select_count_1;
+	(BeamInstr) &ets_select_count_1;
 
     /* Non visual BIF to trap to. */
     memset(&ets_select_continue_exp, 0, sizeof(Export));
@@ -2738,9 +2980,9 @@ void init_db(void)
     ets_select_continue_exp.code[1] = am_atom_put("select_trap",11);
     ets_select_continue_exp.code[2] = 1;
     ets_select_continue_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     ets_select_continue_exp.code[4] = 
-	(Eterm) &ets_select_trap_1;
+	(BeamInstr) &ets_select_trap_1;
 
     /* Non visual BIF to trap to. */
     memset(&ets_delete_continue_exp, 0, sizeof(Export));
@@ -2748,8 +2990,8 @@ void init_db(void)
     ets_delete_continue_exp.code[0] = am_ets;
     ets_delete_continue_exp.code[1] = am_atom_put("delete_trap",11);
     ets_delete_continue_exp.code[2] = 1;
-    ets_delete_continue_exp.code[3] = (Eterm) em_apply_bif;
-    ets_delete_continue_exp.code[4] = (Eterm) &ets_delete_trap;
+    ets_delete_continue_exp.code[3] = (BeamInstr) em_apply_bif;
+    ets_delete_continue_exp.code[4] = (BeamInstr) &ets_delete_trap;
 
     hp = ms_delete_all_buff;
     ms_delete_all = CONS(hp, am_true, NIL);
@@ -2843,9 +3085,9 @@ static int give_away_to_heir(Process* p, DbTable* tb)
 {
     Process* to_proc;
     ErtsProcLocks to_locks = ERTS_PROC_LOCK_MAIN;
-    Eterm buf[5];
+    DeclareTmpHeap(buf,5,p);
     Eterm to_pid;
-    Eterm heir_data;
+    UWord heir_data;
 
     ASSERT(tb->common.owner == p->id);
     ASSERT(is_internal_pid(tb->common.heir));
@@ -2856,12 +3098,10 @@ retry:
 				to_pid, to_locks,
 				ERTS_P2P_FLG_TRY_LOCK);
     if (to_proc == ERTS_PROC_LOCK_BUSY) {
-	db_ref(tb); /* while unlocked */
 	db_unlock(tb,LCK_WRITE);    
 	to_proc = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN,
 				to_pid, to_locks);    
 	db_lock(tb,LCK_WRITE);
-	tb = db_unref(tb);
 	ASSERT(tb != NULL);
     
 	if (tb->common.owner != p->id) {
@@ -2888,6 +3128,7 @@ retry:
 	erts_smp_proc_unlock(to_proc, to_locks);
 	return 0; /* heir dead and pid reused, table still mine */
     }
+    UseTmpHeap(5,p);
     db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
     db_erase_bag_exact2(meta_pid_to_tab, tb->common.owner,
 			make_small(tb->common.slot));
@@ -2899,11 +3140,11 @@ retry:
 		TUPLE2(buf,to_pid,make_small(tb->common.slot)),
 		0);
     db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
-    
+    UnUseTmpHeap(5,p);
     db_unlock(tb,LCK_WRITE);
     heir_data = tb->common.heir_data;
     if (!is_immed(heir_data)) {
-	Eterm* tpv = DBTERM_BUF((DbTerm*)heir_data); /* tuple_val */
+	Eterm* tpv = ((DbTerm*)heir_data)->tpl; /* tuple_val */
 	ASSERT(arityval(*tpv) == 1);
 	heir_data = tpv[1];
     }
@@ -2968,15 +3209,16 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 	    while (state->slots.ix < state->slots.size) {
 		DbTable *tb = NULL;
 		Sint ix = unsigned_val(state->slots.arr[state->slots.ix]);
-		meta_main_tab_lock(ix);
+		erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix);
+		erts_smp_rwmtx_rlock(mmtl);
 		if (!IS_SLOT_FREE(ix)) {
-		    tb = db_ref(GET_ANY_SLOT_TAB(ix));
+		    tb = GET_ANY_SLOT_TAB(ix);
 		    ASSERT(tb);
 		}
-		meta_main_tab_unlock(ix);
+		erts_smp_rwmtx_runlock(mmtl);
 		if (tb) {
 		    int do_yield;
-		    db_lock_take_over_ref(tb, LCK_WRITE);
+		    db_lock(tb, LCK_WRITE);
 		    /* Ownership may have changed since
 		       we looked up the table. */
 		    if (tb->common.owner != pid) {
@@ -3005,7 +3247,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 			    tb->common.status |= DB_DELETE;
 
 			    if (is_atom(tb->common.id))
-				remove_named_tab(tb->common.id);
+				remove_named_tab(tb, 0);
 
 			    free_heir_data(tb);
 			    free_fixations_locked(tb);
@@ -3055,17 +3297,18 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 	    while (state->slots.ix < state->slots.size) {
 		DbTable *tb = NULL;
 		Sint ix = unsigned_val(state->slots.arr[state->slots.ix]);
-		meta_main_tab_lock(ix);
+		erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix);
+		erts_smp_rwmtx_rlock(mmtl);
 		if (IS_SLOT_ALIVE(ix)) {
-		    tb = db_ref(meta_main_tab[ix].u.tb);
+		    tb = meta_main_tab[ix].u.tb;
 		    ASSERT(tb);
 		}
-		meta_main_tab_unlock(ix);
+		erts_smp_rwmtx_runlock(mmtl);
 		if (tb) {
 		    int reds;
 		    DbFixation** pp;
 
-		    db_lock_take_over_ref(tb, LCK_WRITE_REC);
+		    db_lock(tb, LCK_WRITE_REC);
 		    #ifdef ERTS_SMP
 		    erts_smp_mtx_lock(&tb->common.fixlock);
 		    #endif
@@ -3075,7 +3318,8 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 			  pp = &(*pp)->next) {
 			if ((*pp)->pid == pid) {
 			    DbFixation* fix = *pp;
-			    erts_refc_add(&tb->common.fixref,-fix->counter,0);
+			    erts_aint_t diff = -((erts_aint_t) fix->counter);
+			    erts_refc_add(&tb->common.ref,diff,0);
 			    *pp = fix->next;
 			    erts_db_free(ERTS_ALC_T_DB_FIXATION,
 					 tb, fix, sizeof(DbFixation));
@@ -3145,12 +3389,12 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks)
 static void fix_table_locked(Process* p, DbTable* tb)
 {
     DbFixation *fix;
-    Eterm meta_tuple[3];
+    DeclareTmpHeap(meta_tuple,3,p);
 
 #ifdef ERTS_SMP
     erts_smp_mtx_lock(&tb->common.fixlock);
 #endif
-    erts_refc_inc(&tb->common.fixref,1);
+    erts_refc_inc(&tb->common.ref,1);
     fix = tb->common.fixations;
     if (fix == NULL) { 
 	get_now(&(tb->common.megasec),
@@ -3179,12 +3423,15 @@ static void fix_table_locked(Process* p, DbTable* tb)
     erts_smp_mtx_unlock(&tb->common.fixlock);
 #endif
     p->flags |= F_USING_DB;        
+    UseTmpHeap(3,p);
     db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
     if (db_put_hash(meta_pid_to_fixed_tab,
 		    TUPLE2(meta_tuple, p->id, make_small(tb->common.slot)),
 		    0) != DB_ERROR_NONE) {
+	UnUseTmpHeap(3,p);
 	erl_exit(1,"Could not insert ets metadata in safe_fixtable.");
     }	
+    UnUseTmpHeap(3,p);
     db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
 }
 
@@ -3201,7 +3448,7 @@ static void unfix_table_locked(Process* p,  DbTable* tb,
     for (pp = &tb->common.fixations; *pp != NULL; pp = &(*pp)->next) {
 	if ((*pp)->pid == p->id) {
 	    DbFixation* fix = *pp;
-	    erts_refc_dec(&tb->common.fixref,0);
+	    erts_refc_dec(&tb->common.ref,0);
 	    --(fix->counter);
 	    ASSERT(fix->counter >= 0);
 	    if (fix->counter > 0) {
@@ -3227,7 +3474,7 @@ static void unfix_table_locked(Process* p,  DbTable* tb,
 unlocked:
 
     if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status)
-	&& erts_smp_atomic_read(&tb->hash.fixdel) != (long)NULL) {
+	&& erts_smp_atomic_read_nob(&tb->hash.fixdel) != (erts_aint_t)NULL) {
 #ifdef ERTS_SMP
 	if (*kind_p == LCK_READ && tb->common.is_thread_safe) {
 	    /* Must have write lock while purging pseudo-deleted (OTP-8166) */
@@ -3249,6 +3496,8 @@ static void free_fixations_locked(DbTable *tb)
 
     fix = tb->common.fixations;
     while (fix != NULL) {
+	erts_aint_t diff = -((erts_aint_t) fix->counter);
+	erts_refc_add(&tb->common.ref,diff,0);
 	next_fix = fix->next;
 	db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC);
 	db_erase_bag_exact2(meta_pid_to_fixed_tab,
@@ -3264,7 +3513,7 @@ static void free_fixations_locked(DbTable *tb)
     tb->common.fixations = NULL;
 }
 
-static void set_heir(Process* me, DbTable* tb, Eterm heir, Eterm heir_data)
+static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data)
 {	
     tb->common.heir = heir;
     if (heir == am_none) {
@@ -3285,10 +3534,26 @@ static void set_heir(Process* me, DbTable* tb, Eterm heir, Eterm heir_data)
     }
 
     if (!is_immed(heir_data)) {
-	Eterm tmp[2];
-	/* Make a dummy 1-tuple around data to use db_get_term() */
-	heir_data = (Eterm) db_get_term(&tb->common, NULL, 0,
-					TUPLE1(tmp,heir_data));
+	DeclareTmpHeap(tmp,2,me);
+	Eterm wrap_tpl;
+	int size;
+	DbTerm* dbterm;
+	Eterm* top;
+	ErlOffHeap tmp_offheap;
+
+	UseTmpHeap(2,me);
+	/* Make a dummy 1-tuple around data to use DbTerm */
+	wrap_tpl = TUPLE1(tmp,heir_data);
+	size = size_object(wrap_tpl);
+	dbterm = erts_db_alloc(ERTS_ALC_T_DB_HEIR_DATA, (DbTable *)tb,
+			       (sizeof(DbTerm) + sizeof(Eterm)*(size-1)));
+	dbterm->size = size;
+	top = dbterm->tpl;
+	tmp_offheap.first  = NULL;
+	copy_struct(wrap_tpl, size, &top, &tmp_offheap);
+	dbterm->first_oh = tmp_offheap.first;
+	heir_data = (UWord)dbterm;
+	UnUseTmpHeap(2,me);
 	ASSERT(!is_immed(heir_data));
     }
     tb->common.heir_data = heir_data;
@@ -3298,8 +3563,8 @@ static void free_heir_data(DbTable* tb)
 {
     if (tb->common.heir != am_none && !is_immed(tb->common.heir_data)) {
 	DbTerm* p = (DbTerm*) tb->common.heir_data;
-	db_free_term_data(p);
-	erts_db_free(ERTS_ALC_T_DB_TERM, tb, (void *)p,
+	db_cleanup_offheap_comp(p);
+	erts_db_free(ERTS_ALC_T_DB_HEIR_DATA, tb, (void *)p,
 		     sizeof(DbTerm) + (p->size-1)*sizeof(Eterm));
     }
     #ifdef DEBUG
@@ -3307,14 +3572,19 @@ static void free_heir_data(DbTable* tb)
     #endif
 }
 
-static BIF_RETTYPE ets_delete_trap(Process *p, Eterm cont)
+static BIF_RETTYPE ets_delete_trap(BIF_ALIST_1)
 {
+    Process *p = BIF_P;
+    Eterm cont = BIF_ARG_1;
     int trap;
     Eterm* ptr = big_val(cont);
-    DbTable *tb = (DbTable *) ptr[1];
+    DbTable *tb = *((DbTable **) (UWord) (ptr + 1));
 
+#if HALFWORD_HEAP
+    ASSERT(*ptr == make_pos_bignum_header(2));
+#else
     ASSERT(*ptr == make_pos_bignum_header(1));
-
+#endif
     db_lock(tb, LCK_WRITE);
     trap = free_table_cont(p, tb, 0, 1);
     db_unlock(tb, LCK_WRITE);
@@ -3337,6 +3607,7 @@ static int free_table_cont(Process *p,
 			   int clean_meta_tab)
 {
     Eterm result;
+    erts_smp_rwmtx_t *mmtl;
 
 #ifdef HARDDEBUG
     if (!first) {
@@ -3362,9 +3633,16 @@ static int free_table_cont(Process *p,
 		     tb->common.id);
 #endif
 	/* Completely done - we will not get called again. */
-	meta_main_tab_lock(tb->common.slot);
+	mmtl = get_meta_main_tab_lock(tb->common.slot);
+#ifdef ERTS_SMP
+	if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) {
+	    erts_smp_rwmtx_rwunlock(&tb->common.rwlock);
+	    erts_smp_rwmtx_rwlock(mmtl);
+	    erts_smp_rwmtx_rwlock(&tb->common.rwlock);
+	}
+#endif
 	free_slot(tb->common.slot);
-	meta_main_tab_unlock(tb->common.slot);
+	erts_smp_rwmtx_rwunlock(mmtl);
 
 	if (clean_meta_tab) {
 	    db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC);
@@ -3372,7 +3650,7 @@ static int free_table_cont(Process *p,
 				make_small(tb->common.slot));
 	    db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC);
 	}
-	db_unref(tb);
+	schedule_free_dbtable(tb);
 	BUMP_REDS(p, 100);
 	return 0;
     }
@@ -3383,7 +3661,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
     Eterm ret = THE_NON_VALUE;
 
     if (What == am_size) {
-	ret = make_small(erts_smp_atomic_read(&tb->common.nitems));
+	ret = make_small(erts_smp_atomic_read_nob(&tb->common.nitems));
     } else if (What == am_type) {
 	if (tb->common.status & DB_SET)  {
 	    ret = am_set;
@@ -3396,7 +3674,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 	    ret = am_bag;
 	}
     } else if (What == am_memory) {
-	Uint words = (Uint) ((erts_smp_atomic_read(&tb->common.memory_size)
+	Uint words = (Uint) ((erts_smp_atomic_read_nob(&tb->common.memory_size)
 			      + sizeof(Uint)
 			      - 1)
 			     / sizeof(Uint));
@@ -3420,10 +3698,13 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 	ret = erts_this_dist_entry->sysname;
     } else if (What == am_named_table) {
 	ret = is_atom(tb->common.id) ? am_true : am_false;
+    } else if (What == am_compressed) {
+	ret = tb->common.compress ? am_true : am_false;
+    }
     /*
      * For debugging purposes
      */
-    } else if (What == am_data) { 
+    else if (What == am_data) {
 	print_table(ERTS_PRINT_STDOUT, NULL, 1, tb);
 	ret = am_true;
     } else if (What == am_atom_put("fixed",5)) { 
@@ -3431,9 +3712,6 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 	    ret = am_true;
 	else
 	    ret = am_false;
-    } else if (What == am_atom_put("kept_objects",12)) {
-	ret = make_small(IS_HASH_TABLE(tb->common.status)
-			 ? db_kept_items_hash(&tb->hash) : 0);
     } else if (What == am_atom_put("safe_fixed",10)) {
 #ifdef ERTS_SMP
 	erts_smp_mtx_lock(&tb->common.fixlock);
@@ -3475,7 +3753,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 	    Eterm* hp;
 
 	    db_calc_stats_hash(&tb->hash, &stats);
-	    hp = HAlloc(p, 1 + 6 + FLOAT_SIZE_OBJECT*3);
+	    hp = HAlloc(p, 1 + 7 + FLOAT_SIZE_OBJECT*3);
 	    f.fd = stats.avg_chain_len;
 	    avg = make_float(hp);
 	    PUT_DOUBLE(f, hp);
@@ -3490,10 +3768,11 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What)
 	    std_dev_exp = make_float(hp);
 	    PUT_DOUBLE(f, hp);
 	    hp += FLOAT_SIZE_OBJECT;
-	    ret = TUPLE6(hp, make_small(erts_smp_atomic_read(&tb->hash.nactive)),
+	    ret = TUPLE7(hp, make_small(erts_smp_atomic_read_nob(&tb->hash.nactive)),
 			 avg, std_dev_real, std_dev_exp,
 			 make_small(stats.min_chain_len),
-			 make_small(stats.max_chain_len));
+			 make_small(stats.max_chain_len),
+			 make_small(db_kept_items_hash(&tb->hash)));
 	}
 	else {
 	    ret = am_false;
@@ -3509,9 +3788,9 @@ static void print_table(int to, void *to_arg, int show,  DbTable* tb)
 
     tb->common.meth->db_print(to, to_arg, show, tb);
 
-    erts_print(to, to_arg, "Objects: %d\n", (int)erts_smp_atomic_read(&tb->common.nitems));
+    erts_print(to, to_arg, "Objects: %d\n", (int)erts_smp_atomic_read_nob(&tb->common.nitems));
     erts_print(to, to_arg, "Words: %bpu\n",
-	       (Uint) ((erts_smp_atomic_read(&tb->common.memory_size)
+	       (Uint) ((erts_smp_atomic_read_nob(&tb->common.memory_size)
 			+ sizeof(Uint)
 			- 1)
 		       / sizeof(Uint)));
@@ -3537,8 +3816,9 @@ void db_info(int to, void *to_arg, int show)    /* Called by break handler */
 Uint
 erts_get_ets_misc_mem_size(void)
 {
+    ERTS_SMP_MEMORY_BARRIER;
     /* Memory not allocated in ets_alloc */
-    return (Uint) erts_smp_atomic_read(&erts_ets_misc_mem_size);
+    return (Uint) erts_smp_atomic_read_nob(&erts_ets_misc_mem_size);
 }
 
 /* SMP Note: May only be used when system is locked */
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index 7da28fad29..2e5deaf338 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -61,6 +61,7 @@ void erts_db_foreach_offheap(DbTable *,
 
 extern int user_requested_db_max_tabs; /* set in erl_init */
 extern int erts_ets_realloc_always_moves;  /* set in erl_init */
+extern int erts_ets_always_compress;  /* set in erl_init */
 extern Export ets_select_delete_continue_exp;
 extern Export ets_select_count_continue_exp;
 extern Export ets_select_continue_exp;
@@ -82,13 +83,14 @@ Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt);
 
 #define ERTS_DB_ALC_MEM_UPDATE_(TAB, FREE_SZ, ALLOC_SZ)			\
 do {									\
-    long sz__ = ((long) (ALLOC_SZ)) - ((long) (FREE_SZ));		\
+    erts_aint_t sz__ = (((erts_aint_t) (ALLOC_SZ))			\
+			- ((erts_aint_t) (FREE_SZ)));			\
     ASSERT((TAB));							\
-    erts_smp_atomic_add(&(TAB)->common.memory_size, sz__);		\
+    erts_smp_atomic_add_nob(&(TAB)->common.memory_size, sz__);		\
 } while (0)
 
 #define ERTS_ETS_MISC_MEM_ADD(SZ) \
-  erts_smp_atomic_add(&erts_ets_misc_mem_size, (SZ));
+  erts_smp_atomic_add_nob(&erts_ets_misc_mem_size, (SZ));
 
 ERTS_GLB_INLINE void *erts_db_alloc(ErtsAlcType_t type,
 				    DbTable *tab,
@@ -225,7 +227,7 @@ erts_db_free(ErtsAlcType_t type, DbTable *tab, void *ptr, Uint size)
     ERTS_DB_ALC_MEM_UPDATE_(tab, size, 0);
 
     ASSERT(((void *) tab) != ptr
-	   || erts_smp_atomic_read(&tab->common.memory_size) == 0);
+	   || erts_smp_atomic_read_nob(&tab->common.memory_size) == 0);
 
     erts_free(type, ptr);
 }
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 4141f9766b..2fea4671e1 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -105,9 +105,22 @@
 #define NSEG_2   256 /* Size of second segment table */
 #define NSEG_INC 128 /* Number of segments to grow after that */
 
-#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read(&(tb)->segtab))
-#define NACTIVE(tb) ((int)erts_smp_atomic_read(&(tb)->nactive))
-#define NITEMS(tb) ((int)erts_smp_atomic_read(&(tb)->common.nitems))
+#ifdef ERTS_SMP
+#  define DB_USING_FINE_LOCKING(TB) (((TB))->common.type & DB_FINE_LOCKED)
+#else
+#  define DB_USING_FINE_LOCKING(TB) 0
+#endif
+
+#ifdef ETHR_ORDERED_READ_DEPEND
+#define SEGTAB(tb) ((struct segment**) erts_smp_atomic_read_nob(&(tb)->segtab))
+#else
+#define SEGTAB(tb)							\
+    (DB_USING_FINE_LOCKING(tb)						\
+     ? ((struct segment**) erts_smp_atomic_read_ddrb(&(tb)->segtab))	\
+     : ((struct segment**) erts_smp_atomic_read_nob(&(tb)->segtab)))
+#endif
+#define NACTIVE(tb) ((int)erts_smp_atomic_read_nob(&(tb)->nactive))
+#define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems))
 
 #define BUCKET(tb, i) SEGTAB(tb)[(i) >> SEGSZ_EXP]->buckets[(i) & SEGSZ_MASK]
 
@@ -122,11 +135,13 @@
 */
 static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval)
 {
-    Uint mask = erts_smp_atomic_read(&tb->szm);
+    Uint mask = (DB_USING_FINE_LOCKING(tb)
+		 ? erts_smp_atomic_read_acqb(&tb->szm)
+		 : erts_smp_atomic_read_nob(&tb->szm));
     Uint ix = hval & mask; 
-    if (ix >= erts_smp_atomic_read(&tb->nactive)) {
+    if (ix >= erts_smp_atomic_read_nob(&tb->nactive)) {
 	ix &= mask>>1;
-	ASSERT(ix < erts_smp_atomic_read(&tb->nactive));
+	ASSERT(ix < erts_smp_atomic_read_nob(&tb->nactive));
     }
     return ix;
 }
@@ -135,18 +150,20 @@ static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval)
 */
 static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix)
 {
-    long was_next;
-    long exp_next;
+    erts_aint_t was_next;
+    erts_aint_t exp_next;
     FixedDeletion* fixd = (FixedDeletion*) erts_db_alloc(ERTS_ALC_T_DB_FIX_DEL,
 							 (DbTable *) tb,
 							 sizeof(FixedDeletion));
     ERTS_ETS_MISC_MEM_ADD(sizeof(FixedDeletion));
-    fixd->slot = ix;    
-    was_next = erts_smp_atomic_read(&tb->fixdel);    
+    fixd->slot = ix;
+    was_next = erts_smp_atomic_read_acqb(&tb->fixdel);
     do { /* Lockless atomic insertion in linked list: */
 	exp_next = was_next;
 	fixd->next = (FixedDeletion*) exp_next;
-	was_next = erts_smp_atomic_cmpxchg(&tb->fixdel, (long)fixd, exp_next);
+	was_next = erts_smp_atomic_cmpxchg_relb(&tb->fixdel,
+						(erts_aint_t) fixd,
+						exp_next);
     }while (was_next != exp_next);
 }
 
@@ -256,22 +273,16 @@ static ERTS_INLINE Sint next_slot_w(DbTableHash* tb, Uint ix,
 }
 
 
-/* 
- * tplp is an untagged pointer to a tuple we know is large enough 
- * and dth is a pointer to a DbTableHash.   
- */
-#define GETKEY(dth, tplp)   (*((tplp) +  (dth)->common.keypos))
-
 /*
  * Some special binary flags
  */
 #define BIN_FLAG_ALL_OBJECTS         BIN_FLAG_USR1
 
-/*
- * Size calculations
- */
-#define SIZ_OVERHEAD ((sizeof(HashDbTerm)/sizeof(Eterm)) - 1)
-#define SIZ_DBTERM(HDT) (SIZ_OVERHEAD + (HDT)->dbterm.size)
+
+static ERTS_INLINE void free_term(DbTableHash *tb, HashDbTerm* p)
+{
+    db_free_term((DbTable*)tb, p, offsetof(HashDbTerm, dbterm));
+}
 
 /*
  * Local types 
@@ -312,15 +323,27 @@ struct ext_segment {
     struct segment* segtab[1];     /* The segment table */
 };
 #define SIZEOF_EXTSEG(NSEGS) \
-    (sizeof(struct ext_segment) - sizeof(struct segment*) + sizeof(struct segment*)*(NSEGS))
+    (offsetof(struct ext_segment,segtab) + sizeof(struct segment*)*(NSEGS))
 
-#ifdef DEBUG
-#  include <stddef.h> /* offsetof */
+#if defined(DEBUG) || defined(VALGRIND)
 #  define EXTSEG(SEGTAB_PTR) \
     ((struct ext_segment*) (((char*)(SEGTAB_PTR)) - offsetof(struct ext_segment,segtab)))
 #endif
 
 
+static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb,
+				   struct segment** segtab)
+{
+    if (DB_USING_FINE_LOCKING(tb))
+	erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t) segtab);
+    else
+	erts_smp_atomic_set_nob(&tb->segtab, (erts_aint_t) segtab);
+#ifdef VALGRIND
+    tb->top_ptr_to_segment_with_active_segtab = EXTSEG(segtab);
+#endif
+}
+
+
 /* How the table segments relate to each other:
 
     ext_segment:                      ext_segment:              "plain" segment
@@ -358,10 +381,8 @@ static HashDbTerm* search_list(DbTableHash* tb, Eterm key,
 			       HashValue hval, HashDbTerm *list);
 static void shrink(DbTableHash* tb, int nactive);
 static void grow(DbTableHash* tb, int nactive);
-static void free_term(DbTableHash *tb, HashDbTerm* p);
-static Eterm put_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2);
-static HashDbTerm* get_term(DbTableHash* tb, HashDbTerm* old, 
-			    Eterm obj, HashValue hval);
+static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2,
+			   DbTableHash*);
 static int analyze_pattern(DbTableHash *tb, Eterm pattern, 
 			   struct mp_info *mpi);
 
@@ -434,6 +455,9 @@ static ERTS_INLINE void try_shrink(DbTableHash* tb)
     }
 }	
 
+#define EQ_REL(x,y,y_base) \
+    (is_same(x,NULL,y,y_base) || (is_not_both_immed((x),(y)) && eq_rel((x),NULL,(y),y_base)))
+
 /* Is this a live object (not pseodo-deleted) with the specified key? 
 */
 static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b,
@@ -442,7 +466,8 @@ static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b,
     if (b->hvalue != hval) return 0;
     else {
 	Eterm itemKey = GETKEY(tb, b->dbterm.tpl);
-	return EQ(key,itemKey);
+	ASSERT(!is_header(itemKey));
+	return EQ_REL(key, itemKey, b->dbterm.tpl);
     }
 }
 
@@ -454,10 +479,38 @@ static ERTS_INLINE int has_key(DbTableHash* tb, HashDbTerm* b,
     if (b->hvalue != hval && b->hvalue != INVALID_HASH) return 0;
     else {
 	Eterm itemKey = GETKEY(tb, b->dbterm.tpl);
-	return EQ(key,itemKey);
+	ASSERT(!is_header(itemKey));
+	return EQ_REL(key, itemKey, b->dbterm.tpl);
     }
 }
 
+static ERTS_INLINE HashDbTerm* new_dbterm(DbTableHash* tb, Eterm obj)
+{
+    HashDbTerm* p;
+    if (tb->common.compress) {
+	p = db_store_term_comp(&tb->common, NULL, offsetof(HashDbTerm,dbterm), obj);
+    }
+    else {
+	p = db_store_term(&tb->common, NULL, offsetof(HashDbTerm,dbterm), obj);
+    }
+    return p;
+}
+
+static ERTS_INLINE HashDbTerm* replace_dbterm(DbTableHash* tb, HashDbTerm* old,
+					      Eterm obj)
+{
+    HashDbTerm* ret;
+    ASSERT(old != NULL);
+    if (tb->common.compress) {
+	ret = db_store_term_comp(&tb->common, &(old->dbterm), offsetof(HashDbTerm,dbterm), obj);
+    }
+    else {
+	ret = db_store_term(&tb->common, &(old->dbterm), offsetof(HashDbTerm,dbterm), obj);
+    }
+    return ret;
+}
+
+
 
 /*
 ** External interface 
@@ -514,22 +567,24 @@ static void restore_fixdel(DbTableHash* tb, FixedDeletion* fixdel)
 {
     /*int tries = 0;*/
     DEBUG_WAIT();
-    if (erts_smp_atomic_cmpxchg(&tb->fixdel, (long)fixdel,
-				(long)NULL) != (long)NULL) {
+    if (erts_smp_atomic_cmpxchg_relb(&tb->fixdel,
+				     (erts_aint_t) fixdel,
+				     (erts_aint_t) NULL) != (erts_aint_t) NULL) {
 	/* Oboy, must join lists */    
 	FixedDeletion* last = fixdel;
-	long was_tail;
-	long exp_tail;
+	erts_aint_t was_tail;
+	erts_aint_t exp_tail;
 
-	while (last->next != NULL) last = last->next;	
-	was_tail = erts_smp_atomic_read(&tb->fixdel);
+	while (last->next != NULL) last = last->next;
+	was_tail = erts_smp_atomic_read_acqb(&tb->fixdel);
 	do { /* Lockless atomic list insertion */
 	    exp_tail = was_tail;
 	    last->next = (FixedDeletion*) exp_tail;
 	    /*++tries;*/
 	    DEBUG_WAIT();
-	    was_tail = erts_smp_atomic_cmpxchg(&tb->fixdel, (long)fixdel,
-					       exp_tail);
+	    was_tail = erts_smp_atomic_cmpxchg_relb(&tb->fixdel,
+						    (erts_aint_t) fixdel,
+						    exp_tail);
 	}while (was_tail != exp_tail);
     }
     /*erts_fprintf(stderr,"erl_db_hash: restore_fixdel tries=%d\r\n", tries);*/
@@ -546,7 +601,8 @@ void db_unfix_table_hash(DbTableHash *tb)
 		       || (erts_smp_lc_rwmtx_is_rlocked(&tb->common.rwlock)
 			   && !tb->common.is_thread_safe));
 restart:
-    fixdel = (FixedDeletion*) erts_smp_atomic_xchg(&tb->fixdel, (long)NULL);
+    fixdel = (FixedDeletion*) erts_smp_atomic_xchg_acqb(&tb->fixdel,
+							(erts_aint_t) NULL);
     while (fixdel != NULL) {
 	FixedDeletion *fx = fixdel;
 	int ix = fx->slot;
@@ -613,31 +669,37 @@ int db_create_hash(Process *p, DbTable *tbl)
 {
     DbTableHash *tb = &tbl->hash;
 
-    erts_smp_atomic_init(&tb->szm, SEGSZ_MASK);
-    erts_smp_atomic_init(&tb->nactive, SEGSZ);
-    erts_smp_atomic_init(&tb->fixdel, (long)NULL);
-    erts_smp_atomic_init(&tb->segtab, (long) alloc_ext_seg(tb,0,NULL)->segtab);
+    erts_smp_atomic_init_nob(&tb->szm, SEGSZ_MASK);
+    erts_smp_atomic_init_nob(&tb->nactive, SEGSZ);
+    erts_smp_atomic_init_nob(&tb->fixdel, (erts_aint_t)NULL);
+    erts_smp_atomic_init_nob(&tb->segtab, (erts_aint_t)NULL);
+    SET_SEGTAB(tb, alloc_ext_seg(tb,0,NULL)->segtab);
     tb->nsegs = NSEG_1;
     tb->nslots = SEGSZ;
 
-    erts_smp_atomic_init(&tb->is_resizing, 0);
+    erts_smp_atomic_init_nob(&tb->is_resizing, 0);
 #ifdef ERTS_SMP
     if (tb->common.type & DB_FINE_LOCKED) {
+	erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
 	int i;
+	if (tb->common.type & DB_FREQ_READ)
+	    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
 	tb->locks = (DbTableHashFineLocks*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, /* Other type maybe? */ 
 							      (DbTable *) tb,
 							      sizeof(DbTableHashFineLocks));	    	    
 	for (i=0; i<DB_HASH_LOCK_CNT; ++i) {
-	    erts_rwmtx_init_x(&tb->locks->lck_vec[i].lck, "db_hash_slot", make_small(i));
+	    erts_smp_rwmtx_init_opt_x(&tb->locks->lck_vec[i].lck, &rwmtx_opt,
+				      "db_hash_slot", make_small(i));
 	}
 	/* This important property is needed to guarantee that the buckets
     	 * involved in a grow/shrink operation it protected by the same lock:
 	 */
-	ASSERT(erts_smp_atomic_read(&tb->nactive) % DB_HASH_LOCK_CNT == 0);
+	ASSERT(erts_smp_atomic_read_nob(&tb->nactive) % DB_HASH_LOCK_CNT == 0);
     }
     else { /* coarse locking */
 	tb->locks = NULL;
     }
+    ERTS_THR_MEMORY_BARRIER;
 #endif /* ERST_SMP */
     return DB_ERROR_NONE;
 }
@@ -663,9 +725,7 @@ static int db_first_hash(Process *p, DbTable *tbl, Eterm *ret)
 	}
     }
     if (list != NULL) {
-	Eterm key = GETKEY(tb, list->dbterm.tpl);
-	
-	COPY_OBJECT(key, p, ret);
+	*ret = db_copy_key(p, tbl, &list->dbterm);
 	RUNLOCK_HASH(lck);
     }
     else {
@@ -713,7 +773,7 @@ static int db_next_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
 	*ret = am_EOT;
     }
     else {
-	COPY_OBJECT(GETKEY(tb, b->dbterm.tpl), p, ret);
+	*ret = db_copy_key(p, tbl, &b->dbterm);
 	RUNLOCK_HASH(lck);
     }    
     return DB_ERROR_NONE;
@@ -754,13 +814,13 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail)
     if (tb->common.status & DB_SET) {
 	HashDbTerm* bnext = b->next;
 	if (b->hvalue == INVALID_HASH) {
-	    erts_smp_atomic_inc(&tb->common.nitems);
+	    erts_smp_atomic_inc_nob(&tb->common.nitems);
 	}
 	else if (key_clash_fail) {
 	    ret = DB_ERROR_BADKEY;
 	    goto Ldone;
 	}
-	q = get_term(tb, b, obj, hval);
+	q = replace_dbterm(tb, b, obj);
 	q->next = bnext;
 	q->hvalue = hval; /* In case of INVALID_HASH */
 	*bp = q;
@@ -780,9 +840,9 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail)
 	HashDbTerm** qp = bp;
 	q = b;
 	do {
-	    if (eq(make_tuple(q->dbterm.tpl), obj)) {
+	    if (db_eq(&tb->common,obj,&q->dbterm)) {
 		if (q->hvalue == INVALID_HASH) {
-		    erts_smp_atomic_inc(&tb->common.nitems);
+		    erts_smp_atomic_inc_nob(&tb->common.nitems);
 		    q->hvalue = hval;
 		    if (q != b) { /* must move to preserve key insertion order */
 			*qp = q->next;
@@ -799,10 +859,11 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail)
     /*else DB_DUPLICATE_BAG */
 
 Lnew:
-    q = get_term(tb, NULL, obj, hval);
+    q = new_dbterm(tb, obj);
+    q->hvalue = hval;
     q->next = b;
     *bp = q;
-    nitems = erts_smp_atomic_inctest(&tb->common.nitems);
+    nitems = erts_smp_atomic_inc_read_nob(&tb->common.nitems);
     WUNLOCK_HASH(lck);
     {
 	int nactive = NACTIVE(tb);       
@@ -840,7 +901,7 @@ int db_get_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
 		while(b2 != NULL && has_key(tb,b2,key,hval))
 		    b2 = b2->next;
 	    }
-	    copy = put_term_list(p, b1, b2);
+	    copy = build_term_list(p, b1, b2, tb);
 	    CHECK_TABLES();
 	    *ret = copy;
 	    goto done;
@@ -963,13 +1024,10 @@ static int db_get_element_hash(Process *p, DbTable *tbl,
 
     while(b1 != 0) {
 	if (has_live_key(tb,b1,key,hval)) {
-	    Eterm copy;
-
 	    if (ndex > arityval(b1->dbterm.tpl[0])) {
 		retval = DB_ERROR_BADITEM;
 		goto done;
 	    }
-
 	    if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) {
 		HashDbTerm* b;
 		HashDbTerm* b2 = b1->next;
@@ -983,15 +1041,12 @@ static int db_get_element_hash(Process *p, DbTable *tbl,
 		    }
 		    b2 = b2->next;
 		}
-
 		b = b1;
 		while(b != b2) {
 		    if (b->hvalue != INVALID_HASH) {
 			Eterm *hp;
-			Uint sz = size_object(b->dbterm.tpl[ndex])+2;
-			
-			hp = HAlloc(p, sz);
-			copy = copy_struct(b->dbterm.tpl[ndex], sz-2, &hp, &MSO(p));
+			Eterm copy = db_copy_element_from_ets(&tb->common, p,
+							      &b->dbterm, ndex, &hp, 2);
 			elem_list = CONS(hp, copy, elem_list);
 			hp += 2;
 		    }
@@ -1000,8 +1055,8 @@ static int db_get_element_hash(Process *p, DbTable *tbl,
 		*ret = elem_list;
 	    }
 	    else {
-		COPY_OBJECT(b1->dbterm.tpl[ndex], p, &copy);
-		*ret = copy;
+		Eterm* hp;
+		*ret = db_copy_element_from_ets(&tb->common, p, &b1->dbterm, ndex, &hp, 0);
 	    }
 	    retval = DB_ERROR_NONE;
 	    goto done;
@@ -1036,6 +1091,7 @@ int db_erase_bag_exact2(DbTable *tbl, Eterm key, Eterm value)
 
     ASSERT(!IS_FIXED(tb));
     ASSERT((tb->common.status & DB_BAG));
+    ASSERT(!tb->common.compress);
 
     while(b != 0) {
 	if (has_live_key(tb,b,key,hval)) {
@@ -1044,7 +1100,7 @@ int db_erase_bag_exact2(DbTable *tbl, Eterm key, Eterm value)
 		EQ(value, b->dbterm.tpl[2])) {
 		*bp = b->next;
 		free_term(tb, b);
-		erts_smp_atomic_dec(&tb->common.nitems);
+		erts_smp_atomic_dec_nob(&tb->common.nitems);
 		b = *bp;
 		break;
 	    }
@@ -1103,7 +1159,7 @@ int db_erase_hash(DbTable *tbl, Eterm key, Eterm *ret)
     }
     WUNLOCK_HASH(lck);
     if (nitems_diff) {
-	erts_smp_atomic_add(&tb->common.nitems, nitems_diff);
+	erts_smp_atomic_add_nob(&tb->common.nitems, nitems_diff);
 	try_shrink(tb);
     }
     *ret = am_true;
@@ -1135,7 +1191,7 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret)
     while(b != 0) {
 	if (has_live_key(tb,b,key,hval)) {
 	    ++nkeys;
-	    if (eq(object, make_tuple(b->dbterm.tpl))) {
+	    if (db_eq(&tb->common,object, &b->dbterm)) {
 		--nitems_diff;
 		if (nkeys==1 && IS_FIXED(tb)) { /* Pseudo remove */
 		    add_fixed_deletion(tb,ix);
@@ -1162,7 +1218,7 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret)
     }
     WUNLOCK_HASH(lck);
     if (nitems_diff) {
-	erts_smp_atomic_add(&tb->common.nitems, nitems_diff);
+	erts_smp_atomic_add_nob(&tb->common.nitems, nitems_diff);
 	try_shrink(tb);
     }
     *ret = am_true;
@@ -1184,7 +1240,7 @@ static int db_slot_hash(Process *p, DbTable *tbl, Eterm slot_term, Eterm *ret)
     lck = RLOCK_HASH(tb, slot);
     nactive = NACTIVE(tb);
     if (slot < nactive) {
-	*ret = put_term_list(p, BUCKET(tb, slot), 0);
+	*ret = build_term_list(p, BUCKET(tb, slot), 0, tb);
 	retval = DB_ERROR_NONE;
     }
     else if (slot == nactive) {
@@ -1228,8 +1284,6 @@ static int db_select_continue_hash(Process *p,
     int num_left = 1000;
     HashDbTerm *current = 0;
     Eterm match_list;
-    Uint32 dummy;
-    unsigned sz;
     Eterm *hp;
     Eterm match_res;
     Sint got;
@@ -1281,26 +1335,14 @@ static int db_select_continue_hash(Process *p,
     }	  
     for(;;) {
 	if (current->hvalue != INVALID_HASH && 
-	    (match_res = 
-	     db_prog_match(p,mp,
-			   make_tuple(current->dbterm.tpl),
-			   0,&dummy),
+	    (match_res = db_match_dbterm(&tb->common, p, mp, all_objects,
+					 &current->dbterm, &hp, 2),
 	     is_value(match_res))) {
-	    if (all_objects) {
-		hp = HAlloc(p, current->dbterm.size + 2);
-		match_res = copy_shallow(DBTERM_BUF(&current->dbterm),
-					 current->dbterm.size,
-					 &hp,
-					 &MSO(p));
-	    } else {
-		sz = size_object(match_res);
-	    
-		hp = HAlloc(p, sz + 2);
-		match_res = copy_struct(match_res, sz, &hp, &MSO(p));
-	    }
-            match_list = CONS(hp, match_res, match_list);
+
+	    match_list = CONS(hp, match_res, match_list);
 	    ++got;
 	}
+
 	--num_left;
 	save_slot_ix = slot_ix;
 	if ((current = next(tb, (Uint*)&slot_ix, &lck, current)) == NULL) {
@@ -1391,9 +1433,7 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl,
     HashDbTerm *current = 0;
     unsigned current_list_pos = 0;
     Eterm match_list;
-    Uint32 dummy;
     Eterm match_res;
-    unsigned sz;
     Eterm *hp;
     int num_left = 1000;
     Uint got = 0;
@@ -1460,22 +1500,9 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl,
     for(;;) {
 	if (current != NULL) {
 	    if (current->hvalue != INVALID_HASH) {
-		match_res = db_prog_match(p,mpi.mp,
-					  make_tuple(current->dbterm.tpl),
-					  0,&dummy);
+		match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0,
+					    &current->dbterm, &hp, 2);
 		if (is_value(match_res)) {
-		    if (mpi.all_objects) {
-			hp = HAlloc(p, current->dbterm.size + 2);
-			match_res = copy_shallow(DBTERM_BUF(&current->dbterm),
-						 current->dbterm.size,
-						 &hp,
-						 &MSO(p));
-		    } else {
-			sz = size_object(match_res);
-			
-			hp = HAlloc(p, sz + 2);
-			match_res = copy_struct(match_res, sz, &hp, &MSO(p));
-		    }
 		    match_list = CONS(hp, match_res, match_list);
 		    ++got;
 		}
@@ -1590,7 +1617,6 @@ static int db_select_count_hash(Process *p,
     Uint slot_ix = 0;
     HashDbTerm* current = NULL;
     unsigned current_list_pos = 0;
-    Uint32 dummy;
     Eterm *hp;
     int num_left = 1000;
     Uint got = 0;
@@ -1640,8 +1666,8 @@ static int db_select_count_hash(Process *p,
     for(;;) {
 	if (current != NULL) {
 	    if (current->hvalue != INVALID_HASH) {
-		if (db_prog_match(p, mpi.mp, make_tuple(current->dbterm.tpl),
-				  0, &dummy) == am_true) {
+		if (db_match_dbterm(&tb->common, p, mpi.mp, 0,
+				    &current->dbterm, NULL,0) == am_true) {
 		    ++got;
 		}
 		--num_left;
@@ -1709,7 +1735,6 @@ static int db_select_delete_hash(Process *p,
     Uint slot_ix = 0;
     HashDbTerm **current = NULL;
     unsigned current_list_pos = 0;
-    Uint32 dummy;
     Eterm *hp;
     int num_left = 1000;
     Uint got = 0;
@@ -1719,9 +1744,9 @@ static int db_select_delete_hash(Process *p,
     Eterm mpb;
     Eterm egot;
 #ifdef ERTS_SMP
-    int fixated_by_me = tb->common.is_thread_safe ? 0 : 1; /* ToDo: something nicer */
+    erts_aint_t fixated_by_me = tb->common.is_thread_safe ? 0 : 1; /* ToDo: something nicer */
 #else
-    int fixated_by_me = 0;
+    erts_aint_t fixated_by_me = 0;
 #endif
     erts_smp_rwmtx_t* lck;
 
@@ -1790,9 +1815,8 @@ static int db_select_delete_hash(Process *p,
 	} 
 	else {
 	    int did_erase = 0;
-	    if ((db_prog_match(p,mpi.mp,
-			       make_tuple((*current)->dbterm.tpl),
-			       0,&dummy)) == am_true) {
+	    if (db_match_dbterm(&tb->common, p, mpi.mp, 0,
+				&(*current)->dbterm, NULL, 0) == am_true) {
 		if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */
 		    if (slot_ix != last_pseudo_delete) {
 			add_fixed_deletion(tb, slot_ix);
@@ -1805,7 +1829,7 @@ static int db_select_delete_hash(Process *p,
 		    free_term(tb, del);
 		    did_erase = 1;
 		}
-		erts_smp_atomic_dec(&tb->common.nitems);
+		erts_smp_atomic_dec_nob(&tb->common.nitems);
 		++got;
 	    }	    
 	    --num_left;
@@ -1855,7 +1879,6 @@ static int db_select_delete_continue_hash(Process *p,
     Uint slot_ix;
     Uint last_pseudo_delete = (Uint)-1;
     HashDbTerm **current = NULL;
-    Uint32 dummy;
     Eterm *hp;
     int num_left = 1000;
     Uint got;
@@ -1903,8 +1926,8 @@ static int db_select_delete_continue_hash(Process *p,
 	} 
 	else {
 	    int did_erase = 0;
-	    if ((db_prog_match(p,mp,make_tuple((*current)->dbterm.tpl),
-			       0,&dummy)) == am_true) {
+	    if (db_match_dbterm(&tb->common, p, mp, 0,
+				&(*current)->dbterm, NULL, 0) == am_true) {
 		if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */
 		    if (slot_ix != last_pseudo_delete) {
 			add_fixed_deletion(tb, slot_ix);
@@ -1917,7 +1940,7 @@ static int db_select_delete_continue_hash(Process *p,
 		    free_term(tb, del);
 		    did_erase = 1;
 		}
-		erts_smp_atomic_dec(&tb->common.nitems);
+		erts_smp_atomic_dec_nob(&tb->common.nitems);
 		++got;
 	    }
 	    
@@ -1966,7 +1989,6 @@ static int db_select_count_continue_hash(Process *p,
     DbTableHash *tb = &tbl->hash;
     Uint slot_ix;
     HashDbTerm* current;
-    Uint32 dummy;
     Eterm *hp;
     int num_left = 1000;
     Uint got;
@@ -2004,8 +2026,8 @@ static int db_select_count_continue_hash(Process *p,
 		current = current->next;
 		continue;
 	    }
-	    if (db_prog_match(p, mp, make_tuple(current->dbterm.tpl),
-			      0,&dummy) == am_true) {
+	    if (db_match_dbterm(&tb->common, p, mp, 0, &current->dbterm,
+				NULL, 0) == am_true) {
 		++got;
 	    }
 	    --num_left;
@@ -2073,7 +2095,7 @@ int db_mark_all_deleted_hash(DbTable *tbl)
 	    }while(list != NULL);
 	}
     }
-    erts_smp_atomic_set(&tb->common.nitems, 0);    
+    erts_smp_atomic_set_nob(&tb->common.nitems, 0);    
     return DB_ERROR_NONE;
 }
 
@@ -2095,7 +2117,14 @@ static void db_print_hash(int to, void *to_arg, int show, DbTable *tbl)
 	    while(list != 0) {
 		if (list->hvalue == INVALID_HASH)
 		    erts_print(to, to_arg, "*");
-		erts_print(to, to_arg, "%T", make_tuple(list->dbterm.tpl));
+		if (tb->common.compress) {
+		    Eterm key = GETKEY(tb, list->dbterm.tpl);
+		    erts_print(to, to_arg, "key=%R", key, list->dbterm.tpl);
+		}
+		else {
+		    Eterm obj = make_tuple_rel(list->dbterm.tpl,list->dbterm.tpl);
+		    erts_print(to, to_arg, "%R", obj, list->dbterm.tpl);
+		}
 		if (list->next != 0)
 		    erts_print(to, to_arg, ",");
 		list = list->next;
@@ -2117,7 +2146,7 @@ static int db_free_table_continue_hash(DbTable *tbl)
 {
     DbTableHash *tb = &tbl->hash;
     int done;
-    FixedDeletion* fixdel = (FixedDeletion*) erts_smp_atomic_read(&tb->fixdel);
+    FixedDeletion* fixdel = (FixedDeletion*) erts_smp_atomic_read_acqb(&tb->fixdel);
     ERTS_SMP_LC_ASSERT(IS_TAB_WLOCKED(tb));
 
     done = 0;
@@ -2131,11 +2160,11 @@ static int db_free_table_continue_hash(DbTable *tbl)
 		     sizeof(FixedDeletion));
 	ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion));
 	if (++done >= 2*DELETE_RECORD_LIMIT) {
-	    erts_smp_atomic_set(&tb->fixdel, (long)fixdel);
+	    erts_smp_atomic_set_relb(&tb->fixdel, (erts_aint_t)fixdel);
 	    return 0;		/* Not done */
 	}
     }
-    erts_smp_atomic_set(&tb->fixdel, (long)NULL);
+    erts_smp_atomic_set_relb(&tb->fixdel, (erts_aint_t)NULL);
 
     done /= 2;
     while(tb->nslots != 0) {
@@ -2159,7 +2188,7 @@ static int db_free_table_continue_hash(DbTable *tbl)
 	tb->locks = NULL;
     }
 #endif    
-    ASSERT(erts_smp_atomic_read(&tb->common.memory_size) == sizeof(DbTable));
+    ASSERT(erts_smp_atomic_read_nob(&tb->common.memory_size) == sizeof(DbTable));
     return 1;			/* Done */
 }
 
@@ -2184,7 +2213,7 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern,
     HashValue hval = NIL;      
     int num_heads = 0;
     int i;
-    
+
     mpi->lists = mpi->dlists;
     mpi->num_lists = 0;
     mpi->key_given = 1;
@@ -2352,7 +2381,7 @@ static int alloc_seg(DbTableHash *tb)
 	    struct ext_segment* eseg;
 	    eseg = (struct ext_segment*) SEGTAB(tb)[seg_ix-1];
 	    MY_ASSERT(eseg!=NULL && eseg->s.is_ext_segment);
-	    erts_smp_atomic_set(&tb->segtab, (long) eseg->segtab);
+	    SET_SEGTAB(tb, eseg->segtab);	    
 	    tb->nsegs = eseg->nsegs;
 	}
 	ASSERT(seg_ix < tb->nsegs);
@@ -2424,7 +2453,7 @@ static int free_seg(DbTableHash *tb, int free_records)
 	    MY_ASSERT(newtop->s.is_ext_segment);
 	    if (newtop->prev_segtab != NULL) {
 		/* Time to use a smaller segtab */
-		erts_smp_atomic_set(&tb->segtab, (long)newtop->prev_segtab);
+		SET_SEGTAB(tb, newtop->prev_segtab);
 		tb->nsegs = seg_ix;
 		ASSERT(tb->nsegs == EXTSEG(SEGTAB(tb))->nsegs);
 	    }
@@ -2441,7 +2470,7 @@ static int free_seg(DbTableHash *tb, int free_records)
     if (seg_ix > 0) {
 	if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL;
     } else {
-	erts_smp_atomic_set(&tb->segtab, (long)NULL);
+	SET_SEGTAB(tb, NULL);
     }
 #endif
     tb->nslots -= SEGSZ;
@@ -2450,31 +2479,19 @@ static int free_seg(DbTableHash *tb, int free_records)
 }
 
 
-static HashDbTerm* get_term(DbTableHash* tb, HashDbTerm* old, 
-			    Eterm obj, HashValue hval)
-{
-    HashDbTerm* p = db_get_term((DbTableCommon *) tb,
-				(old != NULL) ? &(old->dbterm) : NULL, 
-				((char *) &(old->dbterm)) - ((char *) old),
-				obj);
-    p->hvalue = hval;
-    /*p->next = NULL;*/ /*No Need */
-    return p;
-}
-
-
 /*
 ** Copy terms from ptr1 until ptr2
 ** works for ptr1 == ptr2 == 0  => []
 ** or ptr2 == 0
 */
-static Eterm put_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2)
+static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2,
+			   DbTableHash* tb)
 {
     int sz = 0;
     HashDbTerm* ptr;
     Eterm list = NIL;
     Eterm copy;
-    Eterm *hp;
+    Eterm *hp, *hend;
 
     ptr = ptr1;
     while(ptr != ptr2) {
@@ -2486,26 +2503,42 @@ static Eterm put_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2)
     }
 
     hp = HAlloc(p, sz);
+    hend = hp + sz;
 
     ptr = ptr1;
     while(ptr != ptr2) {
 	if (ptr->hvalue != INVALID_HASH) {
-	    copy = copy_shallow(DBTERM_BUF(&ptr->dbterm), ptr->dbterm.size, &hp, &MSO(p));
+	    copy = db_copy_object_from_ets(&tb->common, &ptr->dbterm, &hp, &MSO(p));
 	    list = CONS(hp, copy, list);
 	    hp  += 2;
 	}
 	ptr = ptr->next;
     }
+    HRelease(p,hend,hp);
+
     return list;
 }
 
-static void free_term(DbTableHash *tb, HashDbTerm* p)
+static ERTS_INLINE int
+begin_resizing(DbTableHash* tb)
 {
-    db_free_term_data(&(p->dbterm));
-    erts_db_free(ERTS_ALC_T_DB_TERM,
-		 (DbTable *) tb,
-		 (void *) p,
-		 SIZ_DBTERM(p)*sizeof(Eterm));
+    if (DB_USING_FINE_LOCKING(tb))
+	return !erts_smp_atomic_xchg_acqb(&tb->is_resizing, 1);
+    else {
+	if (erts_smp_atomic_read_nob(&tb->is_resizing))
+	    return 0;
+	erts_smp_atomic_set_nob(&tb->is_resizing, 1);
+	return 1;
+    }
+}
+
+static ERTS_INLINE void
+done_resizing(DbTableHash* tb)
+{
+    if (DB_USING_FINE_LOCKING(tb))
+	erts_smp_atomic_set_relb(&tb->is_resizing, 0);
+    else
+	erts_smp_atomic_set_nob(&tb->is_resizing, 0);
 }
 
 /* Grow table with one new bucket.
@@ -2520,9 +2553,8 @@ static void grow(DbTableHash* tb, int nactive)
     int from_ix;
     int szm;
 
-    if (erts_smp_atomic_xchg(&tb->is_resizing, 1)) { 
+    if (!begin_resizing(tb))
 	return; /* already in progress */
-    }
     if (NACTIVE(tb) != nactive) {
 	goto abort; /* already done (race) */
     }
@@ -2535,7 +2567,7 @@ static void grow(DbTableHash* tb, int nactive)
     }
     ASSERT(nactive < tb->nslots);
 
-    szm = erts_smp_atomic_read(&tb->szm);
+    szm = erts_smp_atomic_read_nob(&tb->szm);
     if (nactive <= szm) {
 	from_ix = nactive & (szm >> 1);
     } else {
@@ -2552,11 +2584,14 @@ static void grow(DbTableHash* tb, int nactive)
 	WUNLOCK_HASH(lck);
 	goto abort;
     }
-    erts_smp_atomic_inc(&tb->nactive);
+    erts_smp_atomic_inc_nob(&tb->nactive);
     if (from_ix == 0) {
-	erts_smp_atomic_set(&tb->szm, szm);
+	if (DB_USING_FINE_LOCKING(tb))
+	    erts_smp_atomic_set_relb(&tb->szm, szm);
+	else
+	    erts_smp_atomic_set_nob(&tb->szm, szm);
     }
-    erts_smp_atomic_set(&tb->is_resizing, 0);
+    done_resizing(tb);
 
     /* Finally, let's split the bucket. We try to do it in a smart way
        to keep link order and avoid unnecessary updates of next-pointers */
@@ -2588,7 +2623,7 @@ static void grow(DbTableHash* tb, int nactive)
     return;
    
 abort:
-    erts_smp_atomic_set(&tb->is_resizing, 0);
+    done_resizing(tb);
 }
 
 
@@ -2597,13 +2632,12 @@ abort:
 */
 static void shrink(DbTableHash* tb, int nactive)
 {     
-    if (erts_smp_atomic_xchg(&tb->is_resizing, 1)) {
+    if (!begin_resizing(tb))
 	return; /* already in progress */
-    }
     if (NACTIVE(tb) == nactive) {
 	erts_smp_rwmtx_t* lck;
 	int src_ix = nactive - 1;
-	int low_szm = erts_smp_atomic_read(&tb->szm) >> 1;
+	int low_szm = erts_smp_atomic_read_nob(&tb->szm) >> 1;
 	int dst_ix = src_ix & low_szm;
 
 	ASSERT(dst_ix < src_ix);
@@ -2630,9 +2664,9 @@ static void shrink(DbTableHash* tb, int nactive)
 	    *dst_bp = *src_bp;
 	    *src_bp = NULL;
 	    
-	    erts_smp_atomic_set(&tb->nactive, src_ix);
+	    erts_smp_atomic_set_nob(&tb->nactive, src_ix);
 	    if (dst_ix == 0) {
-		erts_smp_atomic_set(&tb->szm, low_szm);
+		erts_smp_atomic_set_relb(&tb->szm, low_szm);
 	    }
 	    WUNLOCK_HASH(lck);
 	    
@@ -2646,7 +2680,7 @@ static void shrink(DbTableHash* tb, int nactive)
 
     }
     /*else already done */
-    erts_smp_atomic_set(&tb->is_resizing, 0);
+    done_resizing(tb);
 }
 
 
@@ -2716,8 +2750,11 @@ static int db_lookup_dbterm_hash(DbTable *tbl, Eterm key, DbUpdateHandle* handle
 	    handle->tb = tbl;
 	    handle->bp = (void**) prevp;
 	    handle->dbterm = &b->dbterm;
-	    handle->new_size = b->dbterm.size;
 	    handle->mustResize = 0;
+	    handle->new_size = b->dbterm.size;
+	#if HALFWORD_HEAP
+	    handle->abs_vec = NULL;
+	#endif
 	    handle->lck = lck;
 	    /* KEEP hval WLOCKED, db_finalize_dbterm_hash will WUNLOCK */
 	    return 1;
@@ -2738,39 +2775,14 @@ static void db_finalize_dbterm_hash(DbUpdateHandle* handle)
     erts_smp_rwmtx_t* lck = (erts_smp_rwmtx_t*) handle->lck;
 
     ERTS_SMP_LC_ASSERT(IS_HASH_WLOCKED(&tbl->hash,lck));  /* locked by db_lookup_dbterm_hash */
-    ASSERT(&oldp->dbterm == handle->dbterm);
 
-    if (handle->mustResize) {
-	Eterm* top;
-	Eterm copy;
-	DbTerm* newDbTerm;
-	HashDbTerm* newp = erts_db_alloc(ERTS_ALC_T_DB_TERM, tbl,
-					 sizeof(HashDbTerm)+sizeof(Eterm)*(handle->new_size-1));    
-	sys_memcpy(newp, oldp, sizeof(HashDbTerm)-sizeof(DbTerm));  /* copy only hashtab header */
-	*(handle->bp) = newp;
-	newDbTerm = &newp->dbterm;
-    
-	newDbTerm->size = handle->new_size;
-	newDbTerm->off_heap.mso = NULL;
-	newDbTerm->off_heap.externals = NULL;
-    #ifndef HYBRID /* FIND ME! */
-	newDbTerm->off_heap.funs = NULL;
-    #endif
-	newDbTerm->off_heap.overhead = 0;
-	
-	/* make a flat copy */
-	top = DBTERM_BUF(newDbTerm);
-	copy = copy_struct(make_tuple(handle->dbterm->tpl),
-			   handle->new_size,
-			   &top, &newDbTerm->off_heap);
-	DBTERM_SET_TPL(newDbTerm,tuple_val(copy));
+    ASSERT((&oldp->dbterm == handle->dbterm) == !(tbl->common.compress && handle->mustResize));
 
+    if (handle->mustResize) {
+	db_finalize_resize(handle, offsetof(HashDbTerm,dbterm));
 	WUNLOCK_HASH(lck);
-		
-	db_free_term_data(handle->dbterm);
-	erts_db_free(ERTS_ALC_T_DB_TERM, tbl,
-		     (void *) (((char *) handle->dbterm) - (sizeof(HashDbTerm) - sizeof(DbTerm))),
-		     sizeof(HashDbTerm) + sizeof(Eterm)*(handle->dbterm->size-1));
+
+	free_term(&tbl->hash, oldp);
     }
     else {
 	WUNLOCK_HASH(lck);
@@ -2779,7 +2791,7 @@ static void db_finalize_dbterm_hash(DbUpdateHandle* handle)
     handle->dbterm = 0;
 #endif
     return;
-}   
+}
 
 static int db_delete_all_objects_hash(Process* p, DbTable* tbl)
 {
@@ -2788,7 +2800,7 @@ static int db_delete_all_objects_hash(Process* p, DbTable* tbl)
     } else {
 	db_free_table_hash(tbl);
 	db_create_hash(p, tbl);
-	erts_smp_atomic_set(&tbl->hash.common.nitems, 0);
+	erts_smp_atomic_set_nob(&tbl->hash.common.nitems, 0);
     }
     return 0;
 }
@@ -2805,7 +2817,11 @@ void db_foreach_offheap_hash(DbTable *tbl,
     for (i = 0; i < nactive; i++) {
 	list = BUCKET(tb,i);
 	while(list != 0) {
-	    (*func)(&(list->dbterm.off_heap), arg);
+	    ErlOffHeap tmp_offheap;
+	    tmp_offheap.first = list->dbterm.first_oh;
+	    tmp_offheap.overhead = 0;
+	    (*func)(&tmp_offheap, arg);
+	    list->dbterm.first_oh = tmp_offheap.first;
 	    list = list->next;
 	}
     }
diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h
index e0285fa5ed..cddd8dfadd 100644
--- a/erts/emulator/beam/erl_db_hash.h
+++ b/erts/emulator/beam/erl_db_hash.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -58,6 +58,9 @@ typedef struct db_table_hash {
 #ifdef ERTS_SMP
     DbTableHashFineLocks* locks;
 #endif
+#ifdef VALGRIND
+    struct ext_segment* top_ptr_to_segment_with_active_segtab;
+#endif
 } DbTableHash;
 
 
diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index b421da591b..312050b931 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -48,11 +48,8 @@
 
 #include "erl_db_tree.h"
 
-
-
-#define GETKEY(dtt, tplp)   (*((tplp) +  (dtt)->common.keypos))
 #define GETKEY_WITH_POS(Keypos, Tplp) (*((Tplp) + Keypos))
-#define NITEMS(tb) ((int)erts_smp_atomic_read(&(tb)->common.nitems))
+#define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems))
 
 /*
 ** A stack of this size is enough for an AVL tree with more than
@@ -87,7 +84,7 @@
 */
 static DbTreeStack* get_static_stack(DbTableTree* tb)
 {
-    if (!erts_smp_atomic_xchg(&tb->is_stack_busy, 1)) {
+    if (!erts_smp_atomic_xchg_acqb(&tb->is_stack_busy, 1)) {
 	return &tb->static_stack;
     }
     return NULL;
@@ -99,7 +96,7 @@ static DbTreeStack* get_static_stack(DbTableTree* tb)
 static DbTreeStack* get_any_stack(DbTableTree* tb)
 {
     DbTreeStack* stack;
-    if (!erts_smp_atomic_xchg(&tb->is_stack_busy, 1)) {
+    if (!erts_smp_atomic_xchg_acqb(&tb->is_stack_busy, 1)) {
 	return &tb->static_stack;
     }
     stack = erts_db_alloc(ERTS_ALC_T_DB_STK, (DbTable *) tb,
@@ -113,8 +110,8 @@ static DbTreeStack* get_any_stack(DbTableTree* tb)
 static void release_stack(DbTableTree* tb, DbTreeStack* stack)
 {
     if (stack == &tb->static_stack) {
-	ASSERT(erts_smp_atomic_read(&tb->is_stack_busy) == 1);
-	erts_smp_atomic_set(&tb->is_stack_busy, 0);
+	ASSERT(erts_smp_atomic_read_nob(&tb->is_stack_busy) == 1);
+	erts_smp_atomic_set_relb(&tb->is_stack_busy, 0);
     }
     else {
 	erts_db_free(ERTS_ALC_T_DB_STK, (DbTable *) tb,
@@ -122,12 +119,41 @@ static void release_stack(DbTableTree* tb, DbTreeStack* stack)
     }
 }
 
-static void reset_static_stack(DbTableTree* tb)
+static ERTS_INLINE void reset_static_stack(DbTableTree* tb)
 {
     tb->static_stack.pos = 0;
     tb->static_stack.slot = 0;
 }
 
+static ERTS_INLINE void free_term(DbTableTree *tb, TreeDbTerm* p)
+{
+    db_free_term((DbTable*)tb, p, offsetof(TreeDbTerm, dbterm));
+}
+
+static ERTS_INLINE TreeDbTerm* new_dbterm(DbTableTree *tb, Eterm obj)
+{
+    TreeDbTerm* p;
+    if (tb->common.compress) {
+	p = db_store_term_comp(&tb->common, NULL, offsetof(TreeDbTerm,dbterm), obj);
+    }
+    else {
+	p = db_store_term(&tb->common, NULL, offsetof(TreeDbTerm,dbterm), obj);
+    }
+    return p;
+}
+static ERTS_INLINE TreeDbTerm* replace_dbterm(DbTableTree *tb, TreeDbTerm* old,
+					      Eterm obj)
+{
+    TreeDbTerm* p;
+    ASSERT(old != NULL);
+    if (tb->common.compress) {
+	p = db_store_term_comp(&tb->common, &(old->dbterm), offsetof(TreeDbTerm,dbterm), obj);
+    }
+    else {
+	p = db_store_term(&tb->common, &(old->dbterm), offsetof(TreeDbTerm,dbterm), obj);
+    }
+    return p;
+}
 
 /*
 ** Some macros for "direction stacks"
@@ -153,7 +179,7 @@ static void reset_static_stack(DbTableTree* tb)
 static TreeDbTerm *traverse_until(TreeDbTerm *t, int *current, int to);
 static void check_slot_pos(DbTableTree *tb);
 static void check_saved_stack(DbTableTree *tb);
-static int check_table_tree(TreeDbTerm *t);
+static int check_table_tree(DbTableTree* tb, TreeDbTerm *t);
 
 #define TREE_DEBUG
 #endif
@@ -168,8 +194,8 @@ static int check_table_tree(TreeDbTerm *t);
 ** Debugging dump
 */
 
-static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t,
-			  int offset);
+static void do_dump_tree2(DbTableTree*, int to, void *to_arg, int show,
+			  TreeDbTerm *t, int offset);
 
 #else
 
@@ -178,12 +204,6 @@ static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t,
 #endif
 
 /*
- * Size calculations
- */
-#define SIZ_OVERHEAD ((sizeof(TreeDbTerm)/sizeof(Eterm)) - 1)
-#define SIZ_DBTERM(TDT) (SIZ_OVERHEAD + (TDT)->dbterm.size)
-
-/*
 ** Datatypes
 */
 
@@ -259,13 +279,10 @@ struct select_delete_context {
 /*
 ** Forward declarations 
 */
-static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key);
+static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key, Eterm* key_base);
 static TreeDbTerm *linkout_object_tree(DbTableTree *tb, 
 				       Eterm object);
 static int do_free_tree_cont(DbTableTree *tb, int num_left);
-static TreeDbTerm* get_term(DbTableTree *tb,
-			    TreeDbTerm* old, 
-			    Eterm obj);
 static void free_term(DbTableTree *tb, TreeDbTerm* p);
 static int balance_left(TreeDbTerm **this); 
 static int balance_right(TreeDbTerm **this); 
@@ -273,15 +290,15 @@ static int delsub(TreeDbTerm **this);
 static TreeDbTerm *slot_search(Process *p, DbTableTree *tb, Sint slot);
 static TreeDbTerm *find_node(DbTableTree *tb, Eterm key);
 static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key);
-static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key);
-static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key);
+static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key, Eterm* kbase);
+static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key, Eterm* kbase);
 static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack*,
 					 Eterm key);
 static TreeDbTerm *find_prev_from_pb_key(DbTableTree *tb, DbTreeStack*,
 					 Eterm key);
 static void traverse_backwards(DbTableTree *tb,
 			       DbTreeStack*,
-			       Eterm lastkey,
+			       Eterm lastkey, Eterm* lk_base,
 			       int (*doit)(DbTableTree *tb,
 					   TreeDbTerm *,
 					   void *,
@@ -289,7 +306,7 @@ static void traverse_backwards(DbTableTree *tb,
 			       void *context); 
 static void traverse_forward(DbTableTree *tb,
 			     DbTreeStack*,
-			     Eterm lastkey,
+			     Eterm lastkey, Eterm* lk_base,
 			     int (*doit)(DbTableTree *tb,
 					 TreeDbTerm *,
 					 void *,
@@ -297,8 +314,8 @@ static void traverse_forward(DbTableTree *tb,
 			     void *context); 
 static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret,
 		     Eterm *partly_bound_key);
-static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key);
-static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done);
+static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_base);
+static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done);
 
 static int analyze_pattern(DbTableTree *tb, Eterm pattern, 
 			   struct mp_info *mpi);
@@ -318,7 +335,6 @@ static int doit_select_delete(DbTableTree *tb,
 			      TreeDbTerm *this,
 			      void *ptr,
 			      int forward);
-static void do_dump_tree(int to, void *to_arg, TreeDbTerm *t);
 
 static int partly_bound_can_match_lesser(Eterm partly_bound_1, 
 					 Eterm partly_bound_2);
@@ -328,8 +344,8 @@ static int do_partly_bound_can_match_lesser(Eterm a, Eterm b,
 					    int *done);
 static int do_partly_bound_can_match_greater(Eterm a, Eterm b, 
 					     int *done);
-static BIF_RETTYPE ets_select_reverse(Process *p, Eterm a1, 
-				      Eterm a2, Eterm a3);
+static BIF_RETTYPE ets_select_reverse(BIF_ALIST_3);
+
 
 /* Method interface functions */
 static int db_first_tree(Process *p, DbTable *tbl, 
@@ -443,9 +459,9 @@ void db_initialize_tree(void)
     ets_select_reverse_exp.code[1] = am_reverse;
     ets_select_reverse_exp.code[2] = 3;
     ets_select_reverse_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     ets_select_reverse_exp.code[4] = 
-	(Eterm) &ets_select_reverse;
+	(BeamInstr) &ets_select_reverse;
     return;
 };
 
@@ -462,7 +478,7 @@ int db_create_tree(Process *p, DbTable *tbl)
 					   sizeof(TreeDbTerm *) * STACK_NEED);
     tb->static_stack.pos = 0;
     tb->static_stack.slot = 0;
-    erts_smp_atomic_init(&tb->is_stack_busy, 0);
+    erts_smp_atomic_init_nob(&tb->is_stack_busy, 0);
     tb->deletion = 0;
     return DB_ERROR_NONE;
 }
@@ -472,9 +488,6 @@ static int db_first_tree(Process *p, DbTable *tbl, Eterm *ret)
     DbTableTree *tb = &tbl->tree;
     DbTreeStack* stack;
     TreeDbTerm *this;
-    Eterm e;
-    Eterm *hp;
-    Uint sz;
 
     if (( this = tb->root ) == NULL) {
 	*ret = am_EOT;
@@ -493,13 +506,7 @@ static int db_first_tree(Process *p, DbTable *tbl, Eterm *ret)
 	stack->slot = 1;
 	release_stack(tb,stack);
     }
-    e = GETKEY(tb, this->dbterm.tpl);
-    sz = size_object(e);
-
-    hp = HAlloc(p, sz);
-
-    *ret = copy_struct(e,sz,&hp,&MSO(p));
-    
+    *ret = db_copy_key(p, tbl, &this->dbterm);
     return DB_ERROR_NONE;
 }
 
@@ -508,26 +515,17 @@ static int db_next_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
     DbTableTree *tb = &tbl->tree;
     DbTreeStack* stack;
     TreeDbTerm *this;
-    Eterm e;
-    Eterm *hp;
-    Uint sz;
 
     if (is_atom(key) && key == am_EOT)
 	return DB_ERROR_BADKEY;
     stack = get_any_stack(tb);
-    this = find_next(tb, stack, key);
+    this = find_next(tb, stack, key, NULL);
     release_stack(tb,stack);
     if (this == NULL) {
 	*ret = am_EOT;
 	return DB_ERROR_NONE;
     }
-    e = GETKEY(tb, this->dbterm.tpl);
-    sz = size_object(e);
-
-    hp = HAlloc(p, sz);
-
-    *ret = copy_struct(e,sz,&hp,&MSO(p));
-    
+    *ret = db_copy_key(p, tbl, &this->dbterm);
     return DB_ERROR_NONE;
 }
 
@@ -536,9 +534,6 @@ static int db_last_tree(Process *p, DbTable *tbl, Eterm *ret)
     DbTableTree *tb = &tbl->tree;
     TreeDbTerm *this;
     DbTreeStack* stack;
-    Eterm e;
-    Eterm *hp;
-    Uint sz;
 
     if (( this = tb->root ) == NULL) {
 	*ret = am_EOT;
@@ -557,13 +552,7 @@ static int db_last_tree(Process *p, DbTable *tbl, Eterm *ret)
 	stack->slot = NITEMS(tb);
 	release_stack(tb,stack);
     }
-    e = GETKEY(tb, this->dbterm.tpl);
-    sz = size_object(e);
-
-    hp = HAlloc(p, sz);
-
-    *ret = copy_struct(e,sz,&hp,&MSO(p));
-    
+    *ret = db_copy_key(p, tbl, &this->dbterm);
     return DB_ERROR_NONE;
 }
 
@@ -572,27 +561,33 @@ static int db_prev_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
     DbTableTree *tb = &tbl->tree;
     TreeDbTerm *this;
     DbTreeStack* stack;
-    Eterm e;
-    Eterm *hp;
-    Uint sz;
 
     if (is_atom(key) && key == am_EOT)
 	return DB_ERROR_BADKEY;
     stack = get_any_stack(tb);
-    this = find_prev(tb, stack, key);
+    this = find_prev(tb, stack, key, NULL);
     release_stack(tb,stack);
     if (this == NULL) {
 	*ret = am_EOT;
 	return DB_ERROR_NONE;
     }
-    e = GETKEY(tb, this->dbterm.tpl);
-    sz = size_object(e);
+    *ret = db_copy_key(p, tbl, &this->dbterm);
+    return DB_ERROR_NONE;
+}
 
-    hp = HAlloc(p, sz);
+static ERTS_INLINE Sint cmp_key(DbTableTree* tb, Eterm key, Eterm* key_base,
+			       TreeDbTerm* obj)
+{
+    return cmp_rel(key, key_base,
+		   GETKEY(tb,obj->dbterm.tpl), obj->dbterm.tpl);
+}
 
-    *ret = copy_struct(e,sz,&hp,&MSO(p));
-    
-    return DB_ERROR_NONE;
+static ERTS_INLINE int cmp_key_eq(DbTableTree* tb, Eterm key, Eterm* key_base,
+				  TreeDbTerm* obj)
+{
+    Eterm obj_key = GETKEY(tb,obj->dbterm.tpl);
+    return is_same(key, key_base, obj_key, obj->dbterm.tpl)
+	|| cmp_rel(key, key_base, obj_key, obj->dbterm.tpl) == 0;
 }
 
 static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail)
@@ -618,16 +613,16 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail)
     for (;;)
 	if (!*this) { /* Found our place */
 	    state = 1;
-	    if (erts_smp_atomic_inctest(&tb->common.nitems) >= TREE_MAX_ELEMENTS) {
-		erts_smp_atomic_dec(&tb->common.nitems);
+	    if (erts_smp_atomic_inc_read_nob(&tb->common.nitems) >= TREE_MAX_ELEMENTS) {
+		erts_smp_atomic_dec_nob(&tb->common.nitems);
 		return DB_ERROR_SYSRES;
 	    }
-	    *this = get_term(tb, NULL, obj);
+	    *this = new_dbterm(tb, obj);
 	    (*this)->balance = 0;
 	    (*this)->left = (*this)->right = NULL;
 	    break;
-	} else if ((c = cmp(key,GETKEY(tb,(*this)->dbterm.tpl))) < 0) { 
-	    /* go left */
+	} else if ((c = cmp_key(tb, key, NULL, *this)) < 0) {
+	    /* go lefts */
 	    dstack[dpos++] = DIR_LEFT;
 	    tstack[tpos++] = this;
 	    this = &((*this)->left);
@@ -636,7 +631,7 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail)
 	    tstack[tpos++] = this;
 	    this = &((*this)->right);
 	} else if (!key_clash_fail) { /* Equal key and this is a set, replace. */
-	    *this = get_term(tb, *this, obj);
+	    *this = replace_dbterm(tb, *this, obj);
 	    break;
 	} else {
 	    return DB_ERROR_BADKEY; /* key already exists */
@@ -714,7 +709,7 @@ static int db_get_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
 {
     DbTableTree *tb = &tbl->tree;
     Eterm copy;
-    Eterm *hp;
+    Eterm *hp, *hend;
     TreeDbTerm *this;
 
     /*
@@ -728,11 +723,11 @@ static int db_get_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
 	*ret = NIL;
     } else {
 	hp = HAlloc(p, this->dbterm.size + 2);
-	copy = copy_shallow(DBTERM_BUF(&this->dbterm), 
-			    this->dbterm.size, 
-			    &hp, 
-			    &MSO(p));
+	hend = hp + this->dbterm.size + 2;
+	copy = db_copy_object_from_ets(&tb->common, &this->dbterm, &hp, &MSO(p));
 	*ret = CONS(hp, copy, NIL);
+	hp += 2;
+	HRelease(p,hend,hp);
     }
     return DB_ERROR_NONE;
 }
@@ -766,18 +761,10 @@ static int db_get_element_tree(Process *p, DbTable *tbl,
     if (this == NULL) {
 	return DB_ERROR_BADKEY;
     } else {
-	Eterm element;
-	Uint sz;
 	if (ndex > arityval(this->dbterm.tpl[0])) {
 	    return DB_ERROR_BADPARAM;
 	}
-	element = this->dbterm.tpl[ndex];
-	sz = size_object(element);
-	hp = HAlloc(p, sz);
-	*ret = copy_struct(element, 
-			   sz, 
-			   &hp, 
-			   &MSO(p));
+	*ret = db_copy_element_from_ets(&tb->common, p, &this->dbterm, ndex, &hp, 0);
     }
     return DB_ERROR_NONE;
 }
@@ -789,7 +776,7 @@ static int db_erase_tree(DbTable *tbl, Eterm key, Eterm *ret)
 
     *ret = am_true;
 
-    if ((res = linkout_tree(tb, key)) != NULL) {
+    if ((res = linkout_tree(tb, key, NULL)) != NULL) {
 	free_term(tb, res);
     }
     return DB_ERROR_NONE;
@@ -815,7 +802,7 @@ static int db_slot_tree(Process *p, DbTable *tbl,
     DbTableTree *tb = &tbl->tree;
     Sint slot;
     TreeDbTerm *st;
-    Eterm *hp;
+    Eterm *hp, *hend;
     Eterm copy;
 
     /*
@@ -847,18 +834,22 @@ static int db_slot_tree(Process *p, DbTable *tbl,
 	return DB_ERROR_UNSPEC;
     }
     hp = HAlloc(p, st->dbterm.size + 2);
-    copy = copy_shallow(DBTERM_BUF(&st->dbterm), 
-			st->dbterm.size, 
-			&hp, 
-			&MSO(p));
+    hend = hp + st->dbterm.size + 2;
+    copy = db_copy_object_from_ets(&tb->common, &st->dbterm, &hp, &MSO(p));
     *ret = CONS(hp, copy, NIL);
+    hp += 2;
+    HRelease(p,hend,hp);
     return DB_ERROR_NONE;
 }
 
 
 
-static BIF_RETTYPE ets_select_reverse(Process *p, Eterm a1, Eterm a2, Eterm a3)
+static BIF_RETTYPE ets_select_reverse(BIF_ALIST_3)
 {
+    Process *p = BIF_P;
+    Eterm a1 = BIF_ARG_1;
+    Eterm a2 = BIF_ARG_2;
+    Eterm a3 = BIF_ARG_3;
     Eterm list;
     Eterm result;
     Eterm* hp;
@@ -981,15 +972,15 @@ static int db_select_continue_tree(Process *p,
     stack = get_any_stack(tb);
     if (chunk_size) {
 	if (reverse) {
-	    traverse_backwards(tb, stack, lastkey, &doit_select_chunk, &sc); 
+	    traverse_backwards(tb, stack, lastkey, NULL, &doit_select_chunk, &sc);
 	} else {
-	    traverse_forward(tb, stack, lastkey, &doit_select_chunk, &sc); 
+	    traverse_forward(tb, stack, lastkey, NULL, &doit_select_chunk, &sc);
 	}
     } else {
 	if (reverse) {
-	    traverse_forward(tb, stack, lastkey, &doit_select, &sc);
+	    traverse_forward(tb, stack, lastkey, NULL, &doit_select, &sc);
 	} else {
-	    traverse_backwards(tb, stack, lastkey, &doit_select, &sc);
+	    traverse_backwards(tb, stack, lastkey, NULL, &doit_select, &sc);
 	}
     }
     release_stack(tb,stack);
@@ -1014,10 +1005,9 @@ static int db_select_continue_tree(Process *p,
 	    }
 
 	    key = GETKEY(tb, sc.lastobj);
-
-	    sz = size_object(key);
+	    sz = size_object_rel(key,sc.lastobj);
 	    hp = HAlloc(p, 9 + sz);
-	    key = copy_struct(key, sz, &hp, &MSO(p));
+	    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
 	    continuation = TUPLE8
 		(hp,
 		 tptr[1],
@@ -1038,8 +1028,8 @@ static int db_select_continue_tree(Process *p,
     key = GETKEY(tb, sc.lastobj);
     if (chunk_size) {
 	if (end_condition != NIL && 
-	    ((!reverse && cmp_partly_bound(end_condition,key) < 0) ||
-	     (reverse && cmp_partly_bound(end_condition,key) > 0))) { 
+	    ((!reverse && cmp_partly_bound(end_condition,key,sc.lastobj) < 0) ||
+	     (reverse && cmp_partly_bound(end_condition,key,sc.lastobj) > 0))) {
 	    /* done anyway */
 	    if (!sc.got) {
 		RET_TO_BIF(am_EOT, DB_ERROR_NONE);
@@ -1051,16 +1041,16 @@ static int db_select_continue_tree(Process *p,
 	}
     } else {
 	if (end_condition != NIL && 
-	    ((!reverse && cmp_partly_bound(end_condition,key) > 0) ||
-	     (reverse && cmp_partly_bound(end_condition,key) < 0))) { 
+	    ((!reverse && cmp_partly_bound(end_condition,key,sc.lastobj) > 0) ||
+	     (reverse && cmp_partly_bound(end_condition,key,sc.lastobj) < 0))) {
 	    /* done anyway */
 	    RET_TO_BIF(sc.accum,DB_ERROR_NONE);
 	}
     }
     /* Not done yet, let's trap. */
-    sz = size_object(key);
+    sz = size_object_rel(key,sc.lastobj);
     hp = HAlloc(p, 9 + sz);
-    key = copy_struct(key, sz, &hp, &MSO(p));
+    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
     continuation = TUPLE8
 	(hp,
 	 tptr[1],
@@ -1081,11 +1071,13 @@ static int db_select_continue_tree(Process *p,
 static int db_select_tree(Process *p, DbTable *tbl, 
 			  Eterm pattern, int reverse, Eterm *ret)
 {
+    /* Strategy: Traverse backwards to build resulting list from tail to head */
     DbTableTree *tb = &tbl->tree;
     DbTreeStack* stack;
     struct select_context sc;
     struct mp_info mpi;
-    Eterm lastkey = NIL;
+    Eterm lastkey = THE_NON_VALUE;
+    Eterm* lk_base = NULL;
     Eterm key;
     Eterm continuation;
     unsigned sz;
@@ -1127,7 +1119,7 @@ static int db_select_tree(Process *p, DbTable *tbl,
     sc.all_objects = mpi.all_objects;
 
     if (!mpi.got_partial && mpi.some_limitation && 
-	cmp(mpi.least,mpi.most) == 0) {
+	CMP(mpi.least,mpi.most) == 0) {
 	doit_select(tb,mpi.save_term,&sc,0 /* direction doesn't matter */);
 	RET_TO_BIF(sc.accum,DB_ERROR_NONE);
     }
@@ -1137,20 +1129,20 @@ static int db_select_tree(Process *p, DbTable *tbl,
 	if (mpi.some_limitation) {
 	    if ((this = find_prev_from_pb_key(tb, stack, mpi.least)) != NULL) {
 		lastkey = GETKEY(tb, this->dbterm.tpl);
+		lk_base = this->dbterm.tpl;
 	    }
 	    sc.end_condition = mpi.most;
 	}
-	
-	traverse_forward(tb, stack, lastkey, &doit_select, &sc);
+	traverse_forward(tb, stack, lastkey, lk_base, &doit_select, &sc);
     } else {
 	if (mpi.some_limitation) {
 	    if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) {
 		lastkey = GETKEY(tb, this->dbterm.tpl);
+		lk_base = this->dbterm.tpl;
 	    }
 	    sc.end_condition = mpi.least;
 	}
-	
-	traverse_backwards(tb, stack, lastkey, &doit_select, &sc);
+	traverse_backwards(tb, stack, lastkey, lk_base, &doit_select, &sc);
     }
     release_stack(tb,stack);
 #ifdef HARDDEBUG
@@ -1163,9 +1155,9 @@ static int db_select_tree(Process *p, DbTable *tbl,
     }
 
     key = GETKEY(tb, sc.lastobj);
-    sz = size_object(key);
+    sz = size_object_rel(key, sc.lastobj);
     hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE);
-    key = copy_struct(key, sz, &hp, &MSO(p));
+    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
     if (mpi.all_objects)
 	(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
     mpb=db_make_mp_binary(p,mpi.mp,&hp);
@@ -1246,7 +1238,7 @@ static int db_select_count_continue_tree(Process *p,
     }
 
     stack = get_any_stack(tb);
-    traverse_backwards(tb, stack, lastkey, &doit_select_count, &sc);
+    traverse_backwards(tb, stack, lastkey, NULL, &doit_select_count, &sc);
     release_stack(tb,stack);
 
     BUMP_REDS(p, 1000 - sc.max);
@@ -1256,12 +1248,12 @@ static int db_select_count_continue_tree(Process *p,
     }	
     key = GETKEY(tb, sc.lastobj);
     if (end_condition != NIL && 
-	(cmp_partly_bound(end_condition,key) > 0)) { 
+	(cmp_partly_bound(end_condition,key,sc.lastobj) > 0)) {
 	/* done anyway */
 	RET_TO_BIF(make_small(sc.got),DB_ERROR_NONE);
     }
     /* Not done yet, let's trap. */
-    sz = size_object(key);
+    sz = size_object_rel(key, sc.lastobj);
     if (IS_USMALL(0, sc.got)) {
 	hp = HAlloc(p, sz + 6);
 	egot = make_small(sc.got);
@@ -1271,7 +1263,7 @@ static int db_select_count_continue_tree(Process *p,
 	egot = uint_to_big(sc.got, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
-    key = copy_struct(key, sz, &hp, &MSO(p));
+    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
     continuation = TUPLE5
 	(hp,
 	 tptr[1],
@@ -1293,7 +1285,8 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
     DbTreeStack* stack;
     struct select_count_context sc;
     struct mp_info mpi;
-    Eterm lastkey = NIL;
+    Eterm lastkey = THE_NON_VALUE;
+    Eterm* lk_base = NULL;
     Eterm key;
     Eterm continuation;
     unsigned sz;
@@ -1334,7 +1327,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
     sc.all_objects = mpi.all_objects;
 
     if (!mpi.got_partial && mpi.some_limitation && 
-	cmp(mpi.least,mpi.most) == 0) {
+	CMP(mpi.least,mpi.most) == 0) {
 	doit_select_count(tb,mpi.save_term,&sc,0 /* dummy */);
 	RET_TO_BIF(erts_make_integer(sc.got,p),DB_ERROR_NONE);
     }
@@ -1343,11 +1336,12 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
     if (mpi.some_limitation) {
 	if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) {
 	    lastkey = GETKEY(tb, this->dbterm.tpl);
+	    lk_base = this->dbterm.tpl;
 	}
 	sc.end_condition = mpi.least;
     }
     
-    traverse_backwards(tb, stack, lastkey, &doit_select_count, &sc);
+    traverse_backwards(tb, stack, lastkey, lk_base, &doit_select_count, &sc);
     release_stack(tb,stack);
     BUMP_REDS(p, 1000 - sc.max);
     if (sc.max > 0) {
@@ -1355,7 +1349,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
     }
 
     key = GETKEY(tb, sc.lastobj);
-    sz = size_object(key);
+    sz = size_object_rel(key, sc.lastobj);
     if (IS_USMALL(0, sc.got)) {
 	hp = HAlloc(p, sz + PROC_BIN_SIZE + 6);
 	egot = make_small(sc.got);
@@ -1365,7 +1359,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl,
 	egot = uint_to_big(sc.got, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
-    key = copy_struct(key, sz, &hp, &MSO(p));
+    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
     if (mpi.all_objects)
 	(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
     mpb = db_make_mp_binary(p,mpi.mp,&hp);
@@ -1395,7 +1389,8 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
     DbTreeStack* stack;
     struct select_context sc;
     struct mp_info mpi;
-    Eterm lastkey = NIL;
+    Eterm lastkey = THE_NON_VALUE;
+    Eterm* lk_base = NULL;
     Eterm key;
     Eterm continuation;
     unsigned sz;
@@ -1437,7 +1432,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
     sc.all_objects = mpi.all_objects;
 
     if (!mpi.got_partial && mpi.some_limitation && 
-	cmp(mpi.least,mpi.most) == 0) {
+	CMP(mpi.least,mpi.most) == 0) {
 	doit_select(tb,mpi.save_term,&sc, 0 /* direction doesn't matter */);
 	if (sc.accum != NIL) {
 	    hp=HAlloc(p, 3);
@@ -1452,20 +1447,20 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
 	if (mpi.some_limitation) {
 	    if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) {
 		lastkey = GETKEY(tb, this->dbterm.tpl);
+		lk_base = this->dbterm.tpl;
 	    }
 	    sc.end_condition = mpi.least;
 	}
-
-	traverse_backwards(tb, stack, lastkey, &doit_select_chunk, &sc);
+	traverse_backwards(tb, stack, lastkey, lk_base, &doit_select_chunk, &sc);
     } else {
 	if (mpi.some_limitation) {
 	    if ((this = find_prev_from_pb_key(tb, stack, mpi.least)) != NULL) {
 		lastkey = GETKEY(tb, this->dbterm.tpl);
+		lk_base = this->dbterm.tpl;
 	    }
 	    sc.end_condition = mpi.most;
 	}
-
-	traverse_forward(tb, stack, lastkey, &doit_select_chunk, &sc);
+	traverse_forward(tb, stack, lastkey, lk_base, &doit_select_chunk, &sc);
     }
     release_stack(tb,stack);
 
@@ -1490,9 +1485,9 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
 	}
 
 	key = GETKEY(tb, sc.lastobj);
-	sz = size_object(key);
+	sz = size_object_rel(key, sc.lastobj);
 	hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE);
-	key = copy_struct(key, sz, &hp, &MSO(p));
+	key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
 	if (mpi.all_objects)
 	    (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
 	mpb = db_make_mp_binary(p,mpi.mp,&hp);
@@ -1515,9 +1510,9 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl,
     }
 
     key = GETKEY(tb, sc.lastobj);
-    sz = size_object(key);
+    sz = size_object_rel(key, sc.lastobj);
     hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE);
-    key = copy_struct(key, sz, &hp, &MSO(p));
+    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL);
 
     if (mpi.all_objects)
 	(mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS;
@@ -1592,8 +1587,8 @@ static int db_select_delete_continue_tree(Process *p,
     sc.max = 1000;
     sc.keypos = tb->common.keypos;
 
-    ASSERT(!erts_smp_atomic_read(&tb->is_stack_busy));
-    traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc);
+    ASSERT(!erts_smp_atomic_read_nob(&tb->is_stack_busy));
+    traverse_backwards(tb, &tb->static_stack, lastkey, NULL, &doit_select_delete, &sc);
 
     BUMP_REDS(p, 1000 - sc.max);
 
@@ -1602,11 +1597,11 @@ static int db_select_delete_continue_tree(Process *p,
     }	
     key = GETKEY(tb, (sc.lastterm)->dbterm.tpl);
     if (end_condition != NIL && 
-	cmp_partly_bound(end_condition,key) > 0) { /* done anyway */
+	cmp_partly_bound(end_condition,key,sc.lastterm->dbterm.tpl) > 0) { /* done anyway */
 	RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE);
     }
     /* Not done yet, let's trap. */
-    sz = size_object(key);
+    sz = size_object_rel(key, sc.lastterm->dbterm.tpl);
     if (IS_USMALL(0, sc.accum)) {
 	hp = HAlloc(p, sz + 6);
 	eaccsum = make_small(sc.accum);
@@ -1616,7 +1611,7 @@ static int db_select_delete_continue_tree(Process *p,
 	eaccsum = uint_to_big(sc.accum, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
-    key = copy_struct(key, sz, &hp, &MSO(p));
+    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastterm->dbterm.tpl, NULL);
     continuation = TUPLE5
 	(hp,
 	 tptr[1],
@@ -1636,7 +1631,8 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
     DbTableTree *tb = &tbl->tree;
     struct select_delete_context sc;
     struct mp_info mpi;
-    Eterm lastkey = NIL;
+    Eterm lastkey = THE_NON_VALUE;
+    Eterm* lk_base = NULL;
     Eterm key;
     Eterm continuation;
     unsigned sz;
@@ -1680,7 +1676,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
     sc.mp = mpi.mp;
 
     if (!mpi.got_partial && mpi.some_limitation && 
-	cmp(mpi.least,mpi.most) == 0) {
+	CMP(mpi.least,mpi.most) == 0) {
 	doit_select_delete(tb,mpi.save_term,&sc, 0 /* direction doesn't 
 						      matter */);
 	RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE);
@@ -1689,11 +1685,12 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
     if (mpi.some_limitation) {
 	if ((this = find_next_from_pb_key(tb, &tb->static_stack, mpi.most)) != NULL) {
 	    lastkey = GETKEY(tb, this->dbterm.tpl);
+	    lk_base = this->dbterm.tpl;
 	}
 	sc.end_condition = mpi.least;
     }
 
-    traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc);
+    traverse_backwards(tb, &tb->static_stack, lastkey, lk_base, &doit_select_delete, &sc);
     BUMP_REDS(p, 1000 - sc.max);
 
     if (sc.max > 0) {
@@ -1701,7 +1698,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
     }
 
     key = GETKEY(tb, (sc.lastterm)->dbterm.tpl);
-    sz = size_object(key);
+    sz = size_object_rel(key, sc.lastterm->dbterm.tpl);
     if (IS_USMALL(0, sc.accum)) {
 	hp = HAlloc(p, sz + PROC_BIN_SIZE + 6);
 	eaccsum = make_small(sc.accum);
@@ -1711,7 +1708,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
 	eaccsum = uint_to_big(sc.accum, hp);
 	hp += BIG_UINT_HEAP_SIZE;
     }
-    key = copy_struct(key, sz, &hp, &MSO(p));
+    key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastterm->dbterm.tpl, NULL);
     mpb = db_make_mp_binary(p,mpi.mp,&hp);
     
     continuation = TUPLE5
@@ -1737,7 +1734,8 @@ static int db_select_delete_tree(Process *p, DbTable *tbl,
 ** Other interface routines (not directly coupled to one bif)
 */
 
-/* Display hash table contents (for dump) */
+
+/* Display tree contents (for dump) */
 static void db_print_tree(int to, void *to_arg, 
 			  int show,
 			  DbTable *tbl)
@@ -1747,13 +1745,12 @@ static void db_print_tree(int to, void *to_arg,
     if (show)
 	erts_print(to, to_arg, "\nTree data dump:\n"
 		   "------------------------------------------------\n");
-    do_dump_tree2(to, to_arg, show, tb->root, 0);
+    do_dump_tree2(&tbl->tree, to, to_arg, show, tb->root, 0);
     if (show)
 	erts_print(to, to_arg, "\n"
 		   "------------------------------------------------\n");
 #else
     erts_print(to, to_arg, "Ordered set (AVL tree), Elements: %d\n", NITEMS(tb));
-    do_dump_tree(to, to_arg, tb->root);
 #endif
 }
 
@@ -1781,7 +1778,7 @@ static int db_free_table_continue_tree(DbTable *tbl)
 		     (DbTable *) tb,
 		     (void *) tb->static_stack.array,
 		     sizeof(TreeDbTerm *) * STACK_NEED);
-	ASSERT(erts_smp_atomic_read(&tb->common.memory_size)
+	ASSERT(erts_smp_atomic_read_nob(&tb->common.memory_size)
 	       == sizeof(DbTable));
     }
     return result;
@@ -1791,7 +1788,7 @@ static int db_delete_all_objects_tree(Process* p, DbTable* tbl)
 {
     db_free_table_tree(tbl);
     db_create_tree(p, tbl);
-    erts_smp_atomic_set(&tbl->tree.common.nitems, 0);
+    erts_smp_atomic_set_nob(&tbl->tree.common.nitems, 0);
     return 0;
 }
 
@@ -1817,15 +1814,19 @@ do_db_tree_foreach_offheap(TreeDbTerm *tdbt,
 			   void (*func)(ErlOffHeap *, void *),
 			   void * arg)
 {
+    ErlOffHeap tmp_offheap;
     if(!tdbt)
 	return;
     do_db_tree_foreach_offheap(tdbt->left, func, arg);
-    (*func)(&(tdbt->dbterm.off_heap), arg);
+    tmp_offheap.first = tdbt->dbterm.first_oh;
+    tmp_offheap.overhead = 0;
+    (*func)(&tmp_offheap, arg);
+    tdbt->dbterm.first_oh = tmp_offheap.first;
     do_db_tree_foreach_offheap(tdbt->right, func, arg);
 }
 
 static TreeDbTerm *linkout_tree(DbTableTree *tb, 
-				Eterm key)
+				Eterm key, Eterm* key_base)
 {
     TreeDbTerm **tstack[STACK_NEED];
     int tpos = 0;
@@ -1848,7 +1849,7 @@ static TreeDbTerm *linkout_tree(DbTableTree *tb,
     for (;;) {
 	if (!*this) { /* Failure */
 	    return NULL;
-	} else if ((c = cmp(key,GETKEY(tb,(*this)->dbterm.tpl))) < 0) { 
+	} else if ((c = cmp_key(tb, key, key_base, *this)) < 0) {
 	    dstack[dpos++] = DIR_LEFT;
 	    tstack[tpos++] = this;
 	    this = &((*this)->left);
@@ -1869,7 +1870,7 @@ static TreeDbTerm *linkout_tree(DbTableTree *tb,
 		tstack[tpos++] = this;
 		state = delsub(this);
 	    }
-	    erts_smp_atomic_dec(&tb->common.nitems);
+	    erts_smp_atomic_dec_nob(&tb->common.nitems);
 	    break;
 	}
     }
@@ -1912,7 +1913,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb,
     for (;;) {
 	if (!*this) { /* Failure */
 	    return NULL;
-	} else if ((c = cmp(key,GETKEY(tb,(*this)->dbterm.tpl))) < 0) { 
+	} else if ((c = cmp_key(tb,key,NULL,*this)) < 0) {
 	    dstack[dpos++] = DIR_LEFT;
 	    tstack[tpos++] = this;
 	    this = &((*this)->left);
@@ -1921,7 +1922,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb,
 	    tstack[tpos++] = this;
 	    this = &((*this)->right);
 	} else { /* Equal key, found the only possible matching object*/
-	    if (!eq(object,make_tuple((*this)->dbterm.tpl))) {
+	    if (!db_eq(&tb->common,object,&(*this)->dbterm)) {
 		return NULL;
 	    }
 	    q = (*this);
@@ -1936,7 +1937,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb,
 		tstack[tpos++] = this;
 		state = delsub(this);
 	    }
-	    erts_smp_atomic_dec(&tb->common.nitems);
+	    erts_smp_atomic_dec_nob(&tb->common.nitems);
 	    break;
 	}
     }
@@ -2065,24 +2066,6 @@ static int analyze_pattern(DbTableTree *tb, Eterm pattern,
     return DB_ERROR_NONE;
 }
 
-static void do_dump_tree(int to, void *to_arg, TreeDbTerm *t)
-{
-    if (t != NULL) {
-	do_dump_tree(to, to_arg, t->left);
-	erts_print(to, to_arg, "%T\n", make_tuple(t->dbterm.tpl));
-	do_dump_tree(to, to_arg, t->right); 
-    }
-}
-
-static void free_term(DbTableTree *tb, TreeDbTerm* p)
-{
-    db_free_term_data(&(p->dbterm));
-    erts_db_free(ERTS_ALC_T_DB_TERM,
-		 (DbTable *) tb,
-		 (void *) p,
-		 SIZ_DBTERM(p)*sizeof(Uint));
-}
-
 static int do_free_tree_cont(DbTableTree *tb, int num_left)
 {
     TreeDbTerm *root;
@@ -2113,17 +2096,6 @@ static int do_free_tree_cont(DbTableTree *tb, int num_left)
     return 1;
 }
 
-static TreeDbTerm* get_term(DbTableTree *tb,
-			    TreeDbTerm* old, 
-			    Eterm obj) 
-{
-    TreeDbTerm* p = db_get_term((DbTableCommon *) tb,
-				(old != NULL) ? &(old->dbterm) : NULL, 
-				((char *) &(old->dbterm)) - ((char *) old),
-				obj);
-    return p;
-}
-
 /*
  * Deletion helpers
  */
@@ -2327,14 +2299,15 @@ done:
  * Find next and previous in sort order
  */
 
-static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, Eterm key)
+static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack,
+			     Eterm key, Eterm* key_base)
 {
     TreeDbTerm *this;
     TreeDbTerm *tmp;
     Sint c;
 
     if(( this = TOP_NODE(stack)) != NULL) {
-	if (!CMP_EQ(GETKEY(tb, this->dbterm.tpl),key)) {
+	if (!cmp_key_eq(tb,key,key_base,this)) {
 	    /* Start from the beginning */
 	    stack->pos = stack->slot = 0;
 	}
@@ -2344,14 +2317,14 @@ static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, Eterm key)
 	    return NULL;
 	for (;;) {
 	    PUSH_NODE(stack, this);
-	    if (( c = cmp(GETKEY(tb, this->dbterm.tpl),key) ) < 0) {
+	    if (( c = cmp_key(tb,key,key_base,this) ) > 0) {
 		if (this->right == NULL) /* We are at the previos 
 					    and the element does
 					    not exist */
 		    break;
 		else
 		    this = this->right;
-	    } else if (c > 0) {
+	    } else if (c < 0) {
 		if (this->left == NULL) /* Done */
 		    return this;
 		else
@@ -2384,14 +2357,15 @@ static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, Eterm key)
     return this;
 }
 
-static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, Eterm key)
+static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack,
+			     Eterm key, Eterm* key_base)
 {
     TreeDbTerm *this;
     TreeDbTerm *tmp;
     Sint c;
 
     if(( this = TOP_NODE(stack)) != NULL) {
-	if (!CMP_EQ(GETKEY(tb, this->dbterm.tpl),key)) {
+	if (!cmp_key_eq(tb,key,key_base,this)) {
 	    /* Start from the beginning */
 	    stack->pos = stack->slot = 0;
 	}
@@ -2401,14 +2375,14 @@ static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, Eterm key)
 	    return NULL;
 	for (;;) {
 	    PUSH_NODE(stack, this);
-	    if (( c = cmp(GETKEY(tb, this->dbterm.tpl),key) ) > 0) {
+	    if (( c = cmp_key(tb,key,key_base,this) ) < 0) {
 		if (this->left == NULL) /* We are at the next 
 					   and the element does
 					   not exist */
 		    break;
 		else
 		    this = this->left;
-	    } else if (c < 0) {
+	    } else if (c > 0) {
 		if (this->right == NULL) /* Done */
 		    return this;
 		else
@@ -2454,7 +2428,8 @@ static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack* stack,
 	return NULL;
     for (;;) {
 	PUSH_NODE(stack, this);
-	if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl)) ) >= 0) {
+	if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl),
+				   this->dbterm.tpl) ) >= 0) {
 	    if (this->right == NULL) {
 		do {
 		    tmp = POP_NODE(stack);
@@ -2487,7 +2462,8 @@ static TreeDbTerm *find_prev_from_pb_key(DbTableTree *tb, DbTreeStack* stack,
 	return NULL;
     for (;;) {
 	PUSH_NODE(stack, this);
-	if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl)) ) <= 0) {
+	if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl),
+				   this->dbterm.tpl) ) <= 0) {
 	    if (this->left == NULL) {
 		do {
 		    tmp = POP_NODE(stack);
@@ -2517,12 +2493,11 @@ static TreeDbTerm *find_node(DbTableTree *tb, Eterm key)
     Sint res;
     DbTreeStack* stack = get_static_stack(tb);
 
-    if(!stack || EMPTY_NODE(stack) 
-       || !CMP_EQ(GETKEY(tb, ( this = TOP_NODE(stack) )->dbterm.tpl), key)) {
+    if(!stack || EMPTY_NODE(stack)
+       || !cmp_key_eq(tb, key, NULL, (this=TOP_NODE(stack)))) {
 
 	this = tb->root;
-	while (this != NULL && 
-	       ( res = cmp(key, GETKEY(tb, this->dbterm.tpl)) ) != 0) {
+	while (this != NULL && (res = cmp_key(tb,key,NULL,this)) != 0) {
 	    if (res < 0)
 		this = this->left;
 	    else
@@ -2544,8 +2519,7 @@ static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key)
     Sint res;
 
     this = &tb->root;
-    while ((*this) != NULL && 
-	   ( res = cmp(key, GETKEY(tb, (*this)->dbterm.tpl)) ) != 0) {
+    while ((*this) != NULL && (res = cmp_key(tb, key, NULL, *this)) != 0) {
 	if (res < 0)
 	    this = &((*this)->left);
 	else
@@ -2565,48 +2539,24 @@ static int db_lookup_dbterm_tree(DbTable *tbl, Eterm key, DbUpdateHandle* handle
 
     handle->tb = tbl;
     handle->dbterm = &(*pp)->dbterm;
+    handle->mustResize = 0;
     handle->bp = (void**) pp;
     handle->new_size = (*pp)->dbterm.size;
-    handle->mustResize = 0;
+#if HALFWORD_HEAP
+    handle->abs_vec = NULL;
+#endif
     return 1;
 }
 
 static void db_finalize_dbterm_tree(DbUpdateHandle* handle)
 {
     if (handle->mustResize) {
-	Eterm* top;
-	Eterm copy;
-	DbTerm* newDbTerm;
-	DbTableTree *tb = &handle->tb->tree;
 	TreeDbTerm* oldp = (TreeDbTerm*) *handle->bp;
-	TreeDbTerm* newp = erts_db_alloc(ERTS_ALC_T_DB_TERM,
-					 handle->tb,
-					 sizeof(TreeDbTerm)+sizeof(Eterm)*(handle->new_size-1));
-	memcpy(newp, oldp, sizeof(TreeDbTerm)-sizeof(DbTerm));  /* copy only tree header */
-	*(handle->bp) = newp;
-	reset_static_stack(tb);
-	newDbTerm = &newp->dbterm;
-    
-	newDbTerm->size = handle->new_size;
-	newDbTerm->off_heap.mso = NULL;
-	newDbTerm->off_heap.externals = NULL;
-    #ifndef HYBRID /* FIND ME! */
-	newDbTerm->off_heap.funs = NULL;
-    #endif
-	newDbTerm->off_heap.overhead = 0;
-	
-	/* make a flat copy */
-	top = DBTERM_BUF(newDbTerm);
-	copy = copy_struct(make_tuple(handle->dbterm->tpl),
-			   handle->new_size,
-			   &top, &newDbTerm->off_heap);
-	DBTERM_SET_TPL(newDbTerm,tuple_val(copy));
-    
-	db_free_term_data(handle->dbterm);
-	erts_db_free(ERTS_ALC_T_DB_TERM,
-		     handle->tb,
-		     (void *) (((char *) handle->dbterm) - (sizeof(TreeDbTerm) - sizeof(DbTerm))),
-		     sizeof(TreeDbTerm) + sizeof(Eterm)*(handle->dbterm->size-1));
+
+	db_finalize_resize(handle, offsetof(TreeDbTerm,dbterm));
+	reset_static_stack(&handle->tb->tree);
+
+	free_term(&handle->tb->tree, oldp);
     }
 #ifdef DEBUG
     handle->dbterm = 0;
@@ -2619,7 +2569,7 @@ static void db_finalize_dbterm_tree(DbUpdateHandle* handle)
  */
 static void traverse_backwards(DbTableTree *tb,
 			       DbTreeStack* stack,
-			       Eterm lastkey,
+			       Eterm lastkey, Eterm* lk_base,
 			       int (*doit)(DbTableTree *,
 					   TreeDbTerm *,
 					   void *,
@@ -2628,7 +2578,7 @@ static void traverse_backwards(DbTableTree *tb,
 {
     TreeDbTerm *this, *next;
 
-    if (lastkey == NIL) {
+    if (lastkey == THE_NON_VALUE) {
 	stack->pos = stack->slot = 0;
 	if (( this = tb->root ) == NULL) {
 	    return;
@@ -2638,15 +2588,16 @@ static void traverse_backwards(DbTableTree *tb,
 	    this = this->right;
 	}
 	this = TOP_NODE(stack);
-	next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl));
+	next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl),
+			 this->dbterm.tpl);
 	if (!((*doit)(tb, this, context, 0)))
 	    return;
     } else {
-	next = find_prev(tb, stack, lastkey);
+	next = find_prev(tb, stack, lastkey, lk_base);
     }
 
     while ((this = next) != NULL) {
-	next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl));
+	next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl);
 	if (!((*doit)(tb, this, context, 0)))
 	    return;
     }
@@ -2657,7 +2608,7 @@ static void traverse_backwards(DbTableTree *tb,
  */
 static void traverse_forward(DbTableTree *tb,
 			     DbTreeStack* stack,
-			     Eterm lastkey,
+			     Eterm lastkey, Eterm* lk_base,
 			     int (*doit)(DbTableTree *,
 					 TreeDbTerm *,
 					 void *,
@@ -2666,7 +2617,7 @@ static void traverse_forward(DbTableTree *tb,
 {
     TreeDbTerm *this, *next;
 
-    if (lastkey == NIL) {
+    if (lastkey == THE_NON_VALUE) {
 	stack->pos = stack->slot = 0;
 	if (( this = tb->root ) == NULL) {
 	    return;
@@ -2676,15 +2627,15 @@ static void traverse_forward(DbTableTree *tb,
 	    this = this->left;
 	}
 	this = TOP_NODE(stack);
-	next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl));
+	next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl);
 	if (!((*doit)(tb, this, context, 1)))
 	    return;
     } else {
-	next = find_next(tb, stack, lastkey);
+	next = find_next(tb, stack, lastkey, lk_base);
     }
 
     while ((this = next) != NULL) {
-	next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl));
+	next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl);
 	if (!((*doit)(tb, this, context, 1)))
 	    return;
     }
@@ -2710,7 +2661,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret,
 	if (( this = find_node(tb, key) ) == NULL) {
 	    return -1;
 	}
-	*ret = this; 
+	*ret = this;
 	return 1;
     } else if (partly_bound != NULL && key != am_Underscore && 
 	       db_is_variable(key) < 0)
@@ -2721,7 +2672,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret,
 
 
 
-static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done)
+static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done)
 {
     Eterm* aa;
     Eterm* bb;
@@ -2735,44 +2686,44 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done)
 	*done = 1;
 	return 0;
     }
-    if (a == b)
+    if (is_same(a,NULL,b,b_base))
 	return 0;
     
     switch (a & _TAG_PRIMARY_MASK) {
     case TAG_PRIMARY_LIST:
 	if (!is_list(b)) {
-	    return cmp(a,b);
+	    return cmp_rel(a,NULL,b,b_base);
 	}
 	aa = list_val(a);
-	bb = list_val(b);
+	bb = list_val_rel(b,b_base);
 	while (1) {
-	    if ((j = do_cmp_partly_bound(*aa++, *bb++, done)) != 0 || *done) 
+	    if ((j = do_cmp_partly_bound(*aa++, *bb++, b_base, done)) != 0 || *done)
 		return j;
-	    if (*aa==*bb)
+	    if (is_same(*aa, NULL, *bb, b_base))
 		return 0;
 	    if (is_not_list(*aa) || is_not_list(*bb))
-		return do_cmp_partly_bound(*aa, *bb, done);
+		return do_cmp_partly_bound(*aa, *bb, b_base, done);
 	    aa = list_val(*aa);
-	    bb = list_val(*bb);
+	    bb = list_val_rel(*bb,b_base);
 	}
     case TAG_PRIMARY_BOXED:
 	if ((b & _TAG_PRIMARY_MASK) != TAG_PRIMARY_BOXED) {
-	    return cmp(a,b);
+	    return cmp_rel(a,NULL,b,b_base);
 	}
 	a_hdr = ((*boxed_val(a)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE;
-	b_hdr = ((*boxed_val(b)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE;
+	b_hdr = ((*boxed_val_rel(b,b_base)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE;
 	if (a_hdr != b_hdr) {
-	    return cmp(a, b);
+	    return cmp_rel(a, NULL, b, b_base);
 	}
 	if (a_hdr == (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE)) {
 	    aa = tuple_val(a);
-	    bb = tuple_val(b);
+	    bb = tuple_val_rel(b, b_base);
 	    /* compare the arities */
 	    i = arityval(*aa);	/* get the arity*/
 	    if (i < arityval(*bb)) return(-1);
 	    if (i > arityval(*bb)) return(1);
 	    while (i--) {
-		if ((j = do_cmp_partly_bound(*++aa, *++bb, done)) != 0 
+		if ((j = do_cmp_partly_bound(*++aa, *++bb, b_base, done)) != 0
 		    || *done) 
 		    return j;
 	    }
@@ -2780,14 +2731,14 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done)
 	}
 	/* Drop through */
       default:
-	  return cmp(a, b);
+	  return cmp_rel(a, NULL, b, b_base);
     }
 }
 
-static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key) 
+static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_base)
 {
     int done = 0;
-    Sint ret = do_cmp_partly_bound(partly_bound_key, bound_key, &done);
+    Sint ret = do_cmp_partly_bound(partly_bound_key, bound_key, bk_base, &done);
 #ifdef HARDDEBUG
     erts_fprintf(stderr,"\ncmp_partly_bound: %T", partly_bound_key);
     if (ret < 0)
@@ -2796,7 +2747,7 @@ static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key)
 	erts_fprintf(stderr," > ");
     else
 	erts_fprintf(stderr," == ");
-    erts_fprintf(stderr,"%T\n",bound_key);
+    erts_fprintf(stderr,"%R\n", bound_key, bk_base);
 #endif
     return ret;
 }
@@ -2883,7 +2834,7 @@ static int do_partly_bound_can_match_lesser(Eterm a, Eterm b,
 
     if (not_eq_tags(a,b)) {
 	*done = 1;
-	return (cmp(a, b) < 0) ? 1 : 0;
+	return (CMP(a, b) < 0) ? 1 : 0;
     }
 
     /* we now know that tags are the same */
@@ -2919,7 +2870,7 @@ static int do_partly_bound_can_match_lesser(Eterm a, Eterm b,
 	    bb = list_val(*bb);
 	}
     default:
-	if((i = cmp(a, b)) != 0) {
+	if((i = CMP(a, b)) != 0) {
 	    *done = 1;
 	}
 	return (i < 0) ? 1 : 0;
@@ -2954,7 +2905,7 @@ static int do_partly_bound_can_match_greater(Eterm a, Eterm b,
 
     if (not_eq_tags(a,b)) {
 	*done = 1;
-	return (cmp(a, b) > 0) ? 1 : 0;
+	return (CMP(a, b) > 0) ? 1 : 0;
     }
 
     /* we now know that tags are the same */
@@ -2990,7 +2941,7 @@ static int do_partly_bound_can_match_greater(Eterm a, Eterm b,
 	    bb = list_val(*bb);
 	}
     default:
-	if((i = cmp(a, b)) != 0) {
+	if((i = CMP(a, b)) != 0) {
 	    *done = 1;
 	}
 	return (i > 0) ? 1 : 0;
@@ -3006,39 +2957,24 @@ static int doit_select(DbTableTree *tb, TreeDbTerm *this, void *ptr,
 {
     struct select_context *sc = (struct select_context *) ptr;
     Eterm ret;
-    Uint32 dummy;
+    Eterm* hp;
 
     sc->lastobj = this->dbterm.tpl;
     
     if (sc->end_condition != NIL && 
 	((forward && 
 	  cmp_partly_bound(sc->end_condition, 
-			   GETKEY_WITH_POS(sc->keypos, 
-					   this->dbterm.tpl)) < 0) ||
+			   GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
+			   this->dbterm.tpl) < 0) ||
 	 (!forward && 
 	  cmp_partly_bound(sc->end_condition, 
-			   GETKEY_WITH_POS(sc->keypos, 
-					   this->dbterm.tpl)) > 0))) {
+			   GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
+			   this->dbterm.tpl) > 0))) {
 	return 0;
     }
-    ret = db_prog_match(sc->p, sc->mp,
-			make_tuple(this->dbterm.tpl), 
-			0, &dummy);
+    ret = db_match_dbterm(&tb->common,sc->p,sc->mp,sc->all_objects,
+			  &this->dbterm, &hp, 2);
     if (is_value(ret)) {
-	Uint sz;
-	Eterm *hp;
-	if (sc->all_objects) {
-	    hp = HAlloc(sc->p, this->dbterm.size + 2);
-	    ret = copy_shallow(DBTERM_BUF(&this->dbterm),
-				     this->dbterm.size,
-				     &hp,
-			             &MSO(sc->p));
-	} else {
-	    sz = size_object(ret);
-	    hp = HAlloc(sc->p, sz + 2);
-	    ret = copy_struct(ret, sz, 
-			      &hp, &MSO(sc->p));
-	}
 	sc->accum = CONS(hp, ret, sc->accum);
     }
     if (MBUF(sc->p)) {
@@ -3059,20 +2995,18 @@ static int doit_select_count(DbTableTree *tb, TreeDbTerm *this, void *ptr,
 {
     struct select_count_context *sc = (struct select_count_context *) ptr;
     Eterm ret;
-    Uint32 dummy;
 
     sc->lastobj = this->dbterm.tpl;
     
     /* Always backwards traversing */
     if (sc->end_condition != NIL && 
 	(cmp_partly_bound(sc->end_condition, 
-			  GETKEY_WITH_POS(sc->keypos, 
-					  this->dbterm.tpl)) > 0)) {
+			  GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
+			  this->dbterm.tpl) > 0)) {
 	return 0;
     }
-    ret = db_prog_match(sc->p, sc->mp,
-			make_tuple(this->dbterm.tpl), 
-			0, &dummy);
+    ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0,
+			  &this->dbterm, NULL, 0);
     if (ret == am_true) {
 	++(sc->got);
     }
@@ -3087,41 +3021,26 @@ static int doit_select_chunk(DbTableTree *tb, TreeDbTerm *this, void *ptr,
 {
     struct select_context *sc = (struct select_context *) ptr;
     Eterm ret;
-    Uint32 dummy;
+    Eterm* hp;
 
     sc->lastobj = this->dbterm.tpl;
     
     if (sc->end_condition != NIL && 
 	((forward && 
 	  cmp_partly_bound(sc->end_condition, 
-			   GETKEY_WITH_POS(sc->keypos, 
-					   this->dbterm.tpl)) < 0) ||
+			   GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
+			   this->dbterm.tpl) < 0) ||
 	 (!forward && 
 	  cmp_partly_bound(sc->end_condition, 
-			   GETKEY_WITH_POS(sc->keypos, 
-					   this->dbterm.tpl)) > 0))) {
+			   GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
+			   this->dbterm.tpl) > 0))) {
 	return 0;
     }
 
-    ret = db_prog_match(sc->p, sc->mp,
-			make_tuple(this->dbterm.tpl), 
-			0, &dummy);
+    ret = db_match_dbterm(&tb->common, sc->p, sc->mp, sc->all_objects,
+			  &this->dbterm, &hp, 2);
     if (is_value(ret)) {
-	Uint sz;
-	Eterm *hp;
-
 	++(sc->got);
-	if (sc->all_objects) {
-	    hp = HAlloc(sc->p, this->dbterm.size + 2);
-	    ret = copy_shallow(DBTERM_BUF(&this->dbterm),
-				     this->dbterm.size,
-				     &hp,
-			             &MSO(sc->p));
-	} else {
-	    sz = size_object(ret);
-	    hp = HAlloc(sc->p, sz + 2);
-	    ret = copy_struct(ret, sz, &hp, &MSO(sc->p));
-	}
 	sc->accum = CONS(hp, ret, sc->accum);
     }
     if (MBUF(sc->p)) {
@@ -3143,7 +3062,6 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr,
 {
     struct select_delete_context *sc = (struct select_delete_context *) ptr;
     Eterm ret;
-    Uint32 dummy;
     Eterm key;
 
     if (sc->erase_lastterm)
@@ -3153,15 +3071,14 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr,
     
     if (sc->end_condition != NIL && 
 	cmp_partly_bound(sc->end_condition, 
-			 GETKEY_WITH_POS(sc->keypos, 
-					 this->dbterm.tpl)) > 0)
+			 GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl),
+			 this->dbterm.tpl) > 0)
 	return 0;
-    ret = db_prog_match(sc->p, sc->mp,
-			make_tuple(this->dbterm.tpl), 
-			0, &dummy);
+    ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0,
+			  &this->dbterm, NULL, 0);
     if (ret == am_true) {
 	key = GETKEY(sc->tb, this->dbterm.tpl);
-	linkout_tree(sc->tb, key);
+	linkout_tree(sc->tb, key, this->dbterm.tpl);
 	sc->erase_lastterm = 1;
 	++sc->accum;
     }
@@ -3172,19 +3089,28 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr,
 }
 
 #ifdef TREE_DEBUG
-static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t,
-			  int offset)
+static void do_dump_tree2(DbTableTree* tb, int to, void *to_arg, int show,
+			  TreeDbTerm *t, int offset)
 {
     if (t == NULL)
-	return 0;
-    do_dump_tree2(to, to_arg, show, t->right, offset + 4);
+	return;
+    do_dump_tree2(tb, to, to_arg, show, t->right, offset + 4);
     if (show) {
-	erts_print(to, to_arg, "%*s%T (addr = %p, bal = %d)\n"
-		   offset, "", make_tuple(t->dbterm.tpl),
+	const char* prefix;
+	Eterm term;
+	if (tb->common.compress) {
+	    prefix = "key=";
+	    term = GETKEY(tb, t->dbterm.tpl);
+	}
+	else {
+	    prefix = "";
+	    term = make_tuple_rel(t->dbterm.tpl,t->dbterm.tpl);
+	}
+	erts_print(to, to_arg, "%*s%s%R (addr = %p, bal = %d)\n",
+		   offset, "", prefix, term, t->dbterm.tpl,
 		   t, t->balance);
     }
-    do_dump_tree2(to, to_arg, show, t->left, offset + 4); 
-    return sum;
+    do_dump_tree2(tb, to, to_arg, show, t->left, offset + 4); 
 }
 
 #endif
@@ -3194,7 +3120,7 @@ static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t,
 void db_check_table_tree(DbTable *tbl)
 {
     DbTableTree *tb = &tbl->tree;
-    check_table_tree(tb->root);
+    check_table_tree(tb, tb->root);
     check_saved_stack(tb);
     check_slot_pos(tb);
 }
@@ -3225,7 +3151,7 @@ static void check_slot_pos(DbTableTree *tb)
 		   "element position %d is really 0x%08X, when stack says "
 		   "it's 0x%08X\n", tb->stack.slot, t, 
 		   tb->stack.array[tb->stack.pos - 1]);
-	do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0);
+	do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0);
     }
 }
 	
@@ -3240,14 +3166,14 @@ static void check_saved_stack(DbTableTree *tb)
      if (t != stack->array[0]) {
 	 erts_fprintf(stderr,"tb->stack[0] is 0x%08X, should be 0x%08X\n",
 		      stack->array[0], t);
-	 do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0);
+	 do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0);
 	 return;
      }
      while (n < stack->pos) {
 	 if (t == NULL) {
 	     erts_fprintf(stderr, "NULL pointer in tree when stack not empty,"
 			" stack depth is %d\n", n);
-	     do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0);
+	     do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0);
 	     return;
 	 }
 	 n++;
@@ -3261,28 +3187,26 @@ static void check_saved_stack(DbTableTree *tb)
 			    "represent child pointer in tree!"
 			    "(left == 0x%08X, right == 0x%08X\n", 
 			    n, tb->stack[n], t->left, t->right);
-		 do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, tb->root, 0);
+		 do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, tb->root, 0);
 		 return;
 	     }
 	 }
      }
 }
 
-static int check_table_tree(TreeDbTerm *t)
+static int check_table_tree(DbTableTree* tb, TreeDbTerm *t)
 {
     int lh, rh;
     if (t == NULL)
 	return 0;
-    lh = check_table_tree(t->left);
-    rh = check_table_tree(t->right);
+    lh = check_table_tree(tb, t->left);
+    rh = check_table_tree(tb, t->right);
     if ((rh - lh) != t->balance) {
 	erts_fprintf(stderr, "Invalid tree balance for this node:\n");
-	erts_fprintf(stderr,"balance = %d, left = 0x%08X, right = 0x%08X\n"
-		     "data = %T",
-		     t->balance, t->left, t->right,
-		     make_tuple(t->dbterm.tpl));
+	erts_fprintf(stderr,"balance = %d, left = 0x%08X, right = 0x%08X\n",
+		     t->balance, t->left, t->right);
 	erts_fprintf(stderr,"\nDump:\n---------------------------------\n");
-	do_dump_tree2(ERTS_PRINT_STDERR, NULL, 1, t, 0);
+	do_dump_tree2(tb, ERTS_PRINT_STDERR, NULL, 1, t, 0);
 	erts_fprintf(stderr,"\n---------------------------------\n");
     }
     return ((rh > lh) ? rh : lh) + 1;
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index 8c373451fd..1aa0cb4552 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1998-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -25,7 +25,6 @@
 #ifdef HAVE_CONFIG_H
 #  include "config.h"
 #endif
-
 #include "sys.h"
 #include "erl_vm.h"
 #include "global.h"
@@ -36,6 +35,7 @@
 #include "bif.h"
 #include "big.h"
 #include "erl_binary.h"
+#include "erl_thr_progress.h"
 
 #include "erl_db_util.h"
 
@@ -58,6 +58,7 @@
 DBIF_TABLE_GUARD | DBIF_TABLE_BODY | DBIF_TRACE_GUARD | DBIF_TRACE_BODY
 
 
+#define HEAP_XTRA 100
 
 /*
 ** Some convenience macros for stacks (DMC == db_match_compile)
@@ -117,6 +118,10 @@ do {									\
 	erts_free(ERTS_ALC_T_DB_MC_STK, (Name).data);			\
 } while (0)
 
+
+#define TermWords(t) (((t) / (sizeof(UWord)/sizeof(Eterm))) + !!((t) % (sizeof(UWord)/sizeof(Eterm))))
+
+
 static ERTS_INLINE Process *
 get_proc(Process *cp, Uint32 cp_locks, Eterm id, Uint32 id_locks)
 {
@@ -226,6 +231,11 @@ typedef enum {
     matchCall2,
     matchCall3,
     matchPushV,
+#if HALFWORD_HEAP
+    matchPushVGuard,    /* First guard-only variable reference */
+#endif
+    matchPushVResult, /* First variable reference in result, or (if HALFWORD)
+                         in guard if also referenced in result */
     matchPushExpr, /* Push the whole expression we're matching ('$_') */
     matchPushArrayAsList, /* Only when parameter is an Array and 
 			     not an erlang term  (DCOMP_TRACE) */
@@ -281,7 +291,7 @@ typedef struct dmc_guard_bif {
 */
 DMC_DECLARE_STACK_TYPE(Eterm);
 
-DMC_DECLARE_STACK_TYPE(Uint);
+DMC_DECLARE_STACK_TYPE(UWord);
 
 DMC_DECLARE_STACK_TYPE(unsigned);
 
@@ -289,11 +299,19 @@ DMC_DECLARE_STACK_TYPE(unsigned);
 ** Data about the heap during compilation
 */
 
+typedef struct DMCVariable {
+    int is_bound;
+    int is_in_body;
+#if HALFWORD_HEAP
+    int first_guard_label;  /* to maybe change from PushVGuard to PushVResult */
+#endif
+} DMCVariable;
+
 typedef struct DMCHeap {
     int size;
-    unsigned def[DMC_DEFAULT_SIZE];
-    unsigned *data;
-    int used;
+    DMCVariable vars_def[DMC_DEFAULT_SIZE];
+    DMCVariable* vars;
+    int vars_used;
 } DMCHeap;
 
 /*
@@ -320,7 +338,6 @@ typedef struct dmc_context {
     Eterm *bodyexpr;
     int num_match;
     int current_match;
-    int eheap_need;
     Uint cflags;
     int is_guard; /* 1 if in guard, 0 if in body */
     int special; /* 1 if the head in the match was a single expression */ 
@@ -343,9 +360,22 @@ typedef struct dmc_context {
 
 #define ERTS_DEFAULT_MS_HEAP_SIZE 128
 
+/* Runtime info about a $-variable
+*/
+typedef struct MatchVariable {
+    Eterm term;
+#ifdef DEBUG
+    Process* proc;
+    Eterm* base;
+#endif
+} MatchVariable;
+
 typedef struct {
     Process process;
-    Eterm *heap;
+    union {
+	Eterm* heap;
+	MatchVariable* variables;   /* first on "heap" */
+    }u;
     Eterm default_heap[ERTS_DEFAULT_MS_HEAP_SIZE];
 } ErtsMatchPseudoProcess;
 
@@ -359,12 +389,7 @@ static ErtsMatchPseudoProcess *match_pseudo_process;
 static ERTS_INLINE void
 cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap)
 {
-    if (mpsp->process.mbuf
-	|| mpsp->process.off_heap.mso
-#ifndef HYBRID /* FIND ME! */
-	|| mpsp->process.off_heap.funs
-#endif
-	|| mpsp->process.off_heap.externals) {
+    if (mpsp->process.mbuf || mpsp->process.off_heap.first) {
 	erts_cleanup_empty_process(&mpsp->process);
     }
 #ifdef DEBUG
@@ -373,16 +398,16 @@ cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap)
     }
 #endif
     if (!keep_heap) {
-	if (mpsp->heap != &mpsp->default_heap[0]) {
+	if (mpsp->u.heap != mpsp->default_heap) {
 	    /* Have to be done *after* call to erts_cleanup_empty_process() */
-	    erts_free(ERTS_ALC_T_DB_MS_RUN_HEAP, (void *) mpsp->heap);
-	    mpsp->heap = &mpsp->default_heap[0];
+	    erts_free(ERTS_ALC_T_DB_MS_RUN_HEAP, (void *) mpsp->u.heap);
+	    mpsp->u.heap = mpsp->default_heap;
 	}
 #ifdef DEBUG
 	else {
 	    int i;
 	    for (i = 0; i < ERTS_DEFAULT_MS_HEAP_SIZE; i++) {
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 		mpsp->default_heap[i] = (Eterm) 0xdeadbeefdeadbeef;
 #else
 		mpsp->default_heap[i] = (Eterm) 0xdeadbeef;
@@ -400,7 +425,7 @@ create_match_pseudo_process(void)
     mpsp = (ErtsMatchPseudoProcess *)erts_alloc(ERTS_ALC_T_DB_MS_PSDO_PROC,
 						sizeof(ErtsMatchPseudoProcess));
     erts_init_empty_process(&mpsp->process);
-    mpsp->heap = &mpsp->default_heap[0];
+    mpsp->u.heap = mpsp->default_heap;
     return mpsp;
 }
 
@@ -424,11 +449,11 @@ get_match_pseudo_process(Process *c_p, Uint heap_size)
     mpsp = match_pseudo_process;
     cleanup_match_pseudo_process(mpsp, 0);
 #endif
-    if (heap_size > ERTS_DEFAULT_MS_HEAP_SIZE)
-	mpsp->heap = (Eterm *) erts_alloc(ERTS_ALC_T_DB_MS_RUN_HEAP,
-					  heap_size*sizeof(Uint));
+    if (heap_size > ERTS_DEFAULT_MS_HEAP_SIZE*sizeof(Eterm)) {
+	mpsp->u.heap = (Eterm*) erts_alloc(ERTS_ALC_T_DB_MS_RUN_HEAP, heap_size);
+    }
     else {
-	ASSERT(mpsp->heap == &mpsp->default_heap[0]);
+	ASSERT(mpsp->u.heap == mpsp->default_heap);
     }
     return mpsp;
 }
@@ -467,28 +492,11 @@ erts_match_set_release_result(Process* c_p)
 
 /* The trace control word. */
 
-static erts_smp_atomic_t trace_control_word;
-
-
-Eterm
-erts_ets_copy_object(Eterm obj, Process* to)
-{
-    Uint size = size_object(obj);
-    Eterm* hp = HAlloc(to, size);
-    Eterm res;
-
-    res = copy_struct(obj, size, &hp, &MSO(to));
-#ifdef DEBUG
-    if (eq(obj, res) == 0) {
-	erl_exit(1, "copy not equal to source\n");
-    }
-#endif
-    return res;
-}
+static erts_smp_atomic32_t trace_control_word;
 
 /* This needs to be here, before the bif table... */
 
-static Eterm db_set_trace_control_word_fake_1(Process *p, Eterm val);
+static Eterm db_set_trace_control_word_fake_1(BIF_ALIST_1);
 
 /*
 ** The table of callable bif's, i e guard bif's and 
@@ -830,50 +838,50 @@ static Uint my_size_object(Eterm t);
 static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap);
 
 /* Guard compilation */
-static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(Uint) *text,
+static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(UWord) *text,
 			     Eterm t);
 static DMCRet dmc_list(DMCContext *context,
 		       DMCHeap *heap,
-		       DMC_STACK_TYPE(Uint) *text,
+		       DMC_STACK_TYPE(UWord) *text,
 		       Eterm t,
 		       int *constant);
 static DMCRet dmc_tuple(DMCContext *context,
 		       DMCHeap *heap,
-		       DMC_STACK_TYPE(Uint) *text,
+		       DMC_STACK_TYPE(UWord) *text,
 		       Eterm t,
 		       int *constant);
 static DMCRet dmc_variable(DMCContext *context,
 			   DMCHeap *heap,
-			   DMC_STACK_TYPE(Uint) *text,
+			   DMC_STACK_TYPE(UWord) *text,
 			   Eterm t,
 			   int *constant);
 static DMCRet dmc_fun(DMCContext *context,
 		      DMCHeap *heap,
-		      DMC_STACK_TYPE(Uint) *text,
+		      DMC_STACK_TYPE(UWord) *text,
 		      Eterm t,
 		      int *constant);
 static DMCRet dmc_expr(DMCContext *context,
 		       DMCHeap *heap,
-		       DMC_STACK_TYPE(Uint) *text,
+		       DMC_STACK_TYPE(UWord) *text,
 		       Eterm t,
 		       int *constant);
 static DMCRet compile_guard_expr(DMCContext *context,
 				    DMCHeap *heap,
-				    DMC_STACK_TYPE(Uint) *text,
+				    DMC_STACK_TYPE(UWord) *text,
 				    Eterm t);
 /* match expression subroutine */
 static DMCRet dmc_one_term(DMCContext *context, 
 			   DMCHeap *heap,
 			   DMC_STACK_TYPE(Eterm) *stack,
-			   DMC_STACK_TYPE(Uint) *text,
+			   DMC_STACK_TYPE(UWord) *text,
 			   Eterm c);
 
 
 #ifdef DMC_DEBUG
 static int test_disassemble_next = 0;
-static void db_match_dis(Binary *prog);
+void db_match_dis(Binary *prog);
 #define TRACE erts_fprintf(stderr,"Trace: %s:%d\n",__FILE__,__LINE__)
-#define FENCE_PATTERN_SIZE 1
+#define FENCE_PATTERN_SIZE (1*sizeof(Uint))
 #define FENCE_PATTERN 0xDEADBEEFUL
 #else
 #define TRACE /* Nothing */
@@ -891,6 +899,8 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace);
 
 static Eterm seq_trace_fake(Process *p, Eterm arg1);
 
+static void db_free_tmp_uncompressed(DbTerm* obj);
+
 
 /*
 ** Interface routines.
@@ -899,14 +909,18 @@ static Eterm seq_trace_fake(Process *p, Eterm arg1);
 /*
 ** Pseudo BIF:s to be callable from the PAM VM.
 */
-
-BIF_RETTYPE db_get_trace_control_word_0(Process *p) 
+BIF_RETTYPE db_get_trace_control_word(Process *p)
 {
-    Uint32 tcw = (Uint32) erts_smp_atomic_read(&trace_control_word);
+    Uint32 tcw = (Uint32) erts_smp_atomic32_read_acqb(&trace_control_word);
     BIF_RET(erts_make_integer((Uint) tcw, p));
 }
 
-BIF_RETTYPE db_set_trace_control_word_1(Process *p, Eterm new) 
+BIF_RETTYPE db_get_trace_control_word_0(BIF_ALIST_0)
+{
+    BIF_RET(db_get_trace_control_word(BIF_P));
+}
+
+BIF_RETTYPE db_set_trace_control_word(Process *p, Eterm new)
 {
     Uint val;
     Uint32 old_tcw;
@@ -914,19 +928,27 @@ BIF_RETTYPE db_set_trace_control_word_1(Process *p, Eterm new)
 	BIF_ERROR(p, BADARG);
     if (val != ((Uint32)val))
 	BIF_ERROR(p, BADARG);
-    
-    old_tcw = (Uint32) erts_smp_atomic_xchg(&trace_control_word, (long) val);
+
+    old_tcw = (Uint32) erts_smp_atomic32_xchg_relb(&trace_control_word,
+						   (erts_aint32_t) val);
     BIF_RET(erts_make_integer((Uint) old_tcw, p));
 }
 
-static Eterm db_set_trace_control_word_fake_1(Process *p, Eterm new) 
+BIF_RETTYPE db_set_trace_control_word_1(BIF_ALIST_1)
+{
+    BIF_RET(db_set_trace_control_word(BIF_P, BIF_ARG_1));
+}
+
+static Eterm db_set_trace_control_word_fake_1(BIF_ALIST_1)
 {
+    Process *p = BIF_P;
+    Eterm new = BIF_ARG_1;
     Uint val;
     if (!term_to_Uint(new, &val))
 	BIF_ERROR(p, BADARG);
     if (val != ((Uint32)val))
 	BIF_ERROR(p, BADARG);
-    BIF_RET(db_get_trace_control_word_0(p));
+    BIF_RET(db_get_trace_control_word(p));
 }
 
 /*
@@ -1179,14 +1201,14 @@ done:
 }
     
 Eterm erts_match_set_run(Process *p, Binary *mpsp, 
-			 Eterm *args, int num_args, 
+			 Eterm *args, int num_args,
+			 enum erts_pam_run_flags in_flags,
 			 Uint32 *return_flags) 
 {
     Eterm ret;
 
-    ret = db_prog_match(p, mpsp,
-			(Eterm) args, 
-			num_args, return_flags);
+    ret = db_prog_match(p, mpsp, NIL, NULL, args, num_args,
+			in_flags, return_flags);
 #if defined(HARDDEBUG)
     if (is_non_value(ret)) {
 	erts_fprintf(stderr, "Failed\n");
@@ -1204,6 +1226,32 @@ Eterm erts_match_set_run(Process *p, Binary *mpsp,
      */
 }
 
+static Eterm erts_match_set_run_ets(Process *p, Binary *mpsp,
+				    Eterm args, int num_args,
+				    Uint32 *return_flags)
+{
+    Eterm ret;
+
+    ret = db_prog_match(p, mpsp, args, NULL, NULL, num_args,
+			ERTS_PAM_CONTIGUOUS_TUPLE | ERTS_PAM_COPY_RESULT,
+			return_flags);
+#if defined(HARDDEBUG)
+    if (is_non_value(ret)) {
+	erts_fprintf(stderr, "Failed\n");
+    } else {
+	erts_fprintf(stderr, "Returning : %T\n", ret);
+    }
+#endif
+    return ret;
+    /* Returns
+     *   THE_NON_VALUE if no match
+     *   am_false      if {message,false} has been called,
+     *   am_true       if {message,_} has not been called or
+     *                 if {message,true} has been called,
+     *   Msg           if {message,Msg} has been called.
+     */
+}
+
 /*
 ** API Used by other erl_db modules.
 */
@@ -1214,7 +1262,7 @@ void db_initialize_util(void){
 	  sizeof(DMCGuardBif), 
 	  (int (*)(const void *, const void *)) &cmp_guard_bif);
     match_pseudo_process_init();
-    erts_smp_atomic_init(&trace_control_word, 0);
+    erts_smp_atomic32_init_nob(&trace_control_word, 0);
 }
 
 
@@ -1245,7 +1293,7 @@ Binary *db_match_compile(Eterm *matchexpr,
 {
     DMCHeap heap;
     DMC_STACK_TYPE(Eterm) stack;
-    DMC_STACK_TYPE(Uint) text;
+    DMC_STACK_TYPE(UWord) text;
     DMCContext context;
     MatchProg *ret = NULL;
     Eterm t;
@@ -1254,7 +1302,6 @@ Binary *db_match_compile(Eterm *matchexpr,
     int structure_checked;
     DMCRet res;
     int current_try_label;
-    Uint max_eheap_need;
     Binary *bp = NULL;
     unsigned clause_start;
 
@@ -1267,27 +1314,24 @@ Binary *db_match_compile(Eterm *matchexpr,
     context.matchexpr = matchexpr;
     context.guardexpr = guards;
     context.bodyexpr = body;
-    context.eheap_need = 0;
     context.err_info = err_info;
     context.cflags = flags;
 
     heap.size = DMC_DEFAULT_SIZE;
-    heap.data = heap.def;
+    heap.vars = heap.vars_def;
 
     /*
     ** Compile the match expression
     */
 restart:
-    heap.used = 0;
-    max_eheap_need = 0;
+    heap.vars_used = 0;
     for (context.current_match = 0; 
 	 context.current_match < num_progs; 
 	 ++context.current_match) { /* This loop is long, 
 				       too long */
-	memset(heap.data, 0, heap.size * sizeof(*heap.data));
+	memset(heap.vars, 0, heap.size * sizeof(*heap.vars));
 	t = context.matchexpr[context.current_match];
 	context.stack_used = 0;
-	context.eheap_need = 0;
 	structure_checked = 0;
 	if (context.current_match < num_progs - 1) {
 	    DMC_PUSH(text,matchTryMeElse);
@@ -1380,7 +1424,7 @@ restart:
     
 	/* 
 	** There is one single top variable in the match expression
-	** iff the text is tho Uint's and the single instruction 
+	** iff the text is two Uint's and the single instruction
 	** is 'matchBind' or it is only a skip.
 	*/
 	context.special = 
@@ -1459,10 +1503,6 @@ restart:
 	if (current_try_label >= 0) {
 	    DMC_POKE(text, current_try_label, DMC_STACK_NUM(text));
 	}
-	/* So, how much eheap did this part of the match program need? */
-	if (context.eheap_need > max_eheap_need) {
-	    max_eheap_need = context.eheap_need;
-	}
     } /* for (context.current_match = 0 ...) */
 
 
@@ -1491,23 +1531,20 @@ restart:
     ** A special case is when the match expression is a single binding 
     ** (i.e '$1'), then the field single_variable is set to 1.
     */
-    bp = erts_create_magic_binary(((sizeof(MatchProg) - sizeof(Uint)) +
-				   (DMC_STACK_NUM(text) * sizeof(Uint))),
+    bp = erts_create_magic_binary(((sizeof(MatchProg) - sizeof(UWord)) +
+				   (DMC_STACK_NUM(text) * sizeof(UWord))),
 				  erts_db_match_prog_destructor);
     ret = Binary2MatchProg(bp);
     ret->saved_program_buf = NULL;
     ret->saved_program = NIL;
     ret->term_save = context.save;
-    ret->num_bindings = heap.used;
+    ret->num_bindings = heap.vars_used;
     ret->single_variable = context.special;
     sys_memcpy(ret->text, DMC_STACK_DATA(text), 
-	       DMC_STACK_NUM(text) * sizeof(Uint));
-    ret->heap_size = ((heap.used * sizeof(Eterm)) +
-		      (max_eheap_need * sizeof(Eterm)) +
-		      (context.stack_need * sizeof(Eterm *)) +
-		      (3 * (FENCE_PATTERN_SIZE * sizeof(Eterm *))));
-    ret->eheap_offset = heap.used + FENCE_PATTERN_SIZE;
-    ret->stack_offset = ret->eheap_offset + max_eheap_need + FENCE_PATTERN_SIZE;
+	       DMC_STACK_NUM(text) * sizeof(UWord));
+    ret->stack_offset = heap.vars_used*sizeof(MatchVariable) + FENCE_PATTERN_SIZE;
+    ret->heap_size = ret->stack_offset + context.stack_need * sizeof(Eterm*) + FENCE_PATTERN_SIZE;
+
 #ifdef DMC_DEBUG
     ret->prog_end = ret->text + DMC_STACK_NUM(text);
 #endif
@@ -1517,17 +1554,16 @@ restart:
      */  
     context.save = NULL;
 error: /* Here is were we land when compilation failed. */
-    while (context.save != NULL) {
-	ErlHeapFragment *ll = context.save->next;
+    if (context.save != NULL) {
 	free_message_buffer(context.save);
-	context.save = ll;
+	context.save = NULL;
     }
     DMC_FREE(stack);
     DMC_FREE(text);
     if (context.copy != NULL) 
 	free_message_buffer(context.copy);
-    if (heap.data != heap.def)
-	erts_free(ERTS_ALC_T_DB_MS_CMPL_HEAP, (void *) heap.data);
+    if (heap.vars != heap.vars_def)
+	erts_free(ERTS_ALC_T_DB_MS_CMPL_HEAP, (void *) heap.vars);
     return bp;
 }
 
@@ -1537,15 +1573,11 @@ error: /* Here is were we land when compilation failed. */
 void erts_db_match_prog_destructor(Binary *bprog)
 {
     MatchProg *prog;
-    ErlHeapFragment *tmp, *ll;
     if (bprog == NULL)
 	return;
     prog = Binary2MatchProg(bprog);
-    tmp = prog->term_save; 
-    while (tmp != NULL) {
-	ll = tmp->next;
-	free_message_buffer(tmp);
-	tmp = ll;
+    if (prog->term_save != NULL) {
+	free_message_buffer(prog->term_save); 
     }
     if (prog->saved_program_buf != NULL)
 	free_message_buffer(prog->saved_program_buf);
@@ -1576,7 +1608,7 @@ erts_match_prog_foreach_offheap(Binary *bprog,
 */
 static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity)
 {
-    Eterm *hp = HAlloc(psp, arity * 2);
+    Eterm *hp = HAllocX(psp, arity * 2, HEAP_XTRA);
     Eterm ret = NIL;
     while (--arity >= 0) {
 	ret = CONS(hp, arr[arity], ret);
@@ -1584,14 +1616,83 @@ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity)
     }
     return ret;
 }
+
+
+#if HALFWORD_HEAP
+struct heap_checkpoint_t
+{
+    Process *p;
+    Eterm* htop;
+    ErlHeapFragment* mbuf;
+    unsigned used_size;
+    ErlOffHeap off_heap;
+};
+
+static void heap_checkpoint_init(Process* p, struct heap_checkpoint_t* hcp)
+{
+    hcp->p = p;
+    hcp->htop = HEAP_TOP(p);
+    hcp->mbuf = MBUF(p);
+    hcp->used_size = hcp->mbuf ? hcp->mbuf->used_size : 0;
+    hcp->off_heap = MSO(p);
+}
+
+static void heap_checkpoint_revert(struct heap_checkpoint_t* hcp)
+{
+    struct erl_off_heap_header* oh = MSO(hcp->p).first;
+
+    if (oh != hcp->off_heap.first) {
+	ASSERT(oh != NULL);
+	if (hcp->off_heap.first) {
+	    while (oh->next != hcp->off_heap.first) {
+		oh = oh->next;
+	    }
+	    oh->next = NULL;
+	}
+	erts_cleanup_offheap(&MSO(hcp->p));
+	MSO(hcp->p) = hcp->off_heap;
+    }
+    if (MBUF(hcp->p) != hcp->mbuf) {
+	ErlHeapFragment* hf = MBUF(hcp->p);
+	ASSERT(hf != NULL);
+	if (hcp->mbuf) {
+	    while (hf->next != hcp->mbuf) {
+		hf = hf->next;
+	    }
+	    hf->next = NULL;
+	}
+	free_message_buffer(MBUF(hcp->p));
+	MBUF(hcp->p) = hcp->mbuf;
+    }
+    if (hcp->mbuf != NULL && hcp->mbuf->used_size != hcp->used_size) {
+	hcp->mbuf->used_size = hcp->used_size;
+    }
+    HEAP_TOP(hcp->p) = hcp->htop;
+}
+#endif /* HALFWORD_HEAP */
+
+static ERTS_INLINE Eterm copy_object_rel(Process* p, Eterm term, Eterm* base)
+{
+    if (!is_immed(term)) {
+	Uint sz = size_object_rel(term, base);
+	Eterm* top = HAllocX(p, sz, HEAP_XTRA);
+	return copy_struct_rel(term, sz, &top, &MSO(p), base, NULL);
+    }
+    return term;
+}
+
+
 /*
 ** Execution of the match program, this is Pam.
 ** May return THE_NON_VALUE, which is a bailout.
-** the para meter 'arity' is only used if 'term' is actually an array,
+** the parameter 'arity' is only used if 'term' is actually an array,
 ** i.e. 'DCOMP_TRACE' was specified 
 */
-Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, 
+Eterm db_prog_match(Process *c_p, Binary *bprog,
+		    Eterm term, Eterm* base,
+		    Eterm *termp,
 		    int arity,
+		    enum erts_pam_run_flags in_flags,
 		    Uint32 *return_flags)
 {
     MatchProg *prog = Binary2MatchProg(bprog);
@@ -1600,8 +1701,9 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term,
     Eterm t;
     Eterm **sp;
     Eterm *esp;
-    Eterm *hp;
-    Uint *pc = prog->text;
+    MatchVariable* variables;
+    BeamInstr *cp;
+    UWord *pc = prog->text;
     Eterm *ehp;
     Eterm ret;
     Uint n = 0; /* To avoid warning. */
@@ -1609,19 +1711,25 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term,
     unsigned do_catch;
     ErtsMatchPseudoProcess *mpsp;
     Process *psp;
+    Process* build_proc;
     Process *tmpp;
     Process *current_scheduled;
     ErtsSchedulerData *esdp;
     Eterm (*bif)(Process*, ...);
+    Eterm bif_args[3];
     int fail_label;
     int atomic_trace;
+#if HALFWORD_HEAP
+    struct heap_checkpoint_t c_p_checkpoint = {};
+#endif
 #ifdef DMC_DEBUG
-    unsigned long *heap_fence;
-    unsigned long *eheap_fence;
-    unsigned long *stack_fence;
+    Uint *heap_fence;
+    Uint *stack_fence;
     Uint save_op;
 #endif /* DMC_DEBUG */
 
+    ASSERT(base==NULL || HALFWORD_HEAP);
+
     mpsp = get_match_pseudo_process(c_p, prog->heap_size);
     psp = &mpsp->process;
 
@@ -1631,34 +1739,35 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term,
     esdp = ERTS_GET_SCHEDULER_DATA_FROM_PROC(c_p);
     ASSERT(esdp != NULL);
     current_scheduled = esdp->current_process;
-    esdp->current_process = psp;
     /* SMP: psp->scheduler_data is set by get_match_pseudo_process */
 
     atomic_trace = 0;
 #define BEGIN_ATOMIC_TRACE(p)                               \
     do {                                                    \
 	if (! atomic_trace) {                               \
+            erts_refc_inc(&bprog->refc, 2);                 \
 	    erts_smp_proc_unlock((p), ERTS_PROC_LOCK_MAIN); \
-	    erts_smp_block_system(0);                       \
+	    erts_smp_thr_progress_block();                  \
             atomic_trace = !0;                              \
 	}                                                   \
     } while (0)
 #define END_ATOMIC_TRACE(p)                               \
     do {                                                  \
 	if (atomic_trace) {                               \
-            erts_smp_release_system();                    \
+            erts_smp_thr_progress_unblock();              \
             erts_smp_proc_lock((p), ERTS_PROC_LOCK_MAIN); \
+            if (erts_refc_dectest(&bprog->refc, 0) == 0) {\
+                erts_bin_free(bprog);                     \
+	    }                                             \
             atomic_trace = 0;                             \
 	}                                                 \
     } while (0)
 
 #ifdef DMC_DEBUG
     save_op = 0;
-    heap_fence =  (unsigned long *) mpsp->heap + prog->eheap_offset - 1;
-    eheap_fence = (unsigned long *) mpsp->heap + prog->stack_offset - 1;
-    stack_fence = (unsigned long *) mpsp->heap + prog->heap_size - 1;
+    heap_fence = (Eterm*)((char*) mpsp->u.heap + prog->stack_offset) - 1;
+    stack_fence = (Eterm*)((char*) mpsp->u.heap + prog->heap_size) - 1;
     *heap_fence = FENCE_PATTERN;
-    *eheap_fence = FENCE_PATTERN;
     *stack_fence = FENCE_PATTERN;
 #endif /* DMC_DEBUG */
 
@@ -1672,36 +1781,48 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term,
 
     *return_flags = 0U;
 
+    variables = mpsp->u.variables;
+#if HALFWORD_HEAP
+    c_p_checkpoint.p = NULL;
+#endif
+
 restart:
     ep = &term;
-    esp = mpsp->heap + prog->stack_offset;
+    esp = (Eterm*)((char*)mpsp->u.heap + prog->stack_offset);
     sp = (Eterm **) esp;
-    hp = mpsp->heap;
-    ehp = mpsp->heap + prog->eheap_offset;
     ret = am_true;
     do_catch = 0;
     fail_label = -1;
+    build_proc = psp;
+    esdp->current_process = psp;
+    ASSERT_HALFWORD(!c_p_checkpoint.p);
+
+#ifdef DEBUG
+    ASSERT(variables == mpsp->u.variables);
+    for (i=0; i<prog->num_bindings; i++) {
+	variables[i].term = THE_NON_VALUE;
+	variables[i].proc = NULL;
+	variables[i].base = base;
+    }
+#endif
 
     for (;;) {
-#ifdef DMC_DEBUG
+
+    #ifdef DMC_DEBUG
 	if (*heap_fence != FENCE_PATTERN) {
 	    erl_exit(1, "Heap fence overwritten in db_prog_match after op "
 		     "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence);
 	}
-	if (*eheap_fence != FENCE_PATTERN) {
-	    erl_exit(1, "Eheap fence overwritten in db_prog_match after op "
-		     "0x%08x, overwritten with 0x%08x.", save_op, 
-		     *eheap_fence);
-	}
 	if (*stack_fence != FENCE_PATTERN) {
 	    erl_exit(1, "Stack fence overwritten in db_prog_match after op "
 		     "0x%08x, overwritten with 0x%08x.", save_op, 
 		     *stack_fence);
 	}
 	save_op = *pc;
-#endif
+    #endif
 	switch (*pc++) {
 	case matchTryMeElse:
+	    ASSERT(fail_label == -1);
 	    fail_label = *pc++;
 	    break;
 	case matchArray: /* only when DCOMP_TRACE, is always first
@@ -1709,16 +1830,17 @@ restart:
 	    n = *pc++;
 	    if ((int) n != arity)
 		FAIL();
-	    ep = (Eterm *) *ep;
+	    ep = termp;
 	    break;
 	case matchArrayBind: /* When the array size is unknown. */
+	    ASSERT(termp);
 	    n = *pc++;
-	    hp[n] = dpm_array_to_list(psp, (Eterm *) term, arity);
+	    variables[n].term = dpm_array_to_list(psp, termp, arity);
 	    break;
 	case matchTuple: /* *ep is a tuple of arity n */
-	    if (!is_tuple(*ep))
+	    if (!is_tuple_rel(*ep,base))
 		FAIL();
-	    ep = tuple_val(*ep);
+	    ep = tuple_val_rel(*ep,base);
 	    n = *pc++;
 	    if (arityval(*ep) != n)
 		FAIL();
@@ -1726,9 +1848,9 @@ restart:
 	    break;
 	case matchPushT: /* *ep is a tuple of arity n, 
 			    push ptr to first element */
-	    if (!is_tuple(*ep))
+	    if (!is_tuple_rel(*ep,base))
 		FAIL();
-	    tp = tuple_val(*ep);
+	    tp = tuple_val_rel(*ep,base);
 	    n = *pc++;
 	    if (arityval(*tp) != n)
 		FAIL();
@@ -1738,12 +1860,12 @@ restart:
 	case matchList:
 	    if (!is_list(*ep))
 		FAIL();
-	    ep = list_val(*ep);
+	    ep = list_val_rel(*ep,base);
 	    break;
 	case matchPushL:
 	    if (!is_list(*ep))
 		FAIL();
-	    *sp++ = list_val(*ep);
+	    *sp++ = list_val_rel(*ep,base);
 	    ++ep;
 	    break;
 	case matchPop:
@@ -1751,52 +1873,61 @@ restart:
 	    break;
 	case matchBind:
 	    n = *pc++;
-	    hp[n] = *ep++;
+	    variables[n].term = *ep++;
 	    break;
 	case matchCmp:
 	    n = *pc++;
-	    if (!eq(hp[n],*ep))
+	    if (!eq_rel(variables[n].term, base, *ep, base))
 		FAIL();
 	    ++ep;
 	    break;
 	case matchEqBin:
 	    t = (Eterm) *pc++;
-	    if (!eq(*ep,t))
+	    if (!eq_rel(t,NULL,*ep,base))
 		FAIL();
 	    ++ep;
 	    break;
 	case matchEqFloat:
-	    if (!is_float(*ep))
+	    if (!is_float_rel(*ep,base))
 		FAIL();
-	    if (memcmp(float_val(*ep) + 1, pc, sizeof(double)))
+	    if (memcmp(float_val_rel(*ep,base) + 1, pc, sizeof(double)))
 		FAIL();
-	    pc += 2;
+	    pc += TermWords(2);
 	    ++ep;
 	    break;
-	case matchEqRef:
-	    if (!is_ref(*ep))
+	case matchEqRef: {
+	    Eterm* epc = (Eterm*)pc;
+	    if (!is_ref_rel(*ep,base))
 		FAIL();
-	    if (!eq(*ep, make_internal_ref(pc)))
+	    if (!eq_rel(make_internal_ref_rel(epc, epc), epc, *ep, base)) {
 		FAIL();
-	    i = thing_arityval(*pc);
-	    pc += i+1;
+	    }
+	    i = thing_arityval(*epc);
+	    pc += TermWords(i+1);
 	    ++ep;
 	    break;
+	}
 	case matchEqBig:
-	    if (!is_big(*ep))
+	    if (!is_big_rel(*ep,base))
 		FAIL();
-	    tp = big_val(*ep);
-	    if (*tp != *pc)
-		FAIL();
-	    i = BIG_ARITY(pc);
-	    while(i--)
-		if (*++tp != *++pc)
+	    tp = big_val_rel(*ep,base);
+	    {
+		Eterm *epc = (Eterm *) pc;
+		if (*tp != *epc)
 		    FAIL();
-	    ++pc;
+		i = BIG_ARITY(epc);
+		pc += TermWords(i+1);
+		while(i--) {
+		    if (*++tp != *++epc) {
+			FAIL();
+		    }
+		}
+	    }
 	    ++ep;
 	    break;
 	case matchEq:
-	    t = (Eterm) *pc++; 
+	    t = (Eterm) *pc++;
+	    ASSERT(is_immed(t));
 	    if (t != *ep++)
 		FAIL();
 	    break;
@@ -1804,25 +1935,32 @@ restart:
 	    ++ep;
 	    break;
 	/* 
-	 * Here comes guard instructions 
+	 * Here comes guard & body instructions
 	 */
 	case matchPushC: /* Push constant */
-	    *esp++ = *pc++;
+	    if ((in_flags & ERTS_PAM_COPY_RESULT)
+		&& do_catch && !is_immed(*pc)) {
+		*esp++ = copy_object(*pc++, c_p);
+	    }
+	    else {
+		*esp++ = *pc++;
+	    }
 	    break;
 	case matchConsA:
-	    ehp[1] = *--esp;
-	    ehp[0] = esp[-1];
+	    ehp = HAllocX(build_proc, 2, HEAP_XTRA);
+	    CDR(ehp) = *--esp;
+	    CAR(ehp) = esp[-1];
 	    esp[-1] = make_list(ehp);
-	    ehp += 2;
 	    break;
 	case matchConsB:
-	    ehp[0] = *--esp;
-	    ehp[1] = esp[-1];
+	    ehp = HAllocX(build_proc, 2, HEAP_XTRA);
+	    CAR(ehp) = *--esp;
+	    CDR(ehp) = esp[-1];
 	    esp[-1] = make_list(ehp);
-	    ehp += 2;
 	    break;
 	case matchMkTuple:
 	    n = *pc++;
+	    ehp = HAllocX(build_proc, n+1, HEAP_XTRA);
 	    t = make_tuple(ehp);
 	    *ehp++ = make_arityval(n);
 	    while (n--) {
@@ -1832,7 +1970,7 @@ restart:
 	    break;
 	case matchCall0:
 	    bif = (Eterm (*)(Process*, ...)) *pc++;
-	    t = (*bif)(psp);
+	    t = (*bif)(build_proc, bif_args);
 	    if (is_non_value(t)) {
 		if (do_catch)
 		    t = FAIL_TERM;
@@ -1843,7 +1981,7 @@ restart:
 	    break;
 	case matchCall1:
 	    bif = (Eterm (*)(Process*, ...)) *pc++;
-	    t = (*bif)(psp, esp[-1]);
+	    t = (*bif)(build_proc, esp-1);
 	    if (is_non_value(t)) {
 		if (do_catch)
 		    t = FAIL_TERM;
@@ -1854,7 +1992,9 @@ restart:
 	    break;
 	case matchCall2:
 	    bif = (Eterm (*)(Process*, ...)) *pc++;
-	    t = (*bif)(psp, esp[-1], esp[-2]);
+	    bif_args[0] = esp[-1];
+	    bif_args[1] = esp[-2];
+	    t = (*bif)(build_proc, bif_args);
 	    if (is_non_value(t)) {
 		if (do_catch)
 		    t = FAIL_TERM;
@@ -1866,7 +2006,10 @@ restart:
 	    break;
 	case matchCall3:
 	    bif = (Eterm (*)(Process*, ...)) *pc++;
-	    t = (*bif)(psp, esp[-1], esp[-2], esp[-3]);
+	    bif_args[0] = esp[-1];
+	    bif_args[1] = esp[-2];
+	    bif_args[2] = esp[-3];
+	    t = (*bif)(build_proc, bif_args);
 	    if (is_non_value(t)) {
 		if (do_catch)
 		    t = FAIL_TERM;
@@ -1876,15 +2019,73 @@ restart:
 	    esp -= 2;
 	    esp[-1] = t;
 	    break;
+
+	#if HALFWORD_HEAP
+	case matchPushVGuard:
+	    if (!base) goto case_matchPushV;
+	    /* Build NULL-based copy on pseudo heap for easy disposal */
+	    n = *pc++;
+	    ASSERT(is_value(variables[n].term));
+	    ASSERT(!variables[n].proc);
+	    variables[n].term = copy_object_rel(psp, variables[n].term, base);
+	    *esp++ = variables[n].term;
+	    #ifdef DEBUG
+	    variables[n].proc = psp;
+	    variables[n].base = NULL;
+	    #endif
+	    break;
+	#endif
+	case matchPushVResult:
+	    if (!(in_flags & ERTS_PAM_COPY_RESULT)) goto case_matchPushV;
+
+	    /* Build (NULL-based) copy on callers heap */
+	#if HALFWORD_HEAP
+	    if (!do_catch && !c_p_checkpoint.p) {
+		heap_checkpoint_init(c_p, &c_p_checkpoint);
+	    }
+        #endif
+	    n = *pc++;
+	    ASSERT(is_value(variables[n].term));
+	    ASSERT(!variables[n].proc);
+	    variables[n].term = copy_object_rel(c_p, variables[n].term, base);
+	    *esp++ = variables[n].term;
+	    #ifdef DEBUG
+	    variables[n].proc = c_p;
+	    variables[n].base = NULL;
+	    #endif
+	    break;
 	case matchPushV:
-	    *esp++ = hp[*pc++];
+	case_matchPushV:
+	    n = *pc++;
+	    ASSERT(is_value(variables[n].term));
+	    ASSERT(!variables[n].base);
+	    *esp++ = variables[n].term;
 	    break;
 	case matchPushExpr:
-	    *esp++ = term;
+	    if (in_flags & ERTS_PAM_COPY_RESULT) {
+		Uint sz;
+		Eterm* top;
+		sz = size_object_rel(term, base);
+		top = HAllocX(build_proc, sz, HEAP_XTRA);
+		if (in_flags & ERTS_PAM_CONTIGUOUS_TUPLE) {
+		    ASSERT(is_tuple_rel(term,base));
+		    *esp++ = copy_shallow_rel(tuple_val_rel(term,base), sz,
+					      &top, &MSO(build_proc), base);
+		}
+		else {
+		    *esp++ = copy_struct_rel(term, sz, &top, &MSO(build_proc),
+					     base, NULL);
+		}
+	    }
+	    else {
+		*esp = term;
+	    }
 	    break;
 	case matchPushArrayAsList:
+	    ASSERT_HALFWORD(base == NULL);
 	    n = arity; /* Only happens when 'term' is an array */
-	    tp = (Eterm *) term;
+	    tp = termp;
+	    ehp = HAllocX(build_proc, n*2, HEAP_XTRA);
 	    *esp++  = make_list(ehp);
 	    while (n--) {
 		*ehp++ = *tp++;
@@ -1897,7 +2098,8 @@ restart:
 	    break;
 	case matchPushArrayAsListU:
 	    /* This instruction is NOT efficient. */
-	    *esp++  = dpm_array_to_list(psp, (Eterm *) term, arity); 
+	    ASSERT_HALFWORD(base == NULL);
+	    *esp++  = dpm_array_to_list(build_proc, termp, arity);
 	    break;
 	case matchTrue:
 	    if (*--esp != am_true)
@@ -1983,7 +2185,8 @@ restart:
 	case matchProcessDump: {
 	    erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0);
 	    print_process_info(ERTS_PRINT_DSBUF, (void *) dsbufp, c_p);
-	    *esp++ = new_binary(psp, (byte *)dsbufp->str, (int)dsbufp->str_len);
+	    *esp++ = new_binary(build_proc, (byte *)dsbufp->str,
+				dsbufp->str_len);
 	    erts_destroy_tmp_dsbuf(dsbufp);
 	    break;
 	}
@@ -2000,7 +2203,11 @@ restart:
 	    *esp++ = am_true;
 	    break;
 	case matchIsSeqTrace:
-	    if (SEQ_TRACE_TOKEN(c_p) != NIL)
+	    if (SEQ_TRACE_TOKEN(c_p) != NIL
+#ifdef USE_VM_PROBES
+		&& SEQ_TRACE_TOKEN(c_p) != am_have_dt_utag
+#endif
+		)
 		*esp++ = am_true;
 	    else
 		*esp++ = am_false;
@@ -2024,32 +2231,31 @@ restart:
 	    --esp;
 	    break;
 	case matchGetSeqToken:
-	    if (SEQ_TRACE_TOKEN(c_p) == NIL) 
+	    if (SEQ_TRACE_TOKEN(c_p) == NIL
+#ifdef USE_VM_PROBES
+		|| SEQ_TRACE_TOKEN(c_p) == am_have_dt_utag
+#endif
+		) 
 		*esp++ = NIL;
 	    else {
+		Eterm sender = SEQ_TRACE_TOKEN_SENDER(c_p);
+		Uint sender_sz = is_immed(sender) ? 0 : size_object(sender);
+		ehp = HAllocX(build_proc, 6 + sender_sz, HEAP_XTRA);
+		if (sender_sz) {
+		    sender = copy_struct(sender, sender_sz, &ehp, &MSO(build_proc));
+		}
  		*esp++ = make_tuple(ehp);
  		ehp[0] = make_arityval(5);
  		ehp[1] = SEQ_TRACE_TOKEN_FLAGS(c_p);
  		ehp[2] = SEQ_TRACE_TOKEN_LABEL(c_p);
  		ehp[3] = SEQ_TRACE_TOKEN_SERIAL(c_p);
- 		ehp[4] = SEQ_TRACE_TOKEN_SENDER(c_p);
+		ehp[4] = sender;
  		ehp[5] = SEQ_TRACE_TOKEN_LASTCNT(c_p);
 		ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5);
 		ASSERT(is_immed(ehp[1]));
 		ASSERT(is_immed(ehp[2]));
 		ASSERT(is_immed(ehp[3]));
 		ASSERT(is_immed(ehp[5]));
-		if(!is_immed(ehp[4])) {
-		    Eterm *sender = &ehp[4];
-		    ehp += 6;
-		    *sender = copy_struct(*sender,
-					  size_object(*sender),
-					  &ehp,
-					  &MSO(psp));
-		}
-		else
-		    ehp += 6;
-
 	    } 
 	    break;
 	case matchEnableTrace:
@@ -2095,17 +2301,17 @@ restart:
 	    }
 	    break;
  	case matchCaller:
- 	    if (!(c_p->cp) || !(hp = find_function_from_pc(c_p->cp))) {
+	    if (!(c_p->cp) || !(cp = find_function_from_pc(c_p->cp))) {
  		*esp++ = am_undefined;
  	    } else {
+		ehp = HAllocX(build_proc, 4, HEAP_XTRA);
  		*esp++ = make_tuple(ehp);
- 		ehp[0] = make_arityval(3);
- 		ehp[1] = hp[0];
- 		ehp[2] = hp[1];
- 		ehp[3] = make_small(hp[2]);
- 		ehp += 4;
- 	    }
- 	    break;
+		ehp[0] = make_arityval(3);
+		ehp[1] = cp[0];
+		ehp[2] = cp[1];
+		ehp[3] = make_small((Uint) cp[2]);
+	    }
+	    break;
 	case matchSilent:
 	    --esp;
 	    if (*esp == am_true) {
@@ -2180,8 +2386,12 @@ restart:
 		}
 	    }
 	    break;
-	case matchCatch:
+	case matchCatch:  /* Match success, now build result */
 	    do_catch = 1;
+	    if (in_flags & ERTS_PAM_COPY_RESULT) {
+		build_proc = c_p;
+		esdp->current_process = c_p;
+	    }
 	    break;
 	case matchHalt:
 	    goto success;
@@ -2190,9 +2400,16 @@ restart:
 	}
     }
 fail:
+#if HALFWORD_HEAP
+    if (c_p_checkpoint.p) {
+	/* Dispose garbage built by guards on caller heap */
+	heap_checkpoint_revert(&c_p_checkpoint);
+	c_p_checkpoint.p = NULL;
+    }
+#endif
     *return_flags = 0U;
-    if (fail_label >= 0) { /* We failed during a "TryMeElse", 
-			      lets restart, with the next match 
+    if (fail_label >= 0) { /* We failed during a "TryMeElse",
+			      lets restart, with the next match
 			      program */
 	pc = (prog->text) + fail_label;
 	cleanup_match_pseudo_process(mpsp, 1);
@@ -2206,11 +2423,6 @@ success:
 	erl_exit(1, "Heap fence overwritten in db_prog_match after op "
 		 "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence);
     }
-    if (*eheap_fence != FENCE_PATTERN) {
-	erl_exit(1, "Eheap fence overwritten in db_prog_match after op "
-		 "0x%08x, overwritten with 0x%08x.", save_op, 
-		 *eheap_fence);
-    }
     if (*stack_fence != FENCE_PATTERN) {
 	erl_exit(1, "Stack fence overwritten in db_prog_match after op "
 		 "0x%08x, overwritten with 0x%08x.", save_op, 
@@ -2221,6 +2433,7 @@ success:
     esdp->current_process = current_scheduled;
 
     END_ATOMIC_TRACE(c_p);
+
     return ret;
 #undef FAIL
 #undef FAIL_TERM
@@ -2232,7 +2445,8 @@ success:
 /*
  * Convert a match program to a "magic" binary to return up to erlang
  */
-Eterm db_make_mp_binary(Process *p, Binary *mp, Eterm **hpp) {
+Eterm db_make_mp_binary(Process *p, Binary *mp, Eterm **hpp)
+{
     return erts_mk_magic_binary_term(hpp, &MSO(p), mp);
 }
 
@@ -2298,13 +2512,13 @@ void db_free_dmc_err_info(DMCErrInfo *ei){
 ** Store bignum in *hpp and increase *hpp accordingly.
 ** *hpp is assumed to be large enough to hold the result.
 */
-Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr)
+Eterm db_add_counter(Eterm** hpp, Wterm counter, Eterm incr)
 {
-    Eterm big_tmp[2];
+    DeclareTmpHeapNoproc(big_tmp,2);
     Eterm res;
     Sint ires;
-    Eterm arg1;
-    Eterm arg2;
+    Wterm arg1;
+    Wterm arg2;
 
     if (is_both_small(counter,incr)) {
 	ires = signed_val(counter) + signed_val(incr);
@@ -2318,6 +2532,7 @@ Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr)
 	}
     }
     else {
+	UseTmpHeapNoproc(2);
 	switch(NUMBER_CODE(counter, incr)) {
 	case SMALL_BIG:
 	    arg1 = small_to_big(signed_val(counter), big_tmp);
@@ -2332,16 +2547,46 @@ Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr)
 	    arg2 = counter;
 	    break;
 	default:
+	    UnUseTmpHeapNoproc(2);
 	    return THE_NON_VALUE;
 	}
 	res = big_plus(arg1, arg2, *hpp);
 	if (is_big(res)) {
 	    *hpp += BIG_NEED_SIZE(big_size(res));
 	}
+	UnUseTmpHeapNoproc(2);
 	return res;
     }
 }
 
+/* Must be called to read elements after db_lookup_dbterm.
+** Will decompress if needed.
+** HEALFWORD_HEAP:
+** 	Will convert from relative to Wterm format if needed.
+**      (but only on top level, tuples and lists will still contain rterms)
+*/
+Wterm db_do_read_element(DbUpdateHandle* handle, Sint position)
+{
+    Eterm elem = handle->dbterm->tpl[position];
+    if (!is_header(elem)) {
+#if HALFWORD_HEAP
+	if (!is_immed(elem)
+	    && !handle->tb->common.compress
+	    && !(handle->abs_vec && handle->abs_vec[position])) {
+	    return rterm2wterm(elem, handle->dbterm->tpl);
+	}
+#endif
+	return elem;
+    }
+
+    ASSERT(((DbTableCommon*)handle->tb)->compress);
+    ASSERT(!handle->mustResize);
+    handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common,
+					       handle->dbterm);
+    handle->mustResize = 1;
+    return handle->dbterm->tpl[position];
+}
+
 /*
 ** Update one element:
 ** handle:   Initialized by db_lookup_dbterm()
@@ -2358,127 +2603,493 @@ void db_do_update_element(DbUpdateHandle* handle,
     Eterm* oldp;
     Uint newval_sz;
     Uint oldval_sz;
+#if HALFWORD_HEAP
+    Eterm* old_base;
+#endif
 
     if (is_both_immed(newval,oldval)) {
 	handle->dbterm->tpl[position] = newval;
+    #ifdef DEBUG_CLONE
+	if (handle->dbterm->debug_clone) {
+	    handle->dbterm->debug_clone[position] = newval;
+	}
+    #endif
 	return;
     }
-    else if (!handle->mustResize && is_boxed(newval)) {
-	newp = boxed_val(newval);
-	switch (*newp & _TAG_HEADER_MASK) {
-	case _TAG_HEADER_POS_BIG:
-	case _TAG_HEADER_NEG_BIG:
-	case _TAG_HEADER_FLOAT:
-	case _TAG_HEADER_HEAP_BIN:	    
-	    newval_sz = header_arity(*newp) + 1;	    
-	    if (is_boxed(oldval)) {
-		oldp = boxed_val(oldval);
-		switch (*oldp & _TAG_HEADER_MASK) {
+    if (!handle->mustResize) {
+	if (handle->tb->common.compress) {
+	    handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common,
+						       handle->dbterm);
+	    handle->mustResize = 1;
+	    oldval = handle->dbterm->tpl[position];
+        #if HALFWORD_HEAP
+	    old_base = NULL;
+	#endif
+	}
+	else {
+	#if HALFWORD_HEAP
+	    ASSERT(!handle->abs_vec);
+	    old_base = handle->dbterm->tpl;
+	#endif
+	    if (is_boxed(newval)) {
+		newp = boxed_val(newval);
+		switch (*newp & _TAG_HEADER_MASK) {
 		case _TAG_HEADER_POS_BIG:
 		case _TAG_HEADER_NEG_BIG:
 		case _TAG_HEADER_FLOAT:
 		case _TAG_HEADER_HEAP_BIN:
-		    oldval_sz = header_arity(*oldp) + 1;
-		    if (oldval_sz == newval_sz) {
-			/* "self contained" terms of same size, do memcpy */
-			sys_memcpy(oldp, newp, newval_sz*sizeof(Eterm));			
-			return;
+		    newval_sz = header_arity(*newp) + 1;
+		    if (is_boxed(oldval)) {
+			oldp = boxed_val_rel(oldval,old_base);
+			switch (*oldp & _TAG_HEADER_MASK) {
+			case _TAG_HEADER_POS_BIG:
+			case _TAG_HEADER_NEG_BIG:
+			case _TAG_HEADER_FLOAT:
+			case _TAG_HEADER_HEAP_BIN:
+			    oldval_sz = header_arity(*oldp) + 1;
+			    if (oldval_sz == newval_sz) {
+				/* "self contained" terms of same size, do memcpy */
+				    sys_memcpy(oldp, newp, newval_sz*sizeof(Eterm));
+				return;
+			    }
+			    goto both_size_set;
+			}
 		    }
-		    goto both_size_set;
+		    goto new_size_set;
 		}
 	    }
-	    goto new_size_set;
 	}
     }
+#if HALFWORD_HEAP
+    else {
+	old_base = (handle->tb->common.compress
+		    || (handle->abs_vec && handle->abs_vec[position])) ?
+	    NULL : handle->dbterm->tpl;
+    }
+#endif
     /* Not possible for simple memcpy or dbterm is already non-contiguous, */
     /* need to realloc... */
 
     newval_sz = is_immed(newval) ? 0 : size_object(newval);
 new_size_set:
-	
-    oldval_sz = is_immed(oldval) ? 0 : size_object(oldval);
+
+    oldval_sz = is_immed(oldval) ? 0 : size_object_rel(oldval,old_base);
 both_size_set:
 
     handle->new_size = handle->new_size - oldval_sz + newval_sz;
 
-    /* write new value in old dbterm, finalize will make a flat copy */	
+    /* write new value in old dbterm, finalize will make a flat copy */
     handle->dbterm->tpl[position] = newval;
     handle->mustResize = 1;
+
+#if HALFWORD_HEAP
+    if (old_base && newval_sz > 0) {
+	ASSERT(!handle->tb->common.compress);
+	if (!handle->abs_vec) {
+	    int i = header_arity(handle->dbterm->tpl[0]);
+	    handle->abs_vec = erts_alloc(ERTS_ALC_T_TMP, (i+1)*sizeof(char));
+	    sys_memset(handle->abs_vec, 0, i+1);
+	    /* abs_vec[0] not used */
+	}
+	handle->abs_vec[position] = 1;
+    }
+#endif
+}
+
+static ERTS_INLINE byte* db_realloc_term(DbTableCommon* tb, void* old,
+					 Uint old_sz, Uint new_sz, Uint offset)
+{
+    byte* ret;
+    if (erts_ets_realloc_always_moves) {
+	ret = erts_db_alloc(ERTS_ALC_T_DB_TERM, (DbTable*)tb, new_sz);
+	sys_memcpy(ret, old, offset);
+	erts_db_free(ERTS_ALC_T_DB_TERM, (DbTable*)tb, old, old_sz);
+    } else {
+	ret = erts_db_realloc(ERTS_ALC_T_DB_TERM, (DbTable*)tb,
+			      old, old_sz, new_sz);
+    }
+    return ret;
+}
+
+/* Allocated size of a compressed dbterm
+*/
+static ERTS_INLINE Uint db_alloced_size_comp(DbTerm* obj)
+{
+    return obj->tpl[arityval(*obj->tpl) + 1];
+}
+
+void db_free_term(DbTable *tb, void* basep, Uint offset)
+{
+    DbTerm* db = (DbTerm*) ((byte*)basep + offset);
+    Uint size;
+    if (tb->common.compress) {
+	db_cleanup_offheap_comp(db);
+	size = db_alloced_size_comp(db);
+    }
+    else {
+	ErlOffHeap tmp_oh;
+	tmp_oh.first = db->first_oh;
+	erts_cleanup_offheap(&tmp_oh);
+	size = offset + offsetof(DbTerm,tpl) + db->size*sizeof(Eterm);
+    }
+    erts_db_free(ERTS_ALC_T_DB_TERM, tb, basep, size);
+}
+
+static ERTS_INLINE Uint align_up(Uint value, Uint pow2)
+{
+    ASSERT((pow2 & (pow2-1)) == 0);
+    return (value + (pow2-1)) & ~(pow2-1);
 }
 
+/* Compressed size of an uncompressed term
+*/
+static Uint db_size_dbterm_comp(DbTableCommon* tb, Eterm obj)
+{
+    Eterm* tpl = tuple_val(obj);
+    int i;
+    Uint size = sizeof(DbTerm)
+	+ arityval(*tpl) * sizeof(Eterm)
+        + sizeof(Uint); /* "alloc_size" */
+
+    for (i = arityval(*tpl); i>0; i--) {
+	if (i != tb->keypos && is_not_immed(tpl[i])) {
+	    size += erts_encode_ext_size_ets(tpl[i]);
+	}
+    }
+    size += size_object(tpl[tb->keypos]) * sizeof(Eterm);
+    return align_up(size, sizeof(Uint));
+}
+
+/* Conversion between top tuple element and pointer to compressed data
+*/
+static ERTS_INLINE Eterm ext2elem(Eterm* tpl, byte* ext)
+{
+    return (((Uint)(ext - (byte*)tpl)) << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER;
+}
+static ERTS_INLINE byte* elem2ext(Eterm* tpl, Uint ix)
+{
+    ASSERT(is_header(tpl[ix]));
+    return (byte*)tpl + (tpl[ix] >> _TAG_PRIMARY_SIZE);
+}
+
+static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest,
+			  Uint alloc_size)
+{
+    ErlOffHeap tmp_offheap;
+    Eterm* src = tuple_val(obj);
+    Eterm* tpl = dest->tpl;
+    Eterm key = src[tb->keypos];
+    int arity = arityval(src[0]);
+    union {
+	Eterm* ep;
+	byte* cp;
+	UWord ui;
+    }top;
+    int i;
+
+    top.ep = tpl+ 1 + arity + 1;
+    tpl[0] = src[0];
+    tpl[arity + 1] = alloc_size;
+
+    tmp_offheap.first = NULL;
+    tpl[tb->keypos] = copy_struct_rel(key, size_object(key), &top.ep, &tmp_offheap, NULL, tpl);
+    dest->first_oh = tmp_offheap.first;
+    for (i=1; i<=arity; i++) {
+	if (i != tb->keypos) {
+	    if (is_immed(src[i])) {
+		tpl[i] = src[i];
+	    }
+	    else {
+		tpl[i] = ext2elem(tpl, top.cp);
+		top.cp = erts_encode_ext_ets(src[i], top.cp, &dest->first_oh);
+	    }
+	}
+    }
+
+#ifdef DEBUG_CLONE
+    {
+	Eterm* dbg_top = erts_alloc(ERTS_ALC_T_DB_TERM, dest->size * sizeof(Eterm));
+	dest->debug_clone = dbg_top;
+	tmp_offheap.first = dest->first_oh;
+	copy_struct_rel(obj, dest->size, &dbg_top, &tmp_offheap, NULL, dbg_top);
+	dest->first_oh = tmp_offheap.first;
+	ASSERT(dbg_top == dest->debug_clone + dest->size);
+    }
+#endif
+    return top.cp;
+}
 
 /*
 ** Copy the object into a possibly new DbTerm, 
 ** offset is the offset of the DbTerm from the start
-** of the sysAllocaed structure, The possibly realloced and copied
+** of the allocated structure, The possibly realloced and copied
 ** structure is returned. Make sure (((char *) old) - offset) is a 
 ** pointer to a ERTS_ALC_T_DB_TERM allocated data area.
 */
-void* db_get_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj)
+void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj)
 {
+    byte* basep;
+    DbTerm* newp;
+    Eterm* top;
     int size = size_object(obj);
-    void *structp = ((char*) old) - offset;
-    DbTerm* p;
-    Eterm copy;
-    Eterm *top;
+    ErlOffHeap tmp_offheap;
 
     if (old != 0) {
-	erts_cleanup_offheap(&old->off_heap);
+	basep = ((byte*) old) - offset;
+	tmp_offheap.first  = old->first_oh;
+	erts_cleanup_offheap(&tmp_offheap);
+	old->first_oh = tmp_offheap.first;
 	if (size == old->size) {
-	    p = old;
-	} else {
+	    newp = old;
+	}
+	else {
 	    Uint new_sz = offset + sizeof(DbTerm) + sizeof(Eterm)*(size-1);
 	    Uint old_sz = offset + sizeof(DbTerm) + sizeof(Eterm)*(old->size-1);
 
-	    if (erts_ets_realloc_always_moves) {
-		void *nstructp = erts_db_alloc(ERTS_ALC_T_DB_TERM,
-					       (DbTable *) tb,
-					       new_sz);
-		memcpy(nstructp,structp,offset);
-		erts_db_free(ERTS_ALC_T_DB_TERM,
-			     (DbTable *) tb,
-			     structp,
-			     old_sz);
-		structp = nstructp;
-	    } else {
-		structp = erts_db_realloc(ERTS_ALC_T_DB_TERM,
-					  (DbTable *) tb,
-					  structp,
-					  old_sz,
-					  new_sz);
-	    }
-	    p = (DbTerm*) ((void *)(((char *) structp) + offset));
+	    basep = db_realloc_term(tb, basep, old_sz, new_sz, offset);
+	    newp = (DbTerm*) (basep + offset);
 	}
     }
     else {
-	structp = erts_db_alloc(ERTS_ALC_T_DB_TERM,
-				(DbTable *) tb,
-				(offset
-				 + sizeof(DbTerm)
-				 + sizeof(Eterm)*(size-1)));
-	p = (DbTerm*) ((void *)(((char *) structp) + offset));
-    }
-    p->size = size;
-    p->off_heap.mso = NULL;
-    p->off_heap.externals = NULL;
-#ifndef HYBRID /* FIND ME! */
-    p->off_heap.funs = NULL;
+	basep = erts_db_alloc(ERTS_ALC_T_DB_TERM, (DbTable *)tb,
+			      (offset + sizeof(DbTerm) + sizeof(Eterm)*(size-1)));
+	newp = (DbTerm*) (basep + offset);
+    }
+    newp->size = size;
+    top = newp->tpl;
+    tmp_offheap.first  = NULL;
+    copy_struct_rel(obj, size, &top, &tmp_offheap, NULL, top);
+    newp->first_oh = tmp_offheap.first;
+#ifdef DEBUG_CLONE
+    newp->debug_clone = NULL;
+#endif
+    return basep;
+}
+
+
+void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj)
+{
+    Uint new_sz = offset + db_size_dbterm_comp(tb, obj);
+    byte* basep;
+    DbTerm* newp;
+#ifdef DEBUG
+    byte* top;
 #endif
-    p->off_heap.overhead = 0;
 
-    top = DBTERM_BUF(p);
-    copy = copy_struct(obj, size, &top, &p->off_heap);
-    DBTERM_SET_TPL(p,tuple_val(copy));
+    ASSERT(tb->compress);
+    if (old != 0) {
+	Uint old_sz = db_alloced_size_comp(old);
+	db_cleanup_offheap_comp(old);
 
-    return structp;
+	basep = ((byte*) old) - offset;
+	if (new_sz == old_sz) {
+	    newp = old;
+	}
+	else {
+	    basep = db_realloc_term(tb, basep, old_sz, new_sz, offset);
+	    newp = (DbTerm*) (basep + offset);
+	}
+    }
+    else {
+	basep = erts_db_alloc(ERTS_ALC_T_DB_TERM, (DbTable*)tb, new_sz);
+	newp = (DbTerm*) (basep + offset);
+    }
+
+    newp->size = size_object(obj);
+#ifdef DEBUG
+    top = 
+#endif
+	copy_to_comp(tb, obj, newp, new_sz);
+    ASSERT(top <= basep + new_sz);
+
+    /* ToDo: Maybe realloc if ((basep+new_sz) - top) > WASTED_SPACE_LIMIT */
+
+    return basep;
+}
+
+
+void db_finalize_resize(DbUpdateHandle* handle, Uint offset)
+{
+    DbTable* tbl = handle->tb;
+    DbTerm* newDbTerm;
+    Uint alloc_sz = offset +
+	(tbl->common.compress ?
+	 db_size_dbterm_comp(&tbl->common, make_tuple(handle->dbterm->tpl)) :
+	 sizeof(DbTerm)+sizeof(Eterm)*(handle->new_size-1));
+    byte* newp = erts_db_alloc(ERTS_ALC_T_DB_TERM, tbl, alloc_sz);
+    byte* oldp = *(handle->bp);
+
+    sys_memcpy(newp, oldp, offset);  /* copy only hash/tree header */
+    *(handle->bp) = newp;
+    newDbTerm = (DbTerm*) (newp + offset);
+    newDbTerm->size = handle->new_size;
+#ifdef DEBUG_CLONE
+    newDbTerm->debug_clone = NULL;
+#endif
+
+    /* make a flat copy */
+
+    if (tbl->common.compress) {
+	copy_to_comp(&tbl->common, make_tuple(handle->dbterm->tpl),
+		     newDbTerm, alloc_sz);
+	db_free_tmp_uncompressed(handle->dbterm);
+    }
+    else {
+	ErlOffHeap tmp_offheap;
+	Eterm* tpl = handle->dbterm->tpl;
+	Eterm* top = newDbTerm->tpl;
+
+	tmp_offheap.first = NULL;
+
+    #if HALFWORD_HEAP
+	if (handle->abs_vec) {
+	    int i, arity = header_arity(handle->dbterm->tpl[0]);
+
+	    top[0] = tpl[0];
+	    top += arity + 1;
+	    for (i=1; i<=arity; i++) {
+		Eterm* src_base = handle->abs_vec[i] ? NULL : tpl;
+
+		newDbTerm->tpl[i] = copy_struct_rel(tpl[i],
+						    size_object_rel(tpl[i],src_base),
+						    &top, &tmp_offheap, src_base,
+						    newDbTerm->tpl);
+	    }
+	    newDbTerm->first_oh = tmp_offheap.first;
+	    ASSERT((byte*)top <= (newp + alloc_sz));
+	    erts_free(ERTS_ALC_T_TMP, handle->abs_vec);
+	}
+	else
+    #endif /* HALFWORD_HEAP */
+	{
+	    copy_struct_rel(make_tuple_rel(tpl,tpl), handle->new_size, &top,
+			    &tmp_offheap, tpl, top);
+	    newDbTerm->first_oh = tmp_offheap.first;
+	    ASSERT((byte*)top == (newp + alloc_sz));
+	}
+    }
+}
+
+Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp,
+			     ErlOffHeap* off_heap)
+{
+    Eterm* hp = *hpp;
+    int i, arity = arityval(bp->tpl[0]);
+
+    hp[0] = bp->tpl[0];
+    *hpp += arity + 1;
+
+    hp[tb->keypos] = copy_struct_rel(bp->tpl[tb->keypos],
+				     size_object_rel(bp->tpl[tb->keypos], bp->tpl),
+				     hpp, off_heap, bp->tpl, NULL);
+    for (i=arity; i>0; i--) {
+	if (i != tb->keypos) {
+	    if (is_immed(bp->tpl[i])) {
+		hp[i] = bp->tpl[i];
+	    }
+	    else {
+		hp[i] = erts_decode_ext_ets(hpp, off_heap,
+					    elem2ext(bp->tpl, i));
+	    }
+	}
+    }
+    ASSERT((*hpp - hp) <= bp->size);
+#ifdef DEBUG_CLONE
+    ASSERT(eq_rel(make_tuple(hp),NULL,make_tuple(bp->debug_clone),bp->debug_clone));
+#endif
+    return make_tuple(hp);
+}
+
+Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p,
+			       DbTerm* obj, Uint pos,
+			       Eterm** hpp, Uint extra)
+{
+    if (is_immed(obj->tpl[pos])) {
+	*hpp = HAlloc(p, extra);
+	return obj->tpl[pos];
+    }
+    if (tb->compress && pos != tb->keypos) {
+	byte* ext = elem2ext(obj->tpl, pos);
+	Sint sz = erts_decode_ext_size_ets(ext, db_alloced_size_comp(obj)) + extra;
+	Eterm* hp = HAlloc(p, sz);
+	Eterm* endp = hp + sz;
+	Eterm copy = erts_decode_ext_ets(&hp, &MSO(p), ext);
+	*hpp = hp;
+	hp += extra;
+	HRelease(p, endp, hp);
+#ifdef DEBUG_CLONE
+	ASSERT(eq_rel(copy, NULL, obj->debug_clone[pos], obj->debug_clone));
+#endif
+	return copy;
+    }
+    else {
+	Uint sz = size_object_rel(obj->tpl[pos], obj->tpl);
+	*hpp = HAlloc(p, sz + extra);
+	return copy_struct_rel(obj->tpl[pos], sz, hpp, &MSO(p), obj->tpl, NULL);
+    }
 }
 
 
-void db_free_term_data(DbTerm* p)
+/* Our own "cleanup_offheap"
+ * as refc-binaries may be unaligned in compressed terms
+*/
+void db_cleanup_offheap_comp(DbTerm* obj)
 {
-    erts_cleanup_offheap(&p->off_heap);
+    union erl_off_heap_ptr u;
+    ProcBin tmp;
+
+    for (u.hdr = obj->first_oh; u.hdr; u.hdr = u.hdr->next) {
+	if ((UWord)u.voidp % sizeof(Uint) != 0) { /* unaligned ptr */
+	    sys_memcpy(&tmp, u.voidp, sizeof(tmp));
+	    /* Warning, must pass (void*)-variable to memcpy. Otherwise it will
+	       cause Bus error on Sparc due to false compile time assumptions
+	       about word aligned memory (type cast is not enough) */
+	    u.pb = &tmp;
+	}
+	switch (thing_subtag(u.hdr->thing_word)) {
+	case REFC_BINARY_SUBTAG:
+	    if (erts_refc_dectest(&u.pb->val->refc, 0) == 0) {
+		erts_bin_free(u.pb->val);
+	    }
+	    break;
+	case FUN_SUBTAG:
+	    ASSERT(u.pb != &tmp);
+	    if (erts_refc_dectest(&u.fun->fe->refc, 0) == 0) {
+		erts_erase_fun_entry(u.fun->fe);
+	    }
+	    break;
+	default:
+	    ASSERT(is_external_header(u.hdr->thing_word));
+	    ASSERT(u.pb != &tmp);
+	    erts_deref_node_entry(u.ext->node);
+	    break;
+	}
+    }
+#ifdef DEBUG_CLONE
+    if (obj->debug_clone != NULL) {
+	erts_free(ERTS_ALC_T_DB_TERM, obj->debug_clone);
+	obj->debug_clone = NULL;
+    }
+#endif
 }
 
+int db_eq_comp(DbTableCommon* tb, Eterm a, DbTerm* b)
+{
+    ErlOffHeap tmp_offheap;
+    Eterm* allocp;
+    Eterm* hp;
+    Eterm tmp_b;
+    int is_eq;
+
+    ASSERT(tb->compress);
+    hp = allocp = erts_alloc(ERTS_ALC_T_TMP, b->size*sizeof(Eterm));
+    tmp_offheap.first = NULL;
+    tmp_b = db_copy_from_comp(tb, b, &hp, &tmp_offheap);
+    is_eq = eq(a,tmp_b);
+    erts_cleanup_offheap(&tmp_offheap);
+    erts_free(ERTS_ALC_T_TMP, allocp);
+    return is_eq;
+}
 
 /*
 ** Check if object represents a "match" variable 
@@ -2606,7 +3217,7 @@ static void add_dmc_err(DMCErrInfo *err_info,
 static DMCRet dmc_one_term(DMCContext *context, 
 			   DMCHeap *heap,
 			   DMC_STACK_TYPE(Eterm) *stack,
-			   DMC_STACK_TYPE(Uint) *text,
+			   DMC_STACK_TYPE(UWord) *text,
 			   Eterm c)
 {
     Sint n;
@@ -2624,7 +3235,7 @@ static DMCRet dmc_one_term(DMCContext *context,
 		** Ouch, big integer in match variable.
 		*/
 		Eterm *save_hp;
-		ASSERT(heap->data == heap->def);
+		ASSERT(heap->vars == heap->vars_def);
 		sz = sz2 = sz3 = 0;
 		for (j = 0; j < context->num_match; ++j) {
 		    sz += size_object(context->matchexpr[j]);
@@ -2662,24 +3273,23 @@ static DMCRet dmc_one_term(DMCContext *context,
 		       may be atoms that changed */
 		    context->matchexpr[j] = context->copy->mem[j];
 		}
-		heap->data = erts_alloc(ERTS_ALC_T_DB_MS_CMPL_HEAP,
-					heap->size*sizeof(unsigned));
-		sys_memset(heap->data, 0, 
-			   heap->size * sizeof(unsigned));
+		heap->vars = erts_alloc(ERTS_ALC_T_DB_MS_CMPL_HEAP,
+					heap->size*sizeof(DMCVariable));
+		sys_memset(heap->vars, 0, heap->size * sizeof(DMCVariable));
 		DMC_CLEAR(*stack);
 		/*DMC_PUSH(*stack,NIL);*/
 		DMC_CLEAR(*text);
 		return retRestart;
 	    }
-	    if (heap->data[n]) { /* already bound ? */
+	    if (heap->vars[n].is_bound) {
 		DMC_PUSH(*text,matchCmp);
 		DMC_PUSH(*text,n);
 	    } else { /* Not bound, bind! */
-		if (n >= heap->used)
-		    heap->used = n + 1;
+		if (n >= heap->vars_used)
+		    heap->vars_used = n + 1;
 		DMC_PUSH(*text,matchBind);
 		DMC_PUSH(*text,n);
-		heap->data[n] = 1;
+		heap->vars[n].is_bound = 1;
 	    }
 	} else if (c == am_Underscore) {
 	    DMC_PUSH(*text, matchSkip);
@@ -2704,27 +3314,84 @@ static DMCRet dmc_one_term(DMCContext *context,
 	    DMC_PUSH(*stack, c);
 	    break;
 	case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE):
-	    n = thing_arityval(*internal_ref_val(c));
+	{
+	    Eterm* ref_val = internal_ref_val(c);
 	    DMC_PUSH(*text, matchEqRef);
-	    DMC_PUSH(*text, *internal_ref_val(c));
-	    for (i = 1; i <= n; ++i) {
-		DMC_PUSH(*text, (Uint) internal_ref_val(c)[i]);
+#if HALFWORD_HEAP
+	    {
+		union {
+		    UWord u;
+		    Uint t[2];
+		} fiddle;
+		ASSERT(thing_arityval(ref_val[0]) == 3);
+		fiddle.t[0] = ref_val[0];
+		fiddle.t[1] = ref_val[1];
+		DMC_PUSH(*text, fiddle.u);
+		fiddle.t[0] = ref_val[2];
+		fiddle.t[1] = ref_val[3];
+		DMC_PUSH(*text, fiddle.u);
+	    }
+#else
+	    n = thing_arityval(ref_val[0]);
+	    for (i = 0; i <= n; ++i) {
+		DMC_PUSH(*text, ref_val[i]);
 	    }
+#endif
 	    break;
+	}
 	case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE):
 	case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE):
-	    n = thing_arityval(*big_val(c));
+	{
+	    Eterm* bval = big_val(c);
+	    n = thing_arityval(bval[0]);
 	    DMC_PUSH(*text, matchEqBig);
-	    DMC_PUSH(*text, *big_val(c));
-	    for (i = 1; i <= n; ++i) {
-		DMC_PUSH(*text, (Uint) big_val(c)[i]);
+#if HALFWORD_HEAP
+	    {
+		union {
+		    UWord u;
+		    Uint t[2];
+		} fiddle;
+		ASSERT(n >= 1);
+		fiddle.t[0] = bval[0];
+		fiddle.t[1] = bval[1];
+		DMC_PUSH(*text, fiddle.u);
+		for (i = 2; i <= n; ++i) {
+		    fiddle.t[0] = bval[i];
+		    if (++i <= n) {
+			fiddle.t[1] = bval[i];
+		    } else {
+			fiddle.t[1] = (Uint) 0;
+		    }
+		    DMC_PUSH(*text, fiddle.u);
+		}
 	    }
+#else
+	    for (i = 0; i <= n; ++i) {
+		DMC_PUSH(*text, (Uint) bval[i]);
+	    }
+#endif
 	    break;
+	}
 	case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE):
 	    DMC_PUSH(*text,matchEqFloat);
+#if HALFWORD_HEAP
+	    {
+		union {
+		    UWord u;
+		    Uint t[2];
+		} fiddle;
+		fiddle.t[0] = float_val(c)[1];
+		fiddle.t[1] = float_val(c)[2];
+		DMC_PUSH(*text, fiddle.u);
+	    }
+#else
 	    DMC_PUSH(*text, (Uint) float_val(c)[1]);
-	    /* XXX: this reads and pushes random junk on ARCH_64 */
+#ifdef ARCH_64
+	    DMC_PUSH(*text, (Uint) 0);
+#else
 	    DMC_PUSH(*text, (Uint) float_val(c)[2]);
+#endif
+#endif
 	    break;
 	default: /* BINARY, FUN, VECTOR, or EXTERNAL */
 	    /*
@@ -2753,7 +3420,7 @@ static DMCRet dmc_one_term(DMCContext *context,
 ** Match guard compilation
 */
 
-static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(Uint) *text,
+static void do_emit_constant(DMCContext *context, DMC_STACK_TYPE(UWord) *text,
 			     Eterm t) 
 {
 	int sz;
@@ -2807,7 +3474,7 @@ add_dmc_err((ContextP)->err_info, String, -1, T, dmcWarning)
 
 static DMCRet dmc_list(DMCContext *context,
 		       DMCHeap *heap,
-		       DMC_STACK_TYPE(Uint) *text,
+		       DMC_STACK_TYPE(UWord) *text,
 		       Eterm t,
 		       int *constant)
 {
@@ -2837,17 +3504,16 @@ static DMCRet dmc_list(DMCContext *context,
 	DMC_PUSH(*text, matchConsB);
     }
     --context->stack_used; /* Two objects on stack becomes one */
-    context->eheap_need += 2;
     return retOk;
 }
 
 static DMCRet dmc_tuple(DMCContext *context,
 		       DMCHeap *heap,
-		       DMC_STACK_TYPE(Uint) *text,
+		       DMC_STACK_TYPE(UWord) *text,
 		       Eterm t,
 		       int *constant)
 {
-    DMC_STACK_TYPE(Uint) instr_save;
+    DMC_STACK_TYPE(UWord) instr_save;
     int all_constant = 1;
     int textpos = DMC_STACK_NUM(*text);
     Eterm *p = tuple_val(t);
@@ -2896,14 +3562,13 @@ static DMCRet dmc_tuple(DMCContext *context,
     DMC_PUSH(*text, matchMkTuple);
     DMC_PUSH(*text, nelems);
     context->stack_used -= (nelems - 1);
-    context->eheap_need += (nelems + 1);
     *constant = 0;
     return retOk;
 }
 
 static DMCRet dmc_whole_expression(DMCContext *context,
 				   DMCHeap *heap,
-				   DMC_STACK_TYPE(Uint) *text,
+				   DMC_STACK_TYPE(UWord) *text,
 				   Eterm t,
 				   int *constant)
 {
@@ -2914,9 +3579,6 @@ static DMCRet dmc_whole_expression(DMCContext *context,
 	} else { 
 	    ASSERT(is_tuple(context->matchexpr
 			    [context->current_match]));
-	    context->eheap_need += 
-		arityval(*(tuple_val(context->matchexpr
-				     [context->current_match]))) * 2;
 	    DMC_PUSH(*text, matchPushArrayAsList);
 	}
     } else {
@@ -2929,20 +3591,55 @@ static DMCRet dmc_whole_expression(DMCContext *context,
     return retOk;
 }
 
+/* Figure out which PushV instruction to use.
+*/
+static void dmc_add_pushv_variant(DMCContext *context, DMCHeap *heap,
+				  DMC_STACK_TYPE(UWord) *text, Uint n)
+{
+    DMCVariable* v = &heap->vars[n];
+    MatchOps instr = matchPushV;
+
+    ASSERT(n < heap->vars_used && v->is_bound);
+    if (context->is_guard) {
+	#if HALFWORD_HEAP
+	if (!v->first_guard_label) {
+	    v->first_guard_label = DMC_STACK_NUM(*text);
+	    ASSERT(v->first_guard_label);
+	    instr = matchPushVGuard; /* may be changed to PushVResult below */
+	}
+	#endif
+    }
+    else { /* body */
+	#if HALFWORD_HEAP
+	if (v->first_guard_label) {
+	    /* Avoid double-copy, copy to result heap at first encounter in guard */
+	    DMC_POKE(*text, v->first_guard_label, matchPushVResult);
+	    v->is_in_body = 1;
+	}
+	#endif
+	if (!v->is_in_body) {
+	    instr = matchPushVResult;
+	    v->is_in_body = 1;
+	}
+    }
+    DMC_PUSH(*text, instr);
+    DMC_PUSH(*text, n);
+}
+
 static DMCRet dmc_variable(DMCContext *context,
 			   DMCHeap *heap,
-			   DMC_STACK_TYPE(Uint) *text,
+			   DMC_STACK_TYPE(UWord) *text,
 			   Eterm t,
 			   int *constant)
 {
     Uint n = db_is_variable(t);
-    ASSERT(n >= 0);
-    if (n >= heap->used) 
-	RETURN_VAR_ERROR("Variable $%d is unbound.", n, context, *constant);
-    if (heap->data[n] == 0U)
+
+    if (n >= heap->vars_used || !heap->vars[n].is_bound) {
 	RETURN_VAR_ERROR("Variable $%d is unbound.", n, context, *constant);
-    DMC_PUSH(*text, matchPushV);
-    DMC_PUSH(*text, n);
+    }
+
+    dmc_add_pushv_variant(context, heap, text, n);
+
     ++context->stack_used;
     if (context->stack_used > context->stack_need)
 	context->stack_need = context->stack_used;
@@ -2952,7 +3649,7 @@ static DMCRet dmc_variable(DMCContext *context,
 
 static DMCRet dmc_all_bindings(DMCContext *context,
 			       DMCHeap *heap,
-			       DMC_STACK_TYPE(Uint) *text,
+			       DMC_STACK_TYPE(UWord) *text,
 			       Eterm t,
 			       int *constant)
 {
@@ -2961,10 +3658,9 @@ static DMCRet dmc_all_bindings(DMCContext *context,
 
     DMC_PUSH(*text, matchPushC);
     DMC_PUSH(*text, NIL);
-    for (i = heap->used - 1; i >= 0; --i) { 
-	if (heap->data[i]) {
-	    DMC_PUSH(*text, matchPushV);
-	    DMC_PUSH(*text, i);
+    for (i = heap->vars_used - 1; i >= 0; --i) {
+	if (heap->vars[i].is_bound) {
+	    dmc_add_pushv_variant(context, heap, text, i);
 	    DMC_PUSH(*text, matchConsB);
 	    heap_used += 2;
 	}
@@ -2972,14 +3668,13 @@ static DMCRet dmc_all_bindings(DMCContext *context,
     ++context->stack_used;
     if ((context->stack_used + 1) > context->stack_need)
 	context->stack_need = (context->stack_used + 1);
-    context->eheap_need += heap_used;
     *constant = 0;
     return retOk;
 }
 
 static DMCRet dmc_const(DMCContext *context,
 		       DMCHeap *heap,
-		       DMC_STACK_TYPE(Uint) *text,
+		       DMC_STACK_TYPE(UWord) *text,
 		       Eterm t,
 		       int *constant)
 {
@@ -2996,7 +3691,7 @@ static DMCRet dmc_const(DMCContext *context,
 
 static DMCRet dmc_and(DMCContext *context,
 		      DMCHeap *heap,
-		      DMC_STACK_TYPE(Uint) *text,
+		      DMC_STACK_TYPE(UWord) *text,
 		      Eterm t,
 		      int *constant)
 {
@@ -3025,7 +3720,7 @@ static DMCRet dmc_and(DMCContext *context,
 
 static DMCRet dmc_or(DMCContext *context,
 		     DMCHeap *heap,
-		     DMC_STACK_TYPE(Uint) *text,
+		     DMC_STACK_TYPE(UWord) *text,
 		     Eterm t,
 		     int *constant)
 {
@@ -3055,7 +3750,7 @@ static DMCRet dmc_or(DMCContext *context,
 
 static DMCRet dmc_andalso(DMCContext *context,
 			  DMCHeap *heap,
-			  DMC_STACK_TYPE(Uint) *text,
+			  DMC_STACK_TYPE(UWord) *text,
 			  Eterm t,
 			  int *constant)
 {
@@ -3104,7 +3799,7 @@ static DMCRet dmc_andalso(DMCContext *context,
 
 static DMCRet dmc_orelse(DMCContext *context,
 			 DMCHeap *heap,
-			 DMC_STACK_TYPE(Uint) *text,
+			 DMC_STACK_TYPE(UWord) *text,
 			 Eterm t,
 			 int *constant)
 {
@@ -3152,7 +3847,7 @@ static DMCRet dmc_orelse(DMCContext *context,
 
 static DMCRet dmc_message(DMCContext *context,
 			  DMCHeap *heap,
-			  DMC_STACK_TYPE(Uint) *text,
+			  DMC_STACK_TYPE(UWord) *text,
 			  Eterm t,
 			  int *constant)
 {
@@ -3194,7 +3889,7 @@ static DMCRet dmc_message(DMCContext *context,
 
 static DMCRet dmc_self(DMCContext *context,
 		     DMCHeap *heap,
-		     DMC_STACK_TYPE(Uint) *text,
+		     DMC_STACK_TYPE(UWord) *text,
 		     Eterm t,
 		     int *constant)
 {
@@ -3214,7 +3909,7 @@ static DMCRet dmc_self(DMCContext *context,
 
 static DMCRet dmc_return_trace(DMCContext *context,
 			       DMCHeap *heap,
-			       DMC_STACK_TYPE(Uint) *text,
+			       DMC_STACK_TYPE(UWord) *text,
 			       Eterm t,
 			       int *constant)
 {
@@ -3244,7 +3939,7 @@ static DMCRet dmc_return_trace(DMCContext *context,
 
 static DMCRet dmc_exception_trace(DMCContext *context,
 			       DMCHeap *heap,
-			       DMC_STACK_TYPE(Uint) *text,
+			       DMC_STACK_TYPE(UWord) *text,
 			       Eterm t,
 			       int *constant)
 {
@@ -3276,7 +3971,7 @@ static DMCRet dmc_exception_trace(DMCContext *context,
 
 static DMCRet dmc_is_seq_trace(DMCContext *context,
 			       DMCHeap *heap,
-			       DMC_STACK_TYPE(Uint) *text,
+			       DMC_STACK_TYPE(UWord) *text,
 			       Eterm t,
 			       int *constant)
 {
@@ -3302,7 +3997,7 @@ static DMCRet dmc_is_seq_trace(DMCContext *context,
 
 static DMCRet dmc_set_seq_token(DMCContext *context,
 				DMCHeap *heap,
-				DMC_STACK_TYPE(Uint) *text,
+				DMC_STACK_TYPE(UWord) *text,
 				Eterm t,
 				int *constant)
 {
@@ -3351,7 +4046,7 @@ static DMCRet dmc_set_seq_token(DMCContext *context,
 
 static DMCRet dmc_get_seq_token(DMCContext *context,
 				DMCHeap *heap,
-				DMC_STACK_TYPE(Uint) *text,
+				DMC_STACK_TYPE(UWord) *text,
 				Eterm t,
 				int *constant)
 {
@@ -3375,10 +4070,6 @@ static DMCRet dmc_get_seq_token(DMCContext *context,
 
     *constant = 0;
     DMC_PUSH(*text, matchGetSeqToken);
-    context->eheap_need += (6 /* A 5-tuple is built */
-			    + EXTERNAL_THING_HEAD_SIZE + 2 /* Sender can
-							      be an external
-							      pid */);
     if (++context->stack_used > context->stack_need)
  	context->stack_need = context->stack_used;
     return retOk;
@@ -3388,7 +4079,7 @@ static DMCRet dmc_get_seq_token(DMCContext *context,
 
 static DMCRet dmc_display(DMCContext *context,
 			  DMCHeap *heap,
-			  DMC_STACK_TYPE(Uint) *text,
+			  DMC_STACK_TYPE(UWord) *text,
 			  Eterm t,
 			  int *constant)
 {
@@ -3428,7 +4119,7 @@ static DMCRet dmc_display(DMCContext *context,
 
 static DMCRet dmc_process_dump(DMCContext *context,
 			       DMCHeap *heap,
-			       DMC_STACK_TYPE(Uint) *text,
+			       DMC_STACK_TYPE(UWord) *text,
 			       Eterm t,
 			       int *constant)
 {
@@ -3458,7 +4149,7 @@ static DMCRet dmc_process_dump(DMCContext *context,
 
 static DMCRet dmc_enable_trace(DMCContext *context,
 			       DMCHeap *heap,
-			       DMC_STACK_TYPE(Uint) *text,
+			       DMC_STACK_TYPE(UWord) *text,
 			       Eterm t,
 			       int *constant)
 {
@@ -3518,7 +4209,7 @@ static DMCRet dmc_enable_trace(DMCContext *context,
 
 static DMCRet dmc_disable_trace(DMCContext *context,
 				DMCHeap *heap,
-				DMC_STACK_TYPE(Uint) *text,
+				DMC_STACK_TYPE(UWord) *text,
 				Eterm t,
 				int *constant)
 {
@@ -3578,7 +4269,7 @@ static DMCRet dmc_disable_trace(DMCContext *context,
 
 static DMCRet dmc_trace(DMCContext *context,
 			DMCHeap *heap,
-			DMC_STACK_TYPE(Uint) *text,
+			DMC_STACK_TYPE(UWord) *text,
 			Eterm t,
 			int *constant)
 {
@@ -3652,7 +4343,7 @@ static DMCRet dmc_trace(DMCContext *context,
 
 static DMCRet dmc_caller(DMCContext *context,
  			 DMCHeap *heap,
- 			 DMC_STACK_TYPE(Uint) *text,
+			 DMC_STACK_TYPE(UWord) *text,
  			 Eterm t,
  			 int *constant)
 {
@@ -3675,7 +4366,6 @@ static DMCRet dmc_caller(DMCContext *context,
     }
     *constant = 0;
     DMC_PUSH(*text, matchCaller); /* Creates binary */
-    context->eheap_need += 4;     /* A 3-tuple is built */
     if (++context->stack_used > context->stack_need)
  	context->stack_need = context->stack_used;
     return retOk;
@@ -3685,7 +4375,7 @@ static DMCRet dmc_caller(DMCContext *context,
   
 static DMCRet dmc_silent(DMCContext *context,
  			 DMCHeap *heap,
- 			 DMC_STACK_TYPE(Uint) *text,
+			 DMC_STACK_TYPE(UWord) *text,
  			 Eterm t,
  			 int *constant)
 {
@@ -3727,7 +4417,7 @@ static DMCRet dmc_silent(DMCContext *context,
 
 static DMCRet dmc_fun(DMCContext *context,
 		       DMCHeap *heap,
-		       DMC_STACK_TYPE(Uint) *text,
+		       DMC_STACK_TYPE(UWord) *text,
 		       Eterm t,
 		       int *constant)
 {
@@ -3844,7 +4534,7 @@ static DMCRet dmc_fun(DMCContext *context,
 	erl_exit(1,"ets:match() internal error, "
 		 "guard with more than 3 arguments.");
     }
-    DMC_PUSH(*text, (Uint) b->biff);
+    DMC_PUSH(*text, (UWord) b->biff);
     context->stack_used -= (((int) a) - 2);
     if (context->stack_used > context->stack_need)
  	context->stack_need = context->stack_used;
@@ -3853,7 +4543,7 @@ static DMCRet dmc_fun(DMCContext *context,
 
 static DMCRet dmc_expr(DMCContext *context,
 		       DMCHeap *heap,
-		       DMC_STACK_TYPE(Uint) *text,
+		       DMC_STACK_TYPE(UWord) *text,
 		       Eterm t,
 		       int *constant)
 {
@@ -3916,7 +4606,7 @@ static DMCRet dmc_expr(DMCContext *context,
     
 static DMCRet compile_guard_expr(DMCContext *context,
 				 DMCHeap *heap,
-				 DMC_STACK_TYPE(Uint) *text,
+				 DMC_STACK_TYPE(UWord) *text,
 				 Eterm l)
 {
     DMCRet ret;
@@ -4031,7 +4721,7 @@ static int match_compact(ErlHeapFragment *expr, DMCErrInfo *err_info)
     DMC_INIT_STACK(heap);
 
     p = expr->mem;
-    i = expr->size;
+    i = expr->used_size;
     while (i--) {
 	if (is_thing(*p)) {
 	    a = thing_arityval(*p);
@@ -4060,7 +4750,7 @@ static int match_compact(ErlHeapFragment *expr, DMCErrInfo *err_info)
     }
 
     p = expr->mem;
-    i = expr->size;
+    i = expr->used_size;
     while (i--) {
 	if (is_thing(*p)) {
 	    a = thing_arityval(*p);
@@ -4230,7 +4920,7 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace)
     Eterm l;
     Uint32 ret_flags;
     Uint sz;
-    Eterm *save_cp;
+    BeamInstr *save_cp;
 
     if (trace && !(is_list(against) || against == NIL)) {
 	return THE_NON_VALUE;
@@ -4271,25 +4961,26 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace)
 		++n;
 		l = CDR(list_val(l));
 	    }
+	    save_cp = p->cp;
+	    p->cp = NULL;
+	    res = erts_match_set_run(p, mps, arr, n,
+				     ERTS_PAM_COPY_RESULT, &ret_flags);
+	    p->cp = save_cp;
 	} else {
 	    n = 0;
-	    arr = (Eterm *) against;
+	    arr = NULL;
+	    res = erts_match_set_run_ets(p, mps, against, n, &ret_flags);
 	}
 	
 	/* We are in the context of a BIF, 
 	   {caller} should return 'undefined' */
-	save_cp = p->cp;
-	p->cp = NULL;
-	res = erts_match_set_run(p, mps, arr, n, &ret_flags);
-	p->cp = save_cp;
 	if (is_non_value(res)) {
 	    res = am_false;
 	}
-	sz = size_object(res);
+	sz = 0;
 	if (ret_flags & MATCH_SET_EXCEPTION_TRACE) sz += 2;
 	if (ret_flags & MATCH_SET_RETURN_TRACE) sz += 2;
 	hp = HAlloc(p, 5 + sz);
-	res = copy_struct(res, sz, &hp, &MSO(p));
 	flg = NIL;
 	if (ret_flags & MATCH_SET_EXCEPTION_TRACE) {
 	    flg = CONS(hp, am_exception_trace, flg);
@@ -4310,21 +5001,76 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace)
 
 static Eterm seq_trace_fake(Process *p, Eterm arg1)
 {
-    Eterm result = seq_trace_info_1(p,arg1);
+    Eterm result = erl_seq_trace_info(p, arg1);
     if (is_tuple(result) && *tuple_val(result) == 2) {
 	return (tuple_val(result))[2];
     }
     return result;
 }
-    
+
+DbTerm* db_alloc_tmp_uncompressed(DbTableCommon* tb, DbTerm* org)
+{
+    ErlOffHeap tmp_offheap;
+    DbTerm* res = erts_alloc(ERTS_ALC_T_TMP,
+			     sizeof(DbTerm) + org->size*sizeof(Eterm));
+    Eterm* hp = res->tpl;
+    tmp_offheap.first = NULL;
+    db_copy_from_comp(tb, org, &hp, &tmp_offheap);
+    res->first_oh = tmp_offheap.first;
+    res->size = org->size;
+#ifdef DEBUG_CLONE
+    res->debug_clone = NULL;
+#endif
+    return res;
+}
+
+void db_free_tmp_uncompressed(DbTerm* obj)
+{
+    ErlOffHeap off_heap;
+    off_heap.first = obj->first_oh;
+    erts_cleanup_offheap(&off_heap);
+#ifdef DEBUG_CLONE
+    ASSERT(obj->debug_clone == NULL);
+#endif
+    erts_free(ERTS_ALC_T_TMP, obj);
+}
+
+Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog,
+			     int all, DbTerm* obj, Eterm** hpp, Uint extra)
+{
+    Uint32 dummy;
+    Eterm* base;
+    Eterm res;
+
+    if (tb->compress) {
+	obj = db_alloc_tmp_uncompressed(tb, obj);
+	base = NULL;
+    }
+    else base = HALFWORD_HEAP ? obj->tpl : NULL;
+
+    res = db_prog_match(c_p, bprog, make_tuple_rel(obj->tpl,base), base, NULL, 0,
+			ERTS_PAM_COPY_RESULT|ERTS_PAM_CONTIGUOUS_TUPLE, &dummy);
+
+    if (is_value(res) && hpp!=NULL) {
+	*hpp = HAlloc(c_p, extra);
+    }
+
+    if (tb->compress) {
+	db_free_tmp_uncompressed(obj);
+    }
+    return res;
+}
+
+
 #ifdef DMC_DEBUG
+
 /*
 ** Disassemble match program
 */
-static void db_match_dis(Binary *bp)
+void db_match_dis(Binary *bp)
 {
     MatchProg *prog = Binary2MatchProg(bp);
-    Uint *t = prog->text;
+    UWord *t = prog->text;
     Uint n;
     Eterm p;
     int first;
@@ -4336,31 +5082,31 @@ static void db_match_dis(Binary *bp)
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("TryMeElse\t%bpu\n", n);
+	    erts_printf("TryMeElse\t%beu\n", n);
 	    break;
 	case matchArray:
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("Array\t%bpu\n", n);
+	    erts_printf("Array\t%beu\n", n);
 	    break;
 	case matchArrayBind:
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("ArrayBind\t%bpu\n", n);
+	    erts_printf("ArrayBind\t%beu\n", n);
 	    break;
 	case matchTuple:
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("Tuple\t%bpu\n", n);
+	    erts_printf("Tuple\t%beu\n", n);
 	    break;
 	case matchPushT:
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("PushT\t%bpu\n", n);
+	    erts_printf("PushT\t%beu\n", n);
 	    break;
 	case matchPushL:
 	    ++t;
@@ -4374,13 +5120,13 @@ static void db_match_dis(Binary *bp)
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("Bind\t%bpu\n", n);
+	    erts_printf("Bind\t%beu\n", n);
 	    break;
 	case matchCmp:
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("Cmp\t%bpu\n", n);
+	    erts_printf("Cmp\t%beu\n", n);
 	    break;
 	case matchEqBin:
 	    ++t;
@@ -4390,41 +5136,48 @@ static void db_match_dis(Binary *bp)
 	    break;
 	case matchEqRef:
 	    ++t;
-	    n = thing_arityval(*t);
-	    ++t;
-	    erts_printf("EqRef\t(%d) {", (int) n);
-	    first = 1;
-	    while (n--) {
-		if (first)
-		    first = 0;
-		else
-		    erts_printf(", ");
-#ifdef ARCH_64
-		erts_printf("0x%016bpx", *t);
+	    {
+		RefThing *rt = (RefThing *) t;
+		int ri;
+		n = thing_arityval(rt->header);
+		erts_printf("EqRef\t(%d) {", (int) n);
+		first = 1;
+		for (ri = 0; ri < n; ++ri) {
+		    if (first)
+			first = 0;
+		    else
+			erts_printf(", ");
+#if defined(ARCH_64) && !HALFWORD_HEAP
+		    erts_printf("0x%016bex", rt->data.ui[ri]);
 #else
-		erts_printf("0x%08bpx", *t);
+		    erts_printf("0x%08bex", rt->data.ui[ri]);
 #endif
-		++t;
+		}
 	    }
+	    t += TermWords(REF_THING_SIZE);
 	    erts_printf("}\n");
 	    break;
 	case matchEqBig:
 	    ++t;
 	    n = thing_arityval(*t);
-	    ++t;
-	    erts_printf("EqBig\t(%d) {", (int) n);
-	    first = 1;
-	    while (n--) {
-		if (first)
-		    first = 0;
-		else
-		    erts_printf(", ");
-#ifdef ARCH_64
-		erts_printf("0x%016bpx", *t);
+	    {
+		Eterm *et = (Eterm *) t;
+		t += TermWords(n+1);
+		erts_printf("EqBig\t(%d) {", (int) n);
+		first = 1;
+		++n;
+		while (n--) {
+		    if (first)
+			first = 0;
+		    else
+			erts_printf(", ");
+#if defined(ARCH_64) && !HALFWORD_HEAP
+		    erts_printf("0x%016bex", *et);
 #else
-		erts_printf("0x%08bpx", *t);
+		    erts_printf("0x%08bex", *et);
 #endif
-		++t;
+		++et;
+		}
 	    }
 	    erts_printf("}\n");
 	    break;
@@ -4432,8 +5185,8 @@ static void db_match_dis(Binary *bp)
 	    ++t;
 	    {
 		double num;
-		memcpy(&num,t, 2 * sizeof(*t));
-		t += 2;
+		memcpy(&num,t,sizeof(double));
+		t += TermWords(2);
 		erts_printf("EqFloat\t%f\n", num);
 	    }
 	    break;
@@ -4473,31 +5226,31 @@ static void db_match_dis(Binary *bp)
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("MkTuple\t%bpu\n", n);
+	    erts_printf("MkTuple\t%beu\n", n);
 	    break;
 	case matchOr:
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("Or\t%bpu\n", n);
+	    erts_printf("Or\t%beu\n", n);
 	    break;
 	case matchAnd:
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("And\t%bpu\n", n);
+	    erts_printf("And\t%beu\n", n);
 	    break;
 	case matchOrElse:
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("OrElse\t%bpu\n", n);
+	    erts_printf("OrElse\t%beu\n", n);
 	    break;
 	case matchAndAlso:
 	    ++t;
 	    n = *t;
 	    ++t;
-	    erts_printf("AndAlso\t%bpu\n", n);
+	    erts_printf("AndAlso\t%beu\n", n);
 	    break;
 	case matchCall0:
 	    ++t;
@@ -4527,7 +5280,19 @@ static void db_match_dis(Binary *bp)
 	    ++t;
 	    n = (Uint) *t;
 	    ++t;
-	    erts_printf("PushV\t%bpu\n", n);
+	    erts_printf("PushV\t%beu\n", n);
+	    break;
+    #if HALFWORD_HEAP
+	case matchPushVGuard:
+	    n = (Uint) *++t;
+	    ++t;
+	    erts_printf("PushVGuard\t%beu\n", n);
+	    break;
+    #endif
+	case matchPushVResult:
+	    n = (Uint) *++t;
+	    ++t;
+	    erts_printf("PushVResult\t%beu\n", n);
 	    break;
 	case matchTrue:
 	    ++t;
@@ -4638,9 +5403,8 @@ static void db_match_dis(Binary *bp)
     }
     erts_printf("}\n");
     erts_printf("num_bindings: %d\n", prog->num_bindings);
-    erts_printf("heap_size: %bpu\n", prog->heap_size);
-    erts_printf("eheap_offset: %bpu\n", prog->eheap_offset);
-    erts_printf("stack_offset: %bpu\n", prog->stack_offset);
+    erts_printf("heap_size: %beu\n", prog->heap_size);
+    erts_printf("stack_offset: %beu\n", prog->stack_offset);
     erts_printf("text: 0x%08x\n", (unsigned long) prog->text);
     erts_printf("stack_size: %d (words)\n", prog->heap_size-prog->stack_offset);
     
diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h
index 4fc7b4f52e..6a96e174e1 100644
--- a/erts/emulator/beam/erl_db_util.h
+++ b/erts/emulator/beam/erl_db_util.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -52,22 +52,27 @@
 				     is broken.*/
 #define DB_ERROR_UNSPEC   -10    /* Unspecified error */
 
+/*#define DEBUG_CLONE*/
 
 /*
  * A datatype for a database entry stored out of a process heap
  */
 typedef struct db_term {
-    ErlOffHeap off_heap;   /* Off heap data for term. */
-    Uint size;		   /* Size of term in "words" */
-    Eterm tpl[1];          /* Untagged "constant pointer" to top tuple */
-                           /* (assumed to be first in buffer) */
+    struct erl_off_heap_header* first_oh; /* Off heap data for term. */
+    Uint size;		   /* Heap size of term in "words" */
+#ifdef DEBUG_CLONE
+    Eterm* debug_clone;    /* An uncompressed copy */
+#endif
+    Eterm tpl[1];          /* Term data. Top tuple always first */
+
+    /* Compression: is_immed and key element are uncompressed.
+       Compressed elements are stored in external format after each other
+       last in dbterm. The top tuple elements contains byte offsets, to
+       the start of the data, tagged as headers.
+       The allocated size of the dbterm in bytes is stored at tpl[arity+1].
+     */
 } DbTerm;
 
-/* "Assign" a value to DbTerm.tpl */
-#define DBTERM_SET_TPL(dbtermPtr,tplPtr) ASSERT((tplPtr)==(dbtermPtr->tpl))
-/* Get start of term buffer */
-#define DBTERM_BUF(dbtermPtr) ((dbtermPtr)->tpl)
-
 union db_table;
 typedef union db_table DbTable;
 
@@ -81,6 +86,9 @@ typedef struct {
     Uint new_size;
     int mustResize;
     void* lck;
+#if HALFWORD_HEAP
+    unsigned char* abs_vec;  /* [i] true if dbterm->tpl[i] is absolute Eterm */
+#endif
 } DbUpdateHandle;
 
 
@@ -186,6 +194,12 @@ typedef struct db_table_method
 
 } DbTableMethod;
 
+typedef struct db_fixation {
+    Eterm pid;
+    Uint counter;
+    struct db_fixation *next;
+} DbFixation;
+
 /*
  * This structure contains data for all different types of database
  * tables. Note that these fields must match the same fields
@@ -194,16 +208,8 @@ typedef struct db_table_method
  * operations may be the same on different types of tables.
  */
 
-typedef struct db_fixation {
-    Eterm pid;
-    Uint counter;
-    struct db_fixation *next;
-} DbFixation;
-
-
 typedef struct db_table_common {
-    erts_refc_t ref;
-    erts_refc_t fixref;       /* fixation counter */
+    erts_refc_t ref;          /* fixation counter and delete counter */
 #ifdef ERTS_SMP
     erts_smp_rwmtx_t rwlock;  /* rw lock on table */
     erts_smp_mtx_t fixlock;   /* Protects fixations,megasec,sec,microsec */
@@ -212,7 +218,7 @@ typedef struct db_table_common {
 #endif
     Eterm owner;              /* Pid of the creator */
     Eterm heir;               /* Pid of the heir */
-    Eterm heir_data;          /* To send in ETS-TRANSFER (is_immed or (DbTerm*) */
+    UWord heir_data;          /* To send in ETS-TRANSFER (is_immed or (DbTerm*) */
     SysTimeval heir_started;  /* To further identify the heir */
     Eterm the_name;           /* an atom */
     Eterm id;                 /* atom | integer */
@@ -226,6 +232,7 @@ typedef struct db_table_common {
     Uint32 status;            /* bit masks defined  below */
     int slot;                 /* slot index in meta_main_tab */
     int keypos;               /* defaults to 1 */
+    int compress;
 } DbTableCommon;
 
 /* These are status bit patterns */
@@ -240,30 +247,78 @@ typedef struct db_table_common {
 #define DB_DUPLICATE_BAG (1 << 8)
 #define DB_ORDERED_SET   (1 << 9)
 #define DB_DELETE        (1 << 10) /* table is being deleted */
+#define DB_FREQ_READ     (1 << 11)
 
-#define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET|DB_FINE_LOCKED)
+#define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET|DB_FINE_LOCKED|DB_FREQ_READ)
 
 #define IS_HASH_TABLE(Status) (!!((Status) & \
 				  (DB_BAG | DB_SET | DB_DUPLICATE_BAG)))
 #define IS_TREE_TABLE(Status) (!!((Status) & \
 				  DB_ORDERED_SET))
-#define NFIXED(T) (erts_refc_read(&(T)->common.fixref,0))
+#define NFIXED(T) (erts_refc_read(&(T)->common.ref,0))
 #define IS_FIXED(T) (NFIXED(T) != 0) 
 
-Eterm erts_ets_copy_object(Eterm, Process*);
+/*
+ * tplp is an untagged pointer to a tuple we know is large enough
+ * and dth is a pointer to a DbTableHash.
+ */
+#define GETKEY(dth, tplp)   (*((tplp) + ((DbTableCommon*)(dth))->keypos))
+
+
+ERTS_GLB_INLINE Eterm db_copy_key(Process* p, DbTable* tb, DbTerm* obj);
+Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp,
+			ErlOffHeap* off_heap);
+int db_eq_comp(DbTableCommon* tb, Eterm a, DbTerm* b);
+DbTerm* db_alloc_tmp_uncompressed(DbTableCommon* tb, DbTerm* org);
+
+ERTS_GLB_INLINE Eterm db_copy_object_from_ets(DbTableCommon* tb, DbTerm* bp,
+					      Eterm** hpp, ErlOffHeap* off_heap);
+ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b);
+Wterm db_do_read_element(DbUpdateHandle* handle, Sint position);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE Eterm db_copy_key(Process* p, DbTable* tb, DbTerm* obj)
+{
+    Eterm key = GETKEY(tb, obj->tpl);
+    if IS_CONST(key) return key;
+    else {
+	Uint size = size_object_rel(key, obj->tpl);
+	Eterm* hp = HAlloc(p, size);
+	Eterm res = copy_struct_rel(key, size, &hp, &MSO(p), obj->tpl, NULL);
+	ASSERT(eq_rel(res,NULL,key,obj->tpl));
+	return res;
+    }
+}
+
+ERTS_GLB_INLINE Eterm db_copy_object_from_ets(DbTableCommon* tb, DbTerm* bp,
+					      Eterm** hpp, ErlOffHeap* off_heap)
+{
+    if (tb->compress) {
+	return db_copy_from_comp(tb, bp, hpp, off_heap);
+    }
+    else {
+	return copy_shallow_rel(bp->tpl, bp->size, hpp, off_heap, bp->tpl);
+    }
+}
+
+ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b)
+{
+    if (!tb->compress) {
+	return eq_rel(a, NULL, make_tuple_rel(b->tpl,b->tpl), b->tpl);
+    }
+    else {
+	return db_eq_comp(tb, a, b);
+    }
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
-/* optimised version of copy_object (normal case? atomic object) */
-#define COPY_OBJECT(obj, p, objp) \
-   if (IS_CONST(obj)) { *(objp) = (obj); } \
-   else { *objp = erts_ets_copy_object(obj, p); }
 
 #define DB_READ  (DB_PROTECTED|DB_PUBLIC)
 #define DB_WRITE DB_PUBLIC
 #define DB_INFO  (DB_PROTECTED|DB_PUBLIC|DB_PRIVATE)
 
-/* tb is an DbTableCommon and obj is an Eterm (tagged) */
-#define TERM_GETKEY(tb, obj) db_getkey((tb)->common.keypos, (obj)) 
-
 #define ONLY_WRITER(P,T) (((T)->common.status & (DB_PRIVATE|DB_PROTECTED)) \
 			  && (T)->common.owner == (P)->id)
 
@@ -271,20 +326,26 @@ Eterm erts_ets_copy_object(Eterm, Process*);
 (T)->common.owner == (P)->id)
 
 /* Function prototypes */
-Eterm db_get_trace_control_word_0(Process *p);
-Eterm db_set_trace_control_word_1(Process *p, Eterm val);
+BIF_RETTYPE db_get_trace_control_word(Process* p);
+BIF_RETTYPE db_set_trace_control_word(Process* p, Eterm tcw);
+BIF_RETTYPE db_get_trace_control_word_0(BIF_ALIST_0);
+BIF_RETTYPE db_set_trace_control_word_1(BIF_ALIST_1);
 
 void db_initialize_util(void);
 Eterm db_getkey(int keypos, Eterm obj);
-void db_free_term_data(DbTerm* p);
-void* db_get_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj);
+void db_cleanup_offheap_comp(DbTerm* p);
+void db_free_term(DbTable *tb, void* basep, Uint offset);
+void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj);
+void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj);
+Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, DbTerm* obj,
+			       Uint pos, Eterm** hpp, Uint extra);
 int db_has_variable(Eterm obj);
 int db_is_variable(Eterm obj);
 void db_do_update_element(DbUpdateHandle* handle,
 			  Sint position,
 			  Eterm newval);
-void db_finalize_update_element(DbUpdateHandle* handle);
-Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr);
+void db_finalize_resize(DbUpdateHandle* handle, Uint offset);
+Eterm db_add_counter(Eterm** hpp, Wterm counter, Eterm incr);
 Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags);
 Binary *db_match_set_compile(Process *p, Eterm matchexpr, 
 			     Uint flags);
@@ -301,12 +362,11 @@ typedef struct match_prog {
     struct erl_heap_fragment *saved_program_buf;
     Eterm saved_program;
     Uint heap_size;          /* size of: heap + eheap + stack */
-    Uint eheap_offset;
     Uint stack_offset;
 #ifdef DMC_DEBUG
-    Uint* prog_end;		/* End of program */
+    UWord* prog_end;		/* End of program */
 #endif
-    Uint text[1];		/* Beginning of program */
+    UWord text[1];		/* Beginning of program */
 } MatchProg;
 
 /*
@@ -366,8 +426,15 @@ Binary *db_match_compile(Eterm *matchexpr, Eterm *guards,
 			 Uint flags, 
 			 DMCErrInfo *err_info);
 /* Returns newly allocated MatchProg binary with refc == 0*/
-Eterm db_prog_match(Process *p, Binary *prog, Eterm term, int arity, 
+
+Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog,
+		      int all, DbTerm* obj, Eterm** hpp, Uint extra);
+
+Eterm db_prog_match(Process *p, Binary *prog, Eterm term, Eterm* base,
+		    Eterm *termp, int arity,
+		    enum erts_pam_run_flags in_flags,
 		    Uint32 *return_flags /* Zeroed on enter */);
+
 /* returns DB_ERROR_NONE if matches, 1 if not matches and some db error on 
    error. */
 DMCErrInfo *db_new_dmc_err_info(void);
diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c
index e5c3c76fdd..2121f72fd2 100644
--- a/erts/emulator/beam/erl_debug.c
+++ b/erts/emulator/beam/erl_debug.c
@@ -25,7 +25,6 @@
 #include "erl_vm.h"
 #include "global.h"
 #include "erl_process.h"
-#include "erl_nmgc.h"
 #include "big.h"
 #include "bif.h"
 #include "beam_catches.h"
@@ -33,34 +32,9 @@
 
 #define WITHIN(ptr, x, y) ((x) <= (ptr) && (ptr) < (y))
 
-#if defined(HYBRID)
-#if defined(INCREMENTAL)
-/* Hybrid + Incremental */
-#define IN_HEAP(p, ptr)                                                 \
-    (WITHIN((ptr), p->heap, p->hend) ||                                 \
-     (OLD_HEAP(p) && WITHIN((ptr), OLD_HEAP(p), OLD_HEND(p))) ||        \
-     WITHIN((ptr), global_heap, global_hend) ||                         \
-     (inc_fromspc && WITHIN((ptr), inc_fromspc, inc_fromend)) ||        \
-     WITHIN((ptr), global_old_heap, global_old_hend))
-
-#define IN_MA(ptr)                                                      \
-    (WITHIN((ptr), global_heap, global_hend) ||                         \
-     (inc_fromspc && WITHIN((ptr), inc_fromspc, inc_fromend)) ||        \
-     WITHIN((ptr), global_old_heap, global_old_hend))
-#else
-/* Hybrid */
-#define IN_HEAP(p, ptr)                                                 \
-    (WITHIN((ptr), p->heap, p->hend) ||                                 \
-     (OLD_HEAP(p) && WITHIN((ptr), OLD_HEAP(p), OLD_HEND(p))) ||        \
-     WITHIN((ptr), global_heap, global_hend) ||                         \
-     (global_old_heap && WITHIN((ptr),global_old_heap,global_old_hend)))
-#endif
-#else
-/* Private */
 #define IN_HEAP(p, ptr)                                                 \
     (WITHIN((ptr), p->heap, p->hend) ||                                 \
      (OLD_HEAP(p) && WITHIN((ptr), OLD_HEAP(p), OLD_HEND(p))))
-#endif
 
 
 #ifdef __GNUC__
@@ -207,11 +181,7 @@ pdisplay1(int to, void *to_arg, Process* p, Eterm obj)
     case FLOAT_DEF: {
 	    FloatDef ff;
 	    GET_DOUBLE(obj, ff);
-#ifdef _OSE_
-	    erts_print(to, to_arg, "%e", ff.fd);
-#else
 	    erts_print(to, to_arg, "%.20e", ff.fd);
-#endif
 	}
 	break;
     case BINARY_DEF:
@@ -235,9 +205,9 @@ pps(Process* p, Eterm* stop)
     }
 
     while(sp >= stop) {
-	erts_print(to, to_arg, "%0*lx: ", PTR_SIZE, (Eterm) sp);
+	erts_print(to, to_arg, "%0*lx: ", PTR_SIZE, (UWord) sp);
 	if (is_catch(*sp)) {
-	    erts_print(to, to_arg, "catch %d", (Uint)catch_pc(*sp));
+	    erts_print(to, to_arg, "catch %ld", (UWord)catch_pc(*sp));
 	} else {
 	    paranoid_display(to, to_arg, p, *sp);
 	}
@@ -265,18 +235,11 @@ static int verify_eterm(Process *p,Eterm element)
             return 1;
 
         for (mbuf = p->mbuf; mbuf; mbuf = mbuf->next) {
-            if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->size)) {
+            if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->used_size)) {
                 return 1;
             }
         }
     }
-#ifdef INCREMENTAL
-    else {
-        if (IN_MA(ptr))
-            return 1;
-    }
-#endif
-
     return 0;
 }
 
@@ -312,7 +275,7 @@ void erts_check_stack(Process *p)
 	if (IN_HEAP(p, ptr))
 	    continue;
 	for (mbuf = p->mbuf; mbuf; mbuf = mbuf->next)
-	    if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->size)) {
+	    if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->used_size)) {
 		in_mbuf = 1;
 		break;
 	    }
@@ -451,51 +414,12 @@ void verify_process(Process *p)
     VERIFY_ETERM("fvalue",p->fvalue);
     VERIFY_ETERM("ftrace",p->ftrace);
 
-#ifdef HYBRID
-    VERIFY_AREA("rrma",p->rrma,p->nrr);
-#endif
-
     VERBOSE(DEBUG_MEMORY,("...done\n"));
 
 #undef VERIFY_AREA
 #undef VERIFY_ETERM
 }
 
-void verify_everything()
-{
-#ifdef HYBRID
-    Uint i;
-    Uint n = erts_num_active_procs;
-
-#ifdef INCREMENTAL_FREE_SIZES_NEEDS_TO_BE_TAGGED_AS_HEADERS_WITH_ARITY
-    INC_Page *page = inc_used_mem;
-#endif
-
-    for (i = 0; i < n; i++) {
-	verify_process(erts_active_procs[i]);
-    }
-
-    erts_check_memory(NULL,global_heap,global_htop);
-
-#ifdef INCREMENTAL_FREE_SIZES_NEEDS_TO_BE_TAGGED_AS_HEADERS_WITH_ARITY
-    while (page)
-    {
-	Eterm *end = page + INC_PAGE_SIZE;
-	Eterm *pos = page->start;
-
-	while( pos <  end) {
-	    Eterm val = *pos++;
-	    if(is_header(val))
-		pos += thing_arityval(val);
-	    else
-		verify_eterm(NULL,val);
-	}
-	page = page->next;
-    }
-#endif
-#endif /* HYBRID */
-}
-
 /*
  * print_untagged_memory will print the contents of given memory area.
  */
@@ -586,83 +510,6 @@ void print_tagged_memory(Eterm *pos, Eterm *end)
     erts_printf("+-%s-+-%s-+\n",dashes,dashes);
 }
 
-#ifdef HYBRID
-void print_ma_info(void)
-{
-    erts_printf("Message Area (start - top - end): "
-                "0x%0*lx - 0x%0*lx - 0x%0*lx\n",
-                PTR_SIZE, (unsigned long)global_heap,
-                PTR_SIZE, (unsigned long)global_htop,
-                PTR_SIZE, (unsigned long)global_hend);
-#ifndef INCREMENTAL
-    erts_printf("  High water: 0x%0*lx   "
-                "Old gen: 0x%0*lx - 0x%0*lx - 0x%0*lx\n",
-                PTR_SIZE, (unsigned long)global_high_water,
-                PTR_SIZE, (unsigned long)global_old_heap,
-                PTR_SIZE, (unsigned long)global_old_htop,
-                PTR_SIZE, (unsigned long)global_old_hend);
-#endif
-}
-
-void print_message_area(void)
-{
-    Eterm *pos = global_heap;
-    Eterm *end = global_htop;
-
-    erts_printf("From: 0x%0*lx  to  0x%0*lx\n",
-                PTR_SIZE,(unsigned long)pos,PTR_SIZE,(unsigned long)end);
-    erts_printf("(Old generation: 0x%0*lx  to 0x%0*lx\n",
-                PTR_SIZE, (unsigned long)global_old_heap,
-                PTR_SIZE, (unsigned long)global_old_hend);
-    erts_printf("| %-*s | %-*s |\n",PTR_SIZE,"Address",PTR_SIZE,"Contents");
-    erts_printf("|-%s-|-%s-|\n",dashes,dashes);
-    while( pos < end ) {
-	Eterm val = pos[0];
-	erts_printf("| 0x%0*lx | 0x%0*lx | ",
-                    PTR_SIZE,(unsigned long)pos,PTR_SIZE,(unsigned long)val);
-	++pos;
-	if( is_arity_value(val) ) {
-	    erts_printf("Arity(%lu)", arityval(val));
-	} else if( is_thing(val) ) {
-	    unsigned int ari = thing_arityval(val);
-	    erts_printf("Thing Arity(%u) Tag(%lu)", ari, thing_subtag(val));
-	    while( ari ) {
-		erts_printf("\n| 0x%0*lx | 0x%0*lx | THING",
-                            PTR_SIZE, (unsigned long)pos,
-                            PTR_SIZE, (unsigned long)*pos);
-		++pos;
-		--ari;
-	    }
-	} else
-	    erts_printf("%.30T", val);
-	erts_printf("\n");
-    }
-    erts_printf("+-%s-+-%s-+\n",dashes,dashes);
-}
-
-void check_message_area()
-{
-    Eterm *pos = global_heap;
-    Eterm *end = global_htop;
-
-    while( pos < end ) {
-	Eterm val = *pos++;
-	if(is_header(val))
-	    pos += thing_arityval(val);
-	else if(!is_immed(val))
-	    if ((ptr_val(val) < global_heap || ptr_val(val) >= global_htop) &&
-                (ptr_val(val) < global_old_heap ||
-                 ptr_val(val) >= global_old_hend))
-            {
-		erts_printf("check_message_area: Stray pointer found\n");
-                print_message_area();
-                erts_printf("Crashing to make it look real...\n");
-                pos = 0;
-            }
-    }
-}
-#endif /* HYBRID */
-
 static void print_process_memory(Process *p);
 static void print_process_memory(Process *p)
 {
@@ -707,19 +554,6 @@ static void print_process_memory(Process *p)
     erts_printf("  Fvalue: 0x%0*lx\n",PTR_SIZE,p->fvalue);
     erts_printf("  Ftrace: 0x%0*lx\n",PTR_SIZE,p->ftrace);
 
-#ifdef HYBRID
-    if (p->nrr > 0) {
-        int i;
-        erts_printf("  Remembered Roots:\n");
-        for (i = 0; i < p->nrr; i++)
-            if (p->rrsrc[i] != NULL)
-                erts_printf("0x%0*lx -> 0x%0*lx\n",
-                            PTR_SIZE, (unsigned long)p->rrsrc[i],
-                            PTR_SIZE, (unsigned long)p->rrma[i]);
-        erts_printf("\n");
-    }
-#endif
-
     erts_printf("+- %-*s -+ 0x%0*lx 0x%0*lx %s-%s-+\n",
                 PTR_SIZE, "Stack",
                 PTR_SIZE, (unsigned long)STACK_TOP(p),
@@ -750,7 +584,7 @@ static void print_process_memory(Process *p)
                     PTR_SIZE, "heap fragments",
                     dashes, dashes, dashes, dashes);
     while (bp) {
-	print_untagged_memory(bp->mem,bp->mem + bp->size);
+	print_untagged_memory(bp->mem,bp->mem + bp->used_size);
         bp = bp->next;
     }
 }
@@ -761,92 +595,6 @@ void print_memory(Process *p)
     if (p != NULL) {
         print_process_memory(p);
     }
-#ifdef HYBRID
-    else {
-        Uint i;
-        Uint n = erts_num_active_procs;
-
-        for (i = 0; i < n; i++) {
-            Process *p = erts_active_procs[i];
-            print_process_memory(p);
-        }
-
-        erts_printf("==================\n");
-        erts_printf("|| Message area ||\n");
-        erts_printf("==================\n");
-        erts_printf("+-%s-+-%s-%s-%s-%s-+\n",
-                    dashes,dashes,dashes,dashes,dashes);
-        erts_printf("| %-*s   | 0x%0*lx - 0x%0*lx - 0x%0*lx%*s|\n",
-                    PTR_SIZE, "Young",
-                    PTR_SIZE, (unsigned long)global_heap,
-                    PTR_SIZE, (unsigned long)global_htop,
-                    PTR_SIZE, (unsigned long)global_hend,
-                    PTR_SIZE, "");
-        erts_printf("+-%s-+-%s-%s-%s-%s-+\n",
-                    dashes,dashes,dashes,dashes,dashes);
-
-        print_untagged_memory(global_heap,global_htop);
-
-
-        erts_printf("+-%s-+-%s-%s-%s-%s-+\n",
-                    dashes,dashes,dashes,dashes,dashes);
-        erts_printf("| %-*s   | 0x%0*lx - 0x%0*lx %*s    |\n",
-                    PTR_SIZE, "Old",
-                    PTR_SIZE, (unsigned long)global_old_heap,
-                    PTR_SIZE, (unsigned long)global_old_hend,
-                    2 * PTR_SIZE, "");
-        erts_printf("+-%s-+-%s-%s-%s-%s-+\n",
-                    dashes,dashes,dashes,dashes,dashes);
-
-#ifdef INCREMENTAL
-        {
-            INC_Page *page = inc_used_mem;
-            /* Genom att g� igenom fri-listan f�rst kan vi markera de
-               omr�den som inte �r allokerade och bara skriva ut de som
-               lever.
-               char markarea[INC_PAGESIZE];
-            */
-
-            while (page) {
-                Eterm *ptr = (Eterm*)page->start;
-                Eterm *end = (Eterm*)page->start + INC_PAGESIZE;
-
-                erts_printf("| %*s   | This: 0x%0*lx   Next: 0x%0*lx %*s|\n",
-                            PTR_SIZE, "",
-                            PTR_SIZE, (unsigned long)page,
-                            PTR_SIZE, (unsigned long)page->next,
-                            2 * PTR_SIZE - 8, "");
-                print_untagged_memory(ptr,end);
-                page = page->next;
-            }
-        }
-
-        {
-            INC_MemBlock *this = inc_free_list;
-
-            erts_printf("-- %-*s --%s-%s-%s-%s-\n",PTR_SIZE+2,"Free list",
-                        dashes,dashes,dashes,dashes);
-            while (this) {
-                erts_printf("Block @ 0x%0*lx sz: %8d prev: 0x%0*lx next: 0x%0*lx\n",
-                            PTR_SIZE, (unsigned long)this,this->size,
-                            PTR_SIZE, (unsigned long)this->prev,
-                            PTR_SIZE, (unsigned long)this->next);
-                this = this->next;
-            }
-            erts_printf("--%s---%s-%s-%s-%s--\n",
-                        dashes,dashes,dashes,dashes,dashes);
-        }
-
-        if (inc_fromspc != NULL) {
-            erts_printf("-- fromspace - 0x%0*lx 0x%0*lx "
-                        "------------------------------\n",
-                        PTR_SIZE, (unsigned long)inc_fromspc,
-                        PTR_SIZE, (unsigned long)inc_fromend);
-            print_untagged_memory(inc_fromspc,inc_fromend);
-        }
-#endif /* INCREMENTAL */
-    }
-#endif /* HYBRID */
 }
 
 void print_memory_info(Process *p)
@@ -873,27 +621,31 @@ void print_memory_info(Process *p)
         erts_printf("|| Memory info ||\n");
         erts_printf("=================\n");
     }
-#ifdef HYBRID
-    erts_printf("|- message area --%s-%s-%s-%s-|\n",
-                dashes,dashes,dashes,dashes);
-    erts_printf("| Young | 0x%0*lx - 0x%0*lx - 0x%0*lx   %*s     |\n",
-                PTR_SIZE, (unsigned long)global_heap,
-                PTR_SIZE, (unsigned long)global_htop,
-                PTR_SIZE, (unsigned long)global_hend,
-                PTR_SIZE, "");
-    erts_printf("| Old   | 0x%0*lx - 0x%0*lx      %*s       |\n",
-                PTR_SIZE, (unsigned long)global_old_heap,
-                PTR_SIZE, (unsigned long)global_old_hend,
-                2 * PTR_SIZE, "");
-#endif
-#ifdef INCREMENTAL
-    if (inc_fromspc != NULL)
-        erts_printf("| Frmsp | 0x%0*lx - 0x%0*lx      %*s  |\n",
-                    PTR_SIZE, (unsigned long)inc_fromspc,
-                    PTR_SIZE, (unsigned long)inc_fromend,
-                    2 * PTR_SIZE, "");
-#endif
     erts_printf("+-----------------%s-%s-%s-%s-+\n",dashes,dashes,dashes,dashes);
 }
+#if !HEAP_ON_C_STACK && defined(DEBUG)
+Eterm *erts_debug_allocate_tmp_heap(int size, Process *p)
+{
+    ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p));
+    int offset = sd->num_tmp_heap_used;
+
+    ASSERT(offset+size <= TMP_HEAP_SIZE);
+    return (sd->tmp_heap)+offset;
+}
+void erts_debug_use_tmp_heap(int size, Process *p)
+{
+    ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p));
+
+    sd->num_tmp_heap_used += size;
+    ASSERT(sd->num_tmp_heap_used <= TMP_HEAP_SIZE);
+}
+void erts_debug_unuse_tmp_heap(int size, Process *p)
+{
+    ErtsSchedulerData *sd = ((p == NULL) ? erts_get_scheduler_data() : ERTS_PROC_GET_SCHDATA(p));
+
+    sd->num_tmp_heap_used -= size;
+    ASSERT(sd->num_tmp_heap_used >= 0);
+}
+#endif
 #endif
 
diff --git a/erts/emulator/beam/erl_debug.h b/erts/emulator/beam/erl_debug.h
index 74f4a00b63..a028a95fef 100644
--- a/erts/emulator/beam/erl_debug.h
+++ b/erts/emulator/beam/erl_debug.h
@@ -1,26 +1,27 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 #ifndef _ERL_DEBUG_H_
 #define _ERL_DEBUG_H_
-
 #ifdef DEBUG
 
+#include "erl_term.h"
+
 #ifdef HIPE
 #include "hipe_debug.h"
 #endif
@@ -41,16 +42,15 @@
 #define DEBUG_DEFAULT      0x0000    /* No flags are set per default         */
 #define DEBUG_SYSTEM       0x0001    /* Misc system info at startup and end  */
 #define DEBUG_PRIVATE_GC   0x0002    /* GC of private heaps                  */
-#define DEBUG_HYBRID_GC    0x0004    /* GC of the message area               */
-#define DEBUG_ALLOCATION   0x0008    /* HAlloc. To find holes in the heap    */
-#define DEBUG_MESSAGES     0x0010    /* Message passing                      */
-#define DEBUG_THREADS      0x0020    /* Thread-related stuff                 */
-#define DEBUG_PROCESSES    0x0040    /* Process creation and removal         */
-#define DEBUG_MEMORY       0x0080    /* Display results of memory checks     */
+#define DEBUG_ALLOCATION   0x0004    /* HAlloc. To find holes in the heap    */
+#define DEBUG_MESSAGES     0x0008    /* Message passing                      */
+#define DEBUG_THREADS      0x0010    /* Thread-related stuff                 */
+#define DEBUG_PROCESSES    0x0020    /* Process creation and removal         */
+#define DEBUG_MEMORY       0x0040    /* Display results of memory checks     */
 
 extern Uint32 verbose;
 
-void upp(byte*, int);
+void upp(byte*, size_t);
 void pat(Eterm);
 void pinfo(void);
 void pp(Process*);
@@ -87,16 +87,14 @@ extern void erts_check_stack(Process *p);
 extern void erts_check_heap(Process *p);
 extern void erts_check_memory(Process *p, Eterm *start, Eterm *end);
 extern void verify_process(Process *p);
-extern void verify_everything(void);
 extern void print_tagged_memory(Eterm *start, Eterm *end);
 extern void print_untagged_memory(Eterm *start, Eterm *end);
 extern void print_memory(Process *p);
 extern void print_memory_info(Process *p);
-
-#ifdef HYBRID
-extern void print_ma_info(void);
-extern void print_message_area(void);
-extern void check_message_area(void);
+#if defined(DEBUG) && !HEAP_ON_C_STACK
+extern Eterm *erts_debug_allocate_tmp_heap(int, Process *);
+extern void erts_debug_use_tmp_heap(int, Process *);
+extern void erts_debug_unuse_tmp_heap(int, Process *);
 #endif
 
 #endif /* _ERL_DEBUG_H_ */
diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h
index 489e74d960..1ae9a211d7 100644
--- a/erts/emulator/beam/erl_driver.h
+++ b/erts/emulator/beam/erl_driver.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -28,6 +28,14 @@
 #  include "config.h"
 #endif
 
+#define ERL_DRV_DEPRECATED_FUNC
+#ifdef __GNUC__
+#  if __GNUC__ >= 3
+#    undef ERL_DRV_DEPRECATED_FUNC
+#    define ERL_DRV_DEPRECATED_FUNC __attribute__((deprecated))
+#  endif
+#endif
+
 #ifdef SIZEOF_CHAR
 #  define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR
 #  undef SIZEOF_CHAR
@@ -48,6 +56,10 @@
 #  define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG
 #  undef SIZEOF_LONG_LONG
 #endif
+#ifdef HALFWORD_HEAP_EMULATOR
+#  define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR
+#  undef HALFWORD_HEAP_EMULATOR
+#endif
 #include "erl_int_sizes_config.h"
 #if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR
 #  error SIZEOF_CHAR mismatch
@@ -65,9 +77,15 @@
 #  error SIZEOF_LONG_LONG mismatch
 #endif
 
+/* This is OK to override by the NIF/driver implementor */
+#if defined(HALFWORD_HEAP_EMULATOR_SAVED__) && !defined(HALFWORD_HEAP_EMULATOR)
+#define HALFWORD_HEAP_EMULATOR HALFWORD_HEAP_EMULATOR_SAVED__
+#endif
+
 #include "erl_drv_nif.h"
 
 #include <stdlib.h>
+#include <string.h>		/* ssize_t on Mac OS X */
 
 #if defined(VXWORKS)
 #  include <ioLib.h>
@@ -117,8 +135,8 @@ typedef struct {
 #define DO_WRITE ERL_DRV_WRITE
 
 #define ERL_DRV_EXTENDED_MARKER		(0xfeeeeeed)
-#define ERL_DRV_EXTENDED_MAJOR_VERSION	1
-#define ERL_DRV_EXTENDED_MINOR_VERSION	5
+#define ERL_DRV_EXTENDED_MAJOR_VERSION	2
+#define ERL_DRV_EXTENDED_MINOR_VERSION	0
 
 /*
  * The emulator will refuse to load a driver with different major
@@ -141,12 +159,46 @@ typedef struct {
 #define ERL_DRV_FLAG_SOFT_BUSY		(1 << 1)
 
 /*
+ * Integer types
+ */
+#if  defined(__WIN32__) && (SIZEOF_VOID_P == 8)
+typedef unsigned __int64 ErlDrvTermData;
+typedef unsigned __int64 ErlDrvUInt;
+typedef signed __int64 ErlDrvSInt;
+#else
+typedef unsigned long ErlDrvTermData;
+typedef unsigned long ErlDrvUInt;
+typedef signed long ErlDrvSInt;
+#endif
+
+#if defined(__WIN32__)
+typedef unsigned __int64 ErlDrvUInt64;
+typedef __int64 ErlDrvSInt64;
+#elif SIZEOF_LONG == 8
+typedef unsigned long ErlDrvUInt64;
+typedef long ErlDrvSInt64;
+#elif SIZEOF_LONG_LONG == 8
+typedef unsigned long long ErlDrvUInt64;
+typedef long long ErlDrvSInt64;
+#else
+#error No 64-bit integer type
+#endif
+
+#if defined(__WIN32__)
+typedef ErlDrvUInt ErlDrvSizeT;
+typedef ErlDrvSInt ErlDrvSSizeT;
+#else
+typedef size_t ErlDrvSizeT;
+typedef ssize_t ErlDrvSSizeT;
+#endif
+
+/*
  * A binary as seen in a driver. Note that a binary should never be
  * altered by the driver when it has been sent to Erlang.
  */
 
 typedef struct erl_drv_binary {
-    long orig_size;        /* total length of binary */
+    ErlDrvSInt orig_size;        /* total length of binary */
     char orig_bytes[1];   /* the data (char instead of byte!) */
 } ErlDrvBinary;
 
@@ -170,26 +222,6 @@ struct erl_drv_event_data {
 #endif
 typedef struct erl_drv_event_data *ErlDrvEventData; /* Event data */
 
-/* 
- * Used in monitors...
- */
-typedef unsigned long ErlDrvTermData;
-typedef unsigned long ErlDrvUInt;
-typedef signed long ErlDrvSInt;
-
-#if defined(__WIN32__)
-typedef unsigned __int64 ErlDrvUInt64;
-typedef __int64 ErlDrvSInt64;
-#elif SIZEOF_LONG == 8
-typedef unsigned long ErlDrvUInt64;
-typedef long ErlDrvSInt64;
-#elif SIZEOF_LONG_LONG == 8
-typedef unsigned long long ErlDrvUInt64;
-typedef long long ErlDrvSInt64;
-#else
-#error No 64-bit integer type
-#endif
-
 /*
  * A driver monitor
  */
@@ -226,7 +258,7 @@ typedef struct {
 
 typedef struct erl_io_vec {
     int vsize;			/* length of vectors */
-    int size;			/* total size in bytes */
+    ErlDrvSizeT size;		/* total size in bytes */
     SysIOVec* iov;
     ErlDrvBinary** binv;
 } ErlIOVec;
@@ -267,12 +299,12 @@ typedef struct erl_drv_entry {
     void (*stop)(ErlDrvData drv_data);
                                 /* called when port is closed, and when the
 				   emulator is halted. */
-    void (*output)(ErlDrvData drv_data, char *buf, int len);
-				/* called when we have output from erlang to 
+    void (*output)(ErlDrvData drv_data, char *buf, ErlDrvSizeT len);
+				/* called when we have output from erlang to
 				   the port */
     void (*ready_input)(ErlDrvData drv_data, ErlDrvEvent event); 
 				/* called when we have input from one of 
-				   the driver's handles) */
+				   the driver's handles */
     void (*ready_output)(ErlDrvData drv_data, ErlDrvEvent event);  
 				/* called when output is possible to one of 
 				   the driver's handles */
@@ -281,10 +313,10 @@ typedef struct erl_drv_entry {
     void (*finish)(void);        /* called before unloading the driver -
 				   DYNAMIC DRIVERS ONLY */
     void *handle;		/* Reserved -- Used by emulator internally */
-    int (*control)(ErlDrvData drv_data, unsigned int command, char *buf, 
-		   int len, char **rbuf, int rlen); 
-				/* "ioctl" for drivers - invoked by 
-				   port_control/3) */
+    ErlDrvSSizeT (*control)(ErlDrvData drv_data, unsigned int command,
+			    char *buf, ErlDrvSizeT len, char **rbuf,
+			    ErlDrvSizeT rlen); /* "ioctl" for drivers - invoked by
+						  port_control/3 */
     void (*timeout)(ErlDrvData drv_data);	/* Handling of timeout in driver */
     void (*outputv)(ErlDrvData drv_data, ErlIOVec *ev);
 				/* called when we have output from erlang
@@ -295,10 +327,12 @@ typedef struct erl_drv_entry {
 				   closed, and there is data in the 
 				   driver queue that needs to be flushed
 				   before 'stop' can be called */
-    int (*call)(ErlDrvData drv_data, unsigned int command, char *buf, 
-		   int len, char **rbuf, int rlen, unsigned int *flags); 
-                                /* Works mostly like 'control', a syncronous
-				   call into the driver. */
+    ErlDrvSSizeT (*call)(ErlDrvData drv_data,
+			 unsigned int command, char *buf, ErlDrvSizeT len,
+			 char **rbuf, ErlDrvSizeT rlen,
+			 unsigned int *flags); /* Works mostly like 'control',
+						  a synchronous
+						  call into the driver. */
     void (*event)(ErlDrvData drv_data, ErlDrvEvent event,
 		  ErlDrvEventData event_data);
                                 /* Called when an event selected by 
@@ -337,11 +371,17 @@ typedef struct erl_drv_entry {
 #ifndef ERL_DRIVER_TYPES_ONLY
 
 #if defined(VXWORKS)
-#  define DRIVER_INIT(DRIVER_NAME) ErlDrvEntry* DRIVER_NAME  ## _init(void)
+#  define DRIVER_INIT(DRIVER_NAME) \
+    ErlDrvEntry* DRIVER_NAME  ## _init(void); \
+    ErlDrvEntry* DRIVER_NAME  ## _init(void)
 #elif defined(__WIN32__)
-#  define DRIVER_INIT(DRIVER_NAME) __declspec(dllexport) ErlDrvEntry* driver_init(void)
+#  define DRIVER_INIT(DRIVER_NAME) \
+    __declspec(dllexport) ErlDrvEntry* driver_init(void); \
+    __declspec(dllexport) ErlDrvEntry* driver_init(void)
 #else 
-#  define DRIVER_INIT(DRIVER_NAME) ErlDrvEntry* driver_init(void)
+#  define DRIVER_INIT(DRIVER_NAME) \
+    ErlDrvEntry* driver_init(void); \
+    ErlDrvEntry* driver_init(void)
 #endif
 
 /*
@@ -350,14 +390,16 @@ typedef struct erl_drv_entry {
 EXTERN int driver_select(ErlDrvPort port, ErlDrvEvent event, int mode, int on);
 EXTERN int driver_event(ErlDrvPort port, ErlDrvEvent event, 
 			ErlDrvEventData event_data);
-EXTERN int driver_output(ErlDrvPort port, char *buf, int len);
-EXTERN int driver_output2(ErlDrvPort port, char *hbuf, int hlen, 
-			  char *buf, int len);
-EXTERN int driver_output_binary(ErlDrvPort port, char *hbuf, int hlen,
-				ErlDrvBinary* bin, int offset, int len);
-EXTERN int driver_outputv(ErlDrvPort port, char* hbuf, int hlen, ErlIOVec *ev,
-			  int skip);
-EXTERN int driver_vec_to_buf(ErlIOVec *ev, char *buf, int len);
+
+EXTERN int driver_output(ErlDrvPort port, char *buf, ErlDrvSizeT len);
+EXTERN int driver_output2(ErlDrvPort port, char *hbuf, ErlDrvSizeT hlen,
+			  char *buf, ErlDrvSizeT len);
+EXTERN int driver_output_binary(ErlDrvPort port, char *hbuf, ErlDrvSizeT hlen,
+				ErlDrvBinary* bin,
+				ErlDrvSizeT offset, ErlDrvSizeT len);
+EXTERN int driver_outputv(ErlDrvPort port, char* hbuf, ErlDrvSizeT hlen,
+			  ErlIOVec *ev, ErlDrvSizeT skip);
+EXTERN ErlDrvSizeT driver_vec_to_buf(ErlIOVec *ev, char *buf, ErlDrvSizeT len);
 EXTERN int driver_set_timer(ErlDrvPort port, unsigned long time);
 EXTERN int driver_cancel_timer(ErlDrvPort port);
 EXTERN int driver_read_timer(ErlDrvPort port, unsigned long *time_left);
@@ -385,9 +427,9 @@ EXTERN int driver_exit (ErlDrvPort port, int err);
 EXTERN ErlDrvPDL driver_pdl_create(ErlDrvPort);
 EXTERN void driver_pdl_lock(ErlDrvPDL);
 EXTERN void driver_pdl_unlock(ErlDrvPDL);
-EXTERN long driver_pdl_get_refc(ErlDrvPDL);
-EXTERN long driver_pdl_inc_refc(ErlDrvPDL);
-EXTERN long driver_pdl_dec_refc(ErlDrvPDL);
+EXTERN ErlDrvSInt driver_pdl_get_refc(ErlDrvPDL);
+EXTERN ErlDrvSInt driver_pdl_inc_refc(ErlDrvPDL);
+EXTERN ErlDrvSInt driver_pdl_dec_refc(ErlDrvPDL);
 
 /*
  * Process monitors
@@ -418,34 +460,34 @@ EXTERN int  get_port_flags(ErlDrvPort port);
  * since the binary is a shared object it MUST be written once.
  */
 
-EXTERN ErlDrvBinary* driver_alloc_binary(int size);
-EXTERN ErlDrvBinary* driver_realloc_binary(ErlDrvBinary *bin, int size);
+EXTERN ErlDrvBinary* driver_alloc_binary(ErlDrvSizeT size);
+EXTERN ErlDrvBinary* driver_realloc_binary(ErlDrvBinary *bin, ErlDrvSizeT size);
 EXTERN void driver_free_binary(ErlDrvBinary *bin);
 
 /* Referenc count on driver binaries */
-EXTERN long driver_binary_get_refc(ErlDrvBinary *dbp);
-EXTERN long driver_binary_inc_refc(ErlDrvBinary *dbp);
-EXTERN long driver_binary_dec_refc(ErlDrvBinary *dbp);
+EXTERN ErlDrvSInt driver_binary_get_refc(ErlDrvBinary *dbp);
+EXTERN ErlDrvSInt driver_binary_inc_refc(ErlDrvBinary *dbp);
+EXTERN ErlDrvSInt driver_binary_dec_refc(ErlDrvBinary *dbp);
 
 /* Allocation interface */
-EXTERN void *driver_alloc(size_t size);
-EXTERN void *driver_realloc(void *ptr, size_t size);
+EXTERN void *driver_alloc(ErlDrvSizeT size);
+EXTERN void *driver_realloc(void *ptr, ErlDrvSizeT size);
 EXTERN void driver_free(void *ptr);
 
 /* Queue interface */
-EXTERN int driver_enq(ErlDrvPort port, char* buf, int len);
-EXTERN int driver_pushq(ErlDrvPort port, char* buf, int len);
-EXTERN int driver_deq(ErlDrvPort port, int size);
-EXTERN int driver_sizeq(ErlDrvPort port);
-EXTERN int driver_enq_bin(ErlDrvPort port, ErlDrvBinary *bin, int offset, 
-			  int len);
-EXTERN int driver_pushq_bin(ErlDrvPort port, ErlDrvBinary *bin, int offset,
-			    int len);
-
-EXTERN int driver_peekqv(ErlDrvPort port, ErlIOVec *ev);
+EXTERN int driver_enq(ErlDrvPort port, char* buf, ErlDrvSizeT len);
+EXTERN int driver_pushq(ErlDrvPort port, char* buf, ErlDrvSizeT len);
+EXTERN ErlDrvSizeT driver_deq(ErlDrvPort port, ErlDrvSizeT size);
+EXTERN ErlDrvSizeT driver_sizeq(ErlDrvPort port);
+EXTERN int driver_enq_bin(ErlDrvPort port, ErlDrvBinary *bin, ErlDrvSizeT offset,
+			  ErlDrvSizeT len);
+EXTERN int driver_pushq_bin(ErlDrvPort port, ErlDrvBinary *bin, ErlDrvSizeT offset,
+			    ErlDrvSizeT len);
+
+EXTERN ErlDrvSizeT driver_peekqv(ErlDrvPort port, ErlIOVec *ev);
 EXTERN SysIOVec* driver_peekq(ErlDrvPort port, int *vlen);
-EXTERN int driver_enqv(ErlDrvPort port, ErlIOVec *ev, int skip);
-EXTERN int driver_pushqv(ErlDrvPort port, ErlIOVec *ev, int skip);
+EXTERN int driver_enqv(ErlDrvPort port, ErlIOVec *ev, ErlDrvSizeT skip);
+EXTERN int driver_pushqv(ErlDrvPort port, ErlIOVec *ev, ErlDrvSizeT skip);
 
 /*
  * Add and remove driver entries.
@@ -572,8 +614,11 @@ EXTERN long driver_async(ErlDrvPort ix,
 			 void* async_data,
 			 void (*async_free)(void*));
 
-
-EXTERN int driver_async_cancel(unsigned int key);
+/*
+ * driver_async_cancel() is deprecated. It is scheduled for removal
+ * in OTP-R16. For more information see the erl_driver(3) documentation.
+ */
+EXTERN int driver_async_cancel(unsigned int key) ERL_DRV_DEPRECATED_FUNC;
 
 /* Locks the driver in the machine "forever", there is
    no unlock function. Note that this is almost never useful, as an open
@@ -604,6 +649,8 @@ EXTERN int erl_drv_getenv(char *key, char *value, size_t *value_size);
 
 #endif
 
+/* also in global.h, but driver's can't include global.h */
+void dtrace_drvport_str(ErlDrvPort port, char *port_buf);
 
 
 
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 50d8c25c46..a49a155701 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2007-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2007-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -24,6 +24,10 @@
 #include "global.h"
 #include <string.h>
 
+#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__)
+#define __DARWIN__ 1
+#endif
+
 #define ERL_DRV_THR_OPTS_SIZE(LAST_FIELD) \
   (((size_t) &((ErlDrvThreadOpts *) 0)->LAST_FIELD) \
    + sizeof(((ErlDrvThreadOpts *) 0)->LAST_FIELD))
@@ -154,7 +158,9 @@ erl_drv_mutex_create(char *name)
 				       (sizeof(ErlDrvMutex)
 					+ (name ? sys_strlen(name) + 1 : 0)));
     if (dmtx) {
-	if (ethr_mutex_init(&dmtx->mtx) != 0) {
+	ethr_mutex_opt opt = ETHR_MUTEX_OPT_DEFAULT_INITER;
+	opt.posix_compliant = 1;
+	if (ethr_mutex_init_opt(&dmtx->mtx, &opt) != 0) {
 	    erts_free(ERTS_ALC_T_DRV_MTX, (void *) dmtx);
 	    dmtx = NULL;
 	}
@@ -186,10 +192,9 @@ int
 erl_drv_mutex_trylock(ErlDrvMutex *dmtx)
 {
 #ifdef USE_THREADS
-    int res = dmtx ? ethr_mutex_trylock(&dmtx->mtx) : EINVAL;
-    if (res != 0 && res != EBUSY)
-	fatal_error(res, "erl_drv_mutex_trylock()");
-    return res;
+    if (!dmtx)
+	fatal_error(EINVAL, "erl_drv_mutex_trylock()");
+    return ethr_mutex_trylock(&dmtx->mtx);
 #else
     return 0;
 #endif
@@ -199,9 +204,9 @@ void
 erl_drv_mutex_lock(ErlDrvMutex *dmtx)
 {
 #ifdef USE_THREADS
-    int res = dmtx ? ethr_mutex_lock(&dmtx->mtx) : EINVAL;
-    if (res != 0)
-	fatal_error(res, "erl_drv_mutex_lock()");
+    if (!dmtx)
+	fatal_error(EINVAL, "erl_drv_mutex_lock()");
+    ethr_mutex_lock(&dmtx->mtx);
 #endif
 }
 
@@ -209,9 +214,9 @@ void
 erl_drv_mutex_unlock(ErlDrvMutex *dmtx)
 {
 #ifdef USE_THREADS
-    int res = dmtx ? ethr_mutex_unlock(&dmtx->mtx) : EINVAL;
-    if (res != 0)
-	fatal_error(res, "erl_drv_mutex_unlock()");
+    if (!dmtx)
+	fatal_error(EINVAL, "erl_drv_mutex_unlock()");
+    ethr_mutex_unlock(&dmtx->mtx);
 #endif
 }
 
@@ -223,7 +228,9 @@ erl_drv_cond_create(char *name)
 				      (sizeof(ErlDrvCond)
 				       + (name ? sys_strlen(name) + 1 : 0)));
     if (dcnd) {
-	if (ethr_cond_init(&dcnd->cnd) != 0) {
+	ethr_cond_opt opt = ETHR_COND_OPT_DEFAULT_INITER;
+	opt.posix_compliant = 1;
+	if (ethr_cond_init_opt(&dcnd->cnd, &opt) != 0) {
 	    erts_free(ERTS_ALC_T_DRV_CND, (void *) dcnd);
 	    dcnd = NULL;
 	}
@@ -256,9 +263,9 @@ void
 erl_drv_cond_signal(ErlDrvCond *dcnd)
 {
 #ifdef USE_THREADS
-    int res = dcnd ? ethr_cond_signal(&dcnd->cnd) : EINVAL;
-    if (res != 0)
-	fatal_error(res, "erl_drv_cond_signal()");
+    if (!dcnd)
+	fatal_error(EINVAL, "erl_drv_cond_signal()");
+    ethr_cond_signal(&dcnd->cnd);
 #endif
 }
 
@@ -266,9 +273,9 @@ void
 erl_drv_cond_broadcast(ErlDrvCond *dcnd)
 {
 #ifdef USE_THREADS
-    int res = dcnd ? ethr_cond_broadcast(&dcnd->cnd) : EINVAL;
-    if (res != 0)
-	fatal_error(res, "erl_drv_cond_broadcast()");
+    if (!dcnd)
+	fatal_error(EINVAL, "erl_drv_cond_broadcast()");
+    ethr_cond_broadcast(&dcnd->cnd);
 #endif
 }
 
@@ -277,18 +284,13 @@ void
 erl_drv_cond_wait(ErlDrvCond *dcnd, ErlDrvMutex *dmtx)
 {
 #ifdef USE_THREADS
-    int res;
     if (!dcnd || !dmtx) {
-	res = EINVAL;
-    error:
-	fatal_error(res, "erl_drv_cond_wait()");
+	fatal_error(EINVAL, "erl_drv_cond_wait()");
     }
     while (1) {
-	res = ethr_cond_wait(&dcnd->cnd, &dmtx->mtx);
+	int res = ethr_cond_wait(&dcnd->cnd, &dmtx->mtx);
 	if (res == 0)
 	    break;
-	if (res != EINTR)
-	    goto error;
     }
 #endif
 }
@@ -333,10 +335,9 @@ int
 erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck)
 {
 #ifdef USE_THREADS
-    int res = drwlck ? ethr_rwmutex_tryrlock(&drwlck->rwmtx) : EINVAL;
-    if (res != 0 && res != EBUSY)
-	fatal_error(res, "erl_drv_rwlock_tryrlock()");
-    return res;
+    if (!drwlck)
+	fatal_error(EINVAL, "erl_drv_rwlock_tryrlock()");
+    return ethr_rwmutex_tryrlock(&drwlck->rwmtx);
 #else
     return 0;
 #endif
@@ -346,9 +347,9 @@ void
 erl_drv_rwlock_rlock(ErlDrvRWLock *drwlck)
 {
 #ifdef USE_THREADS
-    int res = drwlck ? ethr_rwmutex_rlock(&drwlck->rwmtx) : EINVAL;
-    if (res != 0)
-	fatal_error(res, "erl_drv_rwlock_rlock()");
+    if (!drwlck)
+	fatal_error(EINVAL, "erl_drv_rwlock_rlock()");
+    ethr_rwmutex_rlock(&drwlck->rwmtx);
 #endif
 }
 
@@ -356,9 +357,9 @@ void
 erl_drv_rwlock_runlock(ErlDrvRWLock *drwlck)
 {
 #ifdef USE_THREADS
-    int res = drwlck ? ethr_rwmutex_runlock(&drwlck->rwmtx) : EINVAL;
-    if (res != 0)
-	fatal_error(res, "erl_drv_rwlock_runlock()");
+    if (!drwlck)
+	fatal_error(EINVAL, "erl_drv_rwlock_runlock()");
+    ethr_rwmutex_runlock(&drwlck->rwmtx);
 #endif
 }
 
@@ -366,10 +367,9 @@ int
 erl_drv_rwlock_tryrwlock(ErlDrvRWLock *drwlck)
 {
 #ifdef USE_THREADS
-    int res = drwlck ? ethr_rwmutex_tryrwlock(&drwlck->rwmtx) : EINVAL;
-    if (res != 0 && res != EBUSY)
-	fatal_error(res, "erl_drv_rwlock_tryrwlock()");
-    return res;
+    if (!drwlck)
+	fatal_error(EINVAL, "erl_drv_rwlock_tryrwlock()");
+    return ethr_rwmutex_tryrwlock(&drwlck->rwmtx);
 #else
     return 0;
 #endif
@@ -379,9 +379,9 @@ void
 erl_drv_rwlock_rwlock(ErlDrvRWLock *drwlck)
 {
 #ifdef USE_THREADS
-    int res = drwlck ? ethr_rwmutex_rwlock(&drwlck->rwmtx) : EINVAL;
-    if (res != 0)
-	fatal_error(res, "erl_drv_rwlock_rwlock()");
+    if (!drwlck)
+	fatal_error(EINVAL, "erl_drv_rwlock_rwlock()");
+    ethr_rwmutex_rwlock(&drwlck->rwmtx);
 #endif
 }
 
@@ -389,9 +389,9 @@ void
 erl_drv_rwlock_rwunlock(ErlDrvRWLock *drwlck)
 {
 #ifdef USE_THREADS
-    int res = drwlck ? ethr_rwmutex_rwunlock(&drwlck->rwmtx) : EINVAL;
-    if (res != 0)
-	fatal_error(res, "erl_drv_rwlock_rwunlock()");
+    if (!drwlck)
+	fatal_error(EINVAL, "erl_drv_rwlock_rwunlock()");
+    ethr_rwmutex_rwunlock(&drwlck->rwmtx);
 #endif
 }
 
@@ -536,7 +536,7 @@ erl_drv_tsd_get(ErlDrvTSDKey key)
     if (!dtid)
 	return NULL;
 #endif
-    if (ERL_DRV_TSD_LEN__ < key)
+    if (ERL_DRV_TSD_LEN__ <= key)
 	return NULL;
     return ERL_DRV_TSD__[key];
 }
@@ -603,11 +603,7 @@ erl_drv_thread_create(char *name,
 	dtid->name = ((char *) dtid) + sizeof(struct ErlDrvTid_);
 	sys_strcpy(dtid->name, name);
     }
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    res = erts_lcnt_thr_create(&dtid->tid, erl_drv_thread_wrapper, dtid, use_opts);
-#else
     res = ethr_thr_create(&dtid->tid, erl_drv_thread_wrapper, dtid, use_opts);
-#endif
 
     if (res != 0) {
 	erts_free(ERTS_ALC_T_DRV_TID, dtid);
@@ -704,3 +700,64 @@ erl_drv_thread_join(ErlDrvTid tid, void **respp)
 #endif
 }
 
+#if defined(__DARWIN__) && defined(USE_THREADS) && defined(ERTS_SMP)
+extern int erts_darwin_main_thread_pipe[2];
+extern int erts_darwin_main_thread_result_pipe[2];
+
+int erl_drv_stolen_main_thread_join(ErlDrvTid tid, void **respp);
+int erl_drv_steal_main_thread(char *name,
+			      ErlDrvTid *dtid,
+			      void* (*func)(void*),
+			      void* arg,
+			      ErlDrvThreadOpts *opts);
+
+
+int
+erl_drv_stolen_main_thread_join(ErlDrvTid tid, void **respp)
+{
+    void *dummy;
+    void **x;
+    if (respp == NULL)
+	x = &dummy;
+    else
+	x = respp;
+    read(erts_darwin_main_thread_result_pipe[0],x,sizeof(void *));
+    return 0;
+}
+
+int
+erl_drv_steal_main_thread(char *name,
+			  ErlDrvTid *tid,
+			  void* (*func)(void*),
+			  void* arg,
+			  ErlDrvThreadOpts *opts)
+{
+    char buff[sizeof(void* (*)(void*)) + sizeof(void *)];
+    int buff_sz = sizeof(void* (*)(void*)) + sizeof(void *);
+    /*struct ErlDrvTid_ *dtid;
+
+    dtid = erts_alloc_fnf(ERTS_ALC_T_DRV_TID,
+			  (sizeof(struct ErlDrvTid_)
+			   + (name ? sys_strlen(name) + 1 : 0)));
+    if (!dtid)
+	return ENOMEM;
+    memset(dtid,0,sizeof(ErlDrvTid_));
+    dtid->tid = (void * ) -1;
+    dtid->drv_thr = 1;
+    dtid->func = func;
+    dtid->arg = arg;
+    dtid->tsd = NULL;
+    dtid->tsd_len = 0;
+    dtid->name = no_name;
+    *tid = (ErlDrvTid) dtid;
+    */
+    *tid = NULL;
+    /* Ignore options and name... */
+    
+    memcpy(buff,&func,sizeof(void* (*)(void*)));
+    memcpy(buff + sizeof(void* (*)(void*)),&arg,sizeof(void *));
+    write(erts_darwin_main_thread_pipe[1],buff,buff_sz);
+    return 0;
+}
+
+#endif
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index 79e844b315..88947b5536 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2000-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2000-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -37,8 +37,6 @@ static erts_smp_rwmtx_t erts_fun_table_lock;
 #define erts_fun_read_unlock()	erts_smp_rwmtx_runlock(&erts_fun_table_lock)
 #define erts_fun_write_lock()	erts_smp_rwmtx_rwlock(&erts_fun_table_lock)
 #define erts_fun_write_unlock()	erts_smp_rwmtx_rwunlock(&erts_fun_table_lock)
-#define erts_fun_init_lock()	erts_smp_rwmtx_init(&erts_fun_table_lock, \
-						    "fun_tab")
 
 static HashValue fun_hash(ErlFunEntry* obj);
 static int fun_cmp(ErlFunEntry* obj1, ErlFunEntry* obj2);
@@ -50,15 +48,19 @@ static void fun_free(ErlFunEntry* obj);
  * to unloaded_fun[]. The -1 in unloaded_fun[0] will be interpreted
  * as an illegal arity when attempting to call a fun.
  */
-static Eterm unloaded_fun_code[3] = {NIL, -1, 0};
-static Eterm* unloaded_fun = unloaded_fun_code + 2;
+static BeamInstr unloaded_fun_code[3] = {NIL, -1, 0};
+static BeamInstr* unloaded_fun = unloaded_fun_code + 2;
 
 void
 erts_init_fun_table(void)
 {
     HashFunctions f;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+
+    erts_smp_rwmtx_init_opt(&erts_fun_table_lock, &rwmtx_opt, "fun_tab");
 
-    erts_fun_init_lock();
     f.hash = (H_FUN) fun_hash;
     f.cmp  = (HCMP_FUN) fun_cmp;
     f.alloc = (HALLOC_FUN) fun_alloc;
@@ -95,7 +97,7 @@ erts_put_fun_entry(Eterm mod, int uniq, int index)
 {
     ErlFunEntry template;
     ErlFunEntry* fe;
-    long refc;
+    erts_aint_t refc;
     ASSERT(is_atom(mod));
     template.old_uniq = uniq;
     template.old_index = index;
@@ -117,7 +119,7 @@ erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index,
 {
     ErlFunEntry template;
     ErlFunEntry* fe;
-    long refc;
+    erts_aint_t refc;
 
     ASSERT(is_atom(mod));
     template.old_uniq = old_uniq;
@@ -155,7 +157,7 @@ erts_get_fun_entry(Eterm mod, int uniq, int index)
     erts_fun_read_lock();
     ret = (ErlFunEntry *) hash_get(&erts_fun_table, (void*) &template);
     if (ret) {
-	long refc = erts_refc_inctest(&ret->refc, 1);
+	erts_aint_t refc = erts_refc_inctest(&ret->refc, 1);
 	if (refc < 2) /* Pending delete */
 	    erts_refc_inc(&ret->refc, 1);
     }
@@ -192,22 +194,8 @@ erts_erase_fun_entry(ErlFunEntry* fe)
     erts_fun_write_unlock();
 }
 
-#ifndef HYBRID /* FIND ME! */
-void
-erts_cleanup_funs(ErlFunThing* funp)
-{
-    while (funp) {
-	ErlFunEntry* fe = funp->fe;
-	if (erts_refc_dectest(&fe->refc, 0) == 0) {
-	    erts_erase_fun_entry(fe);
-	}
-	funp = funp->next;
-    }
-}
-#endif
-
 void
-erts_cleanup_funs_on_purge(Eterm* start, Eterm* end)
+erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end)
 {
     int limit;
     HashBucket** bucket;
@@ -222,7 +210,7 @@ erts_cleanup_funs_on_purge(Eterm* start, Eterm* end)
 
 	while (b) {
 	    ErlFunEntry* fe = (ErlFunEntry *) b;
-	    Eterm* addr = fe->address;
+	    BeamInstr* addr = fe->address;
 
 	    if (start <= addr && addr < end) {
 		fe->address = unloaded_fun;
@@ -269,7 +257,7 @@ erts_dump_fun_entries(int to, void *to_arg)
 #ifdef HIPE
 	    erts_print(to, to_arg, "Native_address: %p\n", fe->native_address);
 #endif
-	    erts_print(to, to_arg, "Refc: %d\n", erts_refc_read(&fe->refc, 1));
+	    erts_print(to, to_arg, "Refc: %ld\n", erts_refc_read(&fe->refc, 1));
 	    b = b->next;
 	}
     }
diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h
index fb5e75649b..54cfd6aa83 100644
--- a/erts/emulator/beam/erl_fun.h
+++ b/erts/emulator/beam/erl_fun.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2000-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2000-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -33,10 +33,10 @@ typedef struct erl_fun_entry {
     int index;			/* New style index. */
     int old_uniq;		/* Unique number (old_style) */
     int old_index;		/* Old style index */
-    Eterm* address;		/* Pointer to code for fun */
+    BeamInstr* address;		/* Pointer to code for fun */
 
 #ifdef HIPE
-    Eterm* native_address;	/* Native entry code for fun. */
+    UWord* native_address;	/* Native entry code for fun. */
 #endif
 
     Uint arity;			/* The arity of the fun. */
@@ -53,12 +53,10 @@ typedef struct erl_fun_entry {
 
 typedef struct erl_fun_thing {
     Eterm thing_word;		/* Subtag FUN_SUBTAG. */
-#ifndef HYBRID /* FIND ME! */
-    struct erl_fun_thing* next;	/* Next fun in mso list. */
-#endif
     ErlFunEntry* fe;		/* Pointer to fun entry. */
+    struct erl_off_heap_header* next;
 #ifdef HIPE
-    Eterm* native_address;	/* Native code for the fun. */
+    UWord* native_address;	/* Native code for the fun. */
 #endif
     Uint arity;			/* The arity of the fun. */
     Uint num_free;		/* Number of free variables (in env). */
@@ -83,10 +81,8 @@ ErlFunEntry* erts_get_fun_entry2(Eterm mod, int old_uniq, int old_index,
 				byte* uniq, int index, int arity);
 
 void erts_erase_fun_entry(ErlFunEntry* fe);
-#ifndef HYBRID /* FIND ME! */
 void erts_cleanup_funs(ErlFunThing* funp);
-#endif
-void erts_cleanup_funs_on_purge(Eterm* start, Eterm* end);
+void erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end);
 void erts_dump_fun_entries(int, void *);
 
 #endif
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index e9bf37a173..52a6e52e6c 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -33,7 +33,9 @@
 #include "erl_gc.h"
 #if HIPE
 #include "hipe_stack.h"
+#include "hipe_mode_switch.h"
 #endif
+#include "dtrace-wrapper.h"
 
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20
@@ -99,20 +101,18 @@ static Uint combined_message_size(Process* p);
 static void remove_message_buffers(Process* p);
 static int major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl);
 static int minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl);
-static void do_minor(Process *p, int new_sz, Eterm* objv, int nobj);
+static void do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj);
 static Eterm* sweep_rootset(Rootset *rootset, Eterm* htop, char* src, Uint src_size);
 static Eterm* sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size);
 static Eterm* sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
 			     char* src, Uint src_size);
 static Eterm* collect_heap_frags(Process* p, Eterm* heap,
 				 Eterm* htop, Eterm* objv, int nobj);
-static Uint adjust_after_fullsweep(Process *p, int size_before,
+static Uint adjust_after_fullsweep(Process *p, Uint size_before,
 				   int need, Eterm *objv, int nobj);
 static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj);
 static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj);
-static void sweep_proc_bins(Process *p, int fullsweep);
-static void sweep_proc_funs(Process *p, int fullsweep);
-static void sweep_proc_externals(Process *p, int fullsweep);
+static void sweep_off_heap(Process *p, int fullsweep);
 static void offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size);
 static void offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size);
 static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size,
@@ -126,7 +126,7 @@ static void disallow_heap_frag_ref_in_old_heap(Process* p);
 static void disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj);
 #endif
 
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 # define MAX_HEAP_SIZES 154
 #else
 # define MAX_HEAP_SIZES 55
@@ -145,6 +145,16 @@ erts_init_gc(void)
 {
     int i = 0;
 
+    ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word));
+    ASSERT(offsetof(ProcBin,thing_word) == offsetof(ErlFunThing,thing_word));
+    ASSERT(offsetof(ProcBin,thing_word) == offsetof(ExternalThing,header));
+    ASSERT(offsetof(ProcBin,size) == offsetof(struct erl_off_heap_header,size));
+    ASSERT(offsetof(ProcBin,size) == offsetof(ErlSubBin,size));
+    ASSERT(offsetof(ProcBin,size) == offsetof(ErlHeapBin,size));
+    ASSERT(offsetof(ProcBin,next) == offsetof(struct erl_off_heap_header,next));
+    ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next));
+    ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next));
+
     erts_smp_spinlock_init(&info_lck, "gc_info");
     garbage_cols = 0;
     reclaimed = 0;
@@ -206,7 +216,7 @@ erts_next_heap_size(Uint size, Uint offset)
 		low = mid + 1;
 	    }
 	}
-	erl_exit(1, "no next heap size found: %d, offset %d\n", size, offset);
+	erl_exit(1, "no next heap size found: %lu, offset %lu\n", (unsigned long)size, (unsigned long)offset);
     }
     return 0;
 }
@@ -286,25 +296,14 @@ erts_offset_heap_ptr(Eterm* hp, Uint sz, Sint offs,
     offset_heap_ptr(hp, sz, offs, (char *) low, ((char *)high)-((char *)low));
 }
 
+
 #define ptr_within(ptr, low, high) ((ptr) < (high) && (ptr) >= (low))
 
 void
 erts_offset_off_heap(ErlOffHeap *ohp, Sint offs, Eterm* low, Eterm* high)
 {
-    if (ohp->mso && ptr_within((Eterm *)ohp->mso, low, high)) {
-        Eterm** uptr = (Eterm**) (void *) &ohp->mso;
-        *uptr += offs;
-    }
-
-#ifndef HYBRID /* FIND ME! */
-    if (ohp->funs && ptr_within((Eterm *)ohp->funs, low, high)) {
-        Eterm** uptr = (Eterm**) (void *) &ohp->funs;
-        *uptr += offs;
-    }
-#endif
-
-    if (ohp->externals && ptr_within((Eterm *)ohp->externals, low, high)) {
-        Eterm** uptr = (Eterm**) (void *) &ohp->externals;
+    if (ohp->first && ptr_within((Eterm *)ohp->first, low, high)) {
+        Eterm** uptr = (Eterm**) (void *) &ohp->first;
         *uptr += offs;
     }
 }
@@ -317,7 +316,12 @@ erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity)
 
     if (is_non_value(result)) {
 	if (p->freason == TRAP) {
-	    cost = erts_garbage_collect(p, 0, p->def_arg_reg, p->arity);
+	  #if HIPE
+	    if (regs == NULL) {
+		regs = ERTS_PROC_GET_SCHDATA(p)->x_reg_array;
+	    }
+	  #endif
+	    cost = erts_garbage_collect(p, 0, regs, p->arity);
 	} else {
 	    cost = erts_garbage_collect(p, 0, regs, arity);
 	}
@@ -346,7 +350,9 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
     Uint reclaimed_now = 0;
     int done = 0;
     Uint ms1, s1, us1;
-
+#ifdef USE_VM_PROBES
+    DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
+#endif
     if (IS_TRACED_FL(p, F_TRACE_GC)) {
         trace_gc(p, am_gc_start);
     }
@@ -354,12 +360,11 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
     erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
     p->gcstatus = p->status;
     p->status = P_GARBING;
+    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+
     if (erts_system_monitor_long_gc != 0) {
 	get_now(&ms1, &s1, &us1);
     }
-    erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-
-    erts_smp_locked_activity_begin(ERTS_ACTIVITY_GC);
 
     ERTS_CHK_OFFHEAP(p);
 
@@ -367,15 +372,27 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
     if (GEN_GCS(p) >= MAX_GEN_GCS(p)) {
         FLAGS(p) |= F_NEED_FULLSWEEP;
     }
-
+#ifdef USE_VM_PROBES
+    *pidbuf = '\0';
+    if (DTRACE_ENABLED(gc_major_start)
+        || DTRACE_ENABLED(gc_major_end)
+        || DTRACE_ENABLED(gc_minor_start)
+        || DTRACE_ENABLED(gc_minor_end)) {
+        dtrace_proc_str(p, pidbuf);
+    }
+#endif
     /*
      * Test which type of GC to do.
      */
     while (!done) {
 	if ((FLAGS(p) & F_NEED_FULLSWEEP) != 0) {
+	    DTRACE2(gc_major_start, pidbuf, need);
 	    done = major_collection(p, need, objv, nobj, &reclaimed_now);
+	    DTRACE2(gc_major_end, pidbuf, reclaimed_now);
 	} else {
+	    DTRACE2(gc_minor_start, pidbuf, need);
 	    done = minor_collection(p, need, objv, nobj, &reclaimed_now);
+	    DTRACE2(gc_minor_end, pidbuf, reclaimed_now);
 	}
     }
 
@@ -394,8 +411,6 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
         trace_gc(p, am_gc_end);
     }
 
-    erts_smp_locked_activity_end(ERTS_ACTIVITY_GC);
-
     if (erts_system_monitor_long_gc != 0) {
 	Uint ms2, s2, us2;
 	Sint t;
@@ -443,7 +458,15 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
     p->last_old_htop = p->old_htop;
 #endif
 
-    return ((int) (HEAP_TOP(p) - HEAP_START(p)) / 10);
+    /* FIXME: This function should really return an Sint, i.e., a possibly
+       64 bit wide signed integer, but that requires updating all the code
+       that calls it. For now, we just return INT_MAX if the result is too
+       large for an int. */
+    {
+      Sint result = (HEAP_TOP(p) - HEAP_START(p)) / 10;
+      if (result >= INT_MAX) return INT_MAX;
+      else return (int) result;
+    }
 }
 
 /*
@@ -457,7 +480,6 @@ erts_garbage_collect_hibernate(Process* p)
     Eterm* heap;
     Eterm* htop;
     Rootset rootset;
-    int n;
     char* src;
     Uint src_size;
     Uint actual_size;
@@ -472,7 +494,6 @@ erts_garbage_collect_hibernate(Process* p)
     p->gcstatus = p->status;
     p->status = P_GARBING;
     erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-    erts_smp_locked_activity_begin(ERTS_ACTIVITY_GC);
     ErtsGcQuickSanityCheck(p);
     ASSERT(p->mbuf_sz == 0);
     ASSERT(p->mbuf == 0);
@@ -488,7 +509,10 @@ erts_garbage_collect_hibernate(Process* p)
 				    sizeof(Eterm)*heap_size);
     htop = heap;
 
-    n = setup_rootset(p, p->arg_reg, p->arity, &rootset);
+    (void) setup_rootset(p, p->arg_reg, p->arity, &rootset);
+#if HIPE
+    hipe_empty_nstack(p);
+#endif
 
     src = (char *) p->heap;
     src_size = (char *) p->htop - src;
@@ -504,14 +528,8 @@ erts_garbage_collect_hibernate(Process* p)
 
     cleanup_rootset(&rootset);
 
-    if (MSO(p).mso) {
-        sweep_proc_bins(p, 1);
-    }
-    if (MSO(p).funs) {
-        sweep_proc_funs(p, 1);
-    }
-    if (MSO(p).externals) {
-        sweep_proc_externals(p, 1);
+    if (MSO(p).first) {
+        sweep_off_heap(p, 1);
     }
 
     /*
@@ -589,12 +607,13 @@ erts_garbage_collect_hibernate(Process* p)
     erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
     p->status = p->gcstatus;
     erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-    erts_smp_locked_activity_end(ERTS_ACTIVITY_GC);
 }
 
 
 void
-erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
+erts_garbage_collect_literals(Process* p, Eterm* literals,
+			      Uint lit_size,
+			      struct erl_off_heap_header* oh)
 {
     Uint byte_lit_size = sizeof(Eterm)*lit_size;
     Uint old_heap_size;
@@ -605,7 +624,8 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
     char* area;
     Uint area_size;
     Eterm* old_htop;
-    int n;
+    Uint n;
+    struct erl_off_heap_header** prev;
 
     /*
      * Set GC state.
@@ -614,7 +634,6 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
     p->gcstatus = p->status;
     p->status = P_GARBING;
     erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-    erts_smp_locked_activity_begin(ERTS_ACTIVITY_GC);
 
     /*
      * We assume that the caller has already done a major collection
@@ -640,6 +659,9 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
     offset_heap(temp_lit, lit_size, offs, (char *) literals, byte_lit_size);
     offset_heap(p->heap, p->htop - p->heap, offs, (char *) literals, byte_lit_size);
     offset_rootset(p, offs, (char *) literals, byte_lit_size, p->arg_reg, p->arity);
+    if (oh) {
+	oh = (struct erl_off_heap_header *) ((Eterm *)(void *) oh + offs);
+    }
 
     /*
      * Now the literals are placed in memory that is safe to write into,
@@ -667,7 +689,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
 	    case TAG_PRIMARY_BOXED:
 		ptr = boxed_val(gval);
 		val = *ptr;
-                if (IS_MOVED(val)) {
+                if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
                     *g_ptr++ = val;
 		} else if (in_area(ptr, area, area_size)) {
@@ -679,7 +701,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
 	    case TAG_PRIMARY_LIST:
                 ptr = list_val(gval);
                 val = *ptr;
-                if (is_non_value(val)) { /* Moved */
+                if (IS_MOVED_CONS(val)) { /* Moved */
                     *g_ptr++ = ptr[1];
 		} else if (in_area(ptr, area, area_size)) {
                     MOVE_CONS(ptr,val,old_htop,g_ptr++);
@@ -707,6 +729,45 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
     p->old_htop = old_htop;
 
     /*
+     * Prepare to sweep binaries. Since all MSOs on the new heap
+     * must be come before MSOs on the old heap, find the end of
+     * current MSO list and use that as a starting point.
+     */
+
+    if (oh) {
+	prev = &MSO(p).first;
+	while (*prev) {
+	    prev = &(*prev)->next;
+	}
+    }
+
+    /*
+     * Sweep through all binaries in the temporary literal area.
+     */
+
+    while (oh) {
+	if (IS_MOVED_BOXED(oh->thing_word)) {
+	    Binary* bptr;
+	    struct erl_off_heap_header* ptr;
+
+	    ptr = (struct erl_off_heap_header*) boxed_val(oh->thing_word);
+	    ASSERT(thing_subtag(ptr->thing_word) == REFC_BINARY_SUBTAG);
+	    bptr = ((ProcBin*)ptr)->val;
+
+	    /*
+	     * This binary has been copied to the heap.
+	     * We must increment its reference count and
+	     * link it into the MSO list for the process.
+	     */
+
+	    erts_refc_inc(&bptr->refc, 1);
+	    *prev = ptr;
+	    prev = &ptr->next;
+	}
+	oh = oh->next;
+    }
+
+    /*
      * We no longer need this temporary area.
      */
     erts_free(ERTS_ALC_T_TMP, (void *) temp_lit);
@@ -717,7 +778,6 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
     erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
     p->status = p->gcstatus;
     erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-    erts_smp_locked_activity_end(ERTS_ACTIVITY_GC);
 }
 
 static int
@@ -737,7 +797,7 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
          * This improved Estone by more than 1200 estones on my computer
          * (Ultra Sparc 10).
          */
-        size_t new_sz = erts_next_heap_size(HEAP_TOP(p) - HEAP_START(p), 1);
+        Uint new_sz = erts_next_heap_size(HEAP_TOP(p) - HEAP_START(p), 1);
 
         /* Create new, empty old_heap */
         n_old = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_OLD_HEAP,
@@ -752,7 +812,10 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
      * is large enough.
      */
 
-    if (OLD_HEAP(p) && mature <= OLD_HEND(p) - OLD_HTOP(p)) {
+    if (OLD_HEAP(p) &&
+	    ((mature <= OLD_HEND(p) - OLD_HTOP(p)) &&
+	    ((BIN_VHEAP_MATURE(p) < ( BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)))) &&
+	    ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) {
 	ErlMessage *msgp;
 	Uint size_after;
 	Uint need_after;
@@ -863,7 +926,18 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
  * XXX: WARNING: If HiPE starts storing other non-Erlang values on the
  * nstack, such as floats, then this will have to be changed.
  */
-#define offset_nstack(p,offs,area,area_size) offset_heap_ptr(hipe_nstack_start((p)),hipe_nstack_used((p)),(offs),(area),(area_size))
+static ERTS_INLINE void offset_nstack(Process* p, Sint offs,
+				      char* area, Uint area_size)
+{
+    if (p->hipe.nstack) {
+	ASSERT(p->hipe.nsp && p->hipe.nstend);
+	offset_heap_ptr(hipe_nstack_start(p), hipe_nstack_used(p),
+			offs, area, area_size);
+    }
+    else {
+	ASSERT(!p->hipe.nsp && !p->hipe.nstend);
+    }
+}
 
 #else /* !HIPE */
 
@@ -874,12 +948,12 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 #endif /* HIPE */
 
 static void
-do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
+do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 {
     Rootset rootset;            /* Rootset for GC (stack, dictionary, etc). */
     Roots* roots;
     Eterm* n_htop;
-    int n;
+    Uint n;
     Eterm* ptr;
     Eterm val;
     Eterm gval;
@@ -913,7 +987,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
 	    case TAG_PRIMARY_BOXED: {
 		ptr = boxed_val(gval);
                 val = *ptr;
-                if (IS_MOVED(val)) {
+                if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
                     *g_ptr++ = val;
                 } else if (in_area(ptr, heap, mature_size)) {
@@ -929,7 +1003,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
 	    case TAG_PRIMARY_LIST: {
                 ptr = list_val(gval);
                 val = *ptr;
-                if (is_non_value(val)) { /* Moved */
+                if (IS_MOVED_CONS(val)) { /* Moved */
                     *g_ptr++ = ptr[1];
                 } else if (in_area(ptr, heap, mature_size)) {
                     MOVE_CONS(ptr,val,old_htop,g_ptr++);
@@ -972,7 +1046,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
 	    case TAG_PRIMARY_BOXED: {
 		ptr = boxed_val(gval);
 		val = *ptr;
-		if (IS_MOVED(val)) {
+		if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
 		    *n_hp++ = val;
 		} else if (in_area(ptr, heap, mature_size)) {
@@ -987,7 +1061,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
 	    case TAG_PRIMARY_LIST: {
 		ptr = list_val(gval);
 		val = *ptr;
-		if (is_non_value(val)) {
+		if (IS_MOVED_CONS(val)) {
 		    *n_hp++ = ptr[1];
 		} else if (in_area(ptr, heap, mature_size)) {
 		    MOVE_CONS(ptr,val,old_htop,n_hp++);
@@ -1008,7 +1082,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
 			Eterm* origptr = &(mb->orig);
 			ptr = boxed_val(*origptr);
 			val = *ptr;
-			if (IS_MOVED(val)) {
+			if (IS_MOVED_BOXED(val)) {
 			    *origptr = val;
 			    mb->base = binary_bytes(val);
 			} else if (in_area(ptr, heap, mature_size)) {
@@ -1041,15 +1115,8 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
     OLD_HTOP(p) = old_htop;
     HIGH_WATER(p) = (HEAP_START(p) != HIGH_WATER(p)) ? n_heap : n_htop;
 
-    if (MSO(p).mso) {
-        sweep_proc_bins(p, 0);
-    }
-
-    if (MSO(p).funs) {
-        sweep_proc_funs(p, 0);
-    }
-    if (MSO(p).externals) {
-        sweep_proc_externals(p, 0);
+    if (MSO(p).first) {
+	sweep_off_heap(p, 0);
     }
 
 #ifdef HARDDEBUG
@@ -1066,6 +1133,15 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
     sys_memcpy(n_heap + new_sz - n, p->stop, n * sizeof(Eterm));
     p->stop = n_heap + new_sz - n;
 
+#ifdef USE_VM_PROBES
+    if (HEAP_SIZE(p) != new_sz && DTRACE_ENABLED(process_heap_grow)) {
+        DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
+
+        dtrace_proc_str(p, pidbuf);
+        DTRACE3(process_heap_grow, pidbuf, HEAP_SIZE(p), new_sz);
+    }
+#endif
+
     ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
 		   (void*)HEAP_START(p),
 		   HEAP_SIZE(p) * sizeof(Eterm));
@@ -1089,14 +1165,14 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 {
     Rootset rootset;
     Roots* roots;
-    int size_before;
+    Uint size_before;
     Eterm* n_heap;
     Eterm* n_htop;
     char* src = (char *) HEAP_START(p);
     Uint src_size = (char *) HEAP_TOP(p) - src;
     char* oh = (char *) OLD_HEAP(p);
     Uint oh_size = (char *) OLD_HTOP(p) - oh;
-    int n;
+    Uint n;
     Uint new_sz;
     Uint fragments = MBUF_SIZE(p) + combined_message_size(p);
     ErlMessage *msgp;
@@ -1161,7 +1237,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 	    case TAG_PRIMARY_BOXED: {
 		ptr = boxed_val(gval);
 		val = *ptr;
-		if (IS_MOVED(val)) {
+		if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
 		    *g_ptr++ = val;
 		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
@@ -1175,7 +1251,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 	    case TAG_PRIMARY_LIST: {
 		ptr = list_val(gval);
 		val = *ptr;
-		if (is_non_value(val)) {
+		if (IS_MOVED_CONS(val)) {
 		    *g_ptr++ = ptr[1];
 		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
 		    MOVE_CONS(ptr,val,n_htop,g_ptr++);
@@ -1216,7 +1292,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 	    case TAG_PRIMARY_BOXED: {
 		ptr = boxed_val(gval);
 		val = *ptr;
-		if (IS_MOVED(val)) {
+		if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
 		    *n_hp++ = val;
 		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
@@ -1229,7 +1305,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 	    case TAG_PRIMARY_LIST: {
 		ptr = list_val(gval);
 		val = *ptr;
-		if (is_non_value(val)) {
+		if (IS_MOVED_CONS(val)) {
 		    *n_hp++ = ptr[1];
 		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
 		    MOVE_CONS(ptr,val,n_htop,n_hp++);
@@ -1249,7 +1325,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 			origptr = &(mb->orig);
 			ptr = boxed_val(*origptr);
 			val = *ptr;
-			if (IS_MOVED(val)) {
+			if (IS_MOVED_BOXED(val)) {
 			    *origptr = val;
 			    mb->base = binary_bytes(*origptr);
 			} else if (in_area(ptr, src, src_size) ||
@@ -1271,17 +1347,11 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 	}
     }
 
-    if (MSO(p).mso) {
-	sweep_proc_bins(p, 1);
-    }
-    if (MSO(p).funs) {
-	sweep_proc_funs(p, 1);
-    }
-    if (MSO(p).externals) {
-	sweep_proc_externals(p, 1);
+    if (MSO(p).first) {
+	sweep_off_heap(p, 1);
     }
 
-    if (OLD_HEAP(p) != NULL) {
+    if (OLD_HEAP(p) != NULL) {       
 	ERTS_HEAP_FREE(ERTS_ALC_T_OLD_HEAP,
 		       OLD_HEAP(p),
 		       (OLD_HEND(p) - OLD_HEAP(p)) * sizeof(Eterm));
@@ -1293,6 +1363,15 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
     sys_memcpy(n_heap + new_sz - n, p->stop, n * sizeof(Eterm));
     p->stop = n_heap + new_sz - n;
 
+#ifdef USE_VM_PROBES
+    if (HEAP_SIZE(p) != new_sz && DTRACE_ENABLED(process_heap_grow)) {
+        DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
+
+        dtrace_proc_str(p, pidbuf);
+        DTRACE3(process_heap_grow, pidbuf, HEAP_SIZE(p), new_sz);
+    }
+#endif
+
     ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
 		   (void *) HEAP_START(p),
 		   (HEAP_END(p) - HEAP_START(p)) * sizeof(Eterm));
@@ -1305,6 +1384,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
     HIGH_WATER(p) = HEAP_TOP(p);
 
     ErtsGcQuickSanityCheck(p);
+
     /*
      * Copy newly received message onto the end of the new heap.
      */
@@ -1327,10 +1407,10 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 }
 
 static Uint
-adjust_after_fullsweep(Process *p, int size_before, int need, Eterm *objv, int nobj)
+adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int nobj)
 {
-    int wanted, sz, size_after, need_after;
-    int stack_size = STACK_SZ_ON_HEAP(p);
+    Uint wanted, sz, size_after, need_after;
+    Uint stack_size = STACK_SZ_ON_HEAP(p);
     Uint reclaimed_now;
 
     size_after = (HEAP_TOP(p) - HEAP_START(p));
@@ -1392,17 +1472,12 @@ combined_message_size(Process* p)
 static void
 remove_message_buffers(Process* p)
 {
-    ErlHeapFragment* bp = MBUF(p);
-
-    MBUF(p) = NULL;
-    MBUF_SIZE(p) = 0;
-    while (bp != NULL) {
-	ErlHeapFragment* next_bp = bp->next;
-	free_message_buffer(bp);
-	bp = next_bp;
-    }
+    if (MBUF(p) != NULL) {
+	free_message_buffer(MBUF(p));
+	MBUF(p) = NULL;
+    }    
+    MBUF_SIZE(p) = 0;    
 }
-
 #ifdef HARDDEBUG
 
 /*
@@ -1433,12 +1508,12 @@ disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj)
 	case TAG_PRIMARY_BOXED: {
 	    ptr = _unchecked_boxed_val(gval);
 	    val = *ptr;
-	    if (IS_MOVED(val)) {
+	    if (IS_MOVED_BOXED(val)) {
 		ASSERT(is_boxed(val));
 		objv++;
 	    } else {
  		for (qb = mbuf; qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) {
+		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1450,11 +1525,11 @@ disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj)
 	case TAG_PRIMARY_LIST: {
 	    ptr = _unchecked_list_val(gval);
 	    val = *ptr;
-	    if (is_non_value(val)) {
+	    if (IS_MOVED_CONS(val)) {
 		objv++;
 	    } else {
 		for (qb = mbuf; qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) {
+		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1499,7 +1574,7 @@ disallow_heap_frag_ref_in_heap(Process* p)
 	    ptr = _unchecked_boxed_val(val);
 	    if (!in_area(ptr, heap, heap_size)) {
 		for (qb = MBUF(p); qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) {
+		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1509,7 +1584,7 @@ disallow_heap_frag_ref_in_heap(Process* p)
 	    ptr = _unchecked_list_val(val);
 	    if (!in_area(ptr, heap, heap_size)) {
 		for (qb = MBUF(p); qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) {
+		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1551,26 +1626,26 @@ disallow_heap_frag_ref_in_old_heap(Process* p)
 	val = *hp++;
 	switch (primary_tag(val)) {
 	case TAG_PRIMARY_BOXED:
-	    ptr = (Eterm *) val;
+	    ptr = (Eterm *) EXPAND_POINTER(val);
 	    if (!in_area(ptr, old_heap, old_heap_size)) {
 		if (in_area(ptr, new_heap, new_heap_size)) {
 		    abort();
 		}
 		for (qb = MBUF(p); qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) {
+		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
 	    }
 	    break;
 	case TAG_PRIMARY_LIST:
-	    ptr = (Eterm *) val;
+	    ptr = (Eterm *) EXPAND_POINTER(val);
 	    if (!in_area(ptr, old_heap, old_heap_size)) {
 		if (in_area(ptr, new_heap, new_heap_size)) {
 		    abort();
 		}
 		for (qb = MBUF(p); qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) {
+		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1610,7 +1685,7 @@ sweep_rootset(Rootset* rootset, Eterm* htop, char* src, Uint src_size)
 	    case TAG_PRIMARY_BOXED: {
 		ptr = boxed_val(gval);
                 val = *ptr;
-                if (IS_MOVED(val)) {
+                if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
                     *g_ptr++ = val;
                 } else if (in_area(ptr, src, src_size)) {
@@ -1623,7 +1698,7 @@ sweep_rootset(Rootset* rootset, Eterm* htop, char* src, Uint src_size)
 	    case TAG_PRIMARY_LIST: {
                 ptr = list_val(gval);
                 val = *ptr;
-                if (is_non_value(val)) { /* Moved */
+                if (IS_MOVED_CONS(val)) {
                     *g_ptr++ = ptr[1];
                 } else if (in_area(ptr, src, src_size)) {
                     MOVE_CONS(ptr,val,htop,g_ptr++);
@@ -1657,7 +1732,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
 	case TAG_PRIMARY_BOXED: {
 	    ptr = boxed_val(gval);
 	    val = *ptr;
-	    if (IS_MOVED(val)) {
+	    if (IS_MOVED_BOXED(val)) {
 		ASSERT(is_boxed(val));
 		*n_hp++ = val;
 	    } else if (in_area(ptr, src, src_size)) {
@@ -1670,7 +1745,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
 	case TAG_PRIMARY_LIST: {
 	    ptr = list_val(gval);
 	    val = *ptr;
-	    if (is_non_value(val)) {
+	    if (IS_MOVED_CONS(val)) {
 		*n_hp++ = ptr[1];
 	    } else if (in_area(ptr, src, src_size)) {
 		MOVE_CONS(ptr,val,n_htop,n_hp++);
@@ -1690,7 +1765,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
 		    origptr = &(mb->orig);
 		    ptr = boxed_val(*origptr);
 		    val = *ptr;
-		    if (IS_MOVED(val)) {
+		    if (IS_MOVED_BOXED(val)) {
 			*origptr = val;
 			mb->base = binary_bytes(*origptr);
 		    } else if (in_area(ptr, src, src_size)) {
@@ -1722,7 +1797,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr
 	case TAG_PRIMARY_BOXED: {
 	    ptr = boxed_val(gval);
 	    val = *ptr;
-	    if (IS_MOVED(val)) {
+	    if (IS_MOVED_BOXED(val)) {
 		ASSERT(is_boxed(val));
 		*heap_ptr++ = val;
 	    } else if (in_area(ptr, src, src_size)) {
@@ -1735,7 +1810,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr
 	case TAG_PRIMARY_LIST: {
 	    ptr = list_val(gval);
 	    val = *ptr;
-	    if (is_non_value(val)) {
+	    if (IS_MOVED_CONS(val)) {
 		*heap_ptr++ = ptr[1];
 	    } else if (in_area(ptr, src, src_size)) {
 		MOVE_CONS(ptr,val,htop,heap_ptr++);
@@ -1830,28 +1905,6 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop,
     return n_htop;
 }
 
-#ifdef DEBUG
-static Eterm follow_moved(Eterm term)
-{
-    Eterm* ptr;
-    switch (primary_tag(term)) {
-    case TAG_PRIMARY_IMMED1:
-	break;
-    case TAG_PRIMARY_BOXED:
-	ptr = boxed_val(term);
-	if (IS_MOVED(*ptr)) term = *ptr;
-	break;
-    case TAG_PRIMARY_LIST:
-	ptr = list_val(term);
-	if (is_non_value(ptr[0])) term = ptr[1];
-	break;
-    default:
-	abort();
-    }
-    return term;
-}
-#endif
-
 static Uint
 setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
 {
@@ -1887,7 +1940,13 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
 	roots[n].sz = 1;
 	n++;
     }
-
+#ifdef USE_VM_PROBES
+    if (is_not_immed(p->dt_utag)) {
+	roots[n].v = &p->dt_utag;
+	roots[n].sz = 1;
+	n++;
+    }
+#endif
     ASSERT(is_nil(p->tracer_proc) ||
 	   is_internal_pid(p->tracer_proc) ||
 	   is_internal_port(p->tracer_proc));
@@ -1957,8 +2016,8 @@ static void
 grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj)
 {
     Eterm* new_heap;
-    int heap_size = HEAP_TOP(p) - HEAP_START(p);
-    int stack_size = p->hend - p->stop;
+    Uint heap_size = HEAP_TOP(p) - HEAP_START(p);
+    Uint stack_size = p->hend - p->stop;
     Sint offs;
 
     ASSERT(HEAP_SIZE(p) < new_sz);
@@ -1989,6 +2048,16 @@ grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj)
         HEAP_TOP(p) = new_heap + heap_size;
         HEAP_START(p) = new_heap;
     }
+
+#ifdef USE_VM_PROBES
+    if (DTRACE_ENABLED(process_heap_grow)) {
+	DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
+
+        dtrace_proc_str(p, pidbuf);
+	DTRACE3(process_heap_grow, pidbuf, HEAP_SIZE(p), new_sz);
+    }
+#endif
+
     HEAP_SIZE(p) = new_sz;
 }
 
@@ -1996,10 +2065,9 @@ static void
 shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj)
 {
     Eterm* new_heap;
-    int heap_size = HEAP_TOP(p) - HEAP_START(p);
+    Uint heap_size = HEAP_TOP(p) - HEAP_START(p);
     Sint offs;
-
-    int stack_size = p->hend - p->stop;
+    Uint stack_size = p->hend - p->stop;
 
     ASSERT(new_sz < p->heap_sz);
     sys_memmove(p->heap + new_sz - stack_size, p->stop, stack_size *
@@ -2027,11 +2095,21 @@ shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj)
         HEAP_TOP(p) = new_heap + heap_size;
         HEAP_START(p) = new_heap;
     }
+
+#ifdef USE_VM_PROBES
+    if (DTRACE_ENABLED(process_heap_shrink)) {
+	DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
+
+        dtrace_proc_str(p, pidbuf);
+	DTRACE3(process_heap_shrink, pidbuf, HEAP_SIZE(p), new_sz);
+    }
+#endif
+
     HEAP_SIZE(p) = new_sz;
 }
 
-static Uint
-do_next_vheap_size(Uint vheap, Uint vheap_sz) {
+static Uint64
+do_next_vheap_size(Uint64 vheap, Uint64 vheap_sz) {
 
     /*                grow
      *
@@ -2048,131 +2126,53 @@ do_next_vheap_size(Uint vheap, Uint vheap_sz) {
      *          ----------------------
      */
 
-    if (vheap > (Uint) (vheap_sz*3/4)) {
+    if ((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) {
+	Uint64 new_vheap_sz = vheap_sz;
 
-	while(vheap > (Uint) (vheap_sz*3/4)) {
-	    vheap_sz = vheap_sz*2;
+	while((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) {
+	    /* the golden ratio = 1.618 */
+	    new_vheap_sz = (Uint64) vheap_sz * 1.618;
+	    if (new_vheap_sz < vheap_sz ) {
+	        return vheap_sz;
+	    }
+	    vheap_sz = new_vheap_sz;
 	}
 
-	return erts_next_heap_size(vheap_sz, 0);
+	return vheap_sz;
     }
 
-    if (vheap < (Uint) (vheap_sz/4)) {
-	return erts_next_heap_size((Uint) (vheap_sz / 2), 0);
+    if (vheap < (Uint64) (vheap_sz/4)) {
+	return (vheap_sz >> 1);
     }
 
     return vheap_sz;
 
 }
 
-static Uint
-next_vheap_size(Process* p, Uint vheap, Uint vheap_sz) {
-    vheap_sz = do_next_vheap_size(vheap, vheap_sz);
-    return vheap_sz < p->min_vheap_size ? p->min_vheap_size : vheap_sz;
-}
-
-static void
-sweep_proc_externals(Process *p, int fullsweep)
-{
-    ExternalThing** prev;
-    ExternalThing* ptr;
-    char* oh = 0;
-    Uint oh_size = 0;
-
-    if (fullsweep == 0) {
-	oh = (char *) OLD_HEAP(p);
-	oh_size = (char *) OLD_HEND(p) - oh;
-    }
-
-    prev = &MSO(p).externals;
-    ptr = MSO(p).externals;
-
-    while (ptr) {
-        Eterm* ppt = (Eterm *) ptr;
-
-        if (IS_MOVED(*ppt)) {        /* Object is alive */
-            ExternalThing* ro = external_thing_ptr(*ppt);
-
-            *prev = ro;         /* Patch to moved pos */
-            prev = &ro->next;
-            ptr = ro->next;
-        } else if (in_area(ppt, oh, oh_size)) {
-            /*
-             * Object resides on old heap, and we just did a
-             * generational collection - keep object in list.
-             */
-            prev = &ptr->next;
-            ptr = ptr->next;
-        } else {                /* Object has not been moved - deref it */
-	    erts_deref_node_entry(ptr->node);
-            *prev = ptr = ptr->next;
-        }
-    }
-    ASSERT(*prev == NULL);
-}
-
-static void
-sweep_proc_funs(Process *p, int fullsweep)
-{
-    ErlFunThing** prev;
-    ErlFunThing* ptr;
-    char* oh = 0;
-    Uint oh_size = 0;
-
-    if (fullsweep == 0) {
-	oh = (char *) OLD_HEAP(p);
-	oh_size = (char *) OLD_HEND(p) - oh;
-    }
-		      
-    prev = &MSO(p).funs;
-    ptr = MSO(p).funs;
-
-    while (ptr) {
-        Eterm* ppt = (Eterm *) ptr;
-
-        if (IS_MOVED(*ppt)) {        /* Object is alive */
-            ErlFunThing* ro = (ErlFunThing *) fun_val(*ppt);
-
-            *prev = ro;         /* Patch to moved pos */
-            prev = &ro->next;
-            ptr = ro->next;
-        } else if (in_area(ppt, oh, oh_size)) {
-            /*
-             * Object resides on old heap, and we just did a
-             * generational collection - keep object in list.
-             */
-            prev = &ptr->next;
-            ptr = ptr->next;
-        } else {                /* Object has not been moved - deref it */
-            ErlFunEntry* fe = ptr->fe;
-
-            *prev = ptr = ptr->next;
-	    if (erts_refc_dectest(&fe->refc, 0) == 0) {
-                erts_erase_fun_entry(fe);
-            }
-        }
-    }
-    ASSERT(*prev == NULL);
+static Uint64
+next_vheap_size(Process* p, Uint64 vheap, Uint64 vheap_sz) {
+    Uint64 new_vheap_sz = do_next_vheap_size(vheap, vheap_sz);
+    return new_vheap_sz < p->min_vheap_size ? p->min_vheap_size : new_vheap_sz;
 }
 
 struct shrink_cand_data {
-    ProcBin* new_candidates;
-    ProcBin* new_candidates_end;
-    ProcBin* old_candidates;
+    struct erl_off_heap_header* new_candidates;
+    struct erl_off_heap_header* new_candidates_end;
+    struct erl_off_heap_header* old_candidates;
     Uint no_of_candidates;
     Uint no_of_active;
 };
 
 static ERTS_INLINE void
 link_live_proc_bin(struct shrink_cand_data *shrink,
-		   ProcBin ***prevppp,
-		   ProcBin **pbpp,
+		   struct erl_off_heap_header*** prevppp,
+		   struct erl_off_heap_header** currpp,
 		   int new_heap)
 {
-    ProcBin *pbp = *pbpp;
-
-    *pbpp = pbp->next;
+    ProcBin *pbp = (ProcBin*) *currpp;
+    ASSERT(**prevppp == *currpp);
 
+    *currpp = pbp->next;
     if (pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE)) {
 	ASSERT(((pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE))
 		== (PB_ACTIVE_WRITER|PB_IS_WRITABLE))
@@ -2189,15 +2189,16 @@ link_live_proc_bin(struct shrink_cand_data *shrink,
 	    /* Our allocators are 8 byte aligned, i.e., shrinking with
 	       less than 8 bytes will have no real effect */
 	    if (unused >= 8) { /* A shrink candidate; save in candidate list */
+		**prevppp = pbp->next;
 		if (new_heap) {
 		    if (!shrink->new_candidates)
-			shrink->new_candidates_end = pbp;
+			shrink->new_candidates_end = (struct erl_off_heap_header*)pbp;
 		    pbp->next = shrink->new_candidates;
-		    shrink->new_candidates = pbp;
+		    shrink->new_candidates = (struct erl_off_heap_header*)pbp;
 		}
 		else {
 		    pbp->next = shrink->old_candidates;
-		    shrink->old_candidates = pbp;
+		    shrink->old_candidates = (struct erl_off_heap_header*)pbp;
 		}
 		shrink->no_of_candidates++;
 		return;
@@ -2205,83 +2206,117 @@ link_live_proc_bin(struct shrink_cand_data *shrink,
 	}
     }
 
-    /* Not a shrink candidate; keep in original mso list */
-    **prevppp = pbp;
+    /* Not a shrink candidate; keep in original mso list */ 
     *prevppp = &pbp->next;
-
 }
 
 
-static void 
-sweep_proc_bins(Process *p, int fullsweep)
+static void
+sweep_off_heap(Process *p, int fullsweep)
 {
     struct shrink_cand_data shrink = {0};
-    ProcBin** prev;
-    ProcBin* ptr;
-    Binary* bptr;
-    char* oh = NULL;
-    Uint oh_size = 0;
-    Uint bin_vheap = 0;
+    struct erl_off_heap_header* ptr;
+    struct erl_off_heap_header** prev;
+    char* oheap = NULL;
+    Uint oheap_sz = 0;
+    Uint64 bin_vheap = 0;
+#ifdef DEBUG
+    int seen_mature = 0;
+#endif
 
     if (fullsweep == 0) {
-	oh = (char *) OLD_HEAP(p);
-	oh_size = (char *) OLD_HEND(p) - oh;
+	oheap = (char *) OLD_HEAP(p);
+	oheap_sz = (char *) OLD_HEND(p) - oheap;
     }
 
     BIN_OLD_VHEAP(p) = 0;
 
-    prev = &MSO(p).mso;
-    ptr = MSO(p).mso;
+    prev = &MSO(p).first;
+    ptr = MSO(p).first;
 
-    /*
-     * Note: In R7 we no longer force a fullsweep when we find binaries
-     * on the old heap. The reason is that with the introduction of the
-     * bit syntax we can expect binaries to be used a lot more. Note that
-     * in earlier releases a brand new binary (or any other term) could
-     * be put on the old heap during a gen-gc fullsweep, but this is
-     * no longer the case in R7.
+    /* Firts part of the list will reside on the (old) new-heap.
+     * Keep if moved, otherwise deref.
      */
     while (ptr) {
-        Eterm* ppt = (Eterm *) ptr;
-
-        if (IS_MOVED(*ppt)) {        /* Object is alive */
-	    bin_vheap += ptr->size / sizeof(Eterm);
-            ptr = (ProcBin*) binary_val(*ppt);		   
-	    link_live_proc_bin(&shrink,
-			       &prev,
-			       &ptr,
-			       !in_area(ptr, oh, oh_size)); 
-        } else if (in_area(ppt, oh, oh_size)) {
-            /*
-             * Object resides on old heap, and we just did a
-             * generational collection - keep object in list.
-             */
-            BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/
-	    link_live_proc_bin(&shrink, &prev, &ptr, 0); 
-        } else {                /* Object has not been moved - deref it */
-
-            *prev = ptr->next;
-            bptr = ptr->val;
-            if (erts_refc_dectest(&bptr->refc, 0) == 0)
-		erts_bin_free(bptr);
-            ptr = *prev;
-        }
+	if (IS_MOVED_BOXED(ptr->thing_word)) {
+	    ASSERT(!in_area(ptr, oheap, oheap_sz));
+	    *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word);
+	    ASSERT(!IS_MOVED_BOXED(ptr->thing_word));
+	    if (ptr->thing_word == HEADER_PROC_BIN) {
+		int to_new_heap = !in_area(ptr, oheap, oheap_sz);
+		ASSERT(to_new_heap == !seen_mature || (!to_new_heap && (seen_mature=1)));
+		if (to_new_heap) {
+		    bin_vheap += ptr->size / sizeof(Eterm);
+		} else {
+		    BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/
+		}		
+		link_live_proc_bin(&shrink, &prev, &ptr, to_new_heap);
+	    }
+	    else {
+		prev = &ptr->next;
+		ptr = ptr->next;
+	    }
+	}
+	else if (!in_area(ptr, oheap, oheap_sz)) {
+	    /* garbage */
+	    switch (thing_subtag(ptr->thing_word)) {
+	    case REFC_BINARY_SUBTAG:
+		{
+		    Binary* bptr = ((ProcBin*)ptr)->val;	
+		    if (erts_refc_dectest(&bptr->refc, 0) == 0) {
+			erts_bin_free(bptr);
+		    }
+		    break;
+		}
+	    case FUN_SUBTAG:
+		{
+		    ErlFunEntry* fe = ((ErlFunThing*)ptr)->fe;
+		    if (erts_refc_dectest(&fe->refc, 0) == 0) {
+			erts_erase_fun_entry(fe);
+		    }
+		    break;
+		}
+	    default:
+		ASSERT(is_external_header(ptr->thing_word));
+		erts_deref_node_entry(((ExternalThing*)ptr)->node);
+	    }
+	    *prev = ptr = ptr->next;
+	}
+	else break; /* and let old-heap loop continue */
     }
 
-    if (BIN_OLD_VHEAP(p) >= BIN_OLD_VHEAP_SZ(p)) {
-        FLAGS(p) |= F_NEED_FULLSWEEP;
+    /* The rest of the list resides on old-heap, and we just did a
+     * generational collection - keep objects in list.
+     */
+    while (ptr) {
+	ASSERT(in_area(ptr, oheap, oheap_sz));
+	ASSERT(!IS_MOVED_BOXED(ptr->thing_word));       
+	if (ptr->thing_word == HEADER_PROC_BIN) {
+	    BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/
+	    link_live_proc_bin(&shrink, &prev, &ptr, 0);
+	}
+	else {
+	    ASSERT(is_fun_header(ptr->thing_word) ||
+		   is_external_header(ptr->thing_word));
+	    prev = &ptr->next;
+	    ptr = ptr->next;
+	}
     }
 
-    BIN_VHEAP_SZ(p) = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p));
-    BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p), BIN_OLD_VHEAP_SZ(p));
-    MSO(p).overhead = bin_vheap;
+    if (fullsweep) {
+	BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p) + MSO(p).overhead, BIN_OLD_VHEAP_SZ(p));
+    }
+    BIN_VHEAP_SZ(p)     = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p));
+    MSO(p).overhead     = bin_vheap;
+    BIN_VHEAP_MATURE(p) = bin_vheap;
 
     /*
      * If we got any shrink candidates, check them out.
      */
 
     if (shrink.no_of_candidates) {
-	ProcBin *candlist[] = {shrink.new_candidates, shrink.old_candidates};
+	ProcBin *candlist[] = { (ProcBin*)shrink.new_candidates,
+	                        (ProcBin*)shrink.old_candidates };
 	Uint leave_unused = 0;
 	int i;
 
@@ -2293,21 +2328,21 @@ sweep_proc_bins(Process *p, int fullsweep)
 	}
 
 	for (i = 0; i < sizeof(candlist)/sizeof(candlist[0]); i++) {
-
-	    for (ptr = candlist[i]; ptr; ptr = ptr->next) {
-		Uint new_size = ptr->size;
+	    ProcBin* pb;
+	    for (pb = candlist[i]; pb; pb = (ProcBin*)pb->next) {
+		Uint new_size = pb->size;
 
 		if (leave_unused) {
 		    new_size += (new_size * 100) / leave_unused;
 		    /* Our allocators are 8 byte aligned, i.e., shrinking with
 		       less than 8 bytes will have no real effect */
-		    if (new_size + 8 >= ptr->val->orig_size)
+		    if (new_size + 8 >= pb->val->orig_size)
 			continue;
 		}
 
-		ptr->val = erts_bin_realloc(ptr->val, new_size);
-		ptr->val->orig_size = new_size;
-		ptr->bytes = (byte *) ptr->val->orig_bytes;
+		pb->val = erts_bin_realloc(pb->val, new_size);
+		pb->val->orig_size = new_size;
+		pb->bytes = (byte *) pb->val->orig_bytes;
 	    }
 	}
 
@@ -2316,21 +2351,20 @@ sweep_proc_bins(Process *p, int fullsweep)
 	 * We now potentially have the mso list divided into three lists:
 	 * - shrink candidates on new heap (inactive writable with unused data)
 	 * - shrink candidates on old heap (inactive writable with unused data)
-	 * - other binaries (read only + active writable ...)
+	 * - other binaries (read only + active writable ...) + funs and externals
 	 *
 	 * Put them back together: new candidates -> other -> old candidates
 	 * This order will ensure that the list only refers from new
 	 * generation to old and never from old to new *which is important*.
 	 */
 	if (shrink.new_candidates) {
-	    if (prev == &MSO(p).mso) /* empty other binaries list */
+	    if (prev == &MSO(p).first) /* empty other binaries list */
 		prev = &shrink.new_candidates_end->next;
 	    else
-		shrink.new_candidates_end->next = MSO(p).mso;
-	    MSO(p).mso = shrink.new_candidates;
+		shrink.new_candidates_end->next = MSO(p).first;
+	    MSO(p).first = shrink.new_candidates;
 	}
     }
-
     *prev = shrink.old_candidates;
 }
 
@@ -2361,15 +2395,17 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
 	      tari = thing_arityval(val);
 	      switch (thing_subtag(val)) {
 	      case REFC_BINARY_SUBTAG:
+	      case FUN_SUBTAG:
+	      case EXTERNAL_PID_SUBTAG:
+	      case EXTERNAL_PORT_SUBTAG:
+	      case EXTERNAL_REF_SUBTAG:
 		  {
-		      ProcBin* pb = (ProcBin*) hp;
-		      Eterm** uptr = (Eterm **) (void *) &pb->next;
+		      struct erl_off_heap_header* oh = (struct erl_off_heap_header*) hp;
 
-		      if (*uptr && in_area((Eterm *)pb->next, area, area_size)) {
+		      if (in_area(oh->next, area, area_size)) {
+			  Eterm** uptr = (Eterm **) (void *) &oh->next;
 			  *uptr += offs; /* Patch the mso chain */
 		      }
-		      sz -= tari;
-		      hp += tari + 1;
 		  }
 		  break;
 	      case BIN_MATCHSTATE_SUBTAG:
@@ -2380,40 +2416,11 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
 		      mb->orig = offset_ptr(mb->orig, offs);
 		      mb->base = binary_bytes(mb->orig);
 		  }
-		  sz -= tari;
-		  hp += tari + 1;
 		}
 		break;
-	      case FUN_SUBTAG:
-		  {
-		      ErlFunThing* funp = (ErlFunThing *) hp;
-		      Eterm** uptr = (Eterm **) (void *) &funp->next;
-
-		      if (*uptr && in_area((Eterm *)funp->next, area, area_size)) {
-			  *uptr += offs;
-		      }
-		      sz -= tari;
-		      hp += tari + 1;
-		  }
-		  break;
-	      case EXTERNAL_PID_SUBTAG:
-	      case EXTERNAL_PORT_SUBTAG:
-	      case EXTERNAL_REF_SUBTAG:
-		  {
-		      ExternalThing* etp = (ExternalThing *) hp;
-		      Eterm** uptr = (Eterm **) (void *) &etp->next;
-
-		      if (*uptr && in_area((Eterm *)etp->next, area, area_size)) {
-			  *uptr += offs;
-		      }
-		      sz -= tari;
-		      hp += tari + 1;
-		  }
-		  break;
-	      default:
-		  sz -= tari;
-		  hp += tari + 1;
 	      }
+	      sz -= tari;
+	      hp += tari + 1;
 	      break;
 	  }
 	  default:
@@ -2450,18 +2457,8 @@ offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
 static void
 offset_off_heap(Process* p, Sint offs, char* area, Uint area_size)
 {
-    if (MSO(p).mso && in_area((Eterm *)MSO(p).mso, area, area_size)) {
-        Eterm** uptr = (Eterm**) (void *) &MSO(p).mso;
-        *uptr += offs;
-    }
-
-    if (MSO(p).funs && in_area((Eterm *)MSO(p).funs, area, area_size)) {
-        Eterm** uptr = (Eterm**) (void *) &MSO(p).funs;
-        *uptr += offs;
-    }
-
-    if (MSO(p).externals && in_area((Eterm *)MSO(p).externals, area, area_size)) {
-        Eterm** uptr = (Eterm**) (void *) &MSO(p).externals;
+    if (MSO(p).first && in_area((Eterm *)MSO(p).first, area, area_size)) {
+        Eterm** uptr = (Eterm**) (void *) &MSO(p).first;
         *uptr += offs;
     }
 }
@@ -2490,6 +2487,13 @@ offset_mqueue(Process *p, Sint offs, char* area, Uint area_size)
 	if (is_boxed(mesg) && in_area(ptr_val(mesg), area, area_size)) {
 	    ERL_MESSAGE_TOKEN(mp) = offset_ptr(mesg, offs);
         }
+#ifdef USE_VM_PROBES
+	mesg = ERL_MESSAGE_DT_UTAG(mp);
+	if (is_boxed(mesg) && in_area(ptr_val(mesg), area, area_size)) {
+	    ERL_MESSAGE_DT_UTAG(mp) = offset_ptr(mesg, offs);
+        }
+#endif	
+	
         ASSERT((is_nil(ERL_MESSAGE_TOKEN(mp)) ||
 		is_tuple(ERL_MESSAGE_TOKEN(mp)) ||
 		is_atom(ERL_MESSAGE_TOKEN(mp))));
@@ -2509,6 +2513,9 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size,
     offset_heap_ptr(&p->fvalue, 1, offs, area, area_size);
     offset_heap_ptr(&p->ftrace, 1, offs, area, area_size);
     offset_heap_ptr(&p->seq_trace_token, 1, offs, area, area_size);
+#ifdef USE_VM_PROBES
+    offset_heap_ptr(&p->dt_utag, 1, offs, area, area_size);
+#endif
     offset_heap_ptr(&p->group_leader, 1, offs, area, area_size);
     offset_mqueue(p, offs, area, area_size);
     offset_heap_ptr(p->stop, (STACK_START(p) - p->stop), offs, area, area_size);
@@ -2542,7 +2549,7 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop)
         return 1;
     }
     while (bp != NULL) {
-        if (bp->mem <= ptr && ptr < bp->mem + bp->size) {
+        if (bp->mem <= ptr && ptr < bp->mem + bp->used_size) {
             return 1;
         }
         bp = bp->next;
@@ -2556,7 +2563,7 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop)
 		hfp = erts_dist_ext_trailer(mp->data.dist_ext);
 	    else
 		hfp = NULL;
-	    if (hfp && hfp->mem <= ptr && ptr < hfp->mem + hfp->size)
+	    if (hfp && hfp->mem <= ptr && ptr < hfp->mem + hfp->used_size)
 		return 1;
 	}
         mp = mp->next;
@@ -2582,8 +2589,8 @@ do {							\
 		 __FILE__, __LINE__, #EXP);		\
 } while (0)
 
-#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST
-#  define ERTS_EXTERNAL_VISITED_BIT ((Eterm) 1 << 31)
+#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST
+#  define ERTS_OFFHEAP_VISITED_BIT ((Eterm) 1 << 31)
 #endif
 
 
@@ -2593,62 +2600,45 @@ erts_check_off_heap2(Process *p, Eterm *htop)
     Eterm *oheap = (Eterm *) OLD_HEAP(p);
     Eterm *ohtop = (Eterm *) OLD_HTOP(p);
     int old;
-    ProcBin *pb;
-    ErlFunThing *eft;
-    ExternalThing *et;
+    union erl_off_heap_ptr u;
 
     old = 0;
-    for (pb = MSO(p).mso; pb; pb = pb->next) {
-	Eterm *ptr = (Eterm *) pb;
-	long refc = erts_refc_read(&pb->val->refc, 1);
+    for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next) {
+	erts_aint_t refc;
+	switch (thing_subtag(u.hdr->thing_word)) {
+	case REFC_BINARY_SUBTAG:
+	    refc = erts_refc_read(&u.pb->val->refc, 1);		
+	    break;
+	case FUN_SUBTAG:
+	    refc = erts_refc_read(&u.fun->fe->refc, 1);
+	    break;
+	case EXTERNAL_PID_SUBTAG:
+	case EXTERNAL_PORT_SUBTAG:
+	case EXTERNAL_REF_SUBTAG:
+	    refc = erts_refc_read(&u.ext->node->refc, 1);
+	    break;
+	default:
+	    ASSERT(!!"erts_check_off_heap2: Invalid thing_word");
+	}
 	ERTS_CHK_OFFHEAP_ASSERT(refc >= 1);
+#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST
+	ERTS_CHK_OFFHEAP_ASSERT(!(u.hdr->thing_word & ERTS_EXTERNAL_VISITED_BIT));
+	u.hdr->thing_word |= ERTS_OFFHEAP_VISITED_BIT;
+#endif
 	if (old) {
-	    ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop);
+	    ERTS_CHK_OFFHEAP_ASSERT(oheap <= u.ep && u.ep < ohtop);
 	}
-	else if (oheap <= ptr && ptr < ohtop)
+	else if (oheap <= u.ep && u.ep < ohtop)
 	    old = 1;
 	else {
-	    ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop));
+	    ERTS_CHK_OFFHEAP_ASSERT(within2(u.ep, p, htop));
 	}
     }
 
-    old = 0;
-    for (eft = MSO(p).funs; eft; eft = eft->next) {
-	Eterm *ptr = (Eterm *) eft;
-	long refc = erts_refc_read(&eft->fe->refc, 1);
-	ERTS_CHK_OFFHEAP_ASSERT(refc >= 1);
-	if (old)
-	    ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop);
-	else if (oheap <= ptr && ptr < ohtop)
-	    old = 1;
-	else
-	    ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop));
-    }
-
-    old = 0;
-    for (et = MSO(p).externals; et; et = et->next) {
-	Eterm *ptr = (Eterm *) et;
-	long refc = erts_refc_read(&et->node->refc, 1);
-	ERTS_CHK_OFFHEAP_ASSERT(refc >= 1);
-#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST
-	ERTS_CHK_OFFHEAP_ASSERT(!(et->header & ERTS_EXTERNAL_VISITED_BIT));
-#endif
-	if (old)
-	    ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop);
-	else if (oheap <= ptr && ptr < ohtop)
-	    old = 1;
-	else
-	    ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop));
-#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST
-	et->header |= ERTS_EXTERNAL_VISITED_BIT;
-#endif
-    }
-
 #ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST
-    for (et = MSO(p).externals; et; et = et->next)
-	et->header &= ~ERTS_EXTERNAL_VISITED_BIT;
+    for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next)
+	u.hdr->thing_word &= ~ERTS_OFFHEAP_VISITED_BIT;
 #endif
-	
 }
 
 void
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index af55b6363f..1801df359a 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2007-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2007-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -22,11 +22,12 @@
 
 /* GC declarations shared by beam/erl_gc.c and hipe/hipe_gc.c */
 
-#ifdef DEBUG
+#if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF
 #  define HARDDEBUG 1
 #endif
 
-#define IS_MOVED(x)	(!is_header((x)))
+#define IS_MOVED_BOXED(x)	(!is_header((x)))
+#define IS_MOVED_CONS(x)	(is_non_value((x)))
 
 #define MOVE_CONS(PTR,CAR,HTOP,ORIG)					\
 do {									\
@@ -61,7 +62,7 @@ do {									\
 } while(0)
 
 #define in_area(ptr,start,nbytes) \
- ((unsigned long)((char*)(ptr) - (char*)(start)) < (nbytes))
+ ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
 
 extern Uint erts_test_long_gc_sleep;
 
@@ -69,4 +70,28 @@ extern Uint erts_test_long_gc_sleep;
 int within(Eterm *ptr, Process *p);
 #endif
 
+ERTS_GLB_INLINE Eterm follow_moved(Eterm term);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE Eterm follow_moved(Eterm term)
+{
+    Eterm* ptr;
+    switch (primary_tag(term)) {
+    case TAG_PRIMARY_IMMED1:
+	break;
+    case TAG_PRIMARY_BOXED:
+	ptr = boxed_val(term);
+	if (IS_MOVED_BOXED(*ptr)) term = *ptr;
+	break;
+    case TAG_PRIMARY_LIST:
+	ptr = list_val(term);
+	if (IS_MOVED_CONS(ptr[0])) term = ptr[1];
+	break;
+    default:
+	ASSERT(!"strange tag in follow_moved");
+    }
+    return term;
+}
+#endif
+
 #endif /* __ERL_GC_H__ */
diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c
index ea2ba4d55c..e7d4ac2b67 100644
--- a/erts/emulator/beam/erl_goodfit_alloc.c
+++ b/erts/emulator/beam/erl_goodfit_alloc.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -49,30 +49,30 @@
 #define MIN_MBC_FIRST_FREE_SZ		(4*1024)
 
 #define MAX_SUB_MASK_IX \
-  ((((Uint)1) << (NO_OF_BKT_IX_BITS - SUB_MASK_IX_SHIFT)) - 1)
-#define MAX_SUB_BKT_IX ((((Uint)1) << SUB_MASK_IX_SHIFT) - 1)
+  ((((UWord)1) << (NO_OF_BKT_IX_BITS - SUB_MASK_IX_SHIFT)) - 1)
+#define MAX_SUB_BKT_IX ((((UWord)1) << SUB_MASK_IX_SHIFT) - 1)
 #define MAX_BKT_IX (NO_OF_BKTS - 1)
 
-#define MIN_BLK_SZ  UNIT_CEILING(sizeof(GFFreeBlock_t) + sizeof(Uint))
+#define MIN_BLK_SZ  UNIT_CEILING(sizeof(GFFreeBlock_t) + sizeof(UWord))
 
-#define IX2SBIX(IX) ((IX) & (~(~((Uint)0) << SUB_MASK_IX_SHIFT)))
+#define IX2SBIX(IX) ((IX) & (~(~((UWord)0) << SUB_MASK_IX_SHIFT)))
 #define IX2SMIX(IX) ((IX) >> SUB_MASK_IX_SHIFT)
 #define MAKE_BKT_IX(SMIX, SBIX)	\
-  ((((Uint)(SMIX)) << SUB_MASK_IX_SHIFT) | ((Uint)(SBIX)))
+  ((((UWord)(SMIX)) << SUB_MASK_IX_SHIFT) | ((UWord)(SBIX)))
 
 #define SET_BKT_MASK_IX(BM, IX)						\
 do {									\
     int sub_mask_ix__ = IX2SMIX((IX));					\
-    (BM).main |= (((Uint) 1) << sub_mask_ix__);				\
-    (BM).sub[sub_mask_ix__] |= (((Uint)1) << IX2SBIX((IX)));		\
+    (BM).main |= (((UWord) 1) << sub_mask_ix__);				\
+    (BM).sub[sub_mask_ix__] |= (((UWord)1) << IX2SBIX((IX)));		\
 } while (0)
 
 #define UNSET_BKT_MASK_IX(BM, IX)					\
 do {									\
     int sub_mask_ix__ = IX2SMIX((IX));					\
-    (BM).sub[sub_mask_ix__] &= ~(((Uint)1) << IX2SBIX((IX)));		\
+    (BM).sub[sub_mask_ix__] &= ~(((UWord)1) << IX2SBIX((IX)));		\
     if (!(BM).sub[sub_mask_ix__])					\
-	(BM).main &= ~(((Uint)1) << sub_mask_ix__);			\
+	(BM).main &= ~(((UWord)1) << sub_mask_ix__);			\
 } while (0)
 
 /* Buckets ... */
@@ -163,10 +163,10 @@ BKT_MIN_SZ(GFAllctr_t *gfallctr, int ix)
 
 /* Prototypes of callback functions */
 static Block_t *	get_free_block		(Allctr_t *, Uint,
-						 Block_t *, Uint);
-static void		link_free_block		(Allctr_t *, Block_t *);
-static void		unlink_free_block	(Allctr_t *, Block_t *);
-static void		update_last_aux_mbc	(Allctr_t *, Carrier_t *);
+						 Block_t *, Uint, Uint32);
+static void		link_free_block		(Allctr_t *, Block_t *, Uint32);
+static void		unlink_free_block	(Allctr_t *, Block_t *, Uint32);
+static void		update_last_aux_mbc	(Allctr_t *, Carrier_t *, Uint32);
 static Eterm		info_options		(Allctr_t *, char *, int *,
 						 void *, Uint **, Uint *);
 static void		init_atoms		(void);
@@ -190,14 +190,20 @@ erts_gfalc_start(GFAllctr_t *gfallctr,
 		 GFAllctrInit_t *gfinit,
 		 AllctrInit_t *init)
 {
-    GFAllctr_t nulled_state = {{0}};
-    /* {{0}} is used instead of {0}, in order to avoid (an incorrect) gcc
-       warning. gcc warns if {0} is used as initializer of a struct when
-       the first member is a struct (not if, for example, the third member
-       is a struct). */
+    struct {
+	int dummy;
+	GFAllctr_t allctr;
+    } zero = {0};
+    /* The struct with a dummy element first is used in order to avoid (an
+       incorrect) gcc warning. gcc warns if {0} is used as initializer of
+       a struct when the first member is a struct (not if, for example,
+       the third member is a struct). */
+
     Allctr_t *allctr = (Allctr_t *) gfallctr;
 
-    sys_memcpy((void *) gfallctr, (void *) &nulled_state, sizeof(GFAllctr_t));
+    sys_memcpy((void *) gfallctr, (void *) &zero.allctr, sizeof(GFAllctr_t));
+
+    init->sbmbct = 0; /* Small mbc not yet supported by goodfit */
 
     allctr->mbc_header_size		= sizeof(Carrier_t);
     allctr->min_mbc_size		= MIN_MBC_SZ;
@@ -263,8 +269,8 @@ find_bucket(BucketMask_t *bmask, int min_index)
     while(max != min) {					\
 	mid = ((max - min) >> 1) + min;			\
 	if((BitMask)					\
-	   & (~(~((Uint) 0) << (mid + 1)))		\
-	   & (~((Uint) 0) << min))			\
+	   & (~(~((UWord) 0) << (mid + 1)))		\
+	   & (~((UWord) 0) << min))			\
 	    max = mid;					\
 	else						\
 	    min = mid + 1;				\
@@ -272,21 +278,21 @@ find_bucket(BucketMask_t *bmask, int min_index)
     (MinBit) = min
 
 
-    ASSERT(bmask->main < (((Uint) 1) << (MAX_SUB_MASK_IX+1)));
+    ASSERT(bmask->main < (((UWord) 1) << (MAX_SUB_MASK_IX+1)));
 
     sub_mask_ix = IX2SMIX(min_index);
 
-    if ((bmask->main & (~((Uint) 0) << sub_mask_ix)) == 0)
+    if ((bmask->main & (~((UWord) 0) << sub_mask_ix)) == 0)
 	return -1;
 
     /* There exists a non empty bucket; find it... */
 
-    if (bmask->main & (((Uint) 1) << sub_mask_ix)) {
+    if (bmask->main & (((UWord) 1) << sub_mask_ix)) {
 	sub_bkt_ix = IX2SBIX(min_index);
-	if ((bmask->sub[sub_mask_ix] & (~((Uint) 0) << sub_bkt_ix)) == 0) {
+	if ((bmask->sub[sub_mask_ix] & (~((UWord) 0) << sub_bkt_ix)) == 0) {
 	    sub_mask_ix++;
 	    sub_bkt_ix = 0;
-	    if ((bmask->main & (~((Uint) 0)<< sub_mask_ix)) == 0)
+	    if ((bmask->main & (~((UWord) 0)<< sub_mask_ix)) == 0)
 		return -1;
 	}
 	else
@@ -299,17 +305,17 @@ find_bucket(BucketMask_t *bmask, int min_index)
 
     ASSERT(sub_mask_ix <= MAX_SUB_MASK_IX);
     /* Has to be a bit > sub_mask_ix */
-    ASSERT(bmask->main & (~((Uint) 0) << (sub_mask_ix)));
+    ASSERT(bmask->main & (~((UWord) 0) << (sub_mask_ix)));
     GET_MIN_BIT(sub_mask_ix, bmask->main, sub_mask_ix, MAX_SUB_MASK_IX);
 
  find_sub_bkt_ix:
     ASSERT(sub_mask_ix <= MAX_SUB_MASK_IX);
     ASSERT(sub_bkt_ix <= MAX_SUB_BKT_IX);
 
-    if ((bmask->sub[sub_mask_ix] & (((Uint) 1) << sub_bkt_ix)) == 0) {
+    if ((bmask->sub[sub_mask_ix] & (((UWord) 1) << sub_bkt_ix)) == 0) {
 	ASSERT(sub_mask_ix + 1 <= MAX_SUB_BKT_IX);
 	/* Has to be a bit > sub_bkt_ix */
-	ASSERT(bmask->sub[sub_mask_ix] & (~((Uint) 0) << sub_bkt_ix));
+	ASSERT(bmask->sub[sub_mask_ix] & (~((UWord) 0) << sub_bkt_ix));
 
 	GET_MIN_BIT(sub_bkt_ix,
 		    bmask->sub[sub_mask_ix],
@@ -336,7 +342,7 @@ search_bucket(Allctr_t *allctr, int ix, Uint size)
     Uint min_sz;
     Uint blk_sz;
     Uint cand_sz = 0;
-    Uint max_blk_search;
+    UWord max_blk_search;
     GFFreeBlock_t *blk;
     GFFreeBlock_t *cand = NULL;
     int blk_on_lambc;
@@ -379,7 +385,7 @@ search_bucket(Allctr_t *allctr, int ix, Uint size)
 
 static Block_t *
 get_free_block(Allctr_t *allctr, Uint size,
-	       Block_t *cand_blk, Uint cand_size)
+	       Block_t *cand_blk, Uint cand_size, Uint32 flags)
 {
     GFAllctr_t *gfallctr = (GFAllctr_t *) allctr;
     int unsafe_bi, min_bi;
@@ -398,7 +404,7 @@ get_free_block(Allctr_t *allctr, Uint size,
 	if (blk) {
 	    if (cand_blk && cand_size <= BLK_SZ(blk))
 		return NULL; /* cand_blk was better */
-	    unlink_free_block(allctr, blk);
+	    unlink_free_block(allctr, blk, flags);
 	    return blk;
 	}
 	if (min_bi < NO_OF_BKTS - 1) {
@@ -418,14 +424,14 @@ get_free_block(Allctr_t *allctr, Uint size,
     ASSERT(blk);
     if (cand_blk && cand_size <= BLK_SZ(blk))
 	return NULL; /* cand_blk was better */
-    unlink_free_block(allctr, blk);
+    unlink_free_block(allctr, blk, flags);
     return blk;
 }
 
 
 
 static void
-link_free_block(Allctr_t *allctr, Block_t *block)
+link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
 {
     GFAllctr_t *gfallctr = (GFAllctr_t *) allctr;
     GFFreeBlock_t *blk = (GFFreeBlock_t *) block;
@@ -446,7 +452,7 @@ link_free_block(Allctr_t *allctr, Block_t *block)
 }
 
 static void
-unlink_free_block(Allctr_t *allctr, Block_t *block)
+unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
 {
     GFAllctr_t *gfallctr = (GFAllctr_t *) allctr;
     GFFreeBlock_t *blk = (GFFreeBlock_t *) block;
@@ -467,7 +473,7 @@ unlink_free_block(Allctr_t *allctr, Block_t *block)
 }
 
 static void
-update_last_aux_mbc(Allctr_t *allctr, Carrier_t *mbc)
+update_last_aux_mbc(Allctr_t *allctr, Carrier_t *mbc, Uint32 flags)
 {
     GFAllctr_t *gfallctr = (GFAllctr_t *) allctr;
 
@@ -615,9 +621,9 @@ check_block(Allctr_t *allctr, Block_t * blk, int free_block)
 	Uint blk_sz = BLK_SZ(blk);
 	bi = BKT_IX(gfallctr, blk_sz);
 
-	ASSERT(gfallctr->bucket_mask.main & (((Uint) 1) << IX2SMIX(bi)));
+	ASSERT(gfallctr->bucket_mask.main & (((UWord) 1) << IX2SMIX(bi)));
 	ASSERT(gfallctr->bucket_mask.sub[IX2SMIX(bi)]
-	       & (((Uint) 1) << IX2SBIX(bi)));
+	       & (((UWord) 1) << IX2SBIX(bi)));
 		
 	found = 0;
 	for (fblk = gfallctr->buckets[bi]; fblk; fblk = fblk->next)
@@ -648,9 +654,9 @@ check_mbc(Allctr_t *allctr, Carrier_t *mbc)
     int bi;
 
     for(bi = 0; bi < NO_OF_BKTS; bi++) {
-	if ((gfallctr->bucket_mask.main & (((Uint) 1) << IX2SMIX(bi)))
+	if ((gfallctr->bucket_mask.main & (((UWord) 1) << IX2SMIX(bi)))
 	    && (gfallctr->bucket_mask.sub[IX2SMIX(bi)]
-		& (((Uint) 1) << IX2SBIX(bi)))) {
+		& (((UWord) 1) << IX2SBIX(bi)))) {
 	    ASSERT(gfallctr->buckets[bi] != NULL);
 	}
 	else {
diff --git a/erts/emulator/beam/erl_goodfit_alloc.h b/erts/emulator/beam/erl_goodfit_alloc.h
index 3d1b8c01f6..a554a6f466 100644
--- a/erts/emulator/beam/erl_goodfit_alloc.h
+++ b/erts/emulator/beam/erl_goodfit_alloc.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -28,7 +28,7 @@
 typedef struct GFAllctr_t_ GFAllctr_t;
 
 typedef struct {
-    Uint mbsd;
+    UWord mbsd;
 } GFAllctrInit_t;
 
 #define ERTS_DEFAULT_GF_ALLCTR_INIT {                                      \
@@ -49,18 +49,18 @@ Allctr_t *erts_gfalc_start(GFAllctr_t *, GFAllctrInit_t *, AllctrInit_t *);
 #include "erl_alloc_util.h"
 
 #define NO_OF_BKT_IX_BITS (8)
-#ifdef ARCH_64
+#if defined(ARCH_64)
 #  define SUB_MASK_IX_SHIFT (6)
 #else
 #  define SUB_MASK_IX_SHIFT (5)
 #endif
-#define NO_OF_BKTS (((Uint) 1) << NO_OF_BKT_IX_BITS)
-#define NO_OF_SUB_MASKS (NO_OF_BKTS/(((Uint) 1) << SUB_MASK_IX_SHIFT))
+#define NO_OF_BKTS (((UWord) 1) << NO_OF_BKT_IX_BITS)
+#define NO_OF_SUB_MASKS (NO_OF_BKTS/(((UWord) 1) << SUB_MASK_IX_SHIFT))
 
 typedef struct {
-    Uint main;
-    Uint sub[NO_OF_SUB_MASKS];
-} BucketMask_t; 
+    UWord main;
+    UWord sub[NO_OF_SUB_MASKS];
+} BucketMask_t;
 
 typedef struct GFFreeBlock_t_ GFFreeBlock_t;
 struct GFFreeBlock_t_ {
@@ -74,11 +74,11 @@ struct GFAllctr_t_ {
 
     char *		last_aux_mbc_start;
     char *		last_aux_mbc_end;
-    Uint		bkt_max_size_d;
-    Uint		bkt_intrvl_d;
+    UWord		bkt_max_size_d;
+    UWord		bkt_intrvl_d;
     BucketMask_t	bucket_mask;
     GFFreeBlock_t *	buckets[NO_OF_BKTS];
-    Uint 		max_blk_search;
+    UWord 		max_blk_search;
 
 };
 
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index e97ab328cd..02164728fe 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1997-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -34,13 +34,16 @@
 #include "erl_binary.h"
 #include "dist.h"
 #include "erl_mseg.h"
-#include "erl_nmgc.h"
 #include "erl_threads.h"
 #include "erl_bif_timer.h"
 #include "erl_instrument.h"
 #include "erl_printf_term.h"
 #include "erl_misc_utils.h"
 #include "packet_parser.h"
+#include "erl_cpu_topology.h"
+#include "erl_thr_progress.h"
+#include "erl_thr_queue.h"
+#include "erl_async.h"
 
 #ifdef HIPE
 #include "hipe_mode_switch.h"	/* for hipe_mode_switch_init() */
@@ -63,10 +66,15 @@ extern void ConNormalExit(void);
 extern void ConWaitForExit(void);
 #endif
 
+static void erl_init(int ncpu);
+
 #define ERTS_MIN_COMPAT_REL 7
 
+static erts_atomic_t exiting;
+
 #ifdef ERTS_SMP
-erts_smp_atomic_t erts_writing_erl_crash_dump;
+erts_smp_atomic32_t erts_writing_erl_crash_dump;
+erts_tsd_key_t erts_is_crash_dumping_key;
 #else
 volatile int erts_writing_erl_crash_dump = 0;
 #endif
@@ -76,8 +84,6 @@ int erts_initialized = 0;
 static erts_tid_t main_thread;
 #endif
 
-erts_cpu_info_t *erts_cpuinfo;
-
 int erts_use_sender_punish;
 
 /*
@@ -85,7 +91,6 @@ int erts_use_sender_punish;
  */
 
 Uint display_items;	    	/* no of items to display in traces etc */
-Uint display_loads;		/* print info about loaded modules */
 int H_MIN_SIZE;			/* The minimum heap grain */
 int BIN_VH_MIN_SIZE;		/* The minimum binary virtual*/
 
@@ -97,9 +102,7 @@ int erts_backtrace_depth;	/* How many functions to show in a backtrace
 				 * in error codes.
 				 */
 
-int erts_async_max_threads;  /* number of threads for async support */
-int erts_async_thread_suggested_stack_size;
-erts_smp_atomic_t erts_max_gen_gcs;
+erts_smp_atomic32_t erts_max_gen_gcs;
 
 Eterm erts_error_logger_warnings; /* What to map warning logs to, am_error, 
 				     am_info or am_warning, am_error is 
@@ -107,7 +110,6 @@ Eterm erts_error_logger_warnings; /* What to map warning logs to, am_error,
 
 int erts_compat_rel;
 
-static int use_multi_run_queue;
 static int no_schedulers;
 static int no_schedulers_online;
 
@@ -125,6 +127,8 @@ int erts_modified_timing_level;
 
 int erts_no_crash_dump = 0;	/* Use -d to suppress crash dump. */
 
+int erts_no_line_info = 0;	/* -L: Don't load line information */
+
 /*
  * Other global variables.
  */
@@ -149,28 +153,6 @@ Export *erts_delay_trap = NULL;
 
 int erts_use_r9_pids_ports;
 
-#ifdef HYBRID
-Eterm *global_heap;
-Eterm *global_hend;
-Eterm *global_htop;
-Eterm *global_saved_htop;
-Eterm *global_old_heap;
-Eterm *global_old_hend;
-ErlOffHeap erts_global_offheap;
-Uint   global_heap_sz = SH_DEFAULT_SIZE;
-
-#ifndef INCREMENTAL
-Eterm *global_high_water;
-Eterm *global_old_htop;
-#endif
-
-Uint16 global_gen_gcs;
-Uint16 global_max_gen_gcs;
-Uint   global_gc_flags;
-
-Uint   global_heap_min_sz = SH_DEFAULT_SIZE;
-#endif
-
 int ignore_break;
 int replace_intr;
 
@@ -228,32 +210,30 @@ void erl_error(char *fmt, va_list args)
     erts_vfprintf(stderr, fmt, args);
 }
 
-static void early_init(int *argc, char **argv);
+static int early_init(int *argc, char **argv);
 
 void
 erts_short_init(void)
 {
-    early_init(NULL, NULL);
-    erl_init();
+    int ncpu = early_init(NULL, NULL);
+    erl_init(ncpu);
     erts_initialized = 1;
 }
 
-void
-erl_init(void)
+static void
+erl_init(int ncpu)
 {
     init_benchmarking();
 
-#ifdef ERTS_SMP
-    erts_system_block_init();
-#endif
-
     erts_init_monitors();
     erts_init_gc();
-    init_time();
-    erts_init_process();
-    erts_init_scheduling(use_multi_run_queue,
-			 no_schedulers,
+    erts_init_time();
+    erts_init_sys_common_misc();
+    erts_init_process(ncpu);
+    erts_init_scheduling(no_schedulers,
 			 no_schedulers_online);
+    erts_init_cpu_topology(); /* Must be after init_scheduling */
+    erts_alloc_late_init();
 
     H_MIN_SIZE      = erts_next_heap_size(H_MIN_SIZE, 0);
     BIN_VH_MIN_SIZE = erts_next_heap_size(BIN_VH_MIN_SIZE, 0);
@@ -276,122 +256,28 @@ erl_init(void)
     erts_init_node_tables();
     init_dist();
     erl_drv_thr_init();
+    erts_init_async();
     init_io();
-    init_copy();
     init_load();
     erts_init_bif();
     erts_init_bif_chksum();
+    erts_init_bif_binary();
     erts_init_bif_re();
     erts_init_unicode(); /* after RE to get access to PCRE unicode */
     erts_delay_trap = erts_export_put(am_erlang, am_delay_trap, 2);
     erts_late_init_process();
 #if HAVE_ERTS_MSEG
-    erts_mseg_late_init(); /* Must be after timer (init_time()) and thread
+    erts_mseg_late_init(); /* Must be after timer (erts_init_time()) and thread
 			      initializations */
 #endif
 #ifdef HIPE
     hipe_mode_switch_init(); /* Must be after init_load/beam_catches/init */
 #endif
-#ifdef _OSE_
-    erl_sys_init_final();
-#endif
     packet_parser_init();
     erl_nif_init();
 }
 
 static void
-init_shared_memory(int argc, char **argv)
-{
-#ifdef HYBRID
-    int arg_size = 0;
-
-    global_heap_sz = erts_next_heap_size(global_heap_sz,0);
-
-    /* Make sure arguments will fit on the heap, no one else will check! */
-    while (argc--)
-        arg_size += 2 + strlen(argv[argc]);
-    if (global_heap_sz < arg_size)
-        global_heap_sz = erts_next_heap_size(arg_size,1);
-
-#ifndef INCREMENTAL
-    global_heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP,
-					    sizeof(Eterm) * global_heap_sz);
-    global_hend = global_heap + global_heap_sz;
-    global_htop = global_heap;
-    global_high_water = global_heap;
-    global_old_hend = global_old_htop = global_old_heap = NULL;
-#endif
-
-    global_gen_gcs = 0;
-    global_max_gen_gcs = erts_smp_atomic_read(&erts_max_gen_gcs);
-    global_gc_flags = erts_default_process_flags;
-
-    erts_global_offheap.mso = NULL;
-#ifndef HYBRID /* FIND ME! */
-    erts_global_offheap.funs = NULL;
-#endif
-    erts_global_offheap.overhead = 0;
-#endif
-
-#ifdef INCREMENTAL
-    erts_init_incgc();
-#endif
-}
-
-
-/*
- * Create the very first process.
- */
-
-void
-erts_first_process(Eterm modname, void* code, unsigned size, int argc, char** argv)
-{
-    int i;
-    Eterm args;
-    Eterm pid;
-    Eterm* hp;
-    Process parent;
-    Process* p;
-    ErlSpawnOpts so;
-    
-    if (erts_find_function(modname, am_start, 1) == NULL) {
-	char sbuf[256];
-	Atom* ap;
-
-	ap = atom_tab(atom_val(modname));
-	memcpy(sbuf, ap->name, ap->len);
-	sbuf[ap->len] = '\0';
-	erl_exit(5, "No function %s:start/1\n", sbuf);
-    }
-
-    /*
-     * We need a dummy parent process to be able to call erl_create_process().
-     */
-    erts_init_empty_process(&parent);
-    hp = HAlloc(&parent, argc*2 + 4);
-    args = NIL;
-    for (i = argc-1; i >= 0; i--) {
-	int len = sys_strlen(argv[i]);
-	args = CONS(hp, new_binary(&parent, (byte*)argv[i], len), args);
-	hp += 2;
-    }
-    args = CONS(hp, new_binary(&parent, code, size), args);
-    hp += 2;
-    args = CONS(hp, args, NIL);
-
-    so.flags = 0;
-    pid = erl_create_process(&parent, modname, am_start, args, &so);
-    p = process_tab[internal_pid_index(pid)];
-    p->group_leader = pid;
-
-    erts_cleanup_empty_process(&parent);
-}
-
-/*
- * XXX Old way of starting. Hopefully soon obsolete.
- */
-
-static void
 erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** argv)
 {
     int i;
@@ -485,7 +371,7 @@ static void
 load_preloaded(void)
 {
     int i;
-    int res;
+    Eterm res;
     Preload* preload_p;
     Eterm module_name;
     byte* code;
@@ -504,8 +390,9 @@ load_preloaded(void)
 		     name);
 	res = erts_load_module(NULL, 0, NIL, &module_name, code, length);
 	sys_preload_end(&preload_p[i]);
-	if (res < 0)
-	    erl_exit(1,"Failed loading preloaded module %s\n", name);
+	if (res != NIL)
+	    erl_exit(1,"Failed loading preloaded module %s (%T)\n",
+		     name, res);
 	i++;
     }
 }
@@ -547,8 +434,6 @@ void erts_usage(void)
 
     erts_fprintf(stderr, "-K boolean  enable or disable kernel poll\n");
 
-    erts_fprintf(stderr, "-l          turn on auto load tracing\n");
-
     erts_fprintf(stderr, "-M<X> <Y>   memory allocator switches,\n");
     erts_fprintf(stderr, "            see the erts_alloc(3) documentation for more info.\n");
 
@@ -560,10 +445,19 @@ void erts_usage(void)
 	       ERTS_MIN_COMPAT_REL, this_rel_num());
 
     erts_fprintf(stderr, "-r          force ets memory block to be moved on realloc\n");
+    erts_fprintf(stderr, "-rg amount  set reader groups limit\n");
     erts_fprintf(stderr, "-sbt type   set scheduler bind type, valid types are:\n");
     erts_fprintf(stderr, "            u|ns|ts|ps|s|nnts|nnps|tnnps|db\n");
+    erts_fprintf(stderr, "-sbwt val   set scheduler busy wait threshold, valid values are:\n");
+    erts_fprintf(stderr, "            none|very_short|short|medium|long|very_long.\n");
+    erts_fprintf(stderr, "-scl bool   enable/disable compaction of scheduler load,\n");
+    erts_fprintf(stderr, "            see the erl(1) documentation for more info.\n");
     erts_fprintf(stderr, "-sct cput   set cpu topology,\n");
     erts_fprintf(stderr, "            see the erl(1) documentation for more info.\n");
+    erts_fprintf(stderr, "-sws val    set scheduler wakeup strategy, valid values are:\n");
+    erts_fprintf(stderr, "            default|legacy|proposal.\n");
+    erts_fprintf(stderr, "-swt val    set scheduler wakeup threshold, valid values are:\n");
+    erts_fprintf(stderr, "            very_low|low|medium|high|very_high.\n");
     erts_fprintf(stderr, "-sss size   suggested stack size in kilo words for scheduler threads,\n");
     erts_fprintf(stderr, "            valid range is [%d-%d]\n",
 		 ERTS_SCHED_THREAD_MIN_STACK_SIZE,
@@ -585,7 +479,8 @@ void erts_usage(void)
 
     erts_fprintf(stderr, "-W<i|w>     set error logger warnings mapping,\n");
     erts_fprintf(stderr, "            see error_logger documentation for details\n");
-
+    erts_fprintf(stderr, "-zdbbl size set the distribution buffer busy limit in kilobytes\n");
+    erts_fprintf(stderr, "            valid range is [1-%d]\n", INT_MAX/1024);
     erts_fprintf(stderr, "\n");
     erts_fprintf(stderr, "Note that if the emulator is started with erlexec (typically\n");
     erts_fprintf(stderr, "from the erl script), these flags should be specified with +.\n");
@@ -593,7 +488,51 @@ void erts_usage(void)
     erl_exit(-1, "");
 }
 
-static void
+#ifdef USE_THREADS
+/*
+ * allocators for thread lib
+ */
+
+static void *ethr_std_alloc(size_t size)
+{
+    return erts_alloc_fnf(ERTS_ALC_T_ETHR_STD, (Uint) size);
+}
+static void *ethr_std_realloc(void *ptr, size_t size)
+{
+    return erts_realloc_fnf(ERTS_ALC_T_ETHR_STD, ptr, (Uint) size);
+}
+static void ethr_std_free(void *ptr)
+{
+    erts_free(ERTS_ALC_T_ETHR_STD, ptr);
+}
+static void *ethr_sl_alloc(size_t size)
+{
+    return erts_alloc_fnf(ERTS_ALC_T_ETHR_SL, (Uint) size);
+}
+static void *ethr_sl_realloc(void *ptr, size_t size)
+{
+    return erts_realloc_fnf(ERTS_ALC_T_ETHR_SL, ptr, (Uint) size);
+}
+static void ethr_sl_free(void *ptr)
+{
+    erts_free(ERTS_ALC_T_ETHR_SL, ptr);
+}
+static void *ethr_ll_alloc(size_t size)
+{
+    return erts_alloc_fnf(ERTS_ALC_T_ETHR_LL, (Uint) size);
+}
+static void *ethr_ll_realloc(void *ptr, size_t size)
+{
+    return erts_realloc_fnf(ERTS_ALC_T_ETHR_LL, ptr, (Uint) size);
+}
+static void ethr_ll_free(void *ptr)
+{
+    erts_free(ERTS_ALC_T_ETHR_LL, ptr);
+}
+
+#endif
+
+static int
 early_init(int *argc, char **argv) /*
 				   * Only put things here which are
 				   * really important initialize
@@ -606,11 +545,16 @@ early_init(int *argc, char **argv) /*
     int ncpuavail;
     int schdlrs;
     int schdlrs_onln;
-    use_multi_run_queue = 1;
+    int max_main_threads;
+    int max_reader_groups;
+    int reader_groups;
+    char envbuf[21]; /* enough for any 64-bit integer */
+    size_t envbufsz;
+
+    erts_sched_compact_load = 1;
     erts_printf_eterm_func = erts_printf_term;
     erts_disable_tolerant_timeofday = 0;
     display_items = 200;
-    display_loads = 0;
     erts_backtrace_depth = DEFAULT_BACKTRACE_SIZE;
     erts_async_max_threads = 0;
     erts_async_thread_suggested_stack_size = ERTS_ASYNC_THREAD_MIN_STACK_SIZE;
@@ -621,13 +565,11 @@ early_init(int *argc, char **argv) /*
 
     erts_use_sender_punish = 1;
 
-    erts_cpuinfo = erts_cpu_info_create();
-
-#ifdef ERTS_SMP
-    ncpu = erts_get_cpu_configured(erts_cpuinfo);
-    ncpuonln = erts_get_cpu_online(erts_cpuinfo);
-    ncpuavail = erts_get_cpu_available(erts_cpuinfo);
-#else
+    erts_pre_early_init_cpu_topology(&max_reader_groups,
+				     &ncpu,
+				     &ncpuonln,
+				     &ncpuavail);
+#ifndef ERTS_SMP
     ncpu = 1;
     ncpuonln = 1;
     ncpuavail = 1;
@@ -644,17 +586,23 @@ early_init(int *argc, char **argv) /*
     erts_use_r9_pids_ports = 0;
 
     erts_sys_pre_init();
+    erts_atomic_init_nob(&exiting, 0);
+#ifdef ERTS_SMP
+    erts_thr_progress_pre_init();
+#endif
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_init();
 #endif
 #ifdef ERTS_SMP
-    erts_smp_atomic_init(&erts_writing_erl_crash_dump, 0L);
+    erts_smp_atomic32_init_nob(&erts_writing_erl_crash_dump, 0L);
+    erts_tsd_key_create(&erts_is_crash_dumping_key);
 #else
     erts_writing_erl_crash_dump = 0;
 #endif
 
-    erts_smp_atomic_init(&erts_max_gen_gcs, (long)((Uint16) -1));
+    erts_smp_atomic32_init_nob(&erts_max_gen_gcs,
+			       (erts_aint32_t) ((Uint16) -1));
 
     erts_pre_init_process();
 #if defined(USE_THREADS) && !defined(ERTS_SMP)
@@ -673,6 +621,16 @@ early_init(int *argc, char **argv) /*
     schdlrs = no_schedulers;
     schdlrs_onln = no_schedulers_online;
 
+    envbufsz = sizeof(envbuf);
+
+    /* erts_sys_getenv() not initialized yet; need erts_sys_getenv__() */
+    if (erts_sys_getenv__("ERL_THREAD_POOL_SIZE", envbuf, &envbufsz) == 0)
+	erts_async_max_threads = atoi(envbuf);
+    else
+	erts_async_max_threads = 0;
+    if (erts_async_max_threads > ERTS_MAX_NO_OF_ASYNC_THREADS)
+	erts_async_max_threads = ERTS_MAX_NO_OF_ASYNC_THREADS;
+
     if (argc && argv) {
 	int i = 1;
 	while (i < *argc) {
@@ -682,6 +640,38 @@ early_init(int *argc, char **argv) /*
 	    }
 	    if (argv[i][0] == '-') {
 		switch (argv[i][1]) {
+		case 'r': {
+		    char *sub_param = argv[i]+2;
+		    if (has_prefix("g", sub_param)) {
+			char *arg = get_arg(sub_param+1, argv[i+1], &i);
+			if (sscanf(arg, "%d", &max_reader_groups) != 1) {
+			    erts_fprintf(stderr,
+					 "bad reader groups limit: %s\n", arg);
+			    erts_usage();
+			}
+			if (max_reader_groups < 0) {
+			    erts_fprintf(stderr,
+					 "bad reader groups limit: %d\n",
+					 max_reader_groups);
+			    erts_usage();
+			}
+		    }
+		    break;
+		}
+		case 'A': {
+		    /* set number of threads in thread pool */
+		    char *arg = get_arg(argv[i]+2, argv[i+1], &i);
+		    if (((erts_async_max_threads = atoi(arg)) < 0) ||
+			(erts_async_max_threads > ERTS_MAX_NO_OF_ASYNC_THREADS)) {
+			erts_fprintf(stderr,
+				     "bad number of async threads %s\n",
+				     arg);
+			erts_usage();
+			VERBOSE(DEBUG_SYSTEM, ("using %d async-threads\n",
+					       erts_async_max_threads));
+		    }
+		    break;
+		}
 		case 'S' : {
 		    int tot, onln;
 		    char *arg = get_arg(argv[i]+2, argv[i+1], &i);
@@ -741,32 +731,86 @@ early_init(int *argc, char **argv) /*
 	}
     }
 
+#ifndef USE_THREADS
+    erts_async_max_threads = 0;
+#endif
+
 #ifdef ERTS_SMP
     no_schedulers = schdlrs;
     no_schedulers_online = schdlrs_onln;
 
     erts_no_schedulers = (Uint) no_schedulers;
 #endif
+    erts_early_init_scheduling(no_schedulers);
 
+    alloc_opts.ncpu = ncpu;
     erts_alloc_init(argc, argv, &alloc_opts); /* Handles (and removes)
 						 -M flags. */
+    /* Require allocators */
+#ifdef ERTS_SMP
+    /*
+     * Thread progress management:
+     *
+     * * Managed threads:
+     * ** Scheduler threads (see erl_process.c)
+     * ** Aux thread (see erl_process.c)
+     * ** Sys message dispatcher thread (see erl_trace.c)
+     *
+     * * Unmanaged threads that need to register:
+     * ** Async threads (see erl_async.c)
+     */
+    erts_thr_progress_init(no_schedulers,
+			   no_schedulers+2,
+			   erts_async_max_threads);
+#endif
+    erts_thr_q_init();
+    erts_init_utils();
+    erts_early_init_cpu_topology(no_schedulers,
+				 &max_main_threads,
+				 max_reader_groups,
+				 &reader_groups);
 
-    erts_early_init_scheduling(); /* Require allocators */
-    erts_init_utils(); /* Require allocators */
+#ifdef USE_THREADS
+    {
+	erts_thr_late_init_data_t elid = ERTS_THR_LATE_INIT_DATA_DEF_INITER;
+	elid.mem.std.alloc = ethr_std_alloc;
+	elid.mem.std.realloc = ethr_std_realloc;
+	elid.mem.std.free = ethr_std_free;
+	elid.mem.sl.alloc = ethr_sl_alloc;
+	elid.mem.sl.realloc = ethr_sl_realloc;
+	elid.mem.sl.free = ethr_sl_free;
+	elid.mem.ll.alloc = ethr_ll_alloc;
+	elid.mem.ll.realloc = ethr_ll_realloc;
+	elid.mem.ll.free = ethr_ll_free;
+	elid.main_threads = max_main_threads;
+	elid.reader_groups = reader_groups;
+
+	erts_thr_late_init(&elid);
+    }
+#endif
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_late_init();
 #endif
+    
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_late_init();
+#endif
 
 #if defined(HIPE)
     hipe_signal_init();	/* must be done very early */
 #endif
-    erl_sys_init();
 
     erl_sys_args(argc, argv);
 
+    /* Creates threads on Windows that depend on the arguments, so has to be after erl_sys_args */
+    erl_sys_init();
+
     erts_ets_realloc_always_moves = 0;
+    erts_ets_always_compress = 0;
+    erts_dist_buf_busy_limit = ERTS_DE_BUSY_LIMIT;
 
+    return ncpu;
 }
 
 #ifndef ERTS_SMP
@@ -799,9 +843,7 @@ erl_start(int argc, char **argv)
     int have_break_handler = 1;
     char envbuf[21]; /* enough for any 64-bit integer */
     size_t envbufsz;
-    int async_max_threads = erts_async_max_threads;
-
-    early_init(&argc, argv);
+    int ncpu = early_init(&argc, argv);
 
     envbufsz = sizeof(envbuf);
     if (erts_sys_getenv(ERL_MAX_ETS_TABLES_ENV, envbuf, &envbufsz) == 0)
@@ -812,14 +854,16 @@ erl_start(int argc, char **argv)
     envbufsz = sizeof(envbuf);
     if (erts_sys_getenv("ERL_FULLSWEEP_AFTER", envbuf, &envbufsz) == 0) {
 	Uint16 max_gen_gcs = atoi(envbuf);
-	erts_smp_atomic_set(&erts_max_gen_gcs, (long) max_gen_gcs);
+	erts_smp_atomic32_set_nob(&erts_max_gen_gcs,
+				  (erts_aint32_t) max_gen_gcs);
     }
 
-    envbufsz = sizeof(envbuf);
-    if (erts_sys_getenv("ERL_THREAD_POOL_SIZE", envbuf, &envbufsz) == 0) {
-	async_max_threads = atoi(envbuf);
-    }
-    
+#if (defined(__APPLE__) && defined(__MACH__)) || defined(__DARWIN__)
+    /*
+     * The default stack size on MacOS X is too small for pcre.
+     */
+    erts_sched_thread_suggested_stack_size = 256;
+#endif
 
 #ifdef DEBUG
     verbose = DEBUG_DEFAULT;
@@ -858,11 +902,30 @@ erl_start(int argc, char **argv)
 	    VERBOSE(DEBUG_SYSTEM,
                     ("using display items %d\n",display_items));
 	    break;
-
-	case 'l':
-	    display_loads++;
+	case 'f':
+	    if (!strncmp(argv[i],"-fn",3)) {
+		arg = get_arg(argv[i]+3, argv[i+1], &i);
+		switch (*arg) {
+		case 'u':
+		    erts_set_user_requested_filename_encoding(ERL_FILENAME_UTF8);
+		    break;
+		case 'l':
+		    erts_set_user_requested_filename_encoding(ERL_FILENAME_LATIN1);
+		    break;
+		case 'a':
+		    erts_set_user_requested_filename_encoding(ERL_FILENAME_UNKNOWN);
+		default:
+		    erts_fprintf(stderr, "bad filename encoding %s, can be (l,u or a)\n", arg);
+		    erts_usage();
+		}
+		break;
+	    } else {
+		erts_fprintf(stderr, "%s unknown flag %s\n", argv[0], argv[i]);
+		erts_usage();
+	    }
+	case 'L':
+	    erts_no_line_info = 1;
 	    break;
-
 	case 'v':
 #ifdef DEBUG
 	    if (argv[i][2] == '\0') {
@@ -873,7 +936,6 @@ erl_start(int argc, char **argv)
 		    switch (*ch) {
 		    case 's': verbose |= DEBUG_SYSTEM; break;
 		    case 'g': verbose |= DEBUG_PRIVATE_GC; break;
-		    case 'h': verbose |= DEBUG_HYBRID_GC; break;
 		    case 'M': verbose |= DEBUG_MEMORY; break;
 		    case 'a': verbose |= DEBUG_ALLOCATION; break;
 		    case 't': verbose |= DEBUG_THREADS; break;
@@ -886,7 +948,6 @@ erl_start(int argc, char **argv)
             erts_printf("Verbose level: ");
             if (verbose & DEBUG_SYSTEM) erts_printf("SYSTEM ");
             if (verbose & DEBUG_PRIVATE_GC) erts_printf("PRIVATE_GC ");
-            if (verbose & DEBUG_HYBRID_GC) erts_printf("HYBRID_GC ");
             if (verbose & DEBUG_MEMORY) erts_printf("PARANOID_MEMORY ");
 	    if (verbose & DEBUG_ALLOCATION) erts_printf("ALLOCATION ");
 	    if (verbose & DEBUG_THREADS) erts_printf("THREADS ");
@@ -914,12 +975,6 @@ erl_start(int argc, char **argv)
 #ifdef HIPE
 		strcat(tmp, ",HIPE");
 #endif
-#ifdef INCREMENTAL
-		strcat(tmp, ",INCREMENTAL_GC");
-#endif
-#ifdef HYBRID
-                strcat(tmp, ",HYBRID");
-#endif
 		erts_fprintf(stderr, "Erlang ");
 		if (tmp[1]) {
 		    erts_fprintf(stderr, "(%s) ", tmp+1);
@@ -980,15 +1035,20 @@ erl_start(int argc, char **argv)
 	    break;
 
 	case 'e':
-	    /* set maximum number of ets tables */
-	    arg = get_arg(argv[i]+2, argv[i+1], &i);
-	    if (( user_requested_db_max_tabs = atoi(arg) ) < 0) {
-		erts_fprintf(stderr, "bad maximum number of ets tables %s\n", arg);
-		erts_usage();
+	    if (sys_strcmp("c", argv[i]+2) == 0) {
+		erts_ets_always_compress = 1;
+	    }
+	    else {
+		/* set maximum number of ets tables */
+		arg = get_arg(argv[i]+2, argv[i+1], &i);
+		if (( user_requested_db_max_tabs = atoi(arg) ) < 0) {
+		    erts_fprintf(stderr, "bad maximum number of ets tables %s\n", arg);
+		    erts_usage();
+		}
+		VERBOSE(DEBUG_SYSTEM,
+			("using maximum number of ets tables %d\n",
+			 user_requested_db_max_tabs));
 	    }
-	    VERBOSE(DEBUG_SYSTEM,
-                    ("using maximum number of ets tables %d\n",
-                     user_requested_db_max_tabs));
 	    break;
 
 	case 'i':
@@ -1052,7 +1112,7 @@ erl_start(int argc, char **argv)
 	    char *sub_param = argv[i]+2;
 	    if (has_prefix("bt", sub_param)) {
 		arg = get_arg(sub_param+2, argv[i+1], &i);
-		res = erts_init_scheduler_bind_type(arg);
+		res = erts_init_scheduler_bind_type_string(arg);
 		if (res != ERTS_INIT_SCHED_BIND_TYPE_SUCCESS) {
 		    switch (res) {
 		    case ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED:
@@ -1075,9 +1135,32 @@ erl_start(int argc, char **argv)
 		    erts_usage();
 		}
 	    }
+	    else if (has_prefix("bwt", sub_param)) {
+		arg = get_arg(sub_param+3, argv[i+1], &i);
+		if (erts_sched_set_busy_wait_threshold(arg) != 0) {
+		    erts_fprintf(stderr, "bad scheduler busy wait threshold: %s\n",
+				 arg);
+		    erts_usage();
+		}
+		VERBOSE(DEBUG_SYSTEM,
+			("scheduler wakup threshold: %s\n", arg));
+	    }
+	    else if (has_prefix("cl", sub_param)) {
+		arg = get_arg(sub_param+2, argv[i+1], &i);
+		if (sys_strcmp("true", arg) == 0)
+		    erts_sched_compact_load = 1;
+		else if (sys_strcmp("false", arg) == 0)
+		    erts_sched_compact_load = 0;
+		else {
+		    erts_fprintf(stderr,
+				 "bad scheduler compact load value '%s'\n",
+				 arg);
+		    erts_usage();
+		}
+	    }
 	    else if (has_prefix("ct", sub_param)) {
 		arg = get_arg(sub_param+2, argv[i+1], &i);
-		res = erts_init_cpu_topology(arg);
+		res = erts_init_cpu_topology_string(arg);
 		if (res != ERTS_INIT_CPU_TOPOLOGY_OK) {
 		    switch (res) {
 		    case ERTS_INIT_CPU_TOPOLOGY_INVALID_ID:
@@ -1118,12 +1201,28 @@ erl_start(int argc, char **argv)
 		    erts_usage();
 		}
 	    }
-	    else if (sys_strcmp("mrq", sub_param) == 0)
-		use_multi_run_queue = 1;
-	    else if (sys_strcmp("srq", sub_param) == 0)
-		use_multi_run_queue = 0;
 	    else if (sys_strcmp("nsp", sub_param) == 0)
 		erts_use_sender_punish = 0;
+	    else if (sys_strcmp("wt", sub_param) == 0) {
+		arg = get_arg(sub_param+2, argv[i+1], &i);
+		if (erts_sched_set_wakeup_other_thresold(arg) != 0) {
+		    erts_fprintf(stderr, "scheduler wakeup threshold: %s\n",
+				 arg);
+		    erts_usage();
+		}
+		VERBOSE(DEBUG_SYSTEM,
+			("scheduler wakeup threshold: %s\n", arg));
+	    }
+	    else if (sys_strcmp("ws", sub_param) == 0) {
+		arg = get_arg(sub_param+2, argv[i+1], &i);
+		if (erts_sched_set_wakeup_other_type(arg) != 0) {
+		    erts_fprintf(stderr, "scheduler wakeup strategy: %s\n",
+				 arg);
+		    erts_usage();
+		}
+		VERBOSE(DEBUG_SYSTEM,
+			("scheduler wakeup threshold: %s\n", arg));
+	    }
 	    else if (has_prefix("ss", sub_param)) {
 		/* suggested stack size (Kilo Words) for scheduler threads */
 		arg = get_arg(sub_param+2, argv[i+1], &i);
@@ -1206,17 +1305,8 @@ erl_start(int argc, char **argv)
 	    break;
 	}
 
-	case 'A':
-	    /* set number of threads in thread pool */
-	    arg = get_arg(argv[i]+2, argv[i+1], &i);
-	    if (((async_max_threads = atoi(arg)) < 0) ||
-		(async_max_threads > ERTS_MAX_NO_OF_ASYNC_THREADS)) {
-		erts_fprintf(stderr, "bad number of async threads %s\n", arg);
-		erts_usage();
-	    }
-
-	    VERBOSE(DEBUG_SYSTEM, ("using %d async-threads\n",
-				   async_max_threads));
+	case 'A': /* Was handled in early init just read past it */
+	    (void) get_arg(argv[i]+2, argv[i+1], &i);
 	    break;
 
 	case 'a':
@@ -1238,9 +1328,17 @@ erl_start(int argc, char **argv)
 		     erts_async_thread_suggested_stack_size));
 	    break;
 
- 	case 'r':
-	    erts_ets_realloc_always_moves = 1;
+	case 'r': {
+	    char *sub_param = argv[i]+2;
+	    if (has_prefix("g", sub_param)) {
+		get_arg(sub_param+1, argv[i+1], &i);
+		/* already handled */
+	    }
+	    else {
+		erts_ets_realloc_always_moves = 1;
+	    }
 	    break;
+	}
 	case 'n':   /* XXX obsolete */
 	    break;
 	case 'c':
@@ -1270,6 +1368,26 @@ erl_start(int argc, char **argv)
 	    }
 	    break;
 
+	case 'z': {
+	    char *sub_param = argv[i]+2;
+	    int new_limit;
+
+	    if (has_prefix("dbbl", sub_param)) {
+		arg = get_arg(sub_param+4, argv[i+1], &i);
+		new_limit = atoi(arg);
+		if (new_limit < 1 || INT_MAX/1024 < new_limit) {
+		    erts_fprintf(stderr, "Invalid dbbl limit: %d\n", new_limit);
+		    erts_usage();
+		} else {
+		    erts_dist_buf_busy_limit = new_limit*1024;
+		}
+	    } else {
+		erts_fprintf(stderr, "bad -z option %s\n", argv[i]);
+		erts_usage();
+	    }
+	    break;
+        }
+
 	default:
 	    erts_fprintf(stderr, "%s unknown flag %s\n", argv[0], argv[i]);
 	    erts_usage();
@@ -1277,10 +1395,6 @@ erl_start(int argc, char **argv)
 	i++;
     }
 
-#ifdef USE_THREADS
-    erts_async_max_threads = async_max_threads;
-#endif
-
     /* Delayed check of +P flag */
     if (erts_max_processes < ERTS_MIN_PROCESSES
 	|| erts_max_processes > ERTS_MAX_PROCESSES
@@ -1310,9 +1424,8 @@ erl_start(int argc, char **argv)
     boot_argc = argc - i;  /* Number of arguments to init */
     boot_argv = &argv[i];
 
-    erl_init();
+    erl_init(ncpu);
 
-    init_shared_memory(boot_argc, boot_argv);
     load_preloaded();
 
     erts_initialized = 1;
@@ -1325,6 +1438,11 @@ erl_start(int argc, char **argv)
 
     erts_sys_main_thread(); /* May or may not return! */
 #else
+    erts_thr_set_main_status(1, 1);
+#if ERTS_USE_ASYNC_READY_Q
+    erts_get_scheduler_data()->aux_work_data.async_ready.queue
+	= erts_get_async_ready_queue(1);
+#endif
     set_main_stack_size();
     process_main();
 #endif
@@ -1348,8 +1466,31 @@ __decl_noreturn void erts_thr_fatal_error(int err, char *what)
 #endif
 
 static void
-system_cleanup(int exit_code)
+system_cleanup(int flush_async)
 {
+    /*
+     * Make sure only one thread exits the runtime system.
+     */
+    if (erts_atomic_inc_read_nob(&exiting) != 1) {
+	/*
+	 * Another thread is currently exiting the system;
+	 * wait for it to do its job.
+	 */
+#ifdef ERTS_SMP
+	if (erts_thr_progress_is_managed_thread()) {
+	    /*
+	     * The exiting thread might be waiting for
+	     * us to block; need to update status...
+	     */
+	    erts_thr_progress_active(NULL, 0);
+	    erts_thr_progress_prepare_wait(NULL);
+	}
+#endif
+	/* Wait forever... */
+	while (1)
+	    erts_milli_sleep(10000000);
+    }
+
     /* No cleanup wanted if ...
      * 1. we are about to do an abnormal exit
      * 2. we haven't finished initializing, or
@@ -1357,7 +1498,7 @@ system_cleanup(int exit_code)
      *    (in threaded non smp case).
      */
 
-    if (exit_code != 0
+    if (!flush_async
 	|| !erts_initialized
 #if defined(USE_THREADS) && !defined(ERTS_SMP)
 	|| !erts_equal_tids(main_thread, erts_thr_self())
@@ -1369,144 +1510,62 @@ system_cleanup(int exit_code)
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_check_exact(NULL, 0);
 #endif
-    erts_smp_block_system(ERTS_BS_FLG_ALLOW_GC); /* We never release it... */
-#endif
-
-#ifdef HYBRID
-    if (ma_src_stack) erts_free(ERTS_ALC_T_OBJECT_STACK,
-                                (void *)ma_src_stack);
-    if (ma_dst_stack) erts_free(ERTS_ALC_T_OBJECT_STACK,
-                                (void *)ma_dst_stack);
-    if (ma_offset_stack) erts_free(ERTS_ALC_T_OBJECT_STACK,
-                                   (void *)ma_offset_stack);
-    ma_src_stack = NULL;
-    ma_dst_stack = NULL;
-    ma_offset_stack = NULL;
-    erts_cleanup_offheap(&erts_global_offheap);
-#endif
-
-#if defined(HYBRID) && !defined(INCREMENTAL)
-    if (global_heap) {
-	ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
-		       (void*) global_heap,
-		       sizeof(Eterm) * global_heap_sz);
-    }
-    global_heap = NULL;
-#endif
-
-#ifdef INCREMENTAL
-    erts_cleanup_incgc();
 #endif
 
-#if defined(USE_THREADS) && !defined(ERTS_SMP)
-    exit_async();
-#endif
-#if HAVE_ERTS_MSEG
-    erts_mseg_exit();
-#endif
-
-    /*
-     * A lot more cleaning could/should have been done...
-     */
-
+    erts_exit_flush_async();
 }
 
-/*
- * Common exit function, all exits from the system go through here.
- * n <= 0 -> normal exit with status n;
- * n = 127 -> Erlang crash dump produced, exit with status 1;
- * other positive n -> Erlang crash dump and core dump produced.
- */
-
-__decl_noreturn void erl_exit0(char *file, int line, int n, char *fmt,...)
+static __decl_noreturn void __noreturn
+erl_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2)
 {
     unsigned int an;
-    va_list args;
 
-    va_start(args, fmt);
+    system_cleanup(flush_async);
 
     save_statistics();
 
-    system_cleanup(n);
-
     an = abs(n);
 
     if (erts_mtrace_enabled)
 	erts_mtrace_exit((Uint32) an);
 
     /* Produce an Erlang core dump if error */
-    if (n > 0 && erts_initialized &&
-	(erts_no_crash_dump == 0 || n == ERTS_DUMP_EXIT)) {
-	erl_crash_dump_v(file, line, fmt, args); 
+    if (((n > 0 && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT)
+	&& erts_initialized) {
+	erl_crash_dump_v((char*) NULL, 0, fmt, args1);
     }
 
-    /* need to reinitialize va_args thing */
-    va_end(args);
-    va_start(args, fmt);
-
     if (fmt != NULL && *fmt != '\0')
-	  erl_error(fmt, args);	/* Print error message. */
-    va_end(args);
-#ifdef __WIN32__
-    if(n > 0) ConWaitForExit();
-    else ConNormalExit();
-#endif
-#if !defined(__WIN32__) && !defined(VXWORKS) && !defined(_OSE_)
-    sys_tty_reset();
-#endif
+	  erl_error(fmt, args2);	/* Print error message. */
+    sys_tty_reset(n);
 
     if (n == ERTS_INTR_EXIT)
 	exit(0);
-    else if (n == 127)
+    else if (n == ERTS_DUMP_EXIT)
 	ERTS_EXIT_AFTER_DUMP(1);
     else if (n > 0 || n == ERTS_ABORT_EXIT)
         abort();
     exit(an);
 }
 
-__decl_noreturn void erl_exit(int n, char *fmt,...)
+/* Exit without flushing async threads */
+__decl_noreturn void __noreturn erl_exit(int n, char *fmt, ...)
 {
-    unsigned int an;
-    va_list args;
-
-    va_start(args, fmt);
-
-    save_statistics();
-
-    system_cleanup(n);
-
-    an = abs(n);
-
-    if (erts_mtrace_enabled)
-	erts_mtrace_exit((Uint32) an);
-
-    /* Produce an Erlang core dump if error */
-    if (n > 0 && erts_initialized &&
-	(erts_no_crash_dump == 0 || n == ERTS_DUMP_EXIT)) {
-	erl_crash_dump_v((char*) NULL, 0, fmt, args);
-    }
-
-    /* need to reinitialize va_args thing */
-    va_end(args);
-    va_start(args, fmt);
-
-    if (fmt != NULL && *fmt != '\0')
-	  erl_error(fmt, args);	/* Print error message. */
-    va_end(args);
-#ifdef __WIN32__
-    if(n > 0) ConWaitForExit();
-    else ConNormalExit();
-#endif
-#if !defined(__WIN32__) && !defined(VXWORKS) && !defined(_OSE_)
-    sys_tty_reset();
-#endif
-
-    if (n == ERTS_INTR_EXIT)
-	exit(0);
-    else if (n == ERTS_DUMP_EXIT)
-	ERTS_EXIT_AFTER_DUMP(1);
-    else if (n > 0 || n == ERTS_ABORT_EXIT)
-        abort();
-    exit(an);
+    va_list args1, args2;
+    va_start(args1, fmt);
+    va_start(args2, fmt);
+    erl_exit_vv(n, 0, fmt, args1, args2);
+    va_end(args2);
+    va_end(args1);
 }
 
+/* Exit after flushing async threads */
+__decl_noreturn void __noreturn erl_exit_flush_async(int n, char *fmt, ...)
+{
+    va_list args1, args2;
+    va_start(args1, fmt);
+    va_start(args2, fmt);
+    erl_exit_vv(n, 1, fmt, args1, args2);
+    va_end(args2);
+    va_end(args1);
+}
diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c
index 3f022f92b8..963c8b3c58 100644
--- a/erts/emulator/beam/erl_instrument.c
+++ b/erts/emulator/beam/erl_instrument.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -540,18 +540,18 @@ static void dump_memory_map_to_stream(FILE *fp)
 	if (is_internal_pid(bp->pid))
 	    fprintf(fp,
 		    "{%lu, %lu, %lu, {%lu,%lu,%lu}}.\n",
-		    (Uint) bp->type_no,
-		    (Uint) bp->mem,
-		    (Uint) bp->size,
-		    (Uint) pid_channel_no(bp->pid),
-		    (Uint) pid_number(bp->pid),
-		    (Uint) pid_serial(bp->pid));
+		    (UWord) bp->type_no,
+		    (UWord) bp->mem,
+		    (UWord) bp->size,
+		    (UWord) pid_channel_no(bp->pid),
+		    (UWord) pid_number(bp->pid),
+		    (UWord) pid_serial(bp->pid));
 	else
 	    fprintf(fp,
 		    "{%lu, %lu, %lu, undefined}.\n",
-		    (Uint) bp->type_no,
-		    (Uint) bp->mem,
-		    (Uint) bp->size);
+		    (UWord) bp->type_no,
+		    (UWord) bp->mem,
+		    (UWord) bp->size);
     }
 
     if (lock)
@@ -638,7 +638,7 @@ Eterm erts_instr_get_memory_map(Process *proc)
 	    hsz += 4;
 	}
 
-	if ((Uint) bp->mem > MAX_SMALL)
+	if ((UWord) bp->mem > MAX_SMALL)
 	    hsz += BIG_UINT_HEAP_SIZE;
 	if (bp->size > MAX_SMALL)
 	    hsz += BIG_UINT_HEAP_SIZE;
@@ -749,12 +749,12 @@ Eterm erts_instr_get_memory_map(Process *proc)
 #endif
 	type = make_small(bp->type_no);
 
-	if ((Uint) bp->mem > MAX_SMALL) {
-	    ptr = uint_to_big((Uint) bp->mem, hp);
+	if ((UWord) bp->mem > MAX_SMALL) {
+	    ptr = uint_to_big((UWord) bp->mem, hp);
 	    hp += BIG_UINT_HEAP_SIZE;
 	}
 	else
-	    ptr = make_small((Uint) bp->mem);
+	    ptr = make_small((UWord) bp->mem);
 
 	if (bp->size > MAX_SMALL) {
 	    size = uint_to_big(bp->size, hp);
@@ -962,12 +962,12 @@ dump_stat_to_stream(FILE *fp, int begin_max_period)
 
     fprintf(fp,
 	    "{total,[{total,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}]}.\n",
-	    stats->tot.size,
-	    stats->tot.max_size,
-	    stats->tot.max_size_ever,
-	    stats->tot.blocks,
-	    stats->tot.max_blocks,
-	    stats->tot.max_blocks_ever);
+	    (UWord) stats->tot.size,
+	    (UWord) stats->tot.max_size,
+	    (UWord) stats->tot.max_size_ever,
+	    (UWord) stats->tot.blocks,
+	    (UWord) stats->tot.max_blocks,
+	    (UWord) stats->tot.max_blocks_ever);
 
     a_max = 0;
     a_min = ~0;
@@ -992,12 +992,12 @@ dump_stat_to_stream(FILE *fp, int begin_max_period)
 		    "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s",
 		    i == a_min ? "{allocators,\n [" : "  ",
 		    ERTS_ALC_A2AD(i),
-		    stats->a[i].size,
-		    stats->a[i].max_size,
-		    stats->a[i].max_size_ever,
-		    stats->a[i].blocks,
-		    stats->a[i].max_blocks,
-		    stats->a[i].max_blocks_ever,
+		    (UWord) stats->a[i].size,
+		    (UWord) stats->a[i].max_size,
+		    (UWord) stats->a[i].max_size_ever,
+		    (UWord) stats->a[i].blocks,
+		    (UWord) stats->a[i].max_blocks,
+		    (UWord) stats->a[i].max_blocks_ever,
 		    i == a_max ? "]}.\n" : ",\n");
 	}
     }
@@ -1009,12 +1009,12 @@ dump_stat_to_stream(FILE *fp, int begin_max_period)
 		"%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s",
 		i == ERTS_ALC_C_MIN ? "{classes,\n [" : "  ",
 		ERTS_ALC_C2CD(i),
-		stats->c[i].size,
-		stats->c[i].max_size,
-		stats->c[i].max_size_ever,
-		stats->c[i].blocks,
-		stats->c[i].max_blocks,
-		stats->c[i].max_blocks_ever,
+		(UWord) stats->c[i].size,
+		(UWord) stats->c[i].max_size,
+		(UWord) stats->c[i].max_size_ever,
+		(UWord) stats->c[i].blocks,
+		(UWord) stats->c[i].max_blocks,
+		(UWord) stats->c[i].max_blocks_ever,
 		i == ERTS_ALC_C_MAX ? "]}.\n" :  ",\n" );
     }
 
@@ -1025,12 +1025,12 @@ dump_stat_to_stream(FILE *fp, int begin_max_period)
 		"%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s",
 		i == ERTS_ALC_N_MIN ? "{types,\n [" : "  ",
 		ERTS_ALC_N2TD(i),
-		stats->n[i].size,
-		stats->n[i].max_size,
-		stats->n[i].max_size_ever,
-		stats->n[i].blocks,
-		stats->n[i].max_blocks,
-		stats->n[i].max_blocks_ever,
+		(UWord) stats->n[i].size,
+		(UWord) stats->n[i].max_size,
+		(UWord) stats->n[i].max_size_ever,
+		(UWord) stats->n[i].blocks,
+		(UWord) stats->n[i].max_blocks,
+		(UWord) stats->n[i].max_blocks_ever,
 		i == ERTS_ALC_N_MAX ? "]}.\n" :  ",\n" );
     }
 
@@ -1186,6 +1186,8 @@ erts_instr_init(int stat, int map_stat)
     sys_memzero((void *) stats->n, sizeof(Stat_t)*(ERTS_ALC_N_MAX+1));
 
     for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
+	if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i))
+	    continue;
 	if (erts_allctrs_info[i].enabled)
 	    stats->ap[i] = &stats->a[i];
 	else
@@ -1199,6 +1201,8 @@ erts_instr_init(int stat, int map_stat)
 	erts_instr_memory_map = 1;
 	erts_instr_stat = 1;
 	for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
+	    if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i))
+		continue;
 	    erts_allctrs[i].alloc	= map_stat_alloc;
 	    erts_allctrs[i].realloc	= map_stat_realloc;
 	    erts_allctrs[i].free	= map_stat_free;
@@ -1209,6 +1213,8 @@ erts_instr_init(int stat, int map_stat)
     else {
 	erts_instr_stat = 1;
 	for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
+	    if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i))
+		continue;
 	    erts_allctrs[i].alloc	= stat_alloc;
 	    erts_allctrs[i].realloc	= stat_realloc;
 	    erts_allctrs[i].free	= stat_free;
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 074b08ea57..b545ec07c0 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -96,41 +96,36 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"proc_status",				"pid"			},
     {	"proc_tab",				NULL			},
     {   "ports_snapshot",                       NULL                    },
-    {	"db_tab",				"address"		},    
-    {	"db_tab_fix",				"address"		},
     {	"meta_name_tab",	         	"address"		},
     {	"meta_main_tab_slot",			"address"		},
+    {	"db_tab",				"address"		},
+    {	"db_tab_fix",				"address"		},
     {	"meta_main_tab_main",			NULL 			},
     {	"db_hash_slot",				"address"		},
     {	"node_table",				NULL			},
     {	"dist_table",				NULL			},
     {	"sys_tracers",				NULL			},
-    {	"trace_pattern",			NULL			},
     {	"module_tab",				NULL			},
     {	"export_tab",				NULL			},
     {	"fun_tab",				NULL			},
     {	"environ",				NULL			},
 #endif
-    {	"asyncq",				"address"		},
-#ifndef ERTS_SMP
-    {	"async_ready",				NULL			},
-#endif
     {	"efile_drv",				"address"		},
 #if defined(ENABLE_CHILD_WAITER_THREAD) || defined(ERTS_SMP)
     {	"child_status",				NULL			},
 #endif
 #ifdef __WIN32__
-    {	"sys_driver_data_lock",			NULL 			}, 
+    {	"sys_driver_data_lock",			NULL 			},
 #endif
-    {	"drv_ev_state_grow",			NULL,   		}, 
+    {	"drv_ev_state_grow",			NULL,   		},
     {	"drv_ev_state",				"address"		},
     {	"safe_hash",				"address"		},
     {   "pollset_rm_list",                      NULL                    },
-    {   "removed_fd_pre_alloc_lock",            NULL                    },
+    {   "removed_fd_pre_alloc_lock",            "address"               },
     {   "state_prealloc",                       NULL                    },
     {	"schdlr_sspnd",				NULL			},
-    {	"cpu_bind",				NULL			},
     {	"run_queue",				"address"		},
+    {	"cpu_info",				NULL			},
     {	"pollset",				"address"		},
 #ifdef __WIN32__
     {	"pollwaiter",				"address"		},
@@ -139,6 +134,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"alcu_init_atoms",			NULL			},
     {	"mseg_init_atoms",			NULL			},
     {	"drv_tsd",				NULL			},
+    {	"async_enq_mtx",			NULL			},
 #ifdef ERTS_SMP
     {	"sys_msg_q", 				NULL			},
     {	"atom_tab",				NULL			},
@@ -152,9 +148,12 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"mtrace_op",				NULL			},
     {	"instr_x",				NULL			},
     {	"instr",				NULL			},
-    {	"fix_alloc",				"index"			},
     {	"alcu_allocator",			"index"			},
+    {	"sbmbc_alloc",				"index"			},
     {	"mseg",					NULL			},
+#if HALFWORD_HEAP
+    {	"pmmap",				NULL			},
+#endif
 #ifdef ERTS_SMP
     {	"port_task_pre_alloc_lock",		"address"		},
     {	"port_taskq_pre_alloc_lock",		"address"		},
@@ -171,21 +170,21 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"timeofday",				NULL			},
     {	"breakpoints",				NULL			},
     {	"pollsets_lock",			NULL			},
-    {	"async_id",				NULL			},
     {	"pix_lock",				"address"		},
     {	"run_queues_lists",			NULL			},
     {	"sched_stat",				NULL			},
 #endif
-    {	"alloc_thr_ix_lock",			NULL			},
-#ifdef ERTS_SMP
-    {	"proc_lck_wtr_alloc",			NULL 			},
-#endif
+    {	"async_init_mtx",			NULL			},
 #ifdef __WIN32__
 #ifdef DEBUG
     {   "save_ops_lock",                        NULL                    },
 #endif
 #endif
-    {	"mtrace_buf",				NULL			}
+#ifdef	USE_VM_PROBES
+    {   "efile_drv dtrace mutex",               NULL                    },
+#endif
+    {	"mtrace_buf",				NULL			},
+    {	"erts_alloc_hard_debug",		NULL			}
 };
 
 #define ERTS_LOCK_ORDER_SIZE \
@@ -197,6 +196,8 @@ static erts_lc_lock_order_t erts_lock_order[] = {
        & ERTS_LC_FLG_LT_ALL						\
        & ~(ERTS_LC_FLG_LT_SPINLOCK|ERTS_LC_FLG_LT_RWSPINLOCK)))
 
+static __decl_noreturn void  __noreturn lc_abort(void);
+
 static char *
 lock_type(Uint16 flags)
 {
@@ -220,7 +221,7 @@ rw_op_str(Uint16 flags)
 	return " (r)";
     case ERTS_LC_FLG_LO_WRITE:
 	erts_fprintf(stderr, "\nInternal error\n");
-	abort();
+	lc_abort();
     default:
 	break;
     }
@@ -231,7 +232,7 @@ typedef struct erts_lc_locked_lock_t_ erts_lc_locked_lock_t;
 struct erts_lc_locked_lock_t_ {
     erts_lc_locked_lock_t *next;
     erts_lc_locked_lock_t *prev;
-    Eterm extra;
+    UWord extra;
     Sint16 id;
     Uint16 flags;
 };
@@ -269,28 +270,18 @@ static erts_lc_free_block_t *free_blocks;
 #define ERTS_LC_FB_CHUNK_SIZE 10
 #endif
 
-#ifdef ETHR_HAVE_NATIVE_LOCKS
 static ethr_spinlock_t free_blocks_lock;
-#define ERTS_LC_LOCK	ethr_spin_lock
-#define ERTS_LC_UNLOCK	ethr_spin_unlock
-#else
-static ethr_mutex free_blocks_lock;
-#define ERTS_LC_LOCK	ethr_mutex_lock
-#define ERTS_LC_UNLOCK	ethr_mutex_unlock
-#endif
 
 static ERTS_INLINE void
 lc_lock(void)
 {
-    if (ERTS_LC_LOCK(&free_blocks_lock) != 0)
-	abort();
+    ethr_spin_lock(&free_blocks_lock);
 }
 
 static ERTS_INLINE void
 lc_unlock(void)
 {
-    if (ERTS_LC_UNLOCK(&free_blocks_lock) != 0)
-	abort();
+    ethr_spin_unlock(&free_blocks_lock);
 }
 
 static ERTS_INLINE void lc_free(void *p)
@@ -311,7 +302,7 @@ static void *lc_core_alloc(void)
 {
     lc_unlock();
     erts_fprintf(stderr, "Lock checker out of memory!\n");
-    abort();
+    lc_abort();
 }
 
 #else
@@ -325,7 +316,7 @@ static void *lc_core_alloc(void)
 					  * ERTS_LC_FB_CHUNK_SIZE);
     if (!fbs) {
 	erts_fprintf(stderr, "Lock checker failed to allocate memory!\n");
-	abort();
+	lc_abort();
     }
     for (i = 1; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) {
 #ifdef DEBUG
@@ -365,11 +356,11 @@ create_locked_locks(char *thread_name)
 {
     erts_lc_locked_locks_t *l_lcks = malloc(sizeof(erts_lc_locked_locks_t));
     if (!l_lcks)
-	abort();
+	lc_abort();
 
     l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown");
     if (!l_lcks->thread_name)
-	abort();
+	lc_abort();
 
     l_lcks->tid = erts_thr_self();
     l_lcks->required.first = NULL;
@@ -442,12 +433,12 @@ new_locked_lock(erts_lc_lock_t *lck, Uint16 op_flags)
 }
 
 static void
-print_lock2(char *prefix, Sint16 id, Eterm extra, Uint16 flags, char *suffix)
+print_lock2(char *prefix, Sint16 id, Wterm extra, Uint16 flags, char *suffix)
 {
     char *lname = (0 <= id && id < ERTS_LOCK_ORDER_SIZE
 		   ? erts_lock_order[id].name
 		   : "unknown");
-    if (is_boxed(extra))
+    if (is_not_immed(extra))
 	erts_fprintf(stderr,
 		     "%s'%s:%p%s'%s%s",
 		     prefix,
@@ -511,7 +502,7 @@ uninitialized_lock(void)
 {
     erts_fprintf(stderr, "Performing operations on uninitialized lock!\n");
     print_curr_locks(get_my_locked_locks());
-    abort();
+    lc_abort();
 }
 
 static void
@@ -521,7 +512,7 @@ lock_twice(char *prefix, erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck,
     erts_fprintf(stderr, "%s%s", prefix, rw_op_str(op_flags));
     print_lock(" ", lck, " lock which is already locked by thread!\n");
     print_curr_locks(l_lcks);
-    abort();
+    lc_abort();
 }
 
 static void
@@ -531,7 +522,7 @@ unlock_op_mismatch(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck,
     erts_fprintf(stderr, "Unlocking%s ", rw_op_str(op_flags));
     print_lock("", lck, " lock which mismatch previous lock operation!\n");
     print_curr_locks(l_lcks);
-    abort();
+    lc_abort();
 }
 
 static void
@@ -539,7 +530,7 @@ unlock_of_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
 {
     print_lock("Unlocking ", lck, " lock which is not locked by thread!\n");
     print_curr_locks(l_lcks);
-    abort();
+    lc_abort();
 }
 
 static void
@@ -548,7 +539,7 @@ lock_order_violation(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
     print_lock("Lock order violation occured when locking ", lck, "!\n");
     print_curr_locks(l_lcks);
     print_lock_order();
-    abort();
+    lc_abort();
 }
 
 static void
@@ -559,7 +550,7 @@ type_order_violation(char *op, erts_lc_locked_locks_t *l_lcks,
     print_lock(op, lck, "!\n");
     ASSERT(l_lcks);
     print_curr_locks(l_lcks);
-    abort();
+    lc_abort();
 }
 
 static void
@@ -611,7 +602,7 @@ lock_mismatch(erts_lc_locked_locks_t *l_lcks, int exact,
 	}
     }
     print_curr_locks(l_lcks);
-    abort();
+    lc_abort();
 }
 
 static void
@@ -619,7 +610,7 @@ unlock_of_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
 {
     print_lock("Unlocking required ", lck, " lock!\n");
     print_curr_locks(l_lcks);
-    abort();
+    lc_abort();
 }
 
 static void
@@ -627,7 +618,7 @@ unrequire_of_not_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *l
 {
     print_lock("Unrequire on ", lck, " lock not required!\n");
     print_curr_locks(l_lcks);
-    abort();
+    lc_abort();
 }
 
 static void
@@ -635,7 +626,7 @@ require_twice(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
 {
     print_lock("Require on ", lck, " lock already required!\n");
     print_curr_locks(l_lcks);
-    abort();
+    lc_abort();
 }
 
 static void
@@ -643,7 +634,7 @@ required_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
 {
     print_lock("Required ", lck, " lock not locked!\n");
     print_curr_locks(l_lcks);
-    abort();
+    lc_abort();
 }
 
 
@@ -656,13 +647,23 @@ thread_exit_handler(void)
 	    erts_fprintf(stderr,
 			 "Thread exiting while having locked locks!\n");
 	    print_curr_locks(l_lcks);
-	    abort();
+	    lc_abort();
 	}
 	destroy_locked_locks(l_lcks);
 	/* erts_tsd_set(locks_key, NULL); */
     }
 }
 
+static __decl_noreturn void
+lc_abort(void)
+{
+#ifdef __WIN32__
+    DebugBreak();
+#else
+    abort();
+#endif
+}
+
 void
 erts_lc_set_thread_name(char *thread_name)
 {
@@ -674,7 +675,7 @@ erts_lc_set_thread_name(char *thread_name)
 	free((void *) l_lcks->thread_name);
 	l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown");
 	if (!l_lcks->thread_name)
-	    abort();
+	    lc_abort();
     }
 }
 
@@ -684,7 +685,7 @@ erts_lc_assert_failed(char *file, int line, char *assertion)
     erts_fprintf(stderr, "%s:%d: Lock check assertion \"%s\" failed!\n",
 		 file, line, assertion);
     print_curr_locks(get_my_locked_locks());
-    abort();
+    lc_abort();
     return 0;
 }
 
@@ -697,7 +698,7 @@ void erts_lc_fail(char *fmt, ...)
     va_end(args);
     erts_fprintf(stderr, "\n");
     print_curr_locks(get_my_locked_locks());
-    abort();
+    lc_abort();
 }
 
 
@@ -717,7 +718,7 @@ erts_lc_get_lock_order_id(char *name)
 		     "(update erl_lock_check.c)\n",
 		     name);
     }
-    abort();
+    lc_abort();
     return (Sint16) -1;
 }
 
@@ -893,6 +894,25 @@ erts_lc_check_exact(erts_lc_lock_t *have, int have_len)
     }
 }
 
+void
+erts_lc_check_no_locked_of_type(Uint16 flags)
+{
+    erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
+    if (l_lcks) {
+	erts_lc_locked_lock_t *l_lck = l_lcks->locked.first;
+	for (l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next) {
+	    if (l_lck->flags & flags) {
+		erts_fprintf(stderr,
+			     "Locked lock of type %s found which isn't "
+			     "allowed here!\n",
+			     lock_type(l_lck->flags));
+		print_curr_locks(l_lcks);
+		lc_abort();
+	    }
+	}
+    }
+}
+
 int
 erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags)
 {
@@ -952,10 +972,10 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags)
 	/* We only force busy if a lock order violation would occur
 	   and when on an even millisecond. */
 	{
-	    erts_thr_timeval_t time;
-	    erts_thr_time_now(&time);
+	    SysTimeval tv;
+	    sys_gettimeofday(&tv);
 
-	    if ((time.tv_nsec / 1000000) & 1)
+	    if ((tv.tv_usec / 1000) & 1)
 		return 0;
 	}
 #endif
@@ -1231,7 +1251,9 @@ void
 erts_lc_init_lock(erts_lc_lock_t *lck, char *name, Uint16 flags)
 {
     lck->id = erts_lc_get_lock_order_id(name);
-    lck->extra = make_boxed(&lck->extra);
+
+    lck->extra = (UWord) &lck->extra;
+    ASSERT(is_not_immed(lck->extra));
     lck->flags = flags;
     lck->inited = ERTS_LC_INITITALIZED;
 }
@@ -1241,6 +1263,7 @@ erts_lc_init_lock_x(erts_lc_lock_t *lck, char *name, Uint16 flags, Eterm extra)
 {
     lck->id = erts_lc_get_lock_order_id(name);
     lck->extra = extra;
+    ASSERT(is_immed(lck->extra));
     lck->flags = flags;
     lck->inited = ERTS_LC_INITITALIZED;
 }
@@ -1279,13 +1302,8 @@ erts_lc_init(void)
     free_blocks = NULL;
 #endif /* #ifdef ERTS_LC_STATIC_ALLOC */
 
-#ifdef ETHR_HAVE_NATIVE_LOCKS
     if (ethr_spinlock_init(&free_blocks_lock) != 0)
-	abort();
-#else
-    if (ethr_mutex_init(&free_blocks_lock) != 0)
-	abort();
-#endif
+	lc_abort();
 
     erts_tsd_key_create(&locks_key);
 }
diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h
index d5e2ede9ac..b67f36fa06 100644
--- a/erts/emulator/beam/erl_lock_check.h
+++ b/erts/emulator/beam/erl_lock_check.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -39,7 +39,7 @@ typedef struct {
     int inited;
     Sint16 id;
     Uint16 flags;
-    Eterm extra;
+    UWord extra;
 } erts_lc_lock_t;
 
 #define ERTS_LC_INITITALIZED 0x7f7f7f7f
@@ -77,6 +77,7 @@ void erts_lc_check(erts_lc_lock_t *have, int have_len,
 void erts_lc_check_exact(erts_lc_lock_t *have, int have_len);
 void erts_lc_have_locks(int *resv, erts_lc_lock_t *lcks, int len);
 void erts_lc_have_lock_ids(int *resv, int *ids, int len);
+void erts_lc_check_no_locked_of_type(Uint16 flags);
 int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags);
 void erts_lc_trylock_flg(int locked, erts_lc_lock_t *lck, Uint16 op_flags);
 void erts_lc_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags);
diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index 0d7e1335c1..741c0cb08e 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -49,7 +49,7 @@ const char *str_undefined = "undefined";
 
 static ethr_tsd_key lcnt_thr_data_key;
 static int lcnt_n_thr;
-static erts_lcnt_thread_data_t *lcnt_thread_data[1024]; 
+static erts_lcnt_thread_data_t *lcnt_thread_data[4096]; 
 
 /* local functions */
 
@@ -159,15 +159,15 @@ static char* lock_opt(Uint16 flag) {
 }
 
 static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action, char *extra) {
-    long int colls, tries, w_state, r_state;
+    erts_aint_t colls, tries, w_state, r_state;
     erts_lcnt_lock_stats_t *stats = NULL;
     
     char *type;
     int i;
     
     type = lcnt_lock_type(lock->flag);
-    ethr_atomic_read(&lock->r_state, &r_state);
-    ethr_atomic_read(&lock->w_state, &w_state);
+    r_state = ethr_atomic_read(&lock->r_state);
+    w_state = ethr_atomic_read(&lock->w_state);
 
     
     if (lock->flag & flag) {
@@ -240,7 +240,7 @@ void erts_lcnt_init() {
 
     lcnt_lock();
 
-    erts_lcnt_rt_options = ERTS_LCNT_OPT_PROCLOCK;
+    erts_lcnt_rt_options = ERTS_LCNT_OPT_PROCLOCK | ERTS_LCNT_OPT_LOCATION;
     
     eltd = lcnt_thread_data_alloc();
 
@@ -257,6 +257,10 @@ void erts_lcnt_init() {
     erts_lcnt_clear_counters();
 }
 
+void erts_lcnt_late_init() {
+    erts_thr_install_exit_handler(erts_lcnt_thread_exit_handler);
+}
+
 /* list operations */
 
 /* BEGIN ASSUMPTION: lcnt_data_lock taken */
@@ -308,7 +312,7 @@ void erts_lcnt_list_insert(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock)
 }
 
 void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) {
-    
+
     if (lock->next) lock->next->prev = lock->prev;
     if (lock->prev) lock->prev->next = lock->next;
     if (list->head == lock) list->head = lock->next;
@@ -330,6 +334,10 @@ void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) {
 }
 void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) { 
     int i;
+    if (!name) {
+	lock->flag = 0;
+	return;
+    }
     lcnt_lock();
     
     lock->next = NULL;
@@ -359,6 +367,8 @@ void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eter
 void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) {
     erts_lcnt_lock_t *deleted_lock;
 
+    if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
+
     lcnt_lock();
 
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_COPYSAVE) {
@@ -374,6 +384,7 @@ void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) {
     }
     /* delete original */
     erts_lcnt_list_delete(erts_lcnt_data->current_locks, lock);
+    lock->flag = 0;
     
     lcnt_unlock();
 }
@@ -381,19 +392,20 @@ void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) {
 /* lock */
 
 void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) {
-    long r_state = 0, w_state = 0;
+    erts_aint_t r_state = 0, w_state = 0;
     erts_lcnt_thread_data_t *eltd;
     
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
+    if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
 
     eltd = lcnt_get_thread_data();
 
     ASSERT(eltd);
     
-    ethr_atomic_read(&lock->w_state, &w_state);
+    w_state = ethr_atomic_read(&lock->w_state);
     
     if (option & ERTS_LCNT_LO_WRITE) {
-        ethr_atomic_read(&lock->r_state, &r_state);
+        r_state = ethr_atomic_read(&lock->r_state);
         ethr_atomic_inc( &lock->w_state);
     }
     if (option & ERTS_LCNT_LO_READ) {
@@ -414,12 +426,13 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) {
 }
 
 void erts_lcnt_lock(erts_lcnt_lock_t *lock) {
-    long w_state;
+    erts_aint_t w_state;
     erts_lcnt_thread_data_t *eltd;
     
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
+    if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
 
-    ethr_atomic_read(&lock->w_state, &w_state);
+    w_state = ethr_atomic_read(&lock->w_state);
     ethr_atomic_inc( &lock->w_state);
 
     eltd = lcnt_get_thread_data();
@@ -448,6 +461,7 @@ void erts_lcnt_lock_unaquire(erts_lcnt_lock_t *lock) {
     /* should check if this thread was "waiting" */
     
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
+    if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
 
     ethr_atomic_dec( &lock->w_state);
 }
@@ -467,14 +481,15 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line
     erts_lcnt_time_t time_wait;
     erts_lcnt_lock_stats_t *stats;
 #ifdef DEBUG
-    long flowstate;
+    erts_aint_t flowstate;
 #endif
 
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
+    if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
     
 #ifdef DEBUG
     if (!(lock->flag & (ERTS_LCNT_LT_RWMUTEX | ERTS_LCNT_LT_RWSPINLOCK))) {
-    	ethr_atomic_read(&lock->flowstate, &flowstate);
+	flowstate = ethr_atomic_read(&lock->flowstate);
 	ASSERT(flowstate == 0);
     	ethr_atomic_inc( &lock->flowstate);
     }
@@ -485,9 +500,13 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line
     ASSERT(eltd);
 
     /* if lock was in conflict, time it */
-	
-    stats = lcnt_get_lock_stats(lock, file, line);
     
+    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_LOCATION) {
+	stats = lcnt_get_lock_stats(lock, file, line);
+    } else {
+	stats = &lock->stats[0];
+    }
+
     if (eltd->timer_set) {
 	lcnt_time(&timer);
 	
@@ -506,24 +525,26 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line
 
 void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option) {
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
+    if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
     if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_dec(&lock->w_state);
     if (option & ERTS_LCNT_LO_READ ) ethr_atomic_dec(&lock->r_state);
 }
 
 void erts_lcnt_unlock(erts_lcnt_lock_t *lock) {
 #ifdef DEBUG
-    long w_state;  
-    long flowstate;
+    erts_aint_t w_state;  
+    erts_aint_t flowstate;
 #endif
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
+    if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
 #ifdef DEBUG
     /* flowstate */
-    ethr_atomic_read(&lock->flowstate, &flowstate);
+    flowstate = ethr_atomic_read(&lock->flowstate);
     ASSERT(flowstate == 1);
     ethr_atomic_dec( &lock->flowstate);
     
     /* write state */
-    ethr_atomic_read(&lock->w_state, &w_state);
+    w_state = ethr_atomic_read(&lock->w_state);
     ASSERT(w_state > 0)
 #endif
     ethr_atomic_dec(&lock->w_state);
@@ -533,6 +554,7 @@ void erts_lcnt_unlock(erts_lcnt_lock_t *lock) {
 
 void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option) {
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
+    if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
     /* Determine lock_state via res instead of state */
     if (res != EBUSY) {
 	if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_inc(&lock->w_state);
@@ -548,13 +570,14 @@ void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option) {
 void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) {
     /* Determine lock_state via res instead of state */
 #ifdef DEBUG
-    long flowstate;
+    erts_aint_t flowstate;
 #endif 
     if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
+    if (!ERTS_LCNT_LOCK_TYPE(lock)) return;
     if (res != EBUSY) {
 	
 #ifdef DEBUG
-    	ethr_atomic_read(&lock->flowstate, &flowstate);
+	flowstate = ethr_atomic_read(&lock->flowstate);
 	ASSERT(flowstate == 0);
     	ethr_atomic_inc( &lock->flowstate);
 #endif
@@ -570,36 +593,26 @@ void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) {
 
 /* thread operations */
 
-static void *lcnt_thr_init(erts_lcnt_thread_data_t *eltd) {
-    void *(*function)(void *);
-    void *argument;
-    void *res;
-    function = eltd->function;
-    argument = eltd->argument;
-    
-    ethr_tsd_set(lcnt_thr_data_key, eltd);
-    
-    res = (void *)function(argument);
-    free(eltd);
-    return (void *)res;
-}
-
-    
-
-int erts_lcnt_thr_create(ethr_tid *tid, void * (*function)(void *), void *arg, ethr_thr_opts *opts) {
+void erts_lcnt_thread_setup(void) {
     erts_lcnt_thread_data_t *eltd;
-    
+
     lcnt_lock();
     /* lock for thread id global update */
     eltd = lcnt_thread_data_alloc();
     lcnt_unlock();
-    
-    eltd->function = function;
-    eltd->argument = arg;
-
-    return ethr_thr_create(tid, (void *)lcnt_thr_init, (void *)eltd, opts);
+    ASSERT(eltd);
+    ethr_tsd_set(lcnt_thr_data_key, eltd);
 }
 
+void erts_lcnt_thread_exit_handler() {
+    erts_lcnt_thread_data_t *eltd;
+
+    eltd = ethr_tsd_get(lcnt_thr_data_key);
+
+    if (eltd) {
+        free(eltd);
+    }
+}
 
 /* bindings for bifs */
 
diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h
index e3044c371f..690551c71f 100644
--- a/erts/emulator/beam/erl_lock_count.h
+++ b/erts/emulator/beam/erl_lock_count.h
@@ -89,6 +89,7 @@
 #define ERTS_LCNT_OPT_LOCATION  (((Uint16) 1) << 1)
 #define ERTS_LCNT_OPT_PROCLOCK  (((Uint16) 1) << 2)
 #define ERTS_LCNT_OPT_COPYSAVE  (((Uint16) 1) << 3)
+#define ERTS_LCNT_OPT_PORTLOCK  (((Uint16) 1) << 4)
 
 typedef struct {
     unsigned long s;
@@ -155,11 +156,6 @@ typedef struct {
     erts_lcnt_time_t timer;	/* timer         */
     int timer_set;		/* bool          */
     int lock_in_conflict;       /* bool          */
-    
-    /* function pointer */
-    void *(*function)(void *);
-    void *argument;
-    
 } erts_lcnt_thread_data_t;
 
 /* globals */
@@ -169,6 +165,11 @@ extern Uint16 erts_lcnt_rt_options;
 /* function declerations */
 
 void erts_lcnt_init(void);
+void erts_lcnt_late_init(void);
+
+/* thread operations */
+void erts_lcnt_thread_setup(void);
+void erts_lcnt_thread_exit_handler(void);
 
 /* list operations (local)  */
 erts_lcnt_lock_list_t *erts_lcnt_list_init(void);
@@ -194,17 +195,14 @@ void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option);
 void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option);
 void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res);
 
-/* thread operations */
-
-int erts_lcnt_thr_create(ethr_tid *tid, void * (*function)(void *), void *arg, ethr_thr_opts *opts);
-
 /* bif interface */
-
 Uint16 erts_lcnt_set_rt_opt(Uint16 opt);
 Uint16 erts_lcnt_clear_rt_opt(Uint16 opt);
 void   erts_lcnt_clear_counters(void);
 char  *erts_lcnt_lock_type(Uint16 type);
 erts_lcnt_data_t *erts_lcnt_get_data(void);
 
+#define ERTS_LCNT_LOCK_TYPE(lockp)	((lockp)->flag & ERTS_LCNT_LT_ALL)
+
 #endif /* ifdef  ERTS_ENABLE_LOCK_COUNT  */
 #endif /* ifndef ERTS_LOCK_COUNT_H__     */
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index a056fce0c5..919567ab27 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1997-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -29,7 +29,8 @@
 #include "global.h"
 #include "erl_message.h"
 #include "erl_process.h"
-#include "erl_nmgc.h"
+#include "erl_binary.h"
+#include "dtrace-wrapper.h"
 
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message,
 				 ErlMessage,
@@ -42,6 +43,15 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message,
 #undef HARD_DEBUG
 #endif
 
+
+
+
+static ERTS_INLINE int in_heapfrag(const Eterm* ptr, const ErlHeapFragment *bp)
+{
+    return ((unsigned)(ptr - bp->mem) < bp->used_size);
+}
+
+
 void
 init_message(void)
 {
@@ -81,9 +91,12 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size,
 #endif
     ErlHeapFragment* nbp;
 
+    /* ToDo: Make use of 'used_size' to avoid realloc
+	when shrinking just a few words */
+
 #ifdef DEBUG
     {
-	Uint off_sz = size < bp->size ? size : bp->size;
+	Uint off_sz = size < bp->used_size ? size : bp->used_size;
 	for (i = 0; i < brefs_size; i++) {
 	    Eterm *ptr;
 	    if (is_immed(brefs[i]))
@@ -95,12 +108,12 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size,
     }
 #endif
 
-    if (size == bp->size)
+    if (size == bp->used_size)
 	return bp;
 
 #ifdef HARD_DEBUG
     dbg_brefs = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(Eterm *)*brefs_size);
-    dbg_bp = new_message_buffer(bp->size);
+    dbg_bp = new_message_buffer(bp->used_size);
     dbg_hp = dbg_bp->mem;
     dbg_tot_size = 0;
     for (i = 0; i < brefs_size; i++) {
@@ -109,15 +122,15 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size,
 	dbg_brefs[i] = copy_struct(brefs[i], dbg_size, &dbg_hp,
 				   &dbg_bp->off_heap);
     }
-    ASSERT(dbg_tot_size == (size < bp->size ? size : bp->size));
+    ASSERT(dbg_tot_size == (size < bp->used_size ? size : bp->used_size));
 #endif
 
     nbp = (ErlHeapFragment*) ERTS_HEAP_REALLOC(ERTS_ALC_T_HEAP_FRAG,
 					       (void *) bp,
-					       ERTS_HEAP_FRAG_SIZE(bp->size),
+					       ERTS_HEAP_FRAG_SIZE(bp->alloc_size),
 					       ERTS_HEAP_FRAG_SIZE(size));
     if (bp != nbp) {
-	Uint off_sz = size < nbp->size ? size : nbp->size;
+	Uint off_sz = size < nbp->used_size ? size : nbp->used_size;
 	Eterm *sp = &bp->mem[0];
 	Eterm *ep = sp + off_sz;
 	Sint offs = &nbp->mem[0] - sp;
@@ -135,7 +148,7 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size,
 	}
 #endif
     }
-    nbp->size = size;
+    nbp->alloc_size = size;
     nbp->used_size = size;
 
 #ifdef HARD_DEBUG
@@ -152,26 +165,40 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size,
 void
 erts_cleanup_offheap(ErlOffHeap *offheap)
 {
-    if (offheap->mso) {
-	erts_cleanup_mso(offheap->mso);
-    }
-#ifndef HYBRID /* FIND ME! */
-    if (offheap->funs) {
-	erts_cleanup_funs(offheap->funs);
-    }
-#endif
-    if (offheap->externals) {
-	erts_cleanup_externals(offheap->externals);
+    union erl_off_heap_ptr u;
+
+    for (u.hdr = offheap->first; u.hdr; u.hdr = u.hdr->next) {
+	switch (thing_subtag(u.hdr->thing_word)) {
+	case REFC_BINARY_SUBTAG:
+	    if (erts_refc_dectest(&u.pb->val->refc, 0) == 0) {
+		erts_bin_free(u.pb->val);
+	    }
+	    break;
+	case FUN_SUBTAG:
+	    if (erts_refc_dectest(&u.fun->fe->refc, 0) == 0) {
+		erts_erase_fun_entry(u.fun->fe);		    
+	    }
+	    break;
+	default:
+	    ASSERT(is_external_header(u.hdr->thing_word));
+	    erts_deref_node_entry(u.ext->node);
+	    break;
+	}
     }
 }
 
 void
 free_message_buffer(ErlHeapFragment* bp)
 {
-    erts_cleanup_offheap(&bp->off_heap);
-    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG,
-		   (void *) bp,
-		   ERTS_HEAP_FRAG_SIZE(bp->size));
+    ASSERT(bp != NULL);
+    do {
+	ErlHeapFragment* next_bp = bp->next;
+
+	erts_cleanup_offheap(&bp->off_heap);
+	ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp,
+		       ERTS_HEAP_FRAG_SIZE(bp->size));	
+	bp = next_bp;
+    }while (bp != NULL);
 }
 
 static ERTS_INLINE void
@@ -181,43 +208,19 @@ link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp)
 	/* Link the message buffer */
 	bp->next = MBUF(proc);
 	MBUF(proc) = bp;
-	MBUF_SIZE(proc) += bp->size;
+	MBUF_SIZE(proc) += bp->used_size;
 	FLAGS(proc) |= F_FORCE_GC;
 
-	/* Move any binaries into the process */
-	if (bp->off_heap.mso != NULL) {
-	    ProcBin** next_p = &bp->off_heap.mso;
-	    while (*next_p != NULL) {
-		next_p = &((*next_p)->next);
-	    }
-	    *next_p = MSO(proc).mso;
-	    MSO(proc).mso = bp->off_heap.mso;
-	    bp->off_heap.mso = NULL;
-	    MSO(proc).overhead += bp->off_heap.overhead;
-	}
-
-	/* Move any funs into the process */
-#ifndef HYBRID
-	if (bp->off_heap.funs != NULL) {
-	    ErlFunThing** next_p = &bp->off_heap.funs;
-	    while (*next_p != NULL) {
-		next_p = &((*next_p)->next);
-	    }
-	    *next_p = MSO(proc).funs;
-	    MSO(proc).funs = bp->off_heap.funs;
-	    bp->off_heap.funs = NULL;
-	}
-#endif
-
-	/* Move any external things into the process */
-	if (bp->off_heap.externals != NULL) {
-	    ExternalThing** next_p = &bp->off_heap.externals;
+	/* Move any off_heap's into the process */
+	if (bp->off_heap.first != NULL) {
+	    struct erl_off_heap_header** next_p = &bp->off_heap.first;
 	    while (*next_p != NULL) {
 		next_p = &((*next_p)->next);
 	    }
-	    *next_p = MSO(proc).externals;
-	    MSO(proc).externals = bp->off_heap.externals;
-	    bp->off_heap.externals = NULL;
+	    *next_p = MSO(proc).first;
+	    MSO(proc).first = bp->off_heap.first;
+	    bp->off_heap.first = NULL;
+	    OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead);
 	}
     }
 }
@@ -237,12 +240,12 @@ erts_msg_distext2heap(Process *pp,
     Sint sz;
 
     *bpp = NULL;
-    sz = erts_decode_dist_ext_size(dist_extp, 0);
+    sz = erts_decode_dist_ext_size(dist_extp);
     if (sz < 0)
 	goto decode_error;
     if (is_not_nil(*tokenp)) {
 	ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp);
-	tok_sz = heap_frag->size;
+	tok_sz = heap_frag->used_size;
 	sz += tok_sz;
     }
     if (pp)
@@ -283,12 +286,13 @@ erts_msg_distext2heap(Process *pp,
 	erts_cleanup_offheap(&heap_frag->off_heap);
     }
     erts_free_dist_ext_copy(dist_extp);
-    if (*bpp)
+    if (*bpp) {
 	free_message_buffer(*bpp);
+	*bpp = NULL;
+    }    
     else if (hp) {
 	HRelease(pp, hp_end, hp);
     }
-    *bpp = NULL;
     return THE_NON_VALUE;
  }
 
@@ -298,8 +302,6 @@ notify_new_message(Process *receiver)
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS
 		       & erts_proc_lc_my_proc_locks(receiver));
 
-    ACTIVATE(receiver);
-
     switch (receiver->status) {
     case P_GARBING:
 	switch (receiver->gcstatus) {
@@ -331,6 +333,11 @@ erts_queue_dist_message(Process *rcvr,
 			Eterm token)
 {
     ErlMessage* mp;
+#ifdef USE_VM_PROBES
+    Sint tok_label = 0;
+    Sint tok_lastcnt = 0;
+    Sint tok_serial = 0;
+#endif
 #ifdef ERTS_SMP
     ErtsProcLocks need_locks;
 #endif
@@ -372,15 +379,61 @@ erts_queue_dist_message(Process *rcvr,
 	message_free(mp);
 	msg = erts_msg_distext2heap(rcvr, rcvr_locks, &mbuf, &token, dist_ext);
 	if (is_value(msg))
-	    erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token);
+#ifdef USE_VM_PROBES
+            if (DTRACE_ENABLED(message_queued)) {
+                DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
+
+                dtrace_proc_str(rcvr, receiver_name);
+                if (token != NIL && token != am_have_dt_utag) {
+                    tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
+                    tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
+                    tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
+                }
+                DTRACE6(message_queued,
+                        receiver_name, size_object(msg), rcvr->msg.len,
+                        tok_label, tok_lastcnt, tok_serial);
+            }
+#endif
+	    erts_queue_message(rcvr, rcvr_locks, mbuf, msg, token
+#ifdef USE_VM_PROBES
+			       , NIL
+#endif
+			       );
     }
     else {
 	/* Enqueue message on external format */
 
 	ERL_MESSAGE_TERM(mp) = THE_NON_VALUE;
-	ERL_MESSAGE_TOKEN(mp) = token;
+#ifdef USE_VM_PROBES
+	ERL_MESSAGE_DT_UTAG(mp) = NIL;
+	if (token == am_have_dt_utag) {
+	    ERL_MESSAGE_TOKEN(mp) = NIL;
+	} else {
+#endif
+	    ERL_MESSAGE_TOKEN(mp) = token;
+#ifdef USE_VM_PROBES
+	}
+#endif
 	mp->next = NULL;
 
+#ifdef USE_VM_PROBES
+        if (DTRACE_ENABLED(message_queued)) {
+            DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
+
+            dtrace_proc_str(rcvr, receiver_name);
+            if (token != NIL && token != am_have_dt_utag) {
+                tok_label = signed_val(SEQ_TRACE_T_LABEL(token));
+                tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token));
+                tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token));
+            }
+            /*
+             * TODO: We don't know the real size of the external message here.
+             *       -1 will appear to a D script as 4294967295.
+             */
+            DTRACE6(message_queued, receiver_name, -1, rcvr->msg.len + 1,
+                    tok_label, tok_lastcnt, tok_serial);
+        }
+#endif
 	mp->data.dist_ext = dist_ext;
 	LINK_MESSAGE(rcvr, mp);
 
@@ -394,7 +447,11 @@ erts_queue_message(Process* receiver,
 		   ErtsProcLocks *receiver_locks,
 		   ErlHeapFragment* bp,
 		   Eterm message,
-		   Eterm seq_trace_token)
+		   Eterm seq_trace_token
+#ifdef USE_VM_PROBES
+		   , Eterm dt_utag
+#endif
+)
 {
     ErlMessage* mp;
 #ifdef ERTS_SMP
@@ -435,12 +492,14 @@ erts_queue_message(Process* receiver,
 
     ERL_MESSAGE_TERM(mp) = message;
     ERL_MESSAGE_TOKEN(mp) = seq_trace_token;
+#ifdef USE_VM_PROBES
+    ERL_MESSAGE_DT_UTAG(mp) = dt_utag;
+#endif
     mp->next = NULL;
+    mp->data.heap_frag = bp;
 
 #ifdef ERTS_SMP
     if (*receiver_locks & ERTS_PROC_LOCK_MAIN) {
-	mp->data.heap_frag = bp;
-
 	/*
 	 * We move 'in queue' to 'private queue' and place
 	 * message at the end of 'private queue' in order
@@ -453,20 +512,36 @@ erts_queue_message(Process* receiver,
 	LINK_MESSAGE_PRIVQ(receiver, mp);
     }
     else {
-	mp->data.heap_frag = bp;
 	LINK_MESSAGE(receiver, mp);
     }
 #else
-    mp->data.heap_frag = bp;
     LINK_MESSAGE(receiver, mp);
 #endif
 
+#ifdef USE_VM_PROBES
+    if (DTRACE_ENABLED(message_queued)) {
+        DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
+        Sint tok_label = 0;
+        Sint tok_lastcnt = 0;
+        Sint tok_serial = 0;
+
+        dtrace_proc_str(receiver, receiver_name);
+        if (seq_trace_token != NIL && is_tuple(seq_trace_token)) {
+            tok_label = signed_val(SEQ_TRACE_T_LABEL(seq_trace_token));
+            tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(seq_trace_token));
+            tok_serial = signed_val(SEQ_TRACE_T_SERIAL(seq_trace_token));
+        }
+        DTRACE6(message_queued,
+                receiver_name, size_object(message), receiver->msg.len,
+                tok_label, tok_lastcnt, tok_serial);
+    }
+#endif
     notify_new_message(receiver);
 
     if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) {
 	trace_receive(receiver, message);
     }
-    
+
 #ifndef ERTS_SMP
     ERTS_HOLE_CHECK(receiver);
 #endif
@@ -491,23 +566,14 @@ erts_link_mbuf_to_proc(struct process *proc, ErlHeapFragment *bp)
 void
 erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 {
-    /* Unions for typecasts avoids warnings about type-punned pointers and aliasing */
-    union {
-	Uint** upp;
-	ProcBin **pbpp;
-	ErlFunThing **efpp;
-	ExternalThing **etpp;
-    } oh_list_pp, oh_el_next_pp;
-    union {
-	Uint *up;
-	ProcBin *pbp;
-	ErlFunThing *efp;
-	ExternalThing *etp;
-    } oh_el_p;
+    struct erl_off_heap_header* oh;
     Eterm term, token, *fhp, *hp;
     Sint offs;
     Uint sz;
     ErlHeapFragment *bp;
+#ifdef USE_VM_PROBES
+    Eterm utag;
+#endif
 
 #ifdef HARD_DEBUG
     ProcBin *dbg_mso_start = off_heap->mso;
@@ -517,53 +583,70 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
     ErlHeapFragment *dbg_bp;
     Uint *dbg_hp, *dbg_thp_start;
     Uint dbg_term_sz, dbg_token_sz;
+#ifdef USE_VM_PROBES
+    Eterm dbg_utag;
+    Uint dbg_utag_sz;
+#endif
 #endif
 
     bp = msg->data.heap_frag;
     term = ERL_MESSAGE_TERM(msg);
     token = ERL_MESSAGE_TOKEN(msg);
+#ifdef USE_VM_PROBES
+    utag = ERL_MESSAGE_DT_UTAG(msg);
+#endif
     if (!bp) {
+#ifdef USE_VM_PROBES
+	ASSERT(is_immed(term) && is_immed(token) && is_immed(utag));
+#else
 	ASSERT(is_immed(term) && is_immed(token));
+#endif
 	return;
     }
 
 #ifdef HARD_DEBUG
     dbg_term_sz = size_object(term);
     dbg_token_sz = size_object(token);
-    ASSERT(bp->size == dbg_term_sz + dbg_token_sz);
-
-    dbg_bp = new_message_buffer(bp->size);
+    dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz);
+#ifdef USE_VM_PROBES
+    dbg_utag_sz = size_object(utag);
+    dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz + dbg_utag_sz );
+#endif
+    /*ASSERT(dbg_term_sz + dbg_token_sz == erts_msg_used_frag_sz(msg));
+      Copied size may be smaller due to removed SubBins's or garbage.
+      Copied size may be larger due to duplicated shared terms.
+    */
     dbg_hp = dbg_bp->mem;
     dbg_term = copy_struct(term, dbg_term_sz, &dbg_hp, &dbg_bp->off_heap);
     dbg_token = copy_struct(token, dbg_token_sz, &dbg_hp, &dbg_bp->off_heap);
-    dbg_thp_start = *hpp;
+#ifdef USE_VM_PROBES
+    dbg_utag = copy_struct(utag, dbg_utag_sz, &dbg_hp, &dbg_bp->off_heap);
+#endif
+   dbg_thp_start = *hpp;
 #endif
 
-    ASSERT(bp);
-    msg->data.attached = NULL;
-
-    off_heap->overhead += bp->off_heap.overhead;
-    sz = bp->size;
-
-#ifdef DEBUG
-    if (is_not_immed(term)) {
-	ASSERT(bp->mem <= ptr_val(term));
-	ASSERT(bp->mem + bp->size > ptr_val(term));
+    if (bp->next != NULL) {
+	move_multi_frags(hpp, off_heap, bp, msg->m, 
+#ifdef USE_VM_PROBES
+			 3
+#else
+			 2
+#endif
+			 );
+	goto copy_done;
     }
 
-    if (is_not_immed(token)) {
-	ASSERT(bp->mem <= ptr_val(token));
-	ASSERT(bp->mem + bp->size > ptr_val(token));
-    }
-#endif
+    OH_OVERHEAD(off_heap, bp->off_heap.overhead);
+    sz = bp->used_size;
+
+    ASSERT(is_immed(term) || in_heapfrag(ptr_val(term),bp));
+    ASSERT(is_immed(token) || in_heapfrag(ptr_val(token),bp));
 
     fhp = bp->mem;
     hp = *hpp;
     offs = hp - fhp;
 
-    oh_list_pp.upp = NULL;
-    oh_el_next_pp.upp = NULL; /* Shut up compiler warning */
-    oh_el_p.up = NULL; /* Shut up compiler warning */
+    oh = NULL;
     while (sz--) {
 	Uint cpy_sz;
 	Eterm val = *fhp++;
@@ -574,8 +657,7 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 	    break;
 	case TAG_PRIMARY_LIST:
 	case TAG_PRIMARY_BOXED:
-	    ASSERT(bp->mem <= ptr_val(val));
-	    ASSERT(bp->mem + bp->size > ptr_val(val));
+	    ASSERT(in_heapfrag(ptr_val(val), bp));
 	    *hp++ = offset_ptr(val, offs);
 	    break;
 	case TAG_PRIMARY_HEADER:
@@ -584,31 +666,18 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 	    case ARITYVAL_SUBTAG:
 		break;
 	    case REFC_BINARY_SUBTAG:
-		oh_list_pp.pbpp = &off_heap->mso;
-		oh_el_p.up = (hp-1);
-		oh_el_next_pp.pbpp = &(oh_el_p.pbp)->next;
-		cpy_sz = thing_arityval(val);
-		goto cpy_words;
 	    case FUN_SUBTAG:
-#ifndef HYBRID
-		oh_list_pp.efpp = &off_heap->funs;
-		oh_el_p.up = (hp-1);
-		oh_el_next_pp.efpp = &(oh_el_p.efp)->next;
-#endif
-		cpy_sz = thing_arityval(val);
-		goto cpy_words;
 	    case EXTERNAL_PID_SUBTAG:
 	    case EXTERNAL_PORT_SUBTAG:
 	    case EXTERNAL_REF_SUBTAG:
-		oh_list_pp.etpp = &off_heap->externals;
-		oh_el_p.up = (hp-1);
-		oh_el_next_pp.etpp =  &(oh_el_p.etp)->next;
+		oh = (struct erl_off_heap_header*) (hp-1);
 		cpy_sz = thing_arityval(val);
 		goto cpy_words;
 	    default:
 		cpy_sz = header_arity(val);
 
 	    cpy_words:
+		ASSERT(sz >= cpy_sz);
 		sz -= cpy_sz;
 		while (cpy_sz >= 8) {
 		    cpy_sz -= 8;
@@ -631,44 +700,13 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 		case 1: *hp++ = *fhp++;
 		default: break;
 		}
-		if (oh_list_pp.upp) {
-#ifdef HARD_DEBUG
-		    Uint *dbg_old_oh_list_p = *oh_list_pp.upp;
-#endif
+		if (oh) {
 		    /* Add to offheap list */
-		    *oh_el_next_pp.upp = *oh_list_pp.upp;
-		    *oh_list_pp.upp = oh_el_p.up;
-		    ASSERT(*hpp <= oh_el_p.up);
-		    ASSERT(hp > oh_el_p.up);
-#ifdef HARD_DEBUG
-		    switch (val & _HEADER_SUBTAG_MASK) {
-		    case REFC_BINARY_SUBTAG:
-			ASSERT(off_heap->mso == *oh_list_pp.pbpp);
-			ASSERT(off_heap->mso->next
-			       == (ProcBin *) dbg_old_oh_list_p);
-			break;
-#ifndef HYBRID
-		    case FUN_SUBTAG:
-			ASSERT(off_heap->funs == *oh_list_pp.efpp);
-			ASSERT(off_heap->funs->next
-			       == (ErlFunThing *) dbg_old_oh_list_p);
-			break;
-#endif
-		    case EXTERNAL_PID_SUBTAG:
-		    case EXTERNAL_PORT_SUBTAG:
-		    case EXTERNAL_REF_SUBTAG:
-			ASSERT(off_heap->externals
-			       == *oh_list_pp.etpp);
-			ASSERT(off_heap->externals->next
-			       == (ExternalThing *) dbg_old_oh_list_p);
-			break;
-		    default:
-			ASSERT(0);
-		    }
-#endif
-		    oh_list_pp.upp = NULL;
-
-
+		    oh->next = off_heap->first;
+		    off_heap->first = oh;
+		    ASSERT(*hpp <= (Eterm*)oh);
+		    ASSERT(hp > (Eterm*)oh);
+		    oh = NULL;
 		}
 		break;
 	    }
@@ -676,12 +714,11 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 	}
     }
 
-    ASSERT(bp->size == hp - *hpp);
+    ASSERT(bp->used_size == hp - *hpp);
     *hpp = hp;
 
     if (is_not_immed(token)) {
-	ASSERT(bp->mem <= ptr_val(token));
-	ASSERT(bp->mem + bp->size > ptr_val(token));
+	ASSERT(in_heapfrag(ptr_val(token), bp));
 	ERL_MESSAGE_TOKEN(msg) = offset_ptr(token, offs);
 #ifdef HARD_DEBUG
 	ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TOKEN(msg)));
@@ -690,19 +727,30 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
     }
 
     if (is_not_immed(term)) {
-	ASSERT(bp->mem <= ptr_val(term));
-	ASSERT(bp->mem + bp->size > ptr_val(term));
+	ASSERT(in_heapfrag(ptr_val(term),bp));
 	ERL_MESSAGE_TERM(msg) = offset_ptr(term, offs);
 #ifdef HARD_DEBUG
 	ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TERM(msg)));
 	ASSERT(hp > ptr_val(ERL_MESSAGE_TERM(msg)));
 #endif
     }
+#ifdef USE_VM_PROBES
+    if (is_not_immed(utag)) {
+	ASSERT(in_heapfrag(ptr_val(utag), bp));
+	ERL_MESSAGE_DT_UTAG(msg) = offset_ptr(utag, offs);
+#ifdef HARD_DEBUG
+	ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_DT_UTAG(msg)));
+	ASSERT(hp > ptr_val(ERL_MESSAGE_DT_UTAG(msg)));
+#endif
+    }
+#endif
 
+copy_done:
 
 #ifdef HARD_DEBUG
     {
 	int i, j;
+	ErlHeapFragment* frag;
 	{
 	    ProcBin *mso = off_heap->mso;
 	    i = j = 0;
@@ -710,10 +758,12 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 		mso = mso->next;
 		i++;
 	    }
-	    mso = bp->off_heap.mso;
-	    while (mso) {
-		mso = mso->next;
-		j++;
+	    for (frag=bp; frag; frag=frag->next) {
+		mso = frag->off_heap.mso;
+		while (mso) {
+		    mso = mso->next;
+		    j++;
+		}
 	    }
 	    ASSERT(i == j);
 	}
@@ -724,10 +774,12 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 		fun = fun->next;
 		i++;
 	    }
-	    fun = bp->off_heap.funs;
-	    while (fun) {
-		fun = fun->next;
-		j++;
+	    for (frag=bp; frag; frag=frag->next) {
+		fun = frag->off_heap.funs;
+		while (fun) {
+		    fun = fun->next;
+		    j++;
+		}
 	    }
 	    ASSERT(i == j);
 	}
@@ -738,10 +790,12 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 		external = external->next;
 		i++;
 	    }
-	    external = bp->off_heap.externals;
-	    while (external) {
-		external = external->next;
-		j++;
+	    for (frag=bp; frag; frag=frag->next) {
+		external = frag->off_heap.externals;
+		while (external) {
+		    external = external->next;
+		    j++;
+		}
 	    }
 	    ASSERT(i == j);
 	}
@@ -749,21 +803,22 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 #endif
 	    
 
-    bp->off_heap.mso = NULL;
-#ifndef HYBRID
-    bp->off_heap.funs = NULL;
-#endif
-    bp->off_heap.externals = NULL;
+    bp->off_heap.first = NULL;
     free_message_buffer(bp);
+    msg->data.heap_frag = NULL;
 
 #ifdef HARD_DEBUG
     ASSERT(eq(ERL_MESSAGE_TERM(msg), dbg_term));
     ASSERT(eq(ERL_MESSAGE_TOKEN(msg), dbg_token));
+#ifdef USE_VM_PROBES
+    ASSERT(eq(ERL_MESSAGE_DT_UTAG(msg), dbg_utag));
+#endif
     free_message_buffer(dbg_bp);
 #endif
 
 }
 
+
 Uint
 erts_msg_attached_data_size_aux(ErlMessage *msg)
 {
@@ -772,7 +827,7 @@ erts_msg_attached_data_size_aux(ErlMessage *msg)
     ASSERT(msg->data.dist_ext);
     ASSERT(msg->data.dist_ext->heap_size < 0);
 
-    sz = erts_decode_dist_ext_size(msg->data.dist_ext, 0);
+    sz = erts_decode_dist_ext_size(msg->data.dist_ext);
     if (sz < 0) {
 	/* Bad external; remove it */
 	if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) {
@@ -789,7 +844,7 @@ erts_msg_attached_data_size_aux(ErlMessage *msg)
     if (is_not_nil(msg->m[1])) {
 	ErlHeapFragment *heap_frag;
 	heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
-	sz += heap_frag->size;
+	sz += heap_frag->used_size;
     }
     return sz;
 }
@@ -805,7 +860,7 @@ erts_move_msg_attached_data_to_heap(Eterm **hpp, ErlOffHeap *ohp, ErlMessage *ms
 	    ErlHeapFragment *heap_frag;
 	    heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
 	    ERL_MESSAGE_TOKEN(msg) = copy_struct(ERL_MESSAGE_TOKEN(msg),
-						 heap_frag->size,
+						 heap_frag->used_size,
 						 hpp,
 						 ohp);
 	    erts_cleanup_offheap(&heap_frag->off_heap);
@@ -833,85 +888,102 @@ erts_send_message(Process* sender,
     Uint msize;
     ErlHeapFragment* bp = NULL;
     Eterm token = NIL;
-
+#ifdef USE_VM_PROBES
+    DTRACE_CHARBUF(sender_name, 64);
+    DTRACE_CHARBUF(receiver_name, 64);
+    Sint tok_label = 0;
+    Sint tok_lastcnt = 0;
+    Sint tok_serial = 0;
+#endif
     BM_STOP_TIMER(system);
     BM_MESSAGE(message,sender,receiver);
     BM_START_TIMER(send);
 
+ #ifdef USE_VM_PROBES
+    *sender_name = *receiver_name = '\0';
+   if (DTRACE_ENABLED(message_send)) {
+        erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), "%T", sender->id);
+        erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), "%T", receiver->id);
+    }
+#endif
     if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) {
         Eterm* hp;
+	Eterm stoken = SEQ_TRACE_TOKEN(sender);
+	Uint seq_trace_size = 0;
+#ifdef USE_VM_PROBES
+	Uint dt_utag_size = 0;
+	Eterm utag = NIL;
+#endif
 
-        BM_SWAP_TIMER(send,size);
+	BM_SWAP_TIMER(send,size);
 	msize = size_object(message);
-        BM_SWAP_TIMER(size,send);
+	BM_SWAP_TIMER(size,send);
+
+#ifdef USE_VM_PROBES
+	if (stoken != am_have_dt_utag) {
+#endif
+
+	    seq_trace_update_send(sender);
+	    seq_trace_output(stoken, message, SEQ_TRACE_SEND, 
+			     receiver->id, sender);
+	    seq_trace_size = 6; /* TUPLE5 */
+#ifdef USE_VM_PROBES
+	}
+	if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
+	    dt_utag_size = size_object(DT_UTAG(sender));
+	} else if (stoken == am_have_dt_utag ) {
+	    stoken = NIL;
+	}
+#endif
 
-	seq_trace_update_send(sender);
-	seq_trace_output(SEQ_TRACE_TOKEN(sender), message, SEQ_TRACE_SEND, 
-			 receiver->id, sender);
-	bp = new_message_buffer(msize + 6 /* TUPLE5 */);
+	bp = new_message_buffer(msize + seq_trace_size 
+#ifdef USE_VM_PROBES
+				+ dt_utag_size
+#endif
+				);
 	hp = bp->mem;
 
         BM_SWAP_TIMER(send,copy);
-	token = copy_struct(SEQ_TRACE_TOKEN(sender),
-			    6 /* TUPLE5 */,
+	token = copy_struct(stoken,
+			    seq_trace_size,
 			    &hp,
 			    &bp->off_heap);
 
 	message = copy_struct(message, msize, &hp, &bp->off_heap);
+#ifdef USE_VM_PROBES
+	if (DT_UTAG_FLAGS(sender) & DT_UTAG_SPREADING) {
+	    utag = copy_struct(DT_UTAG(sender), dt_utag_size, &hp, &bp->off_heap);
+#ifdef DTRACE_TAG_HARDDEBUG
+	    erts_fprintf(stderr,
+			 "Dtrace -> (%T) Spreading tag (%T) with "
+			 "message %T!\r\n",sender->id, utag, message);
+#endif
+	}
+#endif
         BM_MESSAGE_COPIED(msize);
         BM_SWAP_TIMER(copy,send);
 
+#ifdef USE_VM_PROBES
+        if (DTRACE_ENABLED(message_send)) {
+	    if (stoken != NIL && stoken != am_have_dt_utag) {
+		tok_label = signed_val(SEQ_TRACE_T_LABEL(stoken));
+		tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(stoken));
+		tok_serial = signed_val(SEQ_TRACE_T_SERIAL(stoken));
+	    }
+	    DTRACE6(message_send, sender_name, receiver_name,
+		    msize, tok_label, tok_lastcnt, tok_serial);
+        }
+#endif
         erts_queue_message(receiver,
 			   receiver_locks,
 			   bp,
 			   message,
-			   token);
-        BM_SWAP_TIMER(send,system);
-#ifdef HYBRID
-    } else {
-        ErlMessage* mp = message_alloc();
-        BM_SWAP_TIMER(send,copy);
-#ifdef INCREMENTAL
-        /* TODO: During GC activate processes if the message relies in
-         * the fromspace and the sender is active. During major
-         * collections add the message to the gray stack if it relies
-         * in the old generation and the sender is active and the
-         * receiver is inactive.
-
-        if (!IS_CONST(message) && (ma_gc_flags & GC_CYCLE) &&
-            (ptr_val(message) >= inc_fromspc &&
-            ptr_val(message) < inc_fromend) && INC_IS_ACTIVE(sender))
-            INC_ACTIVATE(receiver);
-        else if (!IS_CONST(message) && (ma_gc_flags & GC_CYCLE) &&
-            (ptr_val(message) >= global_old_heap &&
-            ptr_val(message) < global_old_hend) &&
-            INC_IS_ACTIVE(sender) && !INC_IS_ACTIVE(receiver))
-            Mark message in blackmap and add it to the gray stack
-        */
-
-         if (!IS_CONST(message))
-            INC_ACTIVATE(receiver);
+			   token
+#ifdef USE_VM_PROBES
+			   , utag
 #endif
-        LAZY_COPY(sender,message);
-        BM_SWAP_TIMER(copy,send);
-        ERL_MESSAGE_TERM(mp) = message;
-        ERL_MESSAGE_TOKEN(mp) = NIL;
-        mp->next = NULL;
-	LINK_MESSAGE(receiver, mp);
-        ACTIVATE(receiver);
-
-        if (receiver->status == P_WAITING) {
-            erts_add_to_runq(receiver);
-        } else if (receiver->status == P_SUSPENDED) {
-            receiver->rstatus = P_RUNABLE;
-        }
-        if (IS_TRACED_FL(receiver, F_TRACE_RECEIVE)) {
-            trace_receive(receiver, message);
-        }
-
+			   );
         BM_SWAP_TIMER(send,system);
-        return;
-#else
     } else if (sender == receiver) {
 	/* Drop message if receiver has a pending exit ... */
 #ifdef ERTS_SMP
@@ -933,9 +1005,14 @@ erts_send_message(Process* sender,
 	{
 	    ErlMessage* mp = message_alloc();
 
+            DTRACE6(message_send, sender_name, receiver_name,
+                    size_object(message), tok_label, tok_lastcnt, tok_serial);
 	    mp->data.attached = NULL;
 	    ERL_MESSAGE_TERM(mp) = message;
 	    ERL_MESSAGE_TOKEN(mp) = NIL;
+#ifdef USE_VM_PROBES
+	    ERL_MESSAGE_DT_UTAG(mp) = NIL;
+#endif
 	    mp->next = NULL;
 	    /*
 	     * We move 'in queue' to 'private queue' and place
@@ -967,7 +1044,13 @@ erts_send_message(Process* sender,
 	message = copy_struct(message, msize, &hp, ohp);
 	BM_MESSAGE_COPIED(msz);
 	BM_SWAP_TIMER(copy,send);
-	erts_queue_message(receiver, receiver_locks, bp, message, token);
+        DTRACE6(message_send, sender_name, receiver_name,
+                msize, tok_label, tok_lastcnt, tok_serial);
+	erts_queue_message(receiver, receiver_locks, bp, message, token
+#ifdef USE_VM_PROBES
+			   , NIL
+#endif
+			   );
         BM_SWAP_TIMER(send,system);
 #else
 	ErlMessage* mp = message_alloc();
@@ -987,8 +1070,13 @@ erts_send_message(Process* sender,
 	message = copy_struct(message, msize, &hp, &receiver->off_heap);
 	BM_MESSAGE_COPIED(msize);
         BM_SWAP_TIMER(copy,send);
+        DTRACE6(message_send, sender_name, receiver_name,
+                (uint32_t)msize, tok_label, tok_lastcnt, tok_serial);
 	ERL_MESSAGE_TERM(mp) = message;
 	ERL_MESSAGE_TOKEN(mp) = NIL;
+#ifdef USE_VM_PROBES
+	ERL_MESSAGE_DT_UTAG(mp) = NIL;
+#endif
 	mp->next = NULL;
 	mp->data.attached = NULL;
 	LINK_MESSAGE(receiver, mp);
@@ -1004,7 +1092,6 @@ erts_send_message(Process* sender,
         BM_SWAP_TIMER(send,system);
 #endif /* #ifndef ERTS_SMP */
 	return;
-#endif /* HYBRID */
     }
 }
 
@@ -1027,7 +1114,11 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
     Eterm temptoken;
     ErlHeapFragment* bp = NULL;
 
-    if (token != NIL) {
+    if (token != NIL
+#ifdef USE_VM_PROBES
+	&& token != am_have_dt_utag
+#endif
+	) {
 
 	ASSERT(is_tuple(token));
 	sz_reason = size_object(reason);
@@ -1042,7 +1133,11 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
 	/* the trace token must in this case be updated by the caller */
 	seq_trace_output(token, save, SEQ_TRACE_SEND, to->id, NULL);
 	temptoken = copy_struct(token, sz_token, &hp, &bp->off_heap);
-	erts_queue_message(to, to_locksp, bp, save, temptoken);
+	erts_queue_message(to, to_locksp, bp, save, temptoken
+#ifdef USE_VM_PROBES
+			   , NIL
+#endif
+			   );
     } else {
 	ErlOffHeap *ohp;
 	sz_reason = size_object(reason);
@@ -1059,6 +1154,11 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp,
 		     ? from
 		     : copy_struct(from, sz_from, &hp, ohp));
 	save = TUPLE3(hp, am_EXIT, from_copy, mess);
-	erts_queue_message(to, to_locksp, bp, save, NIL);
+	erts_queue_message(to, to_locksp, bp, save, NIL
+#ifdef USE_VM_PROBES
+			   , NIL
+#endif
+			   );
     }
 }
+
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 5cf7c209bd..3e9a24ee81 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1997-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -28,13 +28,22 @@ struct external_thing_;
  * but is stored outside of any heap.
  */
 
-typedef struct erl_off_heap {
-    struct proc_bin* mso;	/* List of associated binaries. */
-#ifndef HYBRID /* FIND ME! */
-    struct erl_fun_thing* funs;	/* List of funs. */
+struct erl_off_heap_header {
+    Eterm thing_word;
+    Uint size;
+#if HALFWORD_HEAP
+    void* dummy_ptr_padding__;
 #endif
-    struct external_thing_* externals; /* List of external things. */
-    int overhead;		/* Administrative overhead (used to force GC). */
+    struct erl_off_heap_header* next;
+};
+
+#define OH_OVERHEAD(oh, size) do { \
+    (oh)->overhead += size;        \
+} while(0)
+
+typedef struct erl_off_heap {
+    struct erl_off_heap_header* first;
+    Uint64 overhead;     /* Administrative overhead (used to force GC). */
 } ErlOffHeap;
 
 #include "external.h"
@@ -49,7 +58,7 @@ typedef struct erl_heap_fragment ErlHeapFragment;
 struct erl_heap_fragment {
     ErlHeapFragment* next;	/* Next heap fragment */
     ErlOffHeap off_heap;	/* Offset heap data. */
-    unsigned size;		/* Size in words of mem */
+    unsigned alloc_size;	/* Size in (half)words of mem */
     unsigned used_size;         /* With terms to be moved to heap by GC */
     Eterm mem[1];		/* Data */
 };
@@ -61,11 +70,18 @@ typedef struct erl_mesg {
 	ErlHeapFragment *heap_frag;
 	void *attached;
     } data;
+#ifdef USE_VM_PROBES
+    Eterm m[3];                 /* m[0] = message, m[1] = seq trace token, m[3] = dynamic trace user tag */
+#else
     Eterm m[2];			/* m[0] = message, m[1] = seq trace token */
+#endif
 } ErlMessage;
 
 #define ERL_MESSAGE_TERM(mp) ((mp)->m[0])
 #define ERL_MESSAGE_TOKEN(mp) ((mp)->m[1])
+#ifdef USE_VM_PROBES
+#define ERL_MESSAGE_DT_UTAG(mp) ((mp)->m[2])
+#endif
 
 /* Size of default message buffer (erl_message.c) */
 #define ERL_MESSAGE_BUF_SZ 500
@@ -75,6 +91,13 @@ typedef struct {
     ErlMessage** last;  /* point to the last next pointer */
     ErlMessage** save;
     int len;            /* queue length */
+
+    /*
+     * The following two fields are used by the recv_mark/1 and
+     * recv_set/1 instructions.
+     */
+    BeamInstr* mark;		/* address to rec_loop/2 instruction */
+    ErlMessage** saved_last;	/* saved last pointer */
 } ErlMessageQueue;
 
 #ifdef ERTS_SMP
@@ -137,6 +160,7 @@ do {							\
      (p)->msg.len--; \
      if (__mp == NULL) \
          (p)->msg.last = (p)->msg.save; \
+     (p)->msg.mark = 0; \
 } while(0)
 
 /* Reset message save point (after receive match) */
@@ -191,11 +215,9 @@ do {									\
 #define ERTS_INIT_HEAP_FRAG(HEAP_FRAG_P, DATA_WORDS)	\
 do {							\
     (HEAP_FRAG_P)->next = NULL;				\
-    (HEAP_FRAG_P)->size = (DATA_WORDS);			\
+    (HEAP_FRAG_P)->alloc_size = (DATA_WORDS);		\
     (HEAP_FRAG_P)->used_size = (DATA_WORDS);            \
-    (HEAP_FRAG_P)->off_heap.mso = NULL;			\
-    (HEAP_FRAG_P)->off_heap.funs = NULL;		\
-    (HEAP_FRAG_P)->off_heap.externals = NULL;		\
+    (HEAP_FRAG_P)->off_heap.first = NULL; 	        \
     (HEAP_FRAG_P)->off_heap.overhead = 0;		\
 } while (0)
 
@@ -206,7 +228,11 @@ ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint,
 					    Eterm *, Uint);
 void free_message_buffer(ErlHeapFragment *);
 void erts_queue_dist_message(Process*, ErtsProcLocks*, ErtsDistExternal *, Eterm);
-void erts_queue_message(Process*, ErtsProcLocks*, ErlHeapFragment*, Eterm, Eterm);
+void erts_queue_message(Process*, ErtsProcLocks*, ErlHeapFragment*, Eterm, Eterm
+#ifdef USE_VM_PROBES
+		   , Eterm dt_utag
+#endif
+);
 void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm);
 void erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned);
 void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp);
@@ -219,14 +245,25 @@ void erts_move_msg_attached_data_to_heap(Eterm **, ErlOffHeap *, ErlMessage *);
 Eterm erts_msg_distext2heap(Process *, ErtsProcLocks *, ErlHeapFragment **,
 			    Eterm *, ErtsDistExternal *);
 
+ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg);
 ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg)
+{
+    const ErlHeapFragment *bp;
+    Uint sz = 0;
+    for (bp = msg->data.heap_frag; bp!=NULL; bp=bp->next) {
+	sz += bp->used_size;
+    }
+    return sz;
+}
+
 ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg)
 {
     ASSERT(msg->data.attached);
     if (is_value(ERL_MESSAGE_TERM(msg)))
-	return msg->data.heap_frag->size;
+	return erts_msg_used_frag_sz(msg);
     else if (msg->data.dist_ext->heap_size < 0)
 	return erts_msg_attached_data_size_aux(msg);
     else {
@@ -234,7 +271,7 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg)
 	if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) {
 	    ErlHeapFragment *heap_frag;
 	    heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
-	    sz += heap_frag->size;
+	    sz += heap_frag->used_size;
 	}
 	return sz;
     }
diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c
index d873c7a701..1a84950120 100644
--- a/erts/emulator/beam/erl_monitors.c
+++ b/erts/emulator/beam/erl_monitors.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -85,7 +85,7 @@ static ERTS_INLINE int cmp_mon_ref(Eterm ref1, Eterm ref2)
     if (is_ref_thing_header(*b2)) {
 	return 1;
     }
-    return cmp(ref1,ref2);
+    return CMP(ref1,ref2);
 }
 	    
 #define CP_LINK_VAL(To, Hp, From)				\
@@ -125,7 +125,7 @@ static ErtsMonitor *create_monitor(Uint type, Eterm ref, Eterm pid, Eterm name)
      } else {
 	 n = (ErtsMonitor *) erts_alloc(ERTS_ALC_T_MONITOR_LH,
 					mon_size*sizeof(Uint));
-	 erts_smp_atomic_add(&tot_link_lh_size, mon_size*sizeof(Uint));
+	 erts_smp_atomic_add_nob(&tot_link_lh_size, mon_size*sizeof(Uint));
      } 
      hp = n->heap;
 
@@ -156,7 +156,7 @@ static ErtsLink *create_link(Uint type, Eterm pid)
      } else {
 	 n = (ErtsLink *) erts_alloc(ERTS_ALC_T_NLINK_LH,
 				     lnk_size*sizeof(Uint));
-	 erts_smp_atomic_add(&tot_link_lh_size, lnk_size*sizeof(Uint));
+	 erts_smp_atomic_add_nob(&tot_link_lh_size, lnk_size*sizeof(Uint));
      } 
      hp = n->heap;
 
@@ -191,13 +191,13 @@ static ErtsSuspendMonitor *create_suspend_monitor(Eterm pid)
 void
 erts_init_monitors(void)
 {
-    erts_smp_atomic_init(&tot_link_lh_size, 0);
+    erts_smp_atomic_init_nob(&tot_link_lh_size, 0);
 }
 
 Uint
 erts_tot_link_lh_size(void)
 {
-    return (Uint) erts_smp_atomic_read(&tot_link_lh_size);
+    return (Uint) erts_smp_atomic_read_nob(&tot_link_lh_size);
 }
 
 void erts_destroy_monitor(ErtsMonitor *mon)
@@ -222,7 +222,7 @@ void erts_destroy_monitor(ErtsMonitor *mon)
 	erts_free(ERTS_ALC_T_MONITOR_SH, (void *) mon);
     } else {
 	erts_free(ERTS_ALC_T_MONITOR_LH, (void *) mon);
-	erts_smp_atomic_add(&tot_link_lh_size, -1*mon_size*sizeof(Uint));
+	erts_smp_atomic_add_nob(&tot_link_lh_size, -1*mon_size*sizeof(Uint));
     }
 }
     
@@ -244,7 +244,7 @@ void erts_destroy_link(ErtsLink *lnk)
 	erts_free(ERTS_ALC_T_NLINK_SH, (void *) lnk);
     } else {
 	erts_free(ERTS_ALC_T_NLINK_LH, (void *) lnk);
-	erts_smp_atomic_add(&tot_link_lh_size, -1*lnk_size*sizeof(Uint));
+	erts_smp_atomic_add_nob(&tot_link_lh_size, -1*lnk_size*sizeof(Uint));
     }
 }
 
@@ -380,7 +380,7 @@ int erts_add_link(ErtsLink **root, Uint type, Eterm pid)
 	    state = 1;
 	    *this = create_link(type,pid);
 	    break;
-	} else if ((c = cmp(pid,(*this)->pid)) < 0) { 
+	} else if ((c = CMP(pid,(*this)->pid)) < 0) {
 	    /* go left */
 	    dstack[dpos++] = DIR_LEFT;
 	    tstack[tpos++] = this;
@@ -415,7 +415,7 @@ erts_add_or_lookup_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid)
 	    state = 1;
 	    res = *this = create_suspend_monitor(pid);
 	    break;
-	} else if ((c = cmp(pid,(*this)->pid)) < 0) { 
+	} else if ((c = CMP(pid,(*this)->pid)) < 0) {
 	    /* go left */
 	    dstack[dpos++] = DIR_LEFT;
 	    tstack[tpos++] = this;
@@ -453,7 +453,7 @@ ErtsLink *erts_add_or_lookup_link(ErtsLink **root, Uint type, Eterm pid)
 	    *this = create_link(type,pid);
 	    ret = *this;
 	    break;
-	} else if ((c = cmp(pid,(*this)->pid)) < 0) { 
+	} else if ((c = CMP(pid,(*this)->pid)) < 0) {
 	    /* go left */
 	    dstack[dpos++] = DIR_LEFT;
 	    tstack[tpos++] = this;
@@ -663,7 +663,7 @@ ErtsLink *erts_remove_link(ErtsLink **root, Eterm pid)
     for (;;) {
 	if (!*this) { /* Failure */
 	    return NULL;
-	} else if ((c = cmp(pid,(*this)->pid)) < 0) { 
+	} else if ((c = CMP(pid,(*this)->pid)) < 0) {
 	    dstack[dpos++] = DIR_LEFT;
 	    tstack[tpos++] = this;
 	    this = &((*this)->left);
@@ -715,7 +715,7 @@ erts_delete_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid)
     for (;;) {
 	if (!*this) { /* Nothing found */
 	    return;
-	} else if ((c = cmp(pid,(*this)->pid)) < 0) { 
+	} else if ((c = CMP(pid,(*this)->pid)) < 0) {
 	    dstack[dpos++] = DIR_LEFT;
 	    tstack[tpos++] = this;
 	    this = &((*this)->left);
@@ -771,7 +771,7 @@ ErtsLink *erts_lookup_link(ErtsLink *root, Eterm pid)
     Sint c;
 
     for (;;) {
-	if (root == NULL || (c = cmp(pid,root->pid)) == 0) {
+	if (root == NULL || (c = CMP(pid,root->pid)) == 0) {
 	    return root;
 	} else if (c < 0) { 
 	    root = root->left;
@@ -787,7 +787,7 @@ erts_lookup_suspend_monitor(ErtsSuspendMonitor *root, Eterm pid)
     Sint c;
 
     for (;;) {
-	if (root == NULL || (c = cmp(pid,root->pid)) == 0) {
+	if (root == NULL || (c = CMP(pid,root->pid)) == 0) {
 	    return root;
 	} else if (c < 0) { 
 	    root = root->left;
@@ -948,8 +948,10 @@ static void erts_dump_links(ErtsLink *root, int indent)
     erts_destroy_tmp_dsbuf(dsbufp);
 }
 
-Eterm erts_debug_dump_monitors_1(Process *p, Eterm pid)
+Eterm erts_debug_dump_monitors_1(BIF_ALIST_1)
 {
+    Process *p = BIF_P;
+    Eterm pid = BIF_ARG_1;
     Process *rp;
     DistEntry *dep;
     rp = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, pid, ERTS_PROC_LOCK_LINK);
@@ -976,8 +978,10 @@ Eterm erts_debug_dump_monitors_1(Process *p, Eterm pid)
     }
 }
 
-Eterm erts_debug_dump_links_1(Process *p, Eterm pid)
+Eterm erts_debug_dump_links_1(BIF_ALIST_1)
 {
+    Process *p = BIF_P;
+    Eterm pid = BIF_ARG_1;
     Process *rp;
     DistEntry *dep;
     if (is_internal_port(pid)) {
diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c
index 8b8ac2ec80..358c67bf20 100644
--- a/erts/emulator/beam/erl_mtrace.c
+++ b/erts/emulator/beam/erl_mtrace.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -79,7 +79,7 @@ static erts_mtx_t mtrace_buf_mutex;
 #define UI16_SZ					(2)
 #define UI32_SZ					(4)
 #define UI64_SZ					(8)
-#ifdef ARCH_64
+#ifdef ARCH_64 /* XXX:PaN Halfword? (whole file...) */
 #  define UI_SZ					UI64_SZ
 #else
 #  define UI_SZ					UI32_SZ
@@ -188,7 +188,7 @@ check_alloc_entry(byte *sp, byte *ep,
 		  byte tag,
 		  Uint16 ct_no, int ct_no_n,
 		  Uint16 type, int type_n,
-		  Uint res, int res_n,
+		  UWord res, int res_n,
 		  Uint size, int size_n,
 		  Uint32 ti,int ti_n);
 void
@@ -196,8 +196,8 @@ check_realloc_entry(byte *sp, byte *ep,
 		    byte tag,
 		    Uint16 ct_no, int ct_no_n,
 		    Uint16 type, int type_n,
-		    Uint res, int res_n,
-		    Uint ptr, int ptr_n,
+		    UWord res, int res_n,
+		    UWord ptr, int ptr_n,
 		    Uint size, int size_n,
 		    Uint32 ti,int ti_n);
 void
@@ -205,7 +205,7 @@ check_free_entry(byte *sp, byte *ep,
 		 byte tag,
 		 Uint16 ct_no, int ct_no_n,
 		 Uint16 t_no, int t_no_n,
-		 Uint ptr, int ptr_n,
+		 UWord ptr, int ptr_n,
 		 Uint32 ti,int ti_n);
 void
 check_time_inc_entry(byte *sp, byte *ep,
@@ -503,12 +503,6 @@ write_trace_header(char *nodename, char *pid, char *hostname)
 	    case ERTS_ALC_A_SYSTEM:
 		PUT_UI16(tracep, ERTS_MTRACE_SEGMENT_ID);
 		break;
-	    case ERTS_ALC_A_FIXED_SIZE:
-		if (erts_allctrs_info[ERTS_FIX_CORE_ALLOCATOR].enabled)
-		    PUT_UI16(tracep, ERTS_FIX_CORE_ALLOCATOR);
-		else
-		    PUT_UI16(tracep, ERTS_ALC_A_SYSTEM);
-		break;
 	    default:
 		PUT_UI16(tracep, ERTS_MTRACE_SEGMENT_ID);
 		break;
@@ -585,9 +579,7 @@ void erts_mtrace_init(char *receiver, char *nodename)
 	Uint16 port;
 
 	erts_mtx_init(&mtrace_buf_mutex, "mtrace_buf");
-	erts_mtx_set_forksafe(&mtrace_buf_mutex);
 	erts_mtx_init(&mtrace_op_mutex, "mtrace_op");
-	erts_mtx_set_forksafe(&mtrace_op_mutex);
 
 	socket_desc = erts_sock_open();
 	if (socket_desc == ERTS_SOCK_INVALID_SOCKET) {
@@ -785,7 +777,7 @@ write_alloc_entry(byte tag,
 			      tag,
 			      ct_no, ct_no_n,
 			      t_no, t_no_n,
-			      (Uint) res, res_n,
+			      (UWord) res, res_n,
 			      size, size_n,
 			      ti, ti_n);
 #endif
@@ -865,8 +857,8 @@ write_realloc_entry(byte tag,
 				tag,
 				ct_no, ct_no_n,
 				t_no, t_no_n,
-				(Uint) res, res_n,
-				(Uint) ptr, ptr_n,
+				(UWord) res, res_n,
+				(UWord) ptr, ptr_n,
 				size, size_n,
 				ti, ti_n);
 #endif
@@ -934,7 +926,7 @@ write_free_entry(byte tag,
 			     tag,
 			     ct_no, ct_no_n,
 			     t_no, t_no_n,
-			     (Uint) ptr, ptr_n,
+			     (UWord) ptr, ptr_n,
 			     ti, ti_n);
 #endif
 	}
@@ -1135,7 +1127,7 @@ check_alloc_entry(byte *sp, byte *ep,
 		  byte tag,
 		  Uint16 ct_no, int ct_no_n,
 		  Uint16 t_no, int t_no_n,
-		  Uint res, int res_n,
+		  UWord res, int res_n,
 		  Uint size, int size_n,
 		  Uint32 ti,int ti_n)
 {
@@ -1163,8 +1155,8 @@ check_realloc_entry(byte *sp, byte *ep,
 		    byte tag,
 		    Uint16 ct_no, int ct_no_n,
 		    Uint16 t_no, int t_no_n,
-		    Uint res, int res_n,
-		    Uint ptr, int ptr_n,
+		    UWord res, int res_n,
+		    UWord ptr, int ptr_n,
 		    Uint size, int size_n,
 		    Uint32 ti,int ti_n)
 {
@@ -1193,7 +1185,7 @@ check_free_entry(byte *sp, byte *ep,
 		 byte tag,
 		 Uint16 ct_no, int ct_no_n,
 		 Uint16 t_no, int t_no_n,
-		 Uint ptr, int ptr_n,
+		 UWord ptr, int ptr_n,
 		 Uint32 ti,int ti_n)
 {
     byte *p = sp;
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 585a6c1fdf..b323bc7f69 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -32,6 +32,7 @@
 #include "error.h"
 #include "big.h"
 #include "beam_bp.h"
+#include "erl_thr_progress.h"
 
 #include <limits.h>
 #include <stddef.h> /* offsetof */
@@ -48,9 +49,26 @@ struct erl_module_nif {
     struct enif_entry_t* entry;
     erts_refc_t rt_cnt;       /* number of resource types */
     erts_refc_t rt_dtor_cnt;  /* number of resource types with destructors */
-    int is_orphan;            /* if erlang module has been purged */
+    Module* mod;           /* Can be NULL if orphan with dtor-resources left */ 
 };
 
+#ifdef DEBUG
+#  define READONLY_CHECK
+#endif
+#ifdef READONLY_CHECK
+#  define ADD_READONLY_CHECK(ENV,PTR,SIZE) add_readonly_check(ENV,PTR,SIZE)
+static void add_readonly_check(ErlNifEnv*, unsigned char* ptr, unsigned sz);
+#else
+#  define ADD_READONLY_CHECK(ENV,PTR,SIZE) ((void)0)
+#endif
+
+#ifdef DEBUG
+static int is_offheap(const ErlOffHeap* off_heap);
+#endif
+
+#ifdef USE_VM_PROBES
+void dtrace_nifenv_str(ErlNifEnv *, char *);
+#endif
 
 #define MIN_HEAP_FRAG_SZ 200
 static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp);
@@ -67,23 +85,33 @@ static ERTS_INLINE Eterm* alloc_heap(ErlNifEnv* env, unsigned need)
 
 static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp)
 {    
-    unsigned frag_sz;
     env->hp = hp;
     if (env->heap_frag == NULL) {       
 	ASSERT(HEAP_LIMIT(env->proc) == env->hp_end);
 	HEAP_TOP(env->proc) = env->hp;	
     }
     else {
-	HRelease(env->proc, env->hp_end, env->hp);
+	env->heap_frag->used_size = hp - env->heap_frag->mem;
+	ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size);
     }
-    frag_sz = need + MIN_HEAP_FRAG_SZ;
-    hp = erts_heap_alloc(env->proc, frag_sz);
-    env->hp = hp + need;
-    env->hp_end = hp + frag_sz;
+    hp = erts_heap_alloc(env->proc, need, MIN_HEAP_FRAG_SZ);
     env->heap_frag = MBUF(env->proc);
+    env->hp = hp + need;
+    env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size;
+
     return hp;
 }
 
+#if SIZEOF_LONG != ERTS_SIZEOF_ETERM
+static ERTS_INLINE void ensure_heap(ErlNifEnv* env, unsigned may_need)
+{
+    if (env->hp + may_need > env->hp_end) {
+	alloc_heap_heavy(env, may_need, env->hp);
+	env->hp -= may_need;
+    }
+}
+#endif
+
 void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif)
 {
     env->mod_nif = mod_nif;
@@ -106,6 +134,16 @@ static void pre_nif_noproc(ErlNifEnv* env, struct erl_module_nif* mod_nif)
     env->tmp_obj_list = NULL;
 }
 
+/* Temporary object header, auto-deallocated when NIF returns
+ * or when independent environment is cleared.
+ */
+struct enif_tmp_obj_t {
+    struct enif_tmp_obj_t* next;
+    void (*dtor)(struct enif_tmp_obj_t*);
+    ErtsAlcType_t allocator;
+    /*char data[];*/
+};
+
 static ERTS_INLINE void free_tmp_objs(ErlNifEnv* env)
 {
     while (env->tmp_obj_list != NULL) {
@@ -126,8 +164,9 @@ void erts_post_nif(ErlNifEnv* env)
     }
     else {
 	ASSERT(env->hp_end != HEAP_LIMIT(env->proc));
-	ASSERT(env->hp_end - env->hp <= env->heap_frag->size);
-	HRelease(env->proc, env->hp_end, env->hp);
+	ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size);
+	env->heap_frag->used_size = env->hp - env->heap_frag->mem;
+	ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size);
     }
     free_tmp_objs(env);
 }
@@ -141,7 +180,7 @@ static void post_nif_noproc(ErlNifEnv* env)
 
 /* Flush out our cached heap pointers to allow an ordinary HAlloc
 */
-static void enable_halloc(ErlNifEnv* env)
+static void flush_env(ErlNifEnv* env)
 {
     if (env->heap_frag == NULL) {
 	ASSERT(env->hp_end == HEAP_LIMIT(env->proc));
@@ -151,14 +190,15 @@ static void enable_halloc(ErlNifEnv* env)
     }
     else {
 	ASSERT(env->hp_end != HEAP_LIMIT(env->proc));
-	ASSERT(env->hp_end - env->hp <= env->heap_frag->size);
-	HRelease(env->proc, env->hp_end, env->hp);
+	ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size);
+	env->heap_frag->used_size = env->hp - env->heap_frag->mem;
+	ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size);
     }
 }
 
-/* Restore cached heap pointers
+/* Restore cached heap pointers to allow alloc_heap again.
 */
-static void disable_halloc(ErlNifEnv* env)
+static void cache_env(ErlNifEnv* env)
 {
     if (env->heap_frag == NULL) {
 	ASSERT(env->hp_end == HEAP_LIMIT(env->proc));
@@ -168,35 +208,195 @@ static void disable_halloc(ErlNifEnv* env)
     }
     else {
 	ASSERT(env->hp_end != HEAP_LIMIT(env->proc));
-	ASSERT(env->hp_end - env->hp <= env->heap_frag->size);       
+	ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size);       
 	env->heap_frag = MBUF(env->proc);
 	ASSERT(env->heap_frag != NULL);
 	env->hp = env->heap_frag->mem + env->heap_frag->used_size;
-	env->hp_end = env->heap_frag->mem + env->heap_frag->size;
+	env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size;
     }
 }
-
-
 void* enif_priv_data(ErlNifEnv* env)
 {
     return env->mod_nif->priv_data;
 }
 
-void* enif_alloc(ErlNifEnv* env, size_t size)
+void* enif_alloc(size_t size)
 {
     return erts_alloc_fnf(ERTS_ALC_T_NIF, (Uint) size);
 }
 
-void* enif_realloc(ErlNifEnv* env, void* ptr, size_t size)
+void* enif_realloc(void* ptr, size_t size)
 {
     return erts_realloc_fnf(ERTS_ALC_T_NIF, ptr, size);
 }
 
-void enif_free(ErlNifEnv* env, void* ptr)
+void enif_free(void* ptr)
 {
     erts_free(ERTS_ALC_T_NIF, ptr);
 }
 
+struct enif_msg_environment_t
+{
+    ErlNifEnv env;
+    Process phony_proc;
+};
+
+ErlNifEnv* enif_alloc_env(void)
+{
+    struct enif_msg_environment_t* msg_env =
+	erts_alloc_fnf(ERTS_ALC_T_NIF, sizeof(struct enif_msg_environment_t));
+    Eterm* phony_heap = (Eterm*) msg_env; /* dummy non-NULL ptr */
+	
+    msg_env->env.hp = phony_heap; 
+    msg_env->env.hp_end = phony_heap;
+    msg_env->env.heap_frag = NULL;
+    msg_env->env.mod_nif = NULL;
+    msg_env->env.tmp_obj_list = NULL;
+    msg_env->env.proc = &msg_env->phony_proc;
+    memset(&msg_env->phony_proc, 0, sizeof(Process));
+    HEAP_START(&msg_env->phony_proc) = phony_heap;
+    HEAP_TOP(&msg_env->phony_proc) = phony_heap;
+    HEAP_LIMIT(&msg_env->phony_proc) = phony_heap;
+    HEAP_END(&msg_env->phony_proc) = phony_heap;
+    MBUF(&msg_env->phony_proc) = NULL;
+    msg_env->phony_proc.id = ERTS_INVALID_PID;
+#ifdef FORCE_HEAP_FRAGS
+    msg_env->phony_proc.space_verified = 0;
+    msg_env->phony_proc.space_verified_from = NULL;
+#endif
+    return &msg_env->env;
+}
+void enif_free_env(ErlNifEnv* env)
+{
+    enif_clear_env(env);
+    erts_free(ERTS_ALC_T_NIF, env);
+}
+
+static ERTS_INLINE void clear_offheap(ErlOffHeap* oh)
+{
+    oh->first = NULL;
+    oh->overhead = 0;
+}
+
+void enif_clear_env(ErlNifEnv* env)
+{
+    struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)env;
+    Process* p = &menv->phony_proc;
+    ASSERT(p == menv->env.proc);
+    ASSERT(p->id == ERTS_INVALID_PID);
+    ASSERT(MBUF(p) == menv->env.heap_frag);
+    if (MBUF(p) != NULL) {
+	erts_cleanup_offheap(&MSO(p));
+	clear_offheap(&MSO(p));
+	free_message_buffer(MBUF(p));
+	MBUF(p) = NULL;
+	menv->env.heap_frag = NULL;
+    }
+    ASSERT(HEAP_TOP(p) == HEAP_END(p));
+    menv->env.hp = menv->env.hp_end = HEAP_TOP(p);
+    
+    ASSERT(!is_offheap(&MSO(p)));
+    free_tmp_objs(env);
+}
+int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid,
+	      ErlNifEnv* msg_env, ERL_NIF_TERM msg)
+{
+    struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)msg_env;
+    ErtsProcLocks rp_locks = 0;
+    Process* rp;
+    Process* c_p;
+    ErlHeapFragment* frags;
+#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+    ErtsProcLocks rp_had_locks;
+#endif
+    Eterm receiver = to_pid->pid;
+    int flush_me = 0;
+
+    if (env != NULL) {
+	c_p = env->proc;
+	if (receiver == c_p->id) {
+	    rp_locks = ERTS_PROC_LOCK_MAIN;
+	    flush_me = 1;
+	}
+    }
+    else {
+#ifdef ERTS_SMP
+	c_p = NULL;
+#else
+	erl_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM");
+#endif
+    }    
+
+#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+    rp_had_locks = rp_locks;
+#endif
+    rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
+			   receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC);
+    if (rp == NULL) {
+	ASSERT(env == NULL || receiver != c_p->id);
+	return 0;
+    }
+    flush_env(msg_env);
+    frags = menv->env.heap_frag; 
+    ASSERT(frags == MBUF(&menv->phony_proc));
+    if (frags != NULL) {
+	/* Move all offheap's from phony proc to the first fragment.
+	   Quick and dirty, but erts_move_msg_mbuf_to_heap doesn't care. */
+	ASSERT(!is_offheap(&frags->off_heap));
+	frags->off_heap = MSO(&menv->phony_proc);
+	clear_offheap(&MSO(&menv->phony_proc));
+	menv->env.heap_frag = NULL; 
+	MBUF(&menv->phony_proc) = NULL;
+    }
+    ASSERT(!is_offheap(&MSO(&menv->phony_proc)));
+
+    if (flush_me) {	
+	flush_env(env); /* Needed for ERTS_HOLE_CHECK */ 
+    }
+    erts_queue_message(rp, &rp_locks, frags, msg, am_undefined
+#ifdef USE_VM_PROBES
+		       , NIL
+#endif
+		       );
+    if (rp_locks) {	
+	ERTS_SMP_LC_ASSERT(rp_locks == (rp_had_locks | (ERTS_PROC_LOCK_MSGQ | 
+							ERTS_PROC_LOCK_STATUS)));
+	erts_smp_proc_unlock(rp, (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS));
+    }
+    erts_smp_proc_dec_refc(rp);
+    if (flush_me) {
+	cache_env(env);
+    }
+    return 1;
+}
+
+ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term)
+{
+    Uint sz;
+    Eterm* hp;
+    sz = size_object(src_term);
+    hp = alloc_heap(dst_env, sz);
+    return copy_struct(src_term, sz, &hp, &MSO(dst_env->proc));
+}
+
+
+#ifdef DEBUG
+static int is_offheap(const ErlOffHeap* oh)
+{
+    return oh->first != NULL;
+}
+#endif
+
+ErlNifPid* enif_self(ErlNifEnv* caller_env, ErlNifPid* pid)
+{
+    pid->pid = caller_env->proc->id;
+    return pid;
+}
+int enif_get_local_pid(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifPid* pid)
+{
+    return is_internal_pid(term) ? (pid->pid=term, 1) : 0;
+}
+
 int enif_is_atom(ErlNifEnv* env, ERL_NIF_TERM term)
 {
     return is_atom(term);
@@ -232,24 +432,51 @@ int enif_is_ref(ErlNifEnv* env, ERL_NIF_TERM term)
     return is_ref(term);
 }
 
+int enif_is_tuple(ErlNifEnv* env, ERL_NIF_TERM term)
+{
+    return is_tuple(term);
+}
+
+int enif_is_list(ErlNifEnv* env, ERL_NIF_TERM term)
+{
+    return is_list(term) || is_nil(term);
+}
+
+int enif_is_exception(ErlNifEnv* env, ERL_NIF_TERM term)
+{
+    return term == THE_NON_VALUE;
+}
+
+int enif_is_number(ErlNifEnv* env, ERL_NIF_TERM term)
+{
+    return is_number(term);
+}
+
+static ERTS_INLINE int is_proc_bound(ErlNifEnv* env)
+{
+    return env->mod_nif != NULL;
+}
+
 static void aligned_binary_dtor(struct enif_tmp_obj_t* obj)
 {
-    erts_free_aligned_binary_bytes((byte*)obj);
+    erts_free_aligned_binary_bytes_extra((byte*)obj, obj->allocator);
 }
 
 int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin)
 {
+    ErtsAlcType_t allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF;
     union {
 	struct enif_tmp_obj_t* tmp;
 	byte* raw_ptr;
     }u;
     u.tmp = NULL;
-    bin->data = erts_get_aligned_binary_bytes_extra(bin_term, &u.raw_ptr,
+    bin->data = erts_get_aligned_binary_bytes_extra(bin_term, &u.raw_ptr, allocator,
 						    sizeof(struct enif_tmp_obj_t));
     if (bin->data == NULL) {
 	return 0;
     }
     if (u.tmp != NULL) {
+	u.tmp->allocator = allocator;
 	u.tmp->next = env->tmp_obj_list;
 	u.tmp->dtor = &aligned_binary_dtor;
 	env->tmp_obj_list = u.tmp;
@@ -257,18 +484,20 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin)
     bin->bin_term = bin_term;
     bin->size = binary_size(bin_term);
     bin->ref_bin = NULL;
+    ADD_READONLY_CHECK(env, bin->data, bin->size); 
     return 1;
 }
 
 static void tmp_alloc_dtor(struct enif_tmp_obj_t* obj)
 {
-    erts_free(ERTS_ALC_T_TMP,  obj);
+    erts_free(obj->allocator,  obj);
 }
 
 int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin)
 {
     struct enif_tmp_obj_t* tobj;
-    int sz;
+    ErtsAlcType_t allocator;
+    Uint sz;
     if (is_binary(term)) {
 	return enif_inspect_binary(env,term,bin);
     }
@@ -279,11 +508,13 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin)
 	bin->ref_bin = NULL;
 	return 1;
     }
-    if ((sz = io_list_len(term)) < 0) {
+    if (erts_iolist_size(term, &sz)) {
 	return 0;
     }
-    
-    tobj = erts_alloc(ERTS_ALC_T_TMP, sz + sizeof(struct enif_tmp_obj_t));
+
+    allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF;
+    tobj = erts_alloc(allocator, sz + sizeof(struct enif_tmp_obj_t));
+    tobj->allocator = allocator;
     tobj->next = env->tmp_obj_list;
     tobj->dtor = &tmp_alloc_dtor;
     env->tmp_obj_list = tobj;
@@ -293,10 +524,11 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin)
     bin->bin_term = THE_NON_VALUE;
     bin->ref_bin = NULL;
     io_list_to_buf(term, (char*) bin->data, sz);
+    ADD_READONLY_CHECK(env, bin->data, bin->size); 
     return 1;
 }
 
-int enif_alloc_binary(ErlNifEnv* env, unsigned size, ErlNifBinary* bin)
+int enif_alloc_binary(size_t size, ErlNifBinary* bin)
 {
     Binary* refbin;
 
@@ -306,7 +538,7 @@ int enif_alloc_binary(ErlNifEnv* env, unsigned size, ErlNifBinary* bin)
     }
     refbin->flags = BIN_FLAG_DRV; /* BUGBUG: Flag? */
     erts_refc_init(&refbin->refc, 1);
-    refbin->orig_size = (long) size;
+    refbin->orig_size = (SWord) size;
 
     bin->size = size;
     bin->data = (unsigned char*) refbin->orig_bytes;
@@ -315,7 +547,7 @@ int enif_alloc_binary(ErlNifEnv* env, unsigned size, ErlNifBinary* bin)
     return 1;
 }
 
-int enif_realloc_binary(ErlNifEnv* env, ErlNifBinary* bin, unsigned size)
+int enif_realloc_binary(ErlNifBinary* bin, size_t size)
 {
     if (bin->ref_bin != NULL) {
 	Binary* oldbin;
@@ -333,15 +565,15 @@ int enif_realloc_binary(ErlNifEnv* env, ErlNifBinary* bin, unsigned size)
     }
     else {
 	unsigned char* old_data = bin->data;
-	unsigned cpy_sz = (size < bin->size ? size : bin->size);  
-	enif_alloc_binary(env, size, bin);
+	size_t cpy_sz = (size < bin->size ? size : bin->size);  
+	enif_alloc_binary(size, bin);
 	sys_memcpy(bin->data, old_data, cpy_sz); 
     }
     return 1;
 }
 
 
-void enif_release_binary(ErlNifEnv* env, ErlNifBinary* bin)
+void enif_release_binary(ErlNifBinary* bin)
 {
     if (bin->ref_bin != NULL) {
 	Binary* refbin = bin->ref_bin;
@@ -357,14 +589,31 @@ void enif_release_binary(ErlNifEnv* env, ErlNifBinary* bin)
 #endif
 }
 
-int enif_is_identical(ErlNifEnv* env, Eterm lhs, Eterm rhs)
+unsigned char* enif_make_new_binary(ErlNifEnv* env, size_t size,
+				    ERL_NIF_TERM* termp)
+{
+    flush_env(env);
+    *termp = new_binary(env->proc, NULL, size);
+    cache_env(env);
+    return binary_bytes(*termp);
+}
+
+int enif_is_identical(Eterm lhs, Eterm rhs)
 {
     return EQ(lhs,rhs);
 }
 
-int enif_compare(ErlNifEnv* env, Eterm lhs, Eterm rhs)
+int enif_compare(Eterm lhs, Eterm rhs)
 {
-    return cmp(lhs,rhs);
+    Sint result = CMP(lhs,rhs);
+
+    if (result < 0) {
+        return -1;
+    } else if (result > 0) {
+        return 1;
+    }
+
+    return result;
 }
 
 int enif_get_tuple(ErlNifEnv* env, Eterm tpl, int* arity, const Eterm** array)
@@ -425,13 +674,13 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin)
 	pb = (ProcBin *) alloc_heap(env, PROC_BIN_SIZE);
 	pb->thing_word = HEADER_PROC_BIN;
 	pb->size = bptr->orig_size;
-	pb->next = MSO(env->proc).mso;
-	MSO(env->proc).mso = pb;
+	pb->next = MSO(env->proc).first;
+	MSO(env->proc).first = (struct erl_off_heap_header*) pb;
 	pb->val = bptr;
 	pb->bytes = (byte*) bptr->orig_bytes;
 	pb->flags = 0;
 	
-	MSO(env->proc).overhead += pb->size / sizeof(Eterm);
+	OH_OVERHEAD(&(MSO(env->proc)), pb->size / sizeof(Eterm));
 	bin_term = make_binary(pb);	
 	if (erts_refc_read(&bptr->refc, 1) == 1) {
 	    /* Total ownership transfer */
@@ -441,19 +690,20 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin)
 	return bin_term;
     }
     else {
-	enable_halloc(env);
+	flush_env(env);
 	bin->bin_term = new_binary(env->proc, bin->data, bin->size);
-	disable_halloc(env);
+	cache_env(env);
 	return bin->bin_term;
     }
 }
 
 Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term,
-			   unsigned pos, unsigned size)
+			   size_t pos, size_t size)
 {
     ErlSubBin* sb;
     Eterm orig;
     Uint offset, bit_offset, bit_size; 
+#ifdef DEBUG
     unsigned src_size;
 
     ASSERT(is_binary(bin_term));
@@ -461,6 +711,7 @@ Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term,
     ASSERT(pos <= src_size);
     ASSERT(size <= src_size);
     ASSERT(pos + size <= src_size);   
+#endif
     sb = (ErlSubBin*) alloc_heap(env, ERL_SUB_BIN_SIZE);
     ERTS_GET_REAL_BIN(bin_term, orig, offset, bit_offset, bit_size);
     sb->thing_word = HEADER_SUB_BIN;
@@ -479,9 +730,11 @@ Eterm enif_make_badarg(ErlNifEnv* env)
     BIF_ERROR(env->proc, BADARG);
 }
 
-int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len)
+int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len,
+		  ErlNifCharEncoding encoding)
 {
     Atom* ap;
+    ASSERT(encoding == ERL_NIF_LATIN1);
     if (is_not_atom(atom)) {
 	return 0;
     }
@@ -496,9 +749,10 @@ int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len)
 
 int enif_get_int(ErlNifEnv* env, Eterm term, int* ip)
 {
-#if SIZEOF_INT == SIZEOF_VOID_P
+#if SIZEOF_INT ==  ERTS_SIZEOF_ETERM
     return term_to_Sint(term, (Sint*)ip);
-#elif SIZEOF_LONG == SIZEOF_VOID_P
+#elif (SIZEOF_LONG ==  ERTS_SIZEOF_ETERM) || \
+  (SIZEOF_LONG_LONG ==  ERTS_SIZEOF_ETERM)
     Sint i;
     if (!term_to_Sint(term, &i) || i < INT_MIN || i > INT_MAX) {
 	return 0;
@@ -512,9 +766,10 @@ int enif_get_int(ErlNifEnv* env, Eterm term, int* ip)
 
 int enif_get_uint(ErlNifEnv* env, Eterm term, unsigned* ip)
 {
-#if SIZEOF_INT == SIZEOF_VOID_P
+#if SIZEOF_INT == ERTS_SIZEOF_ETERM
     return term_to_Uint(term, (Uint*)ip);
-#elif SIZEOF_LONG == SIZEOF_VOID_P
+#elif (SIZEOF_LONG == ERTS_SIZEOF_ETERM) || \
+  (SIZEOF_LONG_LONG ==  ERTS_SIZEOF_ETERM)
     Uint i;
     if (!term_to_Uint(term, &i) || i > UINT_MAX) {
 	return 0;
@@ -526,8 +781,17 @@ int enif_get_uint(ErlNifEnv* env, Eterm term, unsigned* ip)
 
 int enif_get_long(ErlNifEnv* env, Eterm term, long* ip)
 {
-#if SIZEOF_LONG == SIZEOF_VOID_P
+#if SIZEOF_LONG == ERTS_SIZEOF_ETERM
     return term_to_Sint(term, ip);
+#elif SIZEOF_LONG == 8
+    return term_to_Sint64(term, ip);
+#elif SIZEOF_LONG == SIZEOF_INT
+    int tmp,ret;
+    ret = enif_get_int(env,term,&tmp);
+    if (ret) {
+      *ip = (long) tmp;
+    }
+    return ret;
 #else
 #  error Unknown long word size 
 #endif     
@@ -535,14 +799,36 @@ int enif_get_long(ErlNifEnv* env, Eterm term, long* ip)
 
 int enif_get_ulong(ErlNifEnv* env, Eterm term, unsigned long* ip)
 {
-#if SIZEOF_LONG == SIZEOF_VOID_P
+#if SIZEOF_LONG == ERTS_SIZEOF_ETERM
     return term_to_Uint(term, ip);
+#elif SIZEOF_LONG == 8
+    return term_to_Uint64(term, ip);
+#elif SIZEOF_LONG == SIZEOF_INT
+    int ret;
+    unsigned int tmp;
+    ret = enif_get_uint(env,term,&tmp);
+    if (ret) {
+      *ip = (unsigned long) tmp;
+    }
+    return ret;
 #else
 #  error Unknown long word size 
 #endif     
 }
 
-int enif_get_double(ErlNifEnv* env, Eterm term, double* dp)
+#if HAVE_INT64 && SIZEOF_LONG != 8
+int enif_get_int64(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifSInt64* ip)
+{
+    return term_to_Sint64(term, ip);
+}
+
+int enif_get_uint64(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifUInt64* ip)
+{
+    return term_to_Uint64(term, ip);
+}
+#endif /* HAVE_INT64 && SIZEOF_LONG != 8 */
+
+int enif_get_double(ErlNifEnv* env, ERL_NIF_TERM term, double* dp)
 {
     FloatDef f;
     if (is_not_float(term)) {
@@ -553,6 +839,17 @@ int enif_get_double(ErlNifEnv* env, Eterm term, double* dp)
     return 1;
 }
 
+int enif_get_atom_length(ErlNifEnv* env, Eterm atom, unsigned* len,
+			 ErlNifCharEncoding enc)
+{
+    Atom* ap;
+    ASSERT(enc == ERL_NIF_LATIN1);
+    if (is_not_atom(atom)) return 0;
+    ap = atom_tab(atom_val(atom));
+    *len = ap->len;
+    return 1;
+}
+
 int enif_get_list_cell(ErlNifEnv* env, Eterm term, Eterm* head, Eterm* tail)
 {
     Eterm* val;
@@ -563,37 +860,83 @@ int enif_get_list_cell(ErlNifEnv* env, Eterm term, Eterm* head, Eterm* tail)
     return 1;
 }
 
+int enif_get_list_length(ErlNifEnv* env, Eterm term, unsigned* len)
+{
+    if (is_not_list(term) && is_not_nil(term)) return 0;
+    *len = list_length(term);
+    return 1;
+}
+
 ERL_NIF_TERM enif_make_int(ErlNifEnv* env, int i)
 {
-#if SIZEOF_INT == SIZEOF_VOID_P
+#if SIZEOF_INT == ERTS_SIZEOF_ETERM
     return IS_SSMALL(i) ? make_small(i) : small_to_big(i,alloc_heap(env,2));
-#elif SIZEOF_LONG == SIZEOF_VOID_P
+#elif (SIZEOF_LONG == ERTS_SIZEOF_ETERM) || \
+  (SIZEOF_LONG_LONG == ERTS_SIZEOF_ETERM)
     return make_small(i);
 #endif
 }
 
 ERL_NIF_TERM enif_make_uint(ErlNifEnv* env, unsigned i)
 {
-#if SIZEOF_INT == SIZEOF_VOID_P
+#if SIZEOF_INT == ERTS_SIZEOF_ETERM
     return IS_USMALL(0,i) ? make_small(i) : uint_to_big(i,alloc_heap(env,2));
-#elif SIZEOF_LONG == SIZEOF_VOID_P
+#elif (SIZEOF_LONG ==  ERTS_SIZEOF_ETERM) || \
+  (SIZEOF_LONG_LONG ==  ERTS_SIZEOF_ETERM)
     return make_small(i);
 #endif
 }
 
 ERL_NIF_TERM enif_make_long(ErlNifEnv* env, long i)
 {
-#if SIZEOF_LONG != SIZEOF_VOID_P
-#  error Unknown long word size 
-#endif     
-    return IS_SSMALL(i) ? make_small(i) : small_to_big(i, alloc_heap(env,2));
+    if (IS_SSMALL(i)) {
+	return make_small(i);
+    }
+#if SIZEOF_LONG == ERTS_SIZEOF_ETERM
+    return small_to_big(i, alloc_heap(env,2));
+#elif SIZEOF_LONG_LONG ==  ERTS_SIZEOF_ETERM
+    return make_small(i);
+#elif SIZEOF_LONG == 8
+    ensure_heap(env,3);
+    return erts_sint64_to_big(i, &env->hp);
+#endif
 }
 
 ERL_NIF_TERM enif_make_ulong(ErlNifEnv* env, unsigned long i)
 {
-    return IS_USMALL(0,i) ? make_small(i) : uint_to_big(i,alloc_heap(env,2));
+    if (IS_USMALL(0,i)) {
+	return make_small(i);
+    }
+#if SIZEOF_LONG == ERTS_SIZEOF_ETERM
+    return uint_to_big(i,alloc_heap(env,2));
+#elif SIZEOF_LONG_LONG ==  ERTS_SIZEOF_ETERM
+    return make_small(i);
+#elif SIZEOF_LONG == 8
+    ensure_heap(env,3);
+    return erts_uint64_to_big(i, &env->hp);    
+#endif
 }
 
+#if HAVE_INT64 && SIZEOF_LONG != 8
+ERL_NIF_TERM enif_make_int64(ErlNifEnv* env, ErlNifSInt64 i)
+{
+    Uint* hp;
+    Uint need = 0;
+    erts_bld_sint64(NULL, &need, i);
+    hp = alloc_heap(env, need);
+    return erts_bld_sint64(&hp, NULL, i);
+}
+
+ERL_NIF_TERM enif_make_uint64(ErlNifEnv* env, ErlNifUInt64 i)
+{
+    Uint* hp;
+    Uint need = 0;
+    erts_bld_uint64(NULL, &need, i);
+    hp = alloc_heap(env, need);
+    return erts_bld_uint64(&hp, NULL, i);
+}
+#endif /* HAVE_INT64 && SIZEOF_LONG != 8 */
+
 ERL_NIF_TERM enif_make_double(ErlNifEnv* env, double d)
 {
     Eterm* hp = alloc_heap(env,FLOAT_SIZE_OBJECT);
@@ -605,12 +948,25 @@ ERL_NIF_TERM enif_make_double(ErlNifEnv* env, double d)
 
 ERL_NIF_TERM enif_make_atom(ErlNifEnv* env, const char* name)
 {
-    return am_atom_put(name, sys_strlen(name));
+    return enif_make_atom_len(env, name, sys_strlen(name));
 }
 
-int enif_make_existing_atom(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom)
+ERL_NIF_TERM enif_make_atom_len(ErlNifEnv* env, const char* name, size_t len)
 {
-    return erts_atom_get(name, sys_strlen(name), atom);
+    return am_atom_put(name, len);
+}
+
+int enif_make_existing_atom(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom,
+			    ErlNifCharEncoding enc)
+{
+    return enif_make_existing_atom_len(env, name, sys_strlen(name), atom, enc);
+}
+
+int enif_make_existing_atom_len(ErlNifEnv* env, const char* name, size_t len,
+				ERL_NIF_TERM* atom, ErlNifCharEncoding encoding)
+{
+    ASSERT(encoding == ERL_NIF_LATIN1);
+    return erts_atom_get(name, len, atom);
 }
 
 ERL_NIF_TERM enif_make_tuple(ErlNifEnv* env, unsigned cnt, ...)
@@ -653,21 +1009,26 @@ ERL_NIF_TERM enif_make_list_cell(ErlNifEnv* env, Eterm car, Eterm cdr)
 
 ERL_NIF_TERM enif_make_list(ErlNifEnv* env, unsigned cnt, ...)
 {
-    Eterm* hp = alloc_heap(env,cnt*2);
-    Eterm ret = make_list(hp);
-    Eterm* last = &ret;
-    va_list ap;
-
-    va_start(ap,cnt);
-    while (cnt--) {
-	*last = make_list(hp);
-	*hp = va_arg(ap,Eterm);
-	last = ++hp;
-	++hp;
+    if (cnt == 0) {
+	return NIL;
+    }
+    else {
+	Eterm* hp = alloc_heap(env,cnt*2);
+	Eterm ret = make_list(hp);
+	Eterm* last = &ret;
+	va_list ap;
+
+	va_start(ap,cnt);
+	while (cnt--) {
+	    *last = make_list(hp);
+	    *hp = va_arg(ap,Eterm);
+	    last = ++hp;
+	    ++hp;
+	}
+	va_end(ap);
+	*last = NIL;
+	return ret;
     }
-    va_end(ap);
-    *last = NIL;
-    return ret;
 }
 
 ERL_NIF_TERM enif_make_list_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[], unsigned cnt)
@@ -689,11 +1050,16 @@ ERL_NIF_TERM enif_make_list_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[],
 
 ERL_NIF_TERM enif_make_string(ErlNifEnv* env, const char* string,
 			      ErlNifCharEncoding encoding)
-{    
-    Sint n = sys_strlen(string);    
-    Eterm* hp = alloc_heap(env,n*2);
+{
+    return enif_make_string_len(env, string, sys_strlen(string), encoding);
+}
+
+ERL_NIF_TERM enif_make_string_len(ErlNifEnv* env, const char* string,
+				  size_t len, ErlNifCharEncoding encoding)
+{
+    Eterm* hp = alloc_heap(env,len*2);
     ASSERT(encoding == ERL_NIF_LATIN1);
-    return erts_bld_string_n(&hp,NULL,string,n); 
+    return erts_bld_string_n(&hp,NULL,string,len);
 }
 
 ERL_NIF_TERM enif_make_ref(ErlNifEnv* env)
@@ -707,6 +1073,29 @@ void enif_system_info(ErlNifSysInfo *sip, size_t si_size)
     driver_system_info(sip, si_size);
 }
 
+int enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list) {
+    Eterm *listptr, ret = NIL, *hp;
+
+    if (is_nil(term)) {
+	*list = term;
+        return 1;
+    }
+
+    ret = NIL;
+
+    while (is_not_nil(term)) {
+	if (is_not_list(term)) {
+	    return 0;
+	}
+	hp = alloc_heap(env, 2);
+	listptr = list_val(term);
+	ret = CONS(hp, CAR(listptr), ret);
+	term = CDR(listptr);
+    }
+    *list = ret;
+    return 1;
+}
+
 
 ErlNifMutex* enif_mutex_create(char *name) { return erl_drv_mutex_create(name); }
 void enif_mutex_destroy(ErlNifMutex *mtx) {  erl_drv_mutex_destroy(mtx); }
@@ -764,7 +1153,8 @@ struct enif_resource_type_t
     ErlNifResourceDtor* dtor;      /* user destructor function */
     erts_refc_t refc;  /* num of resources of this type (HOTSPOT warning)
                           +1 for active erl_module_nif */
-    char name[1];
+    Eterm module;
+    Eterm name;
 };
 
 /* dummy node in circular list */
@@ -782,14 +1172,14 @@ typedef struct enif_resource_t
 #define SIZEOF_ErlNifResource(SIZE) (offsetof(ErlNifResource,data) + (SIZE))
 #define DATA_TO_RESOURCE(PTR) ((ErlNifResource*)((char*)(PTR) - offsetof(ErlNifResource,data)))
 
-static ErlNifResourceType* find_resource_type(const char* name)
+static ErlNifResourceType* find_resource_type(Eterm module, Eterm name)
 {
     ErlNifResourceType* type;
     for (type = resource_type_list.next;
 	 type != &resource_type_list;
 	 type = type->next) {
 
-	if (sys_strcmp(type->name, name) == 0) {
+	if (type->module == module && type->name == name) {
 	    return type;
 	}
     }
@@ -797,7 +1187,7 @@ static ErlNifResourceType* find_resource_type(const char* name)
 }
 
 #define in_area(ptr,start,nbytes) \
-    ((unsigned long)((char*)(ptr) - (char*)(start)) < (nbytes))
+    ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
 
 
 static void close_lib(struct erl_module_nif* lib)
@@ -822,33 +1212,42 @@ static void steal_resource_type(ErlNifResourceType* type)
 
     if (type->dtor != NULL
 	&& erts_refc_dectest(&lib->rt_dtor_cnt, 0) == 0
-	&& lib->is_orphan) {
+	&& lib->mod == NULL) {
 	/* last type with destructor gone, close orphan lib */
 
 	close_lib(lib);
     }
     if (erts_refc_dectest(&lib->rt_cnt, 0) == 0
-	&& lib->is_orphan) {
+	&& lib->mod == NULL) {
 	erts_free(ERTS_ALC_T_NIF, lib);
     }
 }
 
 ErlNifResourceType*
-enif_open_resource_type(ErlNifEnv* env, const char* type_name, 
-			  ErlNifResourceDtor* dtor,
-			  enum ErlNifResourceFlags flags,
-			  enum ErlNifResourceFlags* tried)
-{
-    ErlNifResourceType* type = find_resource_type(type_name);
-    enum ErlNifResourceFlags op = flags;
-    ASSERT(erts_smp_is_system_blocked(0));
+enif_open_resource_type(ErlNifEnv* env,
+			const char* module_str, 
+			const char* name_str, 
+			ErlNifResourceDtor* dtor,
+			ErlNifResourceFlags flags,
+			ErlNifResourceFlags* tried)
+{
+    ErlNifResourceType* type = NULL;
+    ErlNifResourceFlags op = flags;
+    Eterm module_am, name_am;
+
+    ASSERT(erts_smp_thr_progress_is_blocking());
+    ASSERT(module_str == NULL); /* for now... */
+    module_am = make_atom(env->mod_nif->mod->module);
+    name_am = enif_make_atom(env, name_str);
+
+    type = find_resource_type(module_am, name_am);
     if (type == NULL) {
 	if (flags & ERL_NIF_RT_CREATE) {
 	    type = erts_alloc(ERTS_ALC_T_NIF,
-			      sizeof(struct enif_resource_type_t) 
-			      + sys_strlen(type_name));
+			      sizeof(struct enif_resource_type_t)); 
 	    type->dtor = dtor;
-	    sys_strcpy(type->name, type_name);
+	    type->module = module_am;
+	    type->name = name_am;
 	    erts_refc_init(&type->refc, 1);
 	    type->owner = env->mod_nif;
 	    type->prev = &resource_type_list;
@@ -896,13 +1295,13 @@ static void nif_resource_dtor(Binary* bin)
     if (erts_refc_dectest(&type->refc, 0) == 0) {
 	ASSERT(type->next == NULL);
 	ASSERT(type->owner != NULL);
-	ASSERT(type->owner->is_orphan);
+	ASSERT(type->owner->mod == NULL);
 	steal_resource_type(type);
 	erts_free(ERTS_ALC_T_NIF, type);
     }
 }
 
-void* enif_alloc_resource(ErlNifEnv* env, ErlNifResourceType* type, unsigned size)
+void* enif_alloc_resource(ErlNifResourceType* type, size_t size)
 {
     Binary* bin = erts_create_magic_binary(SIZEOF_ErlNifResource(size), &nif_resource_dtor);
     ErlNifResource* resource = ERTS_MAGIC_BIN_DATA(bin);
@@ -915,7 +1314,7 @@ void* enif_alloc_resource(ErlNifEnv* env, ErlNifResourceType* type, unsigned siz
     return resource->data;
 }
 
-void enif_release_resource(ErlNifEnv* env, void* obj)
+void enif_release_resource(void* obj)
 {
     ErlNifResource* resource = DATA_TO_RESOURCE(obj);
     ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource);
@@ -929,6 +1328,18 @@ void enif_release_resource(ErlNifEnv* env, void* obj)
     }
 }
 
+void enif_keep_resource(void* obj)
+{
+    ErlNifResource* resource = DATA_TO_RESOURCE(obj);
+    ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource);
+
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor);
+#ifdef DEBUG
+    erts_refc_inc(&resource->nif_refc, 1);
+#endif
+    erts_refc_inc(&bin->binary.refc, 2);
+}
+
 ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj)
 {
     ErlNifResource* resource = DATA_TO_RESOURCE(obj);
@@ -937,15 +1348,30 @@ ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj)
     return erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary);
 }
 
+ERL_NIF_TERM enif_make_resource_binary(ErlNifEnv* env, void* obj,
+				       const void* data, size_t size)
+{
+    Eterm bin = enif_make_resource(env, obj);
+    ProcBin* pb = (ProcBin*) binary_val(bin);
+    pb->bytes = (byte*) data;
+    pb->size = size;
+    return bin;
+}
+
 int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* type,
 		      void** objp)
 {
+    ProcBin* pb;
     Binary* mbin;
     ErlNifResource* resource;
     if (!ERTS_TERM_IS_MAGIC_BINARY(term)) {
 	return 0;
     }
-    mbin = ((ProcBin*) binary_val(term))->val;
+    pb = (ProcBin*) binary_val(term);
+    /*if (pb->size != 0) {	
+	return 0; / * Or should we allow "resource binaries" as handles? * /
+    }*/
+    mbin = pb->val;
     resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(mbin);
     if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor
 	|| resource->type != type) {	
@@ -955,7 +1381,7 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ
     return 1;
 }
 
-unsigned enif_sizeof_resource(ErlNifEnv* env, void* obj)
+size_t enif_sizeof_resource(void* obj)
 {
     ErlNifResource* resource = DATA_TO_RESOURCE(obj);
     Binary* bin = &ERTS_MAGIC_BIN_FROM_DATA(resource)->binary;
@@ -967,37 +1393,22 @@ unsigned enif_sizeof_resource(ErlNifEnv* env, void* obj)
  ***************************************************************************/
 
 
-static Uint** get_func_pp(Eterm* mod_code, Eterm f_atom, unsigned arity)
+static BeamInstr** get_func_pp(BeamInstr* mod_code, Eterm f_atom, unsigned arity)
 {
     int n = (int) mod_code[MI_NUM_FUNCTIONS];
     int j;
     for (j = 0; j < n; ++j) {
-	Uint* code_ptr = (Uint*) mod_code[MI_FUNCTIONS+j];
-	ASSERT(code_ptr[0] == (Uint) BeamOp(op_i_func_info_IaaI));
+	BeamInstr* code_ptr = (BeamInstr*) mod_code[MI_FUNCTIONS+j];
+	ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
 	if (f_atom == ((Eterm) code_ptr[3])
 	    && arity == ((unsigned) code_ptr[4])) {
 
-	    return (Uint**) &mod_code[MI_FUNCTIONS+j];
+	    return (BeamInstr**) &mod_code[MI_FUNCTIONS+j];
 	}
     }
     return NULL;
 }
 
-/*static void refresh_cached_nif_data(Eterm* mod_code,
-				    struct erl_module_nif* mod_nif)
-{
-    int i;
-    for (i=0; i < mod_nif->entry->num_of_funcs; i++) {
-	Eterm f_atom;
-	ErlNifFunc* func = &mod_nif->entry->funcs[i];	
-	Uint* code_ptr;
-	
-	erts_atom_get(func->name, sys_strlen(func->name), &f_atom); 
-	code_ptr = *get_func_pp(mod_code, f_atom, func->arity);
-	code_ptr[5+2] = (Uint) mod_nif->priv_data;
-    }
-}*/
-
 static Eterm mkatom(const char *str)
 {
     return am_atom_put(str, sys_strlen(str));
@@ -1089,7 +1500,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     static const char bad_lib[] = "bad_lib";
     static const char reload[] = "reload";
     static const char upgrade[] = "upgrade";
-    char lib_name[256]; /* BUGBUG: Max-length? */
+    char* lib_name = NULL;
     void* handle = NULL;
     void* init_func;
     ErlNifEntry* entry = NULL;
@@ -1098,22 +1509,28 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     Module* mod;
     Eterm mod_atom;
     Eterm f_atom;
-    Eterm* caller;
+    BeamInstr* caller;
     ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT;
     Eterm ret = am_ok;
     int veto;
     struct erl_module_nif* lib = NULL;
+    int reload_warning = 0;
 
-    len = intlist_to_buf(BIF_ARG_1, lib_name, sizeof(lib_name)-1);
-    if (len < 1) {
-	/*erts_fprintf(stderr, "Invalid library path name '%T'\r\n", BIF_ARG_1);*/
+    len = list_length(BIF_ARG_1);
+    if (len < 0) {
+	BIF_ERROR(BIF_P, BADARG);
+    }
+    lib_name = (char *) erts_alloc(ERTS_ALC_T_TMP, len + 1);
+
+    if (intlist_to_buf(BIF_ARG_1, lib_name, len) != len) {
+	erts_free(ERTS_ALC_T_TMP, lib_name);
 	BIF_ERROR(BIF_P, BADARG);
     }
     lib_name[len] = '\0';
 
     /* Block system (is this the right place to do it?) */
     erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-    erts_smp_block_system(0);
+    erts_smp_thr_progress_block();
 
     /* Find calling module */
     ASSERT(BIF_P->current != NULL);
@@ -1156,7 +1573,13 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	
 	ret = load_nif_error(BIF_P, bad_lib, "Library version (%d.%d) not compatible (with %d.%d).",
 			     entry->major, entry->minor, ERL_NIF_MAJOR_VERSION, ERL_NIF_MINOR_VERSION);
-    }    
+    }   
+    else if (entry->minor >= 1
+	     && sys_strcmp(entry->vm_variant, ERL_NIF_VM_VARIANT) != 0) {
+	ret = load_nif_error(BIF_P, bad_lib, "Library (%s) not compiled for "
+			     "this vm variant (%s).",
+			     entry->vm_variant, ERL_NIF_VM_VARIANT);
+    }
     else if (!erts_is_atom_str((char*)entry->name, mod_atom)) {
 	ret = load_nif_error(BIF_P, bad_lib, "Library module name '%s' does not"
 			     " match calling module '%T'", entry->name, mod_atom);
@@ -1165,7 +1588,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	/*erts_fprintf(stderr, "Found module %T\r\n", mod_atom);*/
     
 	for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) {
-	    Uint** code_pp;
+	    BeamInstr** code_pp;
 	    ErlNifFunc* f = &entry->funcs[i];
 	    if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom)
 		|| (code_pp = get_func_pp(mod->code, f_atom, f->arity))==NULL) { 
@@ -1195,7 +1618,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
     lib->entry = entry;
     erts_refc_init(&lib->rt_cnt, 0);
     erts_refc_init(&lib->rt_dtor_cnt, 0);
-    lib->is_orphan = 0;
+    lib->mod = mod;
     env.mod_nif = lib;
     if (mod->nif != NULL) { /* Reload */
 	int k;
@@ -1231,6 +1654,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	else {
 	    mod->nif->entry = NULL; /* to prevent 'unload' callback */
 	    erts_unload_nif(mod->nif);
+	    reload_warning = 1;
 	}
     }
     else {
@@ -1268,22 +1692,23 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
         mod->nif = lib; 
 	for (i=0; i < entry->num_of_funcs; i++)
 	{
-	    Uint* code_ptr;
+	    BeamInstr* code_ptr;
 	    erts_atom_get(entry->funcs[i].name, sys_strlen(entry->funcs[i].name), &f_atom); 
 	    code_ptr = *get_func_pp(mod->code, f_atom, entry->funcs[i].arity); 
 	    
 	    if (code_ptr[1] == 0) {
-		code_ptr[5+0] = (Uint) BeamOp(op_call_nif);		
+		code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif);
 	    }
 	    else { /* Function traced, patch the original instruction word */
-		BpData* bp = (BpData*) code_ptr[1];
-	        bp->orig_instr = (Uint) BeamOp(op_call_nif); 
+		BpData** bps = (BpData**) code_ptr[1];
+		BpData*  bp  = (BpData*) bps[erts_bp_sched2ix()];
+	        bp->orig_instr = (BeamInstr) BeamOp(op_call_nif);
 	    }	    
-	    code_ptr[5+1] = (Uint) entry->funcs[i].fptr;
-	    code_ptr[5+2] = (Uint) lib;
+	    code_ptr[5+1] = (BeamInstr) entry->funcs[i].fptr;
+	    code_ptr[5+2] = (BeamInstr) lib;
 	}
     }
-    else {    
+    else {
     error:
 	ASSERT(ret != am_ok);
         if (lib != NULL) {
@@ -1295,8 +1720,18 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	erts_sys_ddll_free_error(&errdesc);
     }
 
-    erts_smp_release_system();
+    erts_smp_thr_progress_unblock();
     erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+    erts_free(ERTS_ALC_T_TMP, lib_name);
+
+    if (reload_warning) {
+	erts_dsprintf_buf_t* dsbufp = erts_create_logger_dsbuf();
+	erts_dsprintf(dsbufp,
+		      "Repeated calls to erlang:load_nif from module '%T'.\n\n"
+		      "The NIF reload mechanism is deprecated and must not "
+		      "be used in production systems.\n", mod_atom);
+	erts_send_warning_to_logger(BIF_P->group_leader, dsbufp);
+    }
     BIF_RET(ret);
 }
 
@@ -1306,9 +1741,9 @@ erts_unload_nif(struct erl_module_nif* lib)
 {
     ErlNifResourceType* rt;
     ErlNifResourceType* next;
-    ASSERT(erts_smp_is_system_blocked(0));
+    ASSERT(erts_smp_thr_progress_is_blocking());
     ASSERT(lib != NULL);
-    ASSERT(!lib->is_orphan);
+    ASSERT(lib->mod != NULL);
     for (rt = resource_type_list.next;
 	 rt != &resource_type_list;
 	 rt = next) {
@@ -1338,7 +1773,7 @@ erts_unload_nif(struct erl_module_nif* lib)
     else {
 	ASSERT(erts_refc_read(&lib->rt_cnt, 1) > 0);
     }
-    lib->is_orphan = 1;
+    lib->mod = NULL;   /* orphan lib */
 }	
 
 void erl_nif_init()
@@ -1347,6 +1782,62 @@ void erl_nif_init()
     resource_type_list.prev = &resource_type_list;
     resource_type_list.dtor = NULL;
     resource_type_list.owner = NULL;
-    resource_type_list.name[0] = '\0';
+    resource_type_list.module = THE_NON_VALUE;
+    resource_type_list.name = THE_NON_VALUE;
 }
 
+#ifdef USE_VM_PROBES
+void dtrace_nifenv_str(ErlNifEnv *env, char *process_buf)
+{
+    dtrace_pid_str(env->proc->id, process_buf);
+}
+#endif
+
+#ifdef READONLY_CHECK
+/* Use checksums to assert that NIFs do not write into inspected binaries
+*/
+static void readonly_check_dtor(struct enif_tmp_obj_t*);
+static unsigned calc_checksum(unsigned char* ptr, unsigned size);
+
+struct readonly_check_t
+{
+    struct enif_tmp_obj_t hdr;
+    unsigned char* ptr;
+    unsigned size;
+    unsigned checksum;
+};
+static void add_readonly_check(ErlNifEnv* env, unsigned char* ptr, unsigned sz)
+{
+    ErtsAlcType_t allocator = is_proc_bound(env) ? ERTS_ALC_T_TMP : ERTS_ALC_T_NIF;
+    struct readonly_check_t* obj = erts_alloc(allocator, 
+					      sizeof(struct readonly_check_t));
+    obj->hdr.allocator = allocator;
+    obj->hdr.next = env->tmp_obj_list;
+    env->tmp_obj_list = &obj->hdr;
+    obj->hdr.dtor = &readonly_check_dtor;
+    obj->ptr = ptr;
+    obj->size = sz;
+    obj->checksum = calc_checksum(ptr, sz);    
+}
+static void readonly_check_dtor(struct enif_tmp_obj_t* o)
+{
+    struct readonly_check_t* obj = (struct readonly_check_t*) o;
+    unsigned chksum = calc_checksum(obj->ptr, obj->size);
+    if (chksum != obj->checksum) { 
+	fprintf(stderr, "\r\nReadonly data written by NIF, checksums differ"
+		" %x != %x\r\nABORTING\r\n", chksum, obj->checksum);
+	abort();
+    }
+    erts_free(obj->hdr.allocator,  obj);
+}
+static unsigned calc_checksum(unsigned char* ptr, unsigned size)
+{
+    unsigned i, sum = 0;
+    for (i=0; i<size; i++) {
+	sum ^= ptr[i] << ((i % 4)*8);
+    }
+    return sum;
+}
+
+#endif /* READONLY_CHECK */
+
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 1ccf00293e..e5d99dc4f1 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -23,18 +23,76 @@
 #ifndef __ERL_NIF_H__
 #define __ERL_NIF_H__
 
+
 #include "erl_drv_nif.h"
 
 /* Version history:
 ** 0.1: R13B03
 ** 1.0: R13B04
+** 2.0: R14A
+** 2.1: R14B02 "vm_variant"
+** 2.2: R14B03 enif_is_exception
+** 2.3: R15 enif_make_reverse_list
 */
-#define ERL_NIF_MAJOR_VERSION 1
-#define ERL_NIF_MINOR_VERSION 0
+#define ERL_NIF_MAJOR_VERSION 2
+#define ERL_NIF_MINOR_VERSION 3
 
 #include <stdlib.h>
 
+#ifdef SIZEOF_CHAR
+#  define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR
+#  undef SIZEOF_CHAR
+#endif
+#ifdef SIZEOF_SHORT
+#  define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT
+#  undef SIZEOF_SHORT
+#endif
+#ifdef SIZEOF_INT
+#  define SIZEOF_INT_SAVED__ SIZEOF_INT
+#  undef SIZEOF_INT
+#endif
+#ifdef SIZEOF_LONG
+#  define SIZEOF_LONG_SAVED__ SIZEOF_LONG
+#  undef SIZEOF_LONG
+#endif
+#ifdef SIZEOF_LONG_LONG
+#  define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG
+#  undef SIZEOF_LONG_LONG
+#endif
+#ifdef HALFWORD_HEAP_EMULATOR
+#  define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR
+#  undef HALFWORD_HEAP_EMULATOR
+#endif
+#include "erl_int_sizes_config.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
+typedef unsigned __int64 ErlNifUInt64;
+typedef __int64 ErlNifSInt64;
+#elif SIZEOF_LONG == 8
+typedef unsigned long ErlNifUInt64;
+typedef long ErlNifSInt64;
+#elif SIZEOF_LONG_LONG == 8
+typedef unsigned long long ErlNifUInt64;
+typedef long long ErlNifSInt64;
+#else
+#error No 64-bit integer type
+#endif
+
+#ifdef HALFWORD_HEAP_EMULATOR
+#  define ERL_NIF_VM_VARIANT "beam.halfword" 
+typedef unsigned int ERL_NIF_TERM;
+#else
+#  define ERL_NIF_VM_VARIANT "beam.vanilla" 
+#  if SIZEOF_LONG == SIZEOF_VOID_P
 typedef unsigned long ERL_NIF_TERM;
+#  elif SIZEOF_LONG_LONG == SIZEOF_VOID_P
+typedef unsigned long long ERL_NIF_TERM;
+#  endif
+#endif
 
 struct enif_environment_t;
 typedef struct enif_environment_t ErlNifEnv;
@@ -56,14 +114,15 @@ typedef struct enif_entry_t
     int  (*load)   (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info);
     int  (*reload) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info);
     int  (*upgrade)(ErlNifEnv*, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info);
-    void (*unload) (ErlNifEnv*, void* priv_data);    
+    void (*unload) (ErlNifEnv*, void* priv_data);
+    const char* vm_variant;
 }ErlNifEntry;
 
 
 
 typedef struct
 {
-    unsigned size;
+    size_t size;
     unsigned char* data;
 
     /* Internals (avert your eyes) */
@@ -73,17 +132,22 @@ typedef struct
 
 typedef struct enif_resource_type_t ErlNifResourceType;
 typedef void ErlNifResourceDtor(ErlNifEnv*, void*);
-enum ErlNifResourceFlags
+typedef enum
 {
     ERL_NIF_RT_CREATE = 1,
     ERL_NIF_RT_TAKEOVER = 2
-};
+}ErlNifResourceFlags;
 
 typedef enum
 {
     ERL_NIF_LATIN1 = 1
 }ErlNifCharEncoding;
 
+typedef struct
+{
+    ERL_NIF_TERM pid;  /* internal, may change */
+}ErlNifPid;
+
 typedef ErlDrvSysInfo ErlNifSysInfo;
 
 typedef struct ErlDrvTid_ *ErlNifTid;
@@ -116,8 +180,6 @@ extern TWinDynNifCallbacks WinDynNifCallbacks;
 #endif
 
 
-
-
 #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
 #  define ERL_NIF_INIT_GLOB TWinDynNifCallbacks WinDynNifCallbacks;
 #  define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* nif_init(TWinDynNifCallbacks* callbacks)
@@ -133,8 +195,20 @@ extern TWinDynNifCallbacks WinDynNifCallbacks;
 #endif
 
 
+#ifdef __cplusplus
+}
+#  define ERL_NIF_INIT_PROLOGUE extern "C" {
+#  define ERL_NIF_INIT_EPILOGUE }
+#else
+#  define ERL_NIF_INIT_PROLOGUE
+#  define ERL_NIF_INIT_EPILOGUE
+#endif
+
+
 #define ERL_NIF_INIT(NAME, FUNCS, LOAD, RELOAD, UPGRADE, UNLOAD) \
+ERL_NIF_INIT_PROLOGUE                   \
 ERL_NIF_INIT_GLOB                       \
+ERL_NIF_INIT_DECL(NAME);		\
 ERL_NIF_INIT_DECL(NAME)			\
 {					\
     static ErlNifEntry entry = 		\
@@ -144,11 +218,14 @@ ERL_NIF_INIT_DECL(NAME)			\
 	#NAME,				\
 	sizeof(FUNCS) / sizeof(*FUNCS),	\
 	FUNCS,				\
-	LOAD, RELOAD, UPGRADE, UNLOAD	\
+	LOAD, RELOAD, UPGRADE, UNLOAD,	\
+	ERL_NIF_VM_VARIANT		\
     };                                  \
     ERL_NIF_INIT_BODY;                  \
     return &entry;			\
-}
+}                                       \
+ERL_NIF_INIT_EPILOGUE
+
 
 #endif /* __ERL_NIF_H__ */
 
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index ec07a976b2..6396af09d0 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -21,31 +21,38 @@
 #  error This file should not be included directly
 #endif
 
+/*
+** WARNING: add new ERL_NIF_API_FUNC_DECL entries at the bottom of the list
+** to keep compatibility on Windows!!!
+**
+** And don't forget to increase ERL_NIF_MINOR_VERSION in erl_nif.h
+** when adding functions to the API.
+*/
 #ifdef ERL_NIF_API_FUNC_DECL
 ERL_NIF_API_FUNC_DECL(void*,enif_priv_data,(ErlNifEnv*));
-ERL_NIF_API_FUNC_DECL(void*,enif_alloc,(ErlNifEnv*, size_t size));
-ERL_NIF_API_FUNC_DECL(void,enif_free,(ErlNifEnv*, void* ptr));
+ERL_NIF_API_FUNC_DECL(void*,enif_alloc,(size_t size));
+ERL_NIF_API_FUNC_DECL(void,enif_free,(void* ptr));
 ERL_NIF_API_FUNC_DECL(int,enif_is_atom,(ErlNifEnv*, ERL_NIF_TERM term));
 ERL_NIF_API_FUNC_DECL(int,enif_is_binary,(ErlNifEnv*, ERL_NIF_TERM term));
 ERL_NIF_API_FUNC_DECL(int,enif_is_ref,(ErlNifEnv*, ERL_NIF_TERM term));
 ERL_NIF_API_FUNC_DECL(int,enif_inspect_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, ErlNifBinary* bin));
-ERL_NIF_API_FUNC_DECL(int,enif_alloc_binary,(ErlNifEnv*, unsigned size, ErlNifBinary* bin));
-ERL_NIF_API_FUNC_DECL(int,enif_realloc_binary,(ErlNifEnv*, ErlNifBinary* bin, unsigned size));
-ERL_NIF_API_FUNC_DECL(void,enif_release_binary,(ErlNifEnv*, ErlNifBinary* bin));
+ERL_NIF_API_FUNC_DECL(int,enif_alloc_binary,(size_t size, ErlNifBinary* bin));
+ERL_NIF_API_FUNC_DECL(int,enif_realloc_binary,(ErlNifBinary* bin, size_t size));
+ERL_NIF_API_FUNC_DECL(void,enif_release_binary,(ErlNifBinary* bin));
 ERL_NIF_API_FUNC_DECL(int,enif_get_int,(ErlNifEnv*, ERL_NIF_TERM term, int* ip));
 ERL_NIF_API_FUNC_DECL(int,enif_get_ulong,(ErlNifEnv*, ERL_NIF_TERM term, unsigned long* ip));
 ERL_NIF_API_FUNC_DECL(int,enif_get_double,(ErlNifEnv*, ERL_NIF_TERM term, double* dp));
 ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM* head, ERL_NIF_TERM* tail));
 ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, const ERL_NIF_TERM** array));
-ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ErlNifEnv* env, ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
-ERL_NIF_API_FUNC_DECL(int,enif_compare,(ErlNifEnv* env, ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
+ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
+ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_ulong,(ErlNifEnv* env, unsigned long i));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_double,(ErlNifEnv* env, double d));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_atom,(ErlNifEnv* env, const char* name));
-ERL_NIF_API_FUNC_DECL(int,enif_make_existing_atom,(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom));
+ERL_NIF_API_FUNC_DECL(int,enif_make_existing_atom,(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom, ErlNifCharEncoding));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_tuple,(ErlNifEnv* env, unsigned cnt, ...));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list,(ErlNifEnv* env, unsigned cnt, ...));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list_cell,(ErlNifEnv* env, ERL_NIF_TERM car, ERL_NIF_TERM cdr));
@@ -82,13 +89,13 @@ ERL_NIF_API_FUNC_DECL(int,enif_equal_tids,(ErlNifTid tid1, ErlNifTid tid2));
 ERL_NIF_API_FUNC_DECL(void,enif_thread_exit,(void *resp));
 ERL_NIF_API_FUNC_DECL(int,enif_thread_join,(ErlNifTid, void **respp));
 
-ERL_NIF_API_FUNC_DECL(void*,enif_realloc,(ErlNifEnv*, void* ptr, size_t size));
+ERL_NIF_API_FUNC_DECL(void*,enif_realloc,(void* ptr, size_t size));
 ERL_NIF_API_FUNC_DECL(void,enif_system_info,(ErlNifSysInfo *sip, size_t si_size));
 ERL_NIF_API_FUNC_DECL(int,enif_fprintf,(void/* FILE* */ *filep, const char *format, ...));
 ERL_NIF_API_FUNC_DECL(int,enif_inspect_iolist_as_binary,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifBinary* bin));
-ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_sub_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, unsigned pos, unsigned size));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_sub_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, size_t pos, size_t size));
 ERL_NIF_API_FUNC_DECL(int,enif_get_string,(ErlNifEnv*, ERL_NIF_TERM list, char* buf, unsigned len, ErlNifCharEncoding));
-ERL_NIF_API_FUNC_DECL(int,enif_get_atom,(ErlNifEnv*, ERL_NIF_TERM atom, char* buf, unsigned len));
+ERL_NIF_API_FUNC_DECL(int,enif_get_atom,(ErlNifEnv*, ERL_NIF_TERM atom, char* buf, unsigned len, ErlNifCharEncoding));
 ERL_NIF_API_FUNC_DECL(int,enif_is_fun,(ErlNifEnv*, ERL_NIF_TERM term));
 ERL_NIF_API_FUNC_DECL(int,enif_is_pid,(ErlNifEnv*, ERL_NIF_TERM term));
 ERL_NIF_API_FUNC_DECL(int,enif_is_port,(ErlNifEnv*, ERL_NIF_TERM term));
@@ -99,17 +106,48 @@ ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_long,(ErlNifEnv*, long i));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_tuple_from_array,(ErlNifEnv*, const ERL_NIF_TERM arr[], unsigned cnt));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list_from_array,(ErlNifEnv*, const ERL_NIF_TERM arr[], unsigned cnt));
 ERL_NIF_API_FUNC_DECL(int,enif_is_empty_list,(ErlNifEnv*, ERL_NIF_TERM term));
-ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type,(ErlNifEnv*, const char* type_name, void (*dtor)(ErlNifEnv*,void *), enum ErlNifResourceFlags flags, enum ErlNifResourceFlags* tried));
-ERL_NIF_API_FUNC_DECL(void*,enif_alloc_resource,(ErlNifEnv*, ErlNifResourceType* type, unsigned size));
-ERL_NIF_API_FUNC_DECL(void,enif_release_resource,(ErlNifEnv*, void* obj));
+ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type,(ErlNifEnv*, const char* module_str, const char* name_str, void (*dtor)(ErlNifEnv*,void *), ErlNifResourceFlags flags, ErlNifResourceFlags* tried));
+ERL_NIF_API_FUNC_DECL(void*,enif_alloc_resource,(ErlNifResourceType* type, size_t size));
+ERL_NIF_API_FUNC_DECL(void,enif_release_resource,(void* obj));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_resource,(ErlNifEnv*, void* obj));
 ERL_NIF_API_FUNC_DECL(int,enif_get_resource,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp));
-ERL_NIF_API_FUNC_DECL(unsigned,enif_sizeof_resource,(ErlNifEnv*, void* obj));
+ERL_NIF_API_FUNC_DECL(size_t,enif_sizeof_resource,(void* obj));
+ERL_NIF_API_FUNC_DECL(unsigned char*,enif_make_new_binary,(ErlNifEnv*,size_t size,ERL_NIF_TERM* termp));
+ERL_NIF_API_FUNC_DECL(int,enif_is_list,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_is_tuple,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_get_atom_length,(ErlNifEnv*, ERL_NIF_TERM atom, unsigned* len, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(int,enif_get_list_length,(ErlNifEnv* env, ERL_NIF_TERM term, unsigned* len));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_atom_len,(ErlNifEnv* env, const char* name, size_t len));
+ERL_NIF_API_FUNC_DECL(int, enif_make_existing_atom_len,(ErlNifEnv* env, const char* name, size_t len, ERL_NIF_TERM* atom, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_string_len,(ErlNifEnv* env, const char* string, size_t len, ErlNifCharEncoding));
+ERL_NIF_API_FUNC_DECL(ErlNifEnv*,enif_alloc_env,(void));
+ERL_NIF_API_FUNC_DECL(void,enif_free_env,(ErlNifEnv* env));
+ERL_NIF_API_FUNC_DECL(void,enif_clear_env,(ErlNifEnv* env));
+ERL_NIF_API_FUNC_DECL(int,enif_send,(ErlNifEnv* env, const ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_copy,(ErlNifEnv* dst_env, ERL_NIF_TERM src_term));
+ERL_NIF_API_FUNC_DECL(ErlNifPid*,enif_self,(ErlNifEnv* caller_env, ErlNifPid* pid));
+ERL_NIF_API_FUNC_DECL(int,enif_get_local_pid,(ErlNifEnv* env, ERL_NIF_TERM, ErlNifPid* pid));
+ERL_NIF_API_FUNC_DECL(void,enif_keep_resource,(void* obj));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_resource_binary,(ErlNifEnv*,void* obj,const void* data, size_t size));
+#if SIZEOF_LONG != 8
+ERL_NIF_API_FUNC_DECL(int,enif_get_int64,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifSInt64* ip));
+ERL_NIF_API_FUNC_DECL(int,enif_get_uint64,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifUInt64* ip));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int64,(ErlNifEnv*, ErlNifSInt64));
+ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_uint64,(ErlNifEnv*, ErlNifUInt64));
+#endif
+ERL_NIF_API_FUNC_DECL(int,enif_is_exception,(ErlNifEnv*, ERL_NIF_TERM term));
+ERL_NIF_API_FUNC_DECL(int,enif_make_reverse_list,(ErlNifEnv*, ERL_NIF_TERM term, ERL_NIF_TERM *list));
+ERL_NIF_API_FUNC_DECL(int,enif_is_number,(ErlNifEnv*, ERL_NIF_TERM term));
+
 /*
-** Add last to keep compatibility on Windows!!!
+** Add new entries here to keep compatibility on Windows!!!
 */
 #endif
 
+/*
+** Please keep the ERL_NIF_API_FUNC_MACRO list below in the same order
+** as the ERL_NIF_API_FUNC_DECL list above
+*/
 #ifdef ERL_NIF_API_FUNC_MACRO
 #  define enif_priv_data ERL_NIF_API_FUNC_MACRO(enif_priv_data)
 #  define enif_alloc ERL_NIF_API_FUNC_MACRO(enif_alloc)
@@ -195,9 +233,232 @@ ERL_NIF_API_FUNC_DECL(unsigned,enif_sizeof_resource,(ErlNifEnv*, void* obj));
 #  define enif_make_resource ERL_NIF_API_FUNC_MACRO(enif_make_resource)
 #  define enif_get_resource ERL_NIF_API_FUNC_MACRO(enif_get_resource)
 #  define enif_sizeof_resource ERL_NIF_API_FUNC_MACRO(enif_sizeof_resource)
+#  define enif_make_new_binary ERL_NIF_API_FUNC_MACRO(enif_make_new_binary)
+#  define enif_is_list ERL_NIF_API_FUNC_MACRO(enif_is_list)
+#  define enif_is_tuple ERL_NIF_API_FUNC_MACRO(enif_is_tuple)
+#  define enif_get_atom_length ERL_NIF_API_FUNC_MACRO(enif_get_atom_length)
+#  define enif_get_list_length ERL_NIF_API_FUNC_MACRO(enif_get_list_length)
+#  define enif_make_atom_len ERL_NIF_API_FUNC_MACRO(enif_make_atom_len)
+#  define enif_make_existing_atom_len ERL_NIF_API_FUNC_MACRO(enif_make_existing_atom_len)
+#  define enif_make_string_len ERL_NIF_API_FUNC_MACRO(enif_make_string_len)
+#  define enif_alloc_env ERL_NIF_API_FUNC_MACRO(enif_alloc_env)
+#  define enif_free_env ERL_NIF_API_FUNC_MACRO(enif_free_env)
+#  define enif_clear_env ERL_NIF_API_FUNC_MACRO(enif_clear_env)
+#  define enif_send ERL_NIF_API_FUNC_MACRO(enif_send)
+#  define enif_make_copy ERL_NIF_API_FUNC_MACRO(enif_make_copy)
+#  define enif_self ERL_NIF_API_FUNC_MACRO(enif_self)
+#  define enif_get_local_pid ERL_NIF_API_FUNC_MACRO(enif_get_local_pid)
+#  define enif_keep_resource ERL_NIF_API_FUNC_MACRO(enif_keep_resource)
+#  define enif_make_resource_binary ERL_NIF_API_FUNC_MACRO(enif_make_resource_binary)
+#if  SIZEOF_LONG != 8
+#  define enif_get_int64 ERL_NIF_API_FUNC_MACRO(enif_get_int64)
+#  define enif_get_uint64 ERL_NIF_API_FUNC_MACRO(enif_get_uint64)
+#  define enif_make_int64 ERL_NIF_API_FUNC_MACRO(enif_make_int64)
+#  define enif_make_uint64 ERL_NIF_API_FUNC_MACRO(enif_make_uint64)
+#endif
+
+#  define enif_is_exception ERL_NIF_API_FUNC_MACRO(enif_is_exception)
+#  define enif_make_reverse_list ERL_NIF_API_FUNC_MACRO(enif_make_reverse_list)
+#  define enif_is_number ERL_NIF_API_FUNC_MACRO(enif_is_number)
 
+/*
+** Add new entries here
+*/
 #endif
 
+
+#if defined(__GNUC__) && !(defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
+
+/* Inline functions for compile time type checking of arguments to
+   variadic functions.
+*/
+
+#  define ERL_NIF_INLINE __inline__
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple1(ErlNifEnv* env,
+						    ERL_NIF_TERM e1)
+{
+    return enif_make_tuple(env, 1, e1);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple2(ErlNifEnv* env,
+						    ERL_NIF_TERM e1,
+						    ERL_NIF_TERM e2)
+{
+    return enif_make_tuple(env, 2, e1, e2);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple3(ErlNifEnv* env,
+						    ERL_NIF_TERM e1,
+						    ERL_NIF_TERM e2,
+						    ERL_NIF_TERM e3)
+{
+    return enif_make_tuple(env, 3, e1, e2, e3);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple4(ErlNifEnv* env,
+						    ERL_NIF_TERM e1,
+						    ERL_NIF_TERM e2,
+						    ERL_NIF_TERM e3,
+						    ERL_NIF_TERM e4)
+{
+    return enif_make_tuple(env, 4, e1, e2, e3, e4);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple5(ErlNifEnv* env,
+						    ERL_NIF_TERM e1,
+						    ERL_NIF_TERM e2,
+						    ERL_NIF_TERM e3,
+						    ERL_NIF_TERM e4,
+						    ERL_NIF_TERM e5)
+{
+    return enif_make_tuple(env, 5, e1, e2, e3, e4, e5);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple6(ErlNifEnv* env,
+						    ERL_NIF_TERM e1,
+						    ERL_NIF_TERM e2,
+						    ERL_NIF_TERM e3,
+						    ERL_NIF_TERM e4,
+						    ERL_NIF_TERM e5,
+						    ERL_NIF_TERM e6)
+{
+    return enif_make_tuple(env, 6, e1, e2, e3, e4, e5, e6);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple7(ErlNifEnv* env,
+						    ERL_NIF_TERM e1,
+						    ERL_NIF_TERM e2,
+						    ERL_NIF_TERM e3,
+						    ERL_NIF_TERM e4,
+						    ERL_NIF_TERM e5,
+						    ERL_NIF_TERM e6,
+						    ERL_NIF_TERM e7)
+{
+    return enif_make_tuple(env, 7, e1, e2, e3, e4, e5, e6, e7);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple8(ErlNifEnv* env,
+						    ERL_NIF_TERM e1,
+						    ERL_NIF_TERM e2,
+						    ERL_NIF_TERM e3,
+						    ERL_NIF_TERM e4,
+						    ERL_NIF_TERM e5,
+						    ERL_NIF_TERM e6,
+						    ERL_NIF_TERM e7,
+						    ERL_NIF_TERM e8)
+{
+    return enif_make_tuple(env, 8, e1, e2, e3, e4, e5, e6, e7, e8);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple9(ErlNifEnv* env,
+						    ERL_NIF_TERM e1,
+						    ERL_NIF_TERM e2,
+						    ERL_NIF_TERM e3,
+						    ERL_NIF_TERM e4,
+						    ERL_NIF_TERM e5,
+						    ERL_NIF_TERM e6,
+						    ERL_NIF_TERM e7,
+						    ERL_NIF_TERM e8,
+						    ERL_NIF_TERM e9)
+{
+    return enif_make_tuple(env, 9, e1, e2, e3, e4, e5, e6, e7, e8, e9);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list1(ErlNifEnv* env,
+						   ERL_NIF_TERM e1)
+{
+    return enif_make_list(env, 1, e1);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list2(ErlNifEnv* env,
+						   ERL_NIF_TERM e1,
+						   ERL_NIF_TERM e2)
+{
+    return enif_make_list(env, 2, e1, e2);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list3(ErlNifEnv* env,
+						   ERL_NIF_TERM e1,
+						   ERL_NIF_TERM e2,
+						   ERL_NIF_TERM e3)
+{
+    return enif_make_list(env, 3, e1, e2, e3);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list4(ErlNifEnv* env,
+						   ERL_NIF_TERM e1,
+						   ERL_NIF_TERM e2,
+						   ERL_NIF_TERM e3,
+						   ERL_NIF_TERM e4)
+{
+    return enif_make_list(env, 4, e1, e2, e3, e4);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list5(ErlNifEnv* env,
+						   ERL_NIF_TERM e1,
+						   ERL_NIF_TERM e2,
+						   ERL_NIF_TERM e3,
+						   ERL_NIF_TERM e4,
+						   ERL_NIF_TERM e5)
+{
+    return enif_make_list(env, 5, e1, e2, e3, e4, e5);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list6(ErlNifEnv* env,
+						   ERL_NIF_TERM e1,
+						   ERL_NIF_TERM e2,
+						   ERL_NIF_TERM e3,
+						   ERL_NIF_TERM e4,
+						   ERL_NIF_TERM e5,
+						   ERL_NIF_TERM e6)
+{
+    return enif_make_list(env, 6, e1, e2, e3, e4, e5, e6);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list7(ErlNifEnv* env,
+						   ERL_NIF_TERM e1,
+						   ERL_NIF_TERM e2,
+						   ERL_NIF_TERM e3,
+						   ERL_NIF_TERM e4,
+						   ERL_NIF_TERM e5,
+						   ERL_NIF_TERM e6,
+						   ERL_NIF_TERM e7)
+{
+    return enif_make_list(env, 7, e1, e2, e3, e4, e5, e6, e7);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list8(ErlNifEnv* env,
+						   ERL_NIF_TERM e1,
+						   ERL_NIF_TERM e2,
+						   ERL_NIF_TERM e3,
+						   ERL_NIF_TERM e4,
+						   ERL_NIF_TERM e5,
+						   ERL_NIF_TERM e6,
+						   ERL_NIF_TERM e7,
+						   ERL_NIF_TERM e8)
+{
+    return enif_make_list(env, 8, e1, e2, e3, e4, e5, e6, e7, e8);
+}
+
+static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list9(ErlNifEnv* env,
+						   ERL_NIF_TERM e1,
+						   ERL_NIF_TERM e2,
+						   ERL_NIF_TERM e3,
+						   ERL_NIF_TERM e4,
+						   ERL_NIF_TERM e5,
+						   ERL_NIF_TERM e6,
+						   ERL_NIF_TERM e7,
+						   ERL_NIF_TERM e8,
+						   ERL_NIF_TERM e9)
+{
+    return enif_make_list(env, 9, e1, e2, e3, e4, e5, e6, e7, e8, e9);
+}
+
+#  undef ERL_NIF_INLINE
+
+#else /* fallback with macros */
+
 #ifndef enif_make_list1
 #  define enif_make_list1(ENV,E1) enif_make_list(ENV,1,E1)
 #  define enif_make_list2(ENV,E1,E2) enif_make_list(ENV,2,E1,E2)
@@ -219,7 +480,18 @@ ERL_NIF_API_FUNC_DECL(unsigned,enif_sizeof_resource,(ErlNifEnv*, void* obj));
 #  define enif_make_tuple9(ENV,E1,E2,E3,E4,E5,E6,E7,E8,E9) enif_make_tuple(ENV,9,E1,E2,E3,E4,E5,E6,E7,E8,E9)
 #endif
 
-#ifndef enif_get_data
-#  define enif_get_data enif_priv_data /* deprecated */
+#endif /* __GNUC__ && !WIN32 */
+
+#ifndef enif_make_pid
+
+#  define enif_make_pid(ENV, PID) ((const ERL_NIF_TERM)((PID)->pid))
+
+#if SIZEOF_LONG == 8
+#  define enif_get_int64 enif_get_long
+#  define enif_get_uint64 enif_get_ulong
+#  define enif_make_int64 enif_make_long
+#  define enif_make_uint64 enif_make_ulong
+#endif
+
 #endif
 
diff --git a/erts/emulator/beam/erl_nmgc.c b/erts/emulator/beam/erl_nmgc.c
deleted file mode 100644
index 626d4e295a..0000000000
--- a/erts/emulator/beam/erl_nmgc.c
+++ /dev/null
@@ -1,1402 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-#include "global.h"
-#include "erl_gc.h"
-#include "erl_binary.h"
-#include "erl_nmgc.h"
-#include "erl_debug.h"
-#if HIPE
-#include "hipe_bif0.h" /* for hipe_constants_{start,next} */
-#include "hipe_stack.h"
-#endif
-
-
-#ifdef INCREMENTAL
-/***************************************************************************
- *                                                                         *
- *         Incremental Garbage Collector for the Message Area              *
- *                                                                         *
- ***************************************************************************/
-
-/*
- * The heap pointers are declared in erl_init.c
- * global_heap is the nursery
- * global_old_heap is the old generation
- */
-unsigned char *blackmap = NULL;
-INC_Page *inc_used_mem = NULL;
-INC_MemBlock *inc_free_list = NULL;
-Eterm *inc_fromspc;
-Eterm *inc_fromend;
-Eterm *inc_nursery_scn_ptr;
-Eterm **fwdptrs;
-Eterm *inc_alloc_limit;
-Process *inc_active_proc;
-Process *inc_active_last;
-int   inc_words_to_go;
-
-static Eterm       *inc_last_nursery;
-static int          inc_pages = INC_NoPAGES;
-static INC_Page    *inc_bibop = NULL;
-static int          inc_used_pages;
-
-/* Used when growing the old generation */
-/*
-#define INC_ROOTSAVE 16384
-static Eterm       *root_save[INC_ROOTSAVE];
-static int          roots_saved = 0;
-*/
-
-INC_STORAGE_DECLARATION(,gray);
-
-static void inc_minor_gc(Process *p, int need, Eterm* objv, int nobj);
-static void inc_major_gc(Process *p, int need, Eterm* objv, int nobj);
-
-#ifdef INC_TIME_BASED
-#if USE_PERFCTR
-
-/*
- * This uses the Linux perfctr extension to virtualise the
- * time-stamp counter.
- */
-#include "libperfctr.h"
-static struct vperfctr *vperfctr;
-static double cpu_khz;
-static double tsc_to_cpu_mult;
-
-static void inc_start_hrvtime(void)
-{
-    struct perfctr_info info;
-    struct vperfctr_control control;
-
-    if( vperfctr != NULL )
-        return;
-    vperfctr = vperfctr_open();
-    if( vperfctr == NULL )
-        return;
-    if( vperfctr_info(vperfctr, &info) >= 0 ) {
-        cpu_khz = (double)info.cpu_khz;
-        tsc_to_cpu_mult = (double)(info.tsc_to_cpu_mult ? : 1);
-        if( info.cpu_features & PERFCTR_FEATURE_RDTSC ) {
-            memset(&control, 0, sizeof control);
-            control.cpu_control.tsc_on = 1;
-            if( vperfctr_control(vperfctr, &control) >= 0 )
-                return;
-        }
-    }
-    vperfctr_close(vperfctr);
-    vperfctr = NULL;
-}
-
-#define inc_get_hrvtime() (((double)vperfctr_read_tsc(vperfctr) * tsc_to_cpu_mult) / cpu_khz)
-
-#endif /* USE_PERFCTR */
-#endif /* INC_TIME_BASED */
-
-#ifdef INC_TIME_BASED
-#  define timeslice 1 /* milli seconds */
-#  define WORK_MORE (inc_get_hrvtime() < start_time + timeslice)
-#else
-//#  define inc_min_work 100 /* words */
-#  define inc_min_work global_heap_sz + inc_pages * INC_FULLPAGE /* words */
-#  define WORK_MORE (inc_words_to_go > 0)
-#endif
-
-void erts_init_incgc(void)
-{
-    int i;
-    int size = inc_pages * INC_FULLPAGE;
-
-    /* Young generation */
-    global_heap = (Eterm *)erts_alloc(ERTS_ALC_T_MESSAGE_AREA,
-                                      sizeof(Eterm) * global_heap_sz);
-    global_hend = global_heap + global_heap_sz;
-    global_htop = global_heap;
-    inc_alloc_limit = global_hend;
-
-    /* Fromspace */
-    inc_last_nursery = (Eterm *) erts_alloc(ERTS_ALC_T_MESSAGE_AREA,
-                                            global_heap_sz * sizeof(Eterm));
-    inc_fromspc = inc_fromend = NULL;
-
-    /* Forward-pointers */
-    fwdptrs = erts_alloc(ERTS_ALC_T_MESSAGE_AREA,
-                         global_heap_sz * sizeof(Eterm*));
-    /* Old generation */
-    global_old_heap = (Eterm *)erts_alloc(ERTS_ALC_T_MESSAGE_AREA,
-                                          size * sizeof(Eterm));
-    global_old_hend = global_old_heap + size;
-
-    /* Pages i BiBOP */
-    for (i = 0; i < inc_pages; i++)
-    {
-        INC_Page *this = (INC_Page*)(global_old_heap + i * INC_FULLPAGE);
-        this->next = (INC_Page*)((Eterm*)this + INC_FULLPAGE);
-    }
-
-    inc_bibop = (INC_Page*)global_old_heap;
-    ((INC_Page*)(global_old_heap + (inc_pages - 1) * INC_FULLPAGE))->next =
-        NULL;
-
-    inc_used_mem = inc_bibop;
-    inc_bibop = inc_bibop->next;
-    inc_used_mem->next = NULL;
-    inc_used_pages = 1;
-
-    /* Free-list */
-    inc_free_list = (INC_MemBlock*)inc_used_mem->start;
-    inc_free_list->size = INC_PAGESIZE;
-    inc_free_list->prev = NULL;
-    inc_free_list->next = NULL;
-
-    /* Blackmap */
-    blackmap = (unsigned char*)erts_alloc(ERTS_ALC_T_MESSAGE_AREA,
-                                          INC_FULLPAGE * inc_pages);
-    /* Gray stack */
-    INC_STORAGE_INIT(gray);
-
-    inc_active_proc = NULL;
-    inc_active_last = NULL;
-
-#ifdef INC_TIME_BASED
-    inc_start_hrvtime();
-#endif
-}
-
-void erts_cleanup_incgc(void)
-{
-    INC_STORAGE_ERASE(gray);
-
-    if (inc_fromspc)
-        inc_last_nursery = inc_fromspc;
-
-    erts_free(ERTS_ALC_T_MESSAGE_AREA,(void*)global_heap);
-    erts_free(ERTS_ALC_T_MESSAGE_AREA,(void*)inc_last_nursery);
-    erts_free(ERTS_ALC_T_MESSAGE_AREA,(void*)global_old_heap);
-    erts_free(ERTS_ALC_T_MESSAGE_AREA,(void*)blackmap);
-    erts_free(ERTS_ALC_T_MESSAGE_AREA,(void*)fwdptrs);
-}
-
-void erts_incremental_gc(Process* p, int need, Eterm* objv, int nobj)
-{
-    int repeat_minor;
-#ifdef INC_TIME_BASED
-    double start_time = inc_get_hrvtime();
-    int work_left_before = inc_words_to_go;
-#endif
-    /* Used when growing the fromspace */
-    static char inc_growing_nurs = 0;
-
-    BM_STOP_TIMER(system);
-    //BM_MMU_READ();
-    BM_RESET_TIMER(gc);
-    BM_START_TIMER(gc);
-
-    VERBOSE(DEBUG_HYBRID_GC,
-            ("INCGC: Incremental GC START  Caused by: %T  Need: %d\n",
-             p->id,need));
-
-    ma_gc_flags |= GC_GLOBAL;
-    ma_gc_flags &= ~GC_CYCLE_START;
-
-#ifndef INC_TIME_BASED
-    /* Decide how much work to do this GC stage. The work is meassured
-     * in number of words copied from the young generation to the old
-     * plus number of work marked in the old generation.
-     */
-    if (ma_gc_flags & GC_MAJOR) {
-        int wm = (need > inc_min_work) ? need : inc_min_work;
-        inc_words_to_go = (int)((wm * (((inc_used_pages * INC_PAGESIZE) /
-                                        (double)global_heap_sz) + 1)) + 0.5);
-    }
-    else
-        inc_words_to_go = (need > inc_min_work) ? need : inc_min_work;
-#endif
-
-    do {
-        if (ma_gc_flags & GC_MAJOR) {
-            /* This is a major collection cycle. */
-            inc_major_gc(p,need,objv,nobj);
-        } else if (ma_gc_flags & GC_CYCLE) {
-            /* This is a minor collection cycle. */
-            inc_minor_gc(p,need,objv,nobj);
-        } else {
-            VERBOSE(DEBUG_HYBRID_GC,("INCGC: Collection cycle START\n"));
-            ma_gc_flags |= (GC_CYCLE | GC_CYCLE_START);
-            inc_fromspc = global_heap;
-            inc_fromend = global_htop;
-            global_heap = global_htop = inc_last_nursery;
-            global_hend = global_heap + global_heap_sz;
-            inc_nursery_scn_ptr = global_heap;
-#ifdef INC_TIME_BASED
-            work_left_before = inc_words_to_go = global_heap_sz;
-#endif
-#ifdef DEBUG
-            inc_last_nursery = NULL;
-#endif
-            memset(fwdptrs,0,global_heap_sz * sizeof(Eterm));
-
-            {
-                /* TODO: Alla processer ska v�l egentligen inte aktiveras h�r... */
-                int i;
-                for (i = 0; i < erts_num_active_procs; i++) {
-                    Process *cp = erts_active_procs[i];
-                    INC_ACTIVATE(cp);
-                    cp->scan_top = cp->high_water;
-                }
-            }
-
-            if (ma_gc_flags & GC_NEED_MAJOR) {
-                /* The previous collection cycle caused the old generation to
-                 * overflow.  This collection cycle will therefore be a major
-                 * one.
-                 */
-                BM_COUNT(major_gc_cycles);
-                VERBOSE(DEBUG_HYBRID_GC,("INCGC: MAJOR cycle\n"));
-                inc_major_gc(p,need,objv,nobj);
-            } else {
-                BM_COUNT(minor_gc_cycles);
-                VERBOSE(DEBUG_HYBRID_GC,("INCGC: MINOR cycle\n"));
-                inc_minor_gc(p,need,objv,nobj);
-            }
-        }
-
-        repeat_minor = 0;
-        if (!(ma_gc_flags & GC_CYCLE)) {
-            inc_alloc_limit = global_hend;
-            inc_last_nursery = inc_fromspc;
-            inc_fromspc = inc_fromend = NULL;
-            ASSERT(INC_STORAGE_EMPTY(gray));
-
-            if (inc_growing_nurs) {
-                /*
-                 * The previous collection cycle caused the nursery to
-                 * grow, now we have to grow the from-space as well.
-                 */
-                inc_last_nursery =
-                    (Eterm*) erts_realloc(ERTS_ALC_T_MESSAGE_AREA,
-                                          (void*)inc_last_nursery,
-                                          sizeof(Eterm) * global_heap_sz);
-                inc_growing_nurs = 0;
-            }
-
-            if (global_hend - global_htop <= need) {
-                /*
-                 * Initiate a new GC cycle immediately and, if necessary,
-                 * enlarge the nursery.
-                 */
-                if (global_heap_sz <= need) {
-                    VERBOSE(DEBUG_HYBRID_GC,
-                            ("INCGC: Allocating a larger nursery\n"));
-                    global_heap_sz = erts_next_heap_size(need * 1.5,0);
-                    inc_last_nursery =
-                        (Eterm*) erts_realloc(ERTS_ALC_T_MESSAGE_AREA,
-                                              (void*)inc_last_nursery,
-                                              sizeof(Eterm) * global_heap_sz);
-                    fwdptrs = erts_realloc(ERTS_ALC_T_MESSAGE_AREA,fwdptrs,
-                                           global_heap_sz * sizeof(Eterm*));
-                    inc_growing_nurs = 1;
-                }
-                repeat_minor = 1;
-            }
-
-#ifdef DEBUG
-            /* Fill the from-space with bad things */
-            memset(inc_last_nursery,DEBUG_BAD_BYTE,
-                   global_heap_sz * sizeof(Eterm));
-#endif
-        }
-    } while (repeat_minor);
-
-
-    /* Clean up after garbage collection ********************************/
-
-    if (inc_alloc_limit != global_hend) {
-
-#ifdef INC_TIME_BASED
-        if ((work_left_before - inc_words_to_go) == 0) {
-            inc_alloc_limit = global_htop + need;
-        } else {
-            inc_alloc_limit = (global_hend - global_htop) /
-                (inc_words_to_go / (work_left_before - inc_words_to_go)) +
-                global_htop;
-            if (inc_alloc_limit > global_hend)
-                inc_alloc_limit = global_hend;
-        }
-#else
-        inc_alloc_limit = (Eterm*)(global_htop + (need > inc_min_work) ?
-                                   need : inc_min_work);
-        if (inc_alloc_limit > global_hend)
-            inc_alloc_limit = global_hend;
-#endif
-    }
-
-    ma_gc_flags &= ~GC_GLOBAL;
-
-    /* INC_TIME_BASED: If this fails we have to increase the timeslice! */
-    ASSERT(inc_alloc_limit - global_htop > need);
-
-    BM_STOP_TIMER(gc);
-#ifdef BM_TIMERS
-    minor_global_gc_time += gc_time;
-    if (gc_time > max_global_minor_time)
-        max_global_minor_time = gc_time;
-
-    pause_times[(((gc_time * 1000) < MAX_PAUSE_TIME) ?
-                 (int)(gc_time * 1000) :
-                 MAX_PAUSE_TIME - 1)]++;
-#endif
-    //BM_MMU_INIT();
-    { static long long verif = 0;
-        //erts_printf("innan verify: %d\n",++verif);
-        if (verif==168) print_memory(NULL);
-        verify_everything();
-        //erts_printf("efter verify: %d\n",verif);
-    }
-    BM_START_TIMER(system);
-    VERBOSE(DEBUG_HYBRID_GC,("INCGC: Incremental GC END\n"));
-}
-
-
-/***************************************************************************
- *                                                                         *
- *     Minor collection  - Copy live data from young generation to old     *
- *                                                                         *
- ***************************************************************************/
-
-#define MINOR_SCAN(PTR,END) do {                                        \
-    ASSERT(PTR <= END);                                                 \
-    while (WORK_MORE && PTR < END) {                                    \
-        Eterm val = *PTR;                                               \
-        Eterm *obj_ptr = ptr_val(val);                                  \
-        switch (primary_tag(val)) {                                     \
-        case TAG_PRIMARY_LIST:                                          \
-            if (ptr_within(obj_ptr,inc_fromspc,inc_fromend)) {          \
-                if (INC_IS_FORWARDED(obj_ptr)) {                        \
-                    *PTR = make_list(INC_FORWARD_VALUE(obj_ptr));       \
-                }                                                       \
-                else {                                                  \
-                    Eterm *hp = erts_inc_alloc(2);                      \
-                    INC_STORE(gray,hp,2);                               \
-                    INC_COPY_CONS(obj_ptr,hp,PTR);                      \
-                }                                                       \
-            }                                                           \
-            break;                                                      \
-        case TAG_PRIMARY_BOXED:                                         \
-            if (ptr_within(obj_ptr,inc_fromspc,inc_fromend)) {          \
-                if (INC_IS_FORWARDED(obj_ptr)) {                        \
-                    *PTR = make_boxed(INC_FORWARD_VALUE(obj_ptr));      \
-                }                                                       \
-                else {                                                  \
-                    Eterm *hp = erts_inc_alloc(BOXED_NEED(obj_ptr,*obj_ptr)); \
-                    INC_STORE(gray,hp,BOXED_NEED(obj_ptr,*obj_ptr));    \
-                    INC_COPY_BOXED(obj_ptr,hp,PTR);                     \
-                }                                                       \
-            }                                                           \
-            break;                                                      \
-        case TAG_PRIMARY_HEADER:                                        \
-            switch (val & _TAG_HEADER_MASK) {                           \
-            case ARITYVAL_SUBTAG: break;                                \
-            default: PTR += thing_arityval(val); break;                 \
-            }                                                           \
-            break;                                                      \
-        }                                                               \
-        PTR++;                                                          \
-    }                                                                   \
-} while(0)
-
-
-/* Returns: TRUE (1) if the need is greater than the available space
- * and the garbage collector needs to be restarted immediately. FALSE
- * (0) otherwise.
- */
-static void inc_minor_gc(Process* p, int need, Eterm* objv, int nobj)
-{
-    BM_COUNT(minor_gc_stages);
-
-    /* Start with looking at gray objects found in earlier collection
-     * stages.
-     */
-    VERBOSE(DEBUG_HYBRID_GC,("INCGC: Rescue gray found from nursery\n"));
-    {
-        INC_Object *obj = NULL;
-        Eterm *ptr;
-
-        while (WORK_MORE && !INC_STORAGE_EMPTY(gray)) {
-            obj = INC_STORAGE_GET(gray);
-            if ((*obj->this & _TAG_HEADER_MASK) == FUN_SUBTAG) {
-                ptr = obj->this + thing_arityval(*obj->this) + 1;
-            } else {
-                ptr = obj->this;
-            }
-            MINOR_SCAN(ptr,obj->this + obj->size);
-        }
-        /* TODO: Se f�reg�ende uppdatering av gr� objekt */
-        if (!WORK_MORE && obj != NULL)
-            INC_STORE(gray,obj->this,obj->size);
-    }
-
-    VERBOSE(DEBUG_HYBRID_GC,("INCGC: Scan root-set\n"));
-    while (WORK_MORE && inc_active_proc) {
-        Rootset rootset;
-        Process *cp = inc_active_proc;
-
-        ASSERT(INC_IS_ACTIVE(cp));
-
-        /* TODO: Hur dyrt �r det att bygga nytt rootset varje g�ng? */
-
-        /* TODO: Fundera p� ordningen! Rootset, Heap, Old heap... */
-
-        /* TODO: Scanna stacken fr�n p->send till p->stop! [Brooks84] */
-        /* Notera: Vi GC:ar inte de yngsta objekten - de som allokeras
-           under GC-cykeln. Detta ger ynglingarna en chans att d� innan
-           GC:n b�rjar kopiera dem. [StefanovicMcKinleyMoss@OOPSLA99] */
-
-        /* TODO: N�r rootset �r scannat borde processen inte vara
-           aktiv mer. Den b�r aktiveras i schedule, endast om en
-           process har k�rt beh�ver vi scanna rootset igen. */
-
-        /* MT: In a multithreaded system the process cp needs to be
-         * locked here.
-         */
-
-        if (cp == p)
-            rootset.n = setup_rootset(cp, objv, nobj, &rootset);
-        else
-            rootset.n = setup_rootset(cp, cp->arg_reg, cp->arity, &rootset);
-
-        //MA_GENSWEEP_NSTACK(cp, old_htop, n_htop, objv, nobj);
-
-        while (WORK_MORE && rootset.n--) {
-            Eterm *g_ptr = rootset.v[rootset.n];
-            Uint g_sz = rootset.sz[rootset.n];
-
-            while (WORK_MORE && g_sz--) {
-                Eterm gval = *g_ptr;
-                switch (primary_tag(gval)) {
-                case TAG_PRIMARY_LIST: {
-                    Eterm *ptr = list_val(gval);
-                    if (ptr_within(ptr,inc_fromspc,inc_fromend)) {
-                        if (INC_IS_FORWARDED(ptr)) {
-                            *g_ptr++ = make_list(INC_FORWARD_VALUE(ptr));
-                        }
-                        else {
-                            Eterm *hp = erts_inc_alloc(2);
-                            INC_STORE(gray,hp,2);
-                            INC_COPY_CONS(ptr,hp,g_ptr++);
-                        }
-                    }
-                    else
-                        ++g_ptr;
-                    continue;
-                }
-
-                case TAG_PRIMARY_BOXED: {
-                    Eterm *ptr = boxed_val(gval);
-                    if (ptr_within(ptr,inc_fromspc,inc_fromend)) {
-                        if (INC_IS_FORWARDED(ptr)) {
-                            *g_ptr++ = make_boxed(INC_FORWARD_VALUE(ptr));
-                        }
-                        else {
-                            Eterm *hp = erts_inc_alloc(BOXED_NEED(ptr,*ptr));
-                            INC_STORE(gray,hp,BOXED_NEED(ptr,*ptr));
-                            INC_COPY_BOXED(ptr,hp,g_ptr++);
-                        }
-                    }
-                    else
-                        ++g_ptr;
-                    continue;
-                }
-
-                default:
-                    g_ptr++;
-                    continue;
-                }
-            }
-        }
-
-        restore_one_rootset(cp, &rootset);
-
-        /* MT: cp can be unlocked now. */
-
-        /* VERBOSE(DEBUG_HYBRID_GC,("INCGC: Scan private nursery\n")); */
-        if (cp->scan_top != HEAP_TOP(cp)) {
-            Eterm *ptr = cp->scan_top;
-            MINOR_SCAN(ptr,HEAP_TOP(cp));
-            /* TODO: F�r att spara scan_top h�r m�ste alla ma-pekare
-             * som hittas l�ggas till i cp->rrma.
-             */
-            //cp->scan_top = ptr;
-        }
-
-        /* VERBOSE(DEBUG_HYBRID_GC,("INCGC: Scan heap fragments\n")); */
-        {
-            ErlHeapFragment* bp = MBUF(cp);
-
-            while (WORK_MORE && bp) {
-                Eterm *ptr = bp->mem;
-                if ((ARITH_HEAP(cp) >= bp->mem) &&
-                    (ARITH_HEAP(cp) < bp->mem + bp->size)) {
-                    MINOR_SCAN(ptr,ARITH_HEAP(cp));
-                } else {
-                    MINOR_SCAN(ptr,bp->mem + bp->size);
-                }
-                bp = bp->next;
-            }
-        }
-
-        /* VERBOSE(DEBUG_HYBRID_GC,("INCGC: Scan gray\n")); */
-        {
-            INC_Object *obj = NULL;
-            Eterm *ptr;
-            while (WORK_MORE && !INC_STORAGE_EMPTY(gray)) {
-                obj = INC_STORAGE_GET(gray);
-                if ((*obj->this & _TAG_HEADER_MASK) == FUN_SUBTAG) {
-                    ptr = obj->this + thing_arityval(*obj->this) + 1;
-                } else {
-                    ptr = obj->this;
-                }
-                MINOR_SCAN(ptr,obj->this + obj->size);
-            }
-            /* TODO: INC_STORE(gray,ptr,obj->size-(ptr-obj->this)); Typ.. */
-            if (!WORK_MORE && obj != NULL)
-                INC_STORE(gray,obj->this,obj->size);
-        }
-
-        if (WORK_MORE) {
-            //printf("Rootset after:\r\n");
-            //print_one_rootset(&rootset);
-            INC_DEACTIVATE(cp);
-        }
-    }
-
-    /* Update new pointers in the nursery to new copies in old generation. */
-    VERBOSE(DEBUG_HYBRID_GC,("INCGC: Update nursery\n"));
-    {
-        Eterm *ptr = inc_nursery_scn_ptr;
-        MINOR_SCAN(ptr,global_htop);
-        inc_nursery_scn_ptr = ptr;
-    }
-
-    VERBOSE(DEBUG_HYBRID_GC,("INCGC: Rescue gray found from nursery\n"));
-    {
-        INC_Object *obj = NULL;
-        Eterm *ptr;
-
-        while (WORK_MORE && !INC_STORAGE_EMPTY(gray)) {
-            obj = INC_STORAGE_GET(gray);
-            if ((*obj->this & _TAG_HEADER_MASK) == FUN_SUBTAG) {
-                ptr = obj->this + thing_arityval(*obj->this) + 1;
-            } else {
-                ptr = obj->this;
-            }
-            MINOR_SCAN(ptr,obj->this + obj->size);
-        }
-        /* TODO: Se f�reg�ende uppdatering av gr� objekt */
-        if (!WORK_MORE && obj != NULL)
-            INC_STORE(gray,obj->this,obj->size);
-    }
-
-    /* Atomic phase */
-    VERBOSE(DEBUG_HYBRID_GC,("INCGC: Update copy stack\n"));
-    {
-        Uint i;
-        for (i = 0; i < ma_dst_top; i++) {
-            if (ptr_within(ma_dst_stack[i],inc_fromspc,inc_fromend)) {
-                if (INC_IS_FORWARDED(ma_dst_stack[i]))
-                    ma_dst_stack[i] = INC_FORWARD_VALUE(ma_dst_stack[i]);
-            }
-        }
-    }
-
-    if (WORK_MORE) {
-        VERBOSE(DEBUG_HYBRID_GC,("INCGC: Update offheap-lists\n"));
-        {
-            ExternalThing **prev = &erts_global_offheap.externals;
-            ExternalThing *ptr   = erts_global_offheap.externals;
-
-            /* Atomic phase */
-            VERBOSE(DEBUG_HYBRID_GC,("INCGC: Sweep proc externals\n"));
-            while (ptr) {
-                Eterm *ppt = (Eterm*) ptr;
-
-                if (ptr_within(ppt,global_old_heap,global_old_hend)) {
-                    prev = &ptr->next;
-                    ptr = ptr->next;
-                } else if (ptr_within(ppt, inc_fromspc, inc_fromend) &&
-                           INC_IS_FORWARDED(ppt)) {
-                    ExternalThing *ro = (ExternalThing*)INC_FORWARD_VALUE(ppt);
-                    *prev = ro;         /* Patch to moved pos */
-                    prev = &ro->next;
-                    ptr = ro->next;
-                } else {
-                    erts_deref_node_entry(ptr->node);
-                    *prev = ptr = ptr->next;
-                }
-            }
-            ASSERT(*prev == NULL);
-        }
-
-        {
-            ProcBin **prev = &erts_global_offheap.mso;
-            ProcBin *ptr   = erts_global_offheap.mso;
-
-            /* Atomic phase */
-            VERBOSE(DEBUG_HYBRID_GC,("INCGC: Sweep proc bins\n"));
-            while (ptr) {
-                Eterm *ppt = (Eterm*)ptr;
-
-                if (ptr_within(ppt,global_old_heap,global_old_hend)) {
-                    prev = &ptr->next;
-                    ptr = ptr->next;
-                } else if (ptr_within(ppt, inc_fromspc, inc_fromend) &&
-                           INC_IS_FORWARDED(ppt)) {
-                    ProcBin *ro = (ProcBin*)INC_FORWARD_VALUE(ppt);
-                    *prev = ro;         /* Patch to moved pos */
-                    prev = &ro->next;
-                    ptr = ro->next;
-                } else {
-                    Binary *bptr;
-                    *prev = ptr->next;
-                    bptr = ptr->val;
-                    if (erts_refc_dectest(&bptr->refc, 0) == 0)
-			erts_bin_free(bptr);
-                    ptr = *prev;
-                }
-            }
-            ASSERT(*prev == NULL);
-        }
-
-        VERBOSE(DEBUG_HYBRID_GC,("INCGC: Minor collection cycle END\n"));
-        ma_gc_flags &= ~GC_CYCLE;
-    }
-}
-
-
-
-
-/***************************************************************************
- *                                                                         *
- *   Major collection - CopyMark - Copy young to old, Mark-Sweep old       *
- *                                                                         *
- ***************************************************************************/
-
-#define COPYMARK(PTR,END) do {                                          \
-    ASSERT(PTR <= END);                                                 \
-    while (WORK_MORE && PTR < END) {                                    \
-        Eterm val = *PTR;                                               \
-        Eterm *obj_ptr = ptr_val(val);                                  \
-        switch (primary_tag(val)) {                                     \
-            case TAG_PRIMARY_LIST:                                      \
-                COPYMARK_CONS(obj_ptr,aging_htop,PTR,aging_end); break; \
-            case TAG_PRIMARY_BOXED:                                     \
-                COPYMARK_BOXED(obj_ptr,aging_htop,PTR,aging_end); break; \
-            case TAG_PRIMARY_HEADER:                                    \
-                switch (val & _TAG_HEADER_MASK) {                       \
-                    case ARITYVAL_SUBTAG: break;                        \
-                    default:                                            \
-                        PTR += thing_arityval(val);                     \
-                        break;                                          \
-                }                                                       \
-                break;                                                  \
-            default: break;                                             \
-        }                                                               \
-        PTR++;                                                          \
-    }                                                                   \
-} while(0);
-/* TODO:
-    if (aging_htop + 10 > aging + INC_FULLPAGE) {
-        aging->next = inc_used_mem;
-        inc_used_mem = aging;
-    }
-*/
-
-static void inc_major_gc(Process *p, int need, Eterm* objv, int nobj)
-{
-    Eterm *free_start = NULL;
-    Uint live = 0;
-    Uint old_gen_sz = 0;
-    static INC_Page *aging;
-    static Eterm *aging_htop;
-    static Eterm *aging_end;
-    BM_NEW_TIMER(old_gc);
-
-    BM_SWAP_TIMER(gc,old_gc);
-    BM_COUNT(major_gc_stages);
-
-    VERBOSE(DEBUG_HYBRID_GC,("INCGC: Major collection START\n"));
-
-    ma_gc_flags |= GC_INCLUDE_ALL;
-
-    if (ma_gc_flags & GC_NEED_MAJOR)
-    {
-        INC_Page *page = inc_used_mem;
-
-        ma_gc_flags |= GC_MAJOR;
-        ma_gc_flags &= ~GC_NEED_MAJOR;
-
-        while (page)
-        {
-            memset(blackmap +
-                   ((void*)page - (void*)global_old_heap) / sizeof(void*),
-                   0, INC_FULLPAGE);
-            page = page->next;
-        }
-
-        if (inc_bibop) {
-            aging = inc_bibop;
-            inc_bibop = inc_bibop->next;
-            aging->next = NULL;
-            memset(blackmap +
-                   ((void*)aging - (void*)global_old_heap) / sizeof(void*),
-                   1, INC_FULLPAGE);
-            aging_htop = aging->start;
-            aging_end = aging->start + INC_PAGESIZE;
-        }
-        else {
-            /* There are no free pages.. Either fragmentation is a
-             * problem or we are simply out of memory. Allocation in
-             * the old generation will be done through the free-list
-             * this GC cycle.
-             */
-            aging = NULL;
-            aging_htop = aging_end = NULL;
-        }
-    }
-
-    /* Start with looking at gray objects found in earlier collection
-     * stages.
-     */
-    VERBOSE(DEBUG_HYBRID_GC,("INCGC: Copy-Mark gray\n"));
-    {
-        INC_Object *obj = NULL;
-
-        while (WORK_MORE && !INC_STORAGE_EMPTY(gray)) {
-            Eterm *ptr;
-
-            obj = INC_STORAGE_GET(gray);
-            if ((*obj->this & _TAG_HEADER_MASK) == FUN_SUBTAG) {
-                ptr = obj->this + thing_arityval(*obj->this) + 1;
-            } else {
-                ptr = obj->this;
-            }
-            COPYMARK(ptr,obj->this + obj->size);
-        }
-        /* TODO: Titta p� motsvarande i minor. */
-        if (!WORK_MORE && obj != NULL)
-            INC_STORE(gray,obj->this,obj->size);
-    }
-
-    VERBOSE(DEBUG_HYBRID_GC,("INCGC: Copy-Mark roots\n"));
-    while (WORK_MORE && inc_active_proc)
-    {
-        /* For each process: Scan all areas containing pointers to the
-         * message area. When a process is done here, all it's
-         * message-pointers should be to the old generation.
-         */
-        Rootset rootset;
-        Process *cp = inc_active_proc;
-
-        ASSERT(INC_IS_ACTIVE(cp));
-
-        /* MT: In a multithreaded system the process cp needs to be
-         * locked here.
-         */
-        if (cp == p)
-            rootset.n = setup_rootset(cp, objv, nobj, &rootset);
-        else
-            rootset.n = setup_rootset(cp, cp->arg_reg, cp->arity, &rootset);
-
-        while (WORK_MORE && rootset.n--)
-        {
-            Eterm *ptr = rootset.v[rootset.n];
-            Eterm *end = ptr + rootset.sz[rootset.n];
-
-            while (WORK_MORE && ptr < end) {
-                Eterm val = *ptr;
-                Eterm *obj_ptr = ptr_val(val);
-
-                switch (primary_tag(val)) {
-                    case TAG_PRIMARY_LIST:
-                    {
-                        COPYMARK_CONS(obj_ptr,aging_htop,ptr,aging_end);
-                        break;
-                    }
-
-                    case TAG_PRIMARY_BOXED:
-                    {
-                        COPYMARK_BOXED(obj_ptr,aging_htop,ptr,aging_end);
-                        break;
-                    }
-                }
-                ptr++;
-            }
-        }
-
-#ifdef HIPE
-        /* Atomic phase */
-        VERBOSE(DEBUG_HYBRID_GC,("INCGC: Native stack scan: %T\n",cp->id));
-        aging_htop = ma_fullsweep_nstack(cp,aging_htop,aging_end);
-#endif
-        restore_one_rootset(cp, &rootset);
-
-        /* MT: cp can be unlocked now. But beware!! The message queue
-         * might be updated with new pointers to the fromspace while
-         * we work below. The send operation can not assume that all
-         * active processes will look through their message queue
-         * before deactivating as is the case in non-MT incremental
-         * collection.
-         */
-
-        VERBOSE(DEBUG_HYBRID_GC,("INCGC: Copy-Mark process heap\n"));
-        {
-            Eterm *ptr = cp->scan_top;
-            COPYMARK(ptr,cp->htop);
-            //cp->scan_top = ptr;
-        }
-
-        VERBOSE(DEBUG_HYBRID_GC,("INCGC: Copy-Mark heap fragments\n"));
-        {
-            ErlHeapFragment* bp = MBUF(cp);
-
-            while (WORK_MORE && bp) {
-                Eterm *ptr = bp->mem;
-                Eterm *end;
-
-                if ((ARITH_HEAP(cp) >= bp->mem) &&
-                    (ARITH_HEAP(cp) < bp->mem + bp->size)) {
-                    end = ARITH_HEAP(cp);
-                } else {
-                    end = bp->mem + bp->size;
-                }
-
-                COPYMARK(ptr,end);
-                bp = bp->next;
-            }
-        }
-
-        VERBOSE(DEBUG_HYBRID_GC,("INCGC: Copy-Mark gray stack\n"));
-        {
-            INC_Object *obj = NULL;
-
-            while (WORK_MORE && !INC_STORAGE_EMPTY(gray)) {
-                Eterm *ptr;
-
-                obj = INC_STORAGE_GET(gray);
-                if ((*obj->this & _TAG_HEADER_MASK) == FUN_SUBTAG) {
-                    ptr = obj->this + thing_arityval(*obj->this) + 1;
-                } else {
-                    ptr = obj->this;
-                }
-                COPYMARK(ptr,obj->this + obj->size);
-            }
-            /* TODO: Titta p� motsvarande i minor. */
-            if (!WORK_MORE && obj != NULL)
-                INC_STORE(gray,obj->this,obj->size);
-        }
-
-        if (WORK_MORE) {
-            INC_DEACTIVATE(cp);
-        }
-    }
-
-    VERBOSE(DEBUG_HYBRID_GC,("INCGC: Copy-Mark nursery\n"));
-    {
-        Eterm *ptr = inc_nursery_scn_ptr;
-        COPYMARK(ptr,global_htop);
-        inc_nursery_scn_ptr = ptr;
-    }
-
-    VERBOSE(DEBUG_HYBRID_GC,("INCGC: Copy-Mark gray found in nursery\n"));
-    {
-        INC_Object *obj = NULL;
-
-        while (WORK_MORE && !INC_STORAGE_EMPTY(gray)) {
-            Eterm *ptr;
-
-            obj = INC_STORAGE_GET(gray);
-            if ((*obj->this & _TAG_HEADER_MASK) == FUN_SUBTAG) {
-                ptr = obj->this + thing_arityval(*obj->this) + 1;
-            } else {
-                ptr = obj->this;
-            }
-            COPYMARK(ptr,obj->this + obj->size);
-        }
-        /* TODO: Titta p� motsvarande i minor. */
-        if (!WORK_MORE && obj != NULL)
-            INC_STORE(gray,obj->this,obj->size);
-    }
-
-
-    /**********************************************************************/
-    if (WORK_MORE) {
-        VERBOSE(DEBUG_HYBRID_GC,("INCGC: Sweep phase\n"));
-
-        /* Atomic phase */
-        VERBOSE(DEBUG_HYBRID_GC,("INCGC: Sweep externals in old generation\n"));
-        {
-            ExternalThing** prev = &erts_global_offheap.externals;
-            ExternalThing* ptr = erts_global_offheap.externals;
-
-            while (ptr) {
-                Eterm* ppt = (Eterm *) ptr;
-
-                if ((ptr_within(ppt, global_old_heap, global_old_hend) &&
-                     blackmap[ppt - global_old_heap] == 0) ||
-                    (ptr_within(ppt, inc_fromspc, inc_fromend) &&
-                     !INC_IS_FORWARDED(ppt)))
-                {
-                    erts_deref_node_entry(ptr->node);
-                    *prev = ptr = ptr->next;
-                } else if (ptr_within(ppt, inc_fromspc, inc_fromend)) {
-                    ExternalThing* ro = (ExternalThing*)INC_FORWARD_VALUE(ppt);
-                    *prev = ro;         /* Patch to moved pos */
-                    prev = &ro->next;
-                    ptr = ro->next;
-                } else {
-                    prev = &ptr->next;
-                    ptr = ptr->next;
-                }
-            }
-            ASSERT(*prev == NULL);
-        }
-
-        /* Atomic phase */
-        VERBOSE(DEBUG_HYBRID_GC,("INCGC: Sweep refc bins in old generation\n"));
-        {
-            ProcBin** prev = &erts_global_offheap.mso;
-            ProcBin*  ptr  = erts_global_offheap.mso;
-
-            while (ptr) {
-                Eterm *ppt = (Eterm*)ptr;
-
-                if ((ptr_within(ppt, global_old_heap, global_old_hend) &&
-                     blackmap[ppt - global_old_heap] == 0) ||
-                    (ptr_within(ppt, inc_fromspc, inc_fromend) &&
-                     !INC_IS_FORWARDED(ppt)))
-                {
-                    Binary* bptr;
-                    *prev = ptr->next;
-                    bptr = ptr->val;
-                    if (erts_refc_dectest(&bptr->refc, 0) == 0)
-			erts_bin_free(bptr);
-                    ptr = *prev;
-                } else if (ptr_within(ppt, inc_fromspc, inc_fromend)) {
-                    ProcBin* ro = (ProcBin*)INC_FORWARD_VALUE(ppt);
-                    *prev = ro;         /* Patch to moved pos */
-                    prev = &ro->next;
-                    ptr = ro->next;
-                } else {
-                    prev = &ptr->next;
-                    ptr = ptr->next;
-                }
-            }
-            ASSERT(*prev == NULL);
-        }
-
-        /* TODO: Currently atomic phase - Can not be later of course. */
-        VERBOSE(DEBUG_HYBRID_GC,("INCGC: Sweep old generation\n"));
-        {
-            INC_Page *page = inc_used_mem;
-            INC_Page *prev = NULL;
-            inc_free_list = NULL;
-
-            while (page) {
-                int scavenging = 0;
-                int n = page->start - global_old_heap;
-                int stop = n + INC_PAGESIZE;
-
-                old_gen_sz += INC_PAGESIZE;
-                while (n < stop) {
-                    if (blackmap[n] != 0) {
-                        if (scavenging) {
-                            Eterm *ptr = global_old_heap + n;
-                            scavenging = 0;
-                            if ((ptr - free_start) * sizeof(Eterm) >=
-                                sizeof(INC_MemBlock))
-                            {
-                                INC_MemBlock *new = (INC_MemBlock*)free_start;
-                                new->size = ptr - free_start;
-                                new->prev = NULL;
-                                new->next = inc_free_list;
-                                if (inc_free_list)
-                                    inc_free_list->prev = new;
-                                inc_free_list = new;
-                            }
-                        }
-                        if (blackmap[n] == 255) {
-                            unsigned int size =
-                                *(unsigned int*)(((long)&blackmap[n]+4) & ~3);
-                            live += size;
-                            n += size;
-                        }
-                        else {
-                            live += blackmap[n];
-                            n += blackmap[n];
-                        }
-                    }
-                    else if (!scavenging) {
-                        free_start = global_old_heap + n;
-                        scavenging = 1;
-                        n++;
-                    }
-                    else {
-                        n++;
-                    }
-                }
-
-                if (scavenging) {
-                    if ((global_old_heap + n - free_start) * sizeof(Eterm) >
-                        sizeof(INC_MemBlock))
-                    {
-                        INC_MemBlock *new = (INC_MemBlock*)free_start;
-                        new->size = global_old_heap + n - free_start;
-                        new->prev = NULL;
-                        new->next = inc_free_list;
-                        if (inc_free_list)
-                            inc_free_list->prev = new;
-                        inc_free_list = new;
-                    }
-                    else if (free_start == page->start) {
-                        INC_Page *next = page->next;
-
-                        if (prev)
-                            prev->next = page->next;
-                        else
-                            inc_used_mem = page->next;
-
-                        page->next = inc_bibop;
-                        inc_bibop = page;
-                        inc_used_pages--;
-                        page = next;
-                        continue;
-                    }
-                }
-                prev = page;
-                page = page->next;
-            }
-        }
-    }
-
-    ASSERT(inc_bibop);
-    /*
-      This code is not expected to work right now.
-    if (!inc_bibop) {
-        int i;
-        int new_pages = inc_pages * 2;
-        int size = sizeof(Eterm) * new_pages * INC_FULLPAGE;
-        Eterm *new_heap = erts_alloc(ERTS_ALC_T_MESSAGE_AREA,size);
-        Eterm *new_hend = new_heap + size;
-        Eterm *new_htop;
-        Eterm *last_page_end;
-        INC_Page *new_used_mem;
-        INC_Page *page;
-
-        erts_printf("The last page has been allocated..\n");
-        erts_printf("We need to copy things!\n");
-
-        / * Create new, bigger bag of pages * /
-        for (i = 0; i < new_pages; i++)
-        {
-            INC_Page *this =
-              (INC_Page*)(new_heap + i * INC_FULLPAGE);
-            this->next = (INC_Page*)((Eterm*)this + INC_FULLPAGE);
-        }
-        inc_bibop = (INC_Page*)new_heap;
-        ((INC_Page*)(new_heap + (new_pages - 1) *
-                           INC_FULLPAGE))->next = NULL;
-
-        new_used_mem = inc_bibop;
-        inc_bibop = inc_bibop->next;
-        new_used_mem->next = NULL;
-
-        / * Move stuff from old bag to new * /
-        inc_free_list = NULL;
-        new_htop = new_used_mem->start;
-        last_page_end = new_htop + INC_PAGESIZE;
-        page = inc_used_mem;
-        while (page)
-        {
-            Eterm *ptr = page->start;
-            Eterm *page_end = ptr + INC_PAGESIZE;
-            int n = offsetof(INC_Page,start) / sizeof(void*) +
-              ((Eterm*)page - global_old_heap);
-            while (ptr < page_end)
-            {
-                if (blackmap[n] > 0)
-                {
-                    if (last_page_end - new_htop < blackmap[n])
-                    {
-                        INC_Page *new_page = inc_bibop;
-                        inc_bibop = inc_bibop->next;
-                        new_page->next = new_used_mem;
-                        new_used_mem = new_page;
-                        new_htop = new_page->start;
-                        last_page_end = new_htop + INC_PAGESIZE;
-                    }
-
-                    memcpy(new_htop,ptr,blackmap[n] * sizeof(Eterm));
-                    for (i = 0; i < blackmap[n]; i++)
-                    {
-                        *ptr++ = (Eterm)new_htop++;
-                    }
-                  //new_htop += blackmap[n];
-                  //ptr += blackmap[n];
-                    / *
-                    if (blackmap[n] == 255) Do the right thing...
-                    * /
-                    n += blackmap[n];
-                }
-                else
-                {
-                    n++; ptr++;
-                }
-            }
-            page = page->next;
-        }
-
-        page = inc_used_mem;
-        while (page)
-        {
-            Eterm *ptr = page->start;
-            Eterm *page_end = ptr + INC_PAGESIZE;
-
-            / * TODO: If inc_used_mem is sorted in address order, this
-             * pass can be done at the same time as copying. * /
-            while (ptr < page_end)
-            {
-                if (ptr_within(ptr_val(*ptr),global_old_heap,global_old_hend))
-                {
-                    *ptr = *((Eterm*)ptr_val(*ptr));
-                }
-                ptr++;
-            }
-            page = page->next;
-        }
-
-        printf("Restore rootset after heap move. Roots: %d\r\n",roots_saved);
-        while (roots_saved--)
-        {
-            Eterm *ptr = root_save[roots_saved];
-            *ptr = *((Eterm*)ptr_val(*ptr));
-        }
-
-        erts_free(ERTS_ALC_T_MESSAGE_AREA,(void*)global_old_heap);
-
-        global_old_heap = new_heap;
-        global_old_hend = new_hend;
-        inc_used_mem = new_used_mem;
-        inc_pages = new_pages;
-
-        if ((last_page_end - new_htop) * sizeof(Eterm) >=
-            sizeof(INC_MemBlock))
-        {
-            inc_free_list = (INC_MemBlock*)(new_htop);
-            inc_free_list->size = last_page_end - new_htop;
-            inc_free_list->prev = NULL;
-            inc_free_list->next = NULL;
-        }
-    }
-    */
-
-    /* I vilka l�gen kan vi vilja sl�nga p� en extra sida.. ( < 25% kvar?)
-    if ()
-    {
-        INC_Page *new_page = inc_bibop;
-        INC_MemBlock *new_free =
-          (INC_MemBlock*)new_page->start;
-
-        VERBOSE(DEBUG_HYBRID_GC,("INCGC: Fetching new page\n"));
-        inc_bibop = inc_bibop->next;
-
-        new_page->next = inc_used_mem;
-        if (inc_used_mem)
-            inc_used_mem->prev = new_page;
-        inc_used_mem = new_page;
-
-        // kolla detta med normal sidstorlek! old_gen_sz += INC_PAGESIZE;
-        //BM_SWAP_TIMER(gc,misc1);
-        memset(blackmap +
-               ((void*)new_page - (void*)global_old_heap) / sizeof(void*),
-               0, INC_FULLPAGE);
-        //BM_SWAP_TIMER(misc1,gc);
-
-        new_free->prev = NULL;
-        new_free->next = inc_free_list;
-        new_free->size = INC_PAGESIZE;
-        if (inc_free_list)
-            inc_free_list->prev = new_free;
-        inc_free_list = new_free;
-        //printf("Snatched a new page @ 0x%08x\r\n",(int)new_page);
-        //print_free_list();
-        found = new_free;
-    }
-    */
-
-    VERBOSE(DEBUG_HYBRID_GC,("INCGC: Update copy stack\n"));
-    {
-        Uint i;
-        for (i = 0; i < ma_dst_top; i++) {
-            if (ptr_within(ma_dst_stack[i],inc_fromspc,inc_fromend)) {
-                if (INC_IS_FORWARDED(ma_dst_stack[i]))
-                    ma_dst_stack[i] = INC_FORWARD_VALUE(ma_dst_stack[i]);
-            }
-        }
-    }
-
-    if (WORK_MORE)
-    {
-        int size_left = INC_PAGESIZE - (aging_htop - aging->start);
-
-        if (size_left > sizeof(INC_MemBlock))
-        {
-            ((INC_MemBlock*)aging_htop)->size = size_left;
-            ((INC_MemBlock*)aging_htop)->prev = NULL;
-            ((INC_MemBlock*)aging_htop)->next = inc_free_list;
-            if (inc_free_list)
-                inc_free_list->prev = (INC_MemBlock*)aging_htop;
-            inc_free_list = (INC_MemBlock*)aging_htop;
-        }
-        aging->next = inc_used_mem;
-        inc_used_mem = aging;
-        inc_used_pages++;
-
-        ma_gc_flags &= ~GC_MAJOR;
-        ma_gc_flags &= ~GC_CYCLE;
-
-        VERBOSE(DEBUG_HYBRID_GC,("INCGC: Major collection cycle END\n"));
-    }
-
-    ma_gc_flags &= ~GC_INCLUDE_ALL;
-
-    BM_STOP_TIMER(old_gc);
-#ifdef BM_TIMER
-    major_global_gc_time += old_gc_time;
-    if (old_gc_time > max_global_major_time)
-      max_global_major_time = old_gc_time;
-
-    if ((old_gc_time * 1000) < MAX_PAUSE_TIME)
-        pause_times_old[(int)(old_gc_time * 1000)]++;
-    else
-        pause_times_old[MAX_PAUSE_TIME - 1]++;
-#endif
-    BM_START_TIMER(gc);
-}
-
-
-
-/***************************************************************************
- *                                                                         *
- * Allocation in the old generation. Used in minor colection and when      *
- * copying the rest of a message after a GC.                               *
- *                                                                         *
- ***************************************************************************/
-
-
-Eterm *erts_inc_alloc(int need)
-{
-    INC_MemBlock *this = inc_free_list;
-
-    ASSERT(need < INC_PAGESIZE);
-    while (this && (this->size) < need)
-    {
-        this = this->next;
-    }
-
-    if (!this)
-    {
-        /* If a free block large enough is not found, a new page is
-         * allocated. GC_NEED_MAJOR is set so that the next garbage
-         * collection cycle will be a major one, that is, both
-         * generations will be garbage collected.
-         */
-        INC_Page *new_page = inc_bibop;
-        INC_MemBlock *new_free = (INC_MemBlock*)new_page->start;
-
-        if (new_page)
-        {
-            VERBOSE(DEBUG_HYBRID_GC,
-                    ("INCGC: Allocation grabs a new page\n"));
-            inc_bibop = inc_bibop->next;
-            new_page->next = inc_used_mem;
-            inc_used_mem = new_page;
-            inc_used_pages++;
-
-            new_free->prev = NULL;
-            new_free->next = inc_free_list;
-                new_free->size = INC_PAGESIZE;
-            if (inc_free_list)
-                inc_free_list->prev = new_free;
-            inc_free_list = new_free;
-
-            this = new_free;
-            if (!(ma_gc_flags & GC_MAJOR))
-                ma_gc_flags |= GC_NEED_MAJOR;
-        }
-        else
-        {
-            erl_exit(-1, "inc_alloc ran out of pages!\n");
-        }
-    }
-
-    if (((this->size) - need) * sizeof(Eterm) >= sizeof(INC_MemBlock))
-    {
-        INC_MemBlock *rest = (INC_MemBlock*)((Eterm*)this + need);
-
-        /* The order here IS important! */
-        rest->next = this->next;
-
-        if (rest->next)
-            rest->next->prev = rest;
-
-        rest->prev = this->prev;
-
-        if (rest->prev)
-            rest->prev->next = rest;
-        else
-            inc_free_list = rest;
-
-        rest->size = this->size - need;
-    }
-    else
-    {
-        if (this->prev)
-            this->prev->next = this->next;
-        else
-            inc_free_list = this->next;
-
-        if (this->next)
-          this->next->prev = this->prev;
-    }
-
-    if (ma_gc_flags & GC_MAJOR) {
-        if (need > 254) {
-            blackmap[(Eterm*)this - global_old_heap] = 255;
-            *(int*)((long)(&blackmap[(Eterm*)this - global_old_heap]+4) & ~3) =
-                need;
-        } else
-            blackmap[(Eterm*)this - global_old_heap] = need;
-    }
-    return (Eterm*)this;
-}
-#endif /* INCREMENTAL */
diff --git a/erts/emulator/beam/erl_nmgc.h b/erts/emulator/beam/erl_nmgc.h
deleted file mode 100644
index b207dd37fa..0000000000
--- a/erts/emulator/beam/erl_nmgc.h
+++ /dev/null
@@ -1,364 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-
-#ifndef __ERL_NMGC_H__
-#define __ERL_NMGC_H__
-
-#ifdef INCREMENTAL
-#include <stddef.h>      /* offsetof() */
-#include "erl_process.h"
-
-#define INC_FULLPAGE (INC_PAGESIZE + offsetof(INC_Page,start) / sizeof(void*))
-
-#define BOXED_NEED(PTR,HDR)                                             \
-  (((HDR) & _HEADER_SUBTAG_MASK) == SUB_BINARY_SUBTAG ?                 \
-    header_arity(HDR) + 2 :                                             \
-   ((HDR) & _HEADER_SUBTAG_MASK) == FUN_SUBTAG ?                        \
-    header_arity(HDR) + ((ErlFunThing*)(PTR))->num_free + 2 :           \
-   header_arity(HDR) + 1)
-
-
-#define INC_DECREASE_WORK(n) inc_words_to_go -= (n);
-
-#define INC_COPY_CONS(FROM,TO,PTR)					\
-do {									\
-    TO[0] = FROM[0];							\
-    TO[1] = FROM[1];							\
-    INC_MARK_FORWARD(FROM,TO);						\
-    *(PTR) = make_list(TO);						\
-    INC_DECREASE_WORK(2);						\
-    (TO) += 2;								\
-} while(0)
-
-#define INC_COPY_BOXED(FROM,TO,PTR)					\
-do {									\
-    Sint nelts;								\
-    Eterm hdr = *(FROM);						\
-									\
-    ASSERT(is_header(hdr));						\
-    INC_MARK_FORWARD(FROM,TO);						\
-    *(PTR) = make_boxed(TO);						\
-    *(TO)++ = *(FROM)++;       						\
-    nelts = header_arity(hdr);						\
-    switch ((hdr) & _HEADER_SUBTAG_MASK) {				\
-    case SUB_BINARY_SUBTAG: nelts++; break;				\
-    case FUN_SUBTAG: nelts+=((ErlFunThing*)(FROM-1))->num_free+1; break;\
-    }									\
-    INC_DECREASE_WORK(nelts + 1);					\
-    while (nelts--)							\
-	*(TO)++ = *(FROM)++;						\
-} while(0)
-
-
-/* Things copied to the old generation are not marked in the blackmap. 
- * This is ok since the page they are copied to (aging) is not part of
- * the sweep.
- */
-#define COPYMARK_CONS(FROM,TO,PTR,LIMIT)                                \
-do {							                \
-    if (ptr_within(FROM,inc_fromspc,inc_fromend)) {                     \
-        if (INC_IS_FORWARDED(FROM)) {                                   \
-            *PTR = make_list(INC_FORWARD_VALUE(FROM));                  \
-        } else if (TO + 2 <= LIMIT) {                                   \
-            INC_STORE(gray,TO,2);                                       \
-            INC_COPY_CONS(FROM,TO,PTR);                                 \
-        } else {                                                        \
-            Eterm *hp = erts_inc_alloc(2);                              \
-            INC_STORE(gray,hp,2);                                       \
-            INC_COPY_CONS(FROM,hp,PTR);                                 \
-        }                                                               \
-    } else if (ptr_within(FROM,global_old_heap,global_old_hend) &&     	\
-               (blackmap[FROM - global_old_heap] == 0)) {               \
-        blackmap[FROM - global_old_heap] = 2;                           \
-        INC_DECREASE_WORK(2);                                           \
-        INC_STORE(gray,FROM,2);                                         \
-    }                                                                   \
-} while(0)
-
-#define COPYMARK_BOXED(FROM,TO,PTR,LIMIT)                               \
-do {							                \
-    if (ptr_within(FROM,inc_fromspc,inc_fromend)) {                     \
-        int size = BOXED_NEED(FROM,*FROM);                              \
-        if (INC_IS_FORWARDED(FROM)) {                                   \
-            *PTR = make_boxed(INC_FORWARD_VALUE(FROM));                 \
-        } else if (TO + size <= LIMIT) {                                \
-            INC_STORE(gray,TO,size);                                    \
-            INC_COPY_BOXED(FROM,TO,PTR);                                \
-        } else {                                                        \
-            Eterm *hp = erts_inc_alloc(size);                           \
-            INC_STORE(gray,hp,size);                                    \
-            INC_COPY_BOXED(FROM,hp,PTR);                                \
-        }                                                               \
-    } else if (ptr_within(FROM,global_old_heap,global_old_hend) &&     	\
-               (blackmap[FROM - global_old_heap] == 0)) {               \
-        int size = BOXED_NEED(FROM,*FROM);                              \
-        if (size > 254) {                                               \
-            blackmap[FROM - global_old_heap] = 255;                     \
-            *(int*)((long)(&blackmap[FROM -                             \
-                                     global_old_heap] + 4) & ~3) = size; \
-        } else                                                          \
-            blackmap[FROM - global_old_heap] = size;                    \
-        INC_DECREASE_WORK(size);                                        \
-        INC_STORE(gray,FROM,size);                                      \
-    }                                                                   \
-} while(0)
-
-#define INC_MARK_FORWARD(ptr,dst) fwdptrs[(ptr) - inc_fromspc] = (dst);
-#define INC_IS_FORWARDED(ptr) (fwdptrs[(ptr) - inc_fromspc] != 0)
-#define INC_FORWARD_VALUE(ptr) fwdptrs[(ptr) - inc_fromspc]
-
-/* Note for BM_TIMER: Active timer should always be 'system' when IncAlloc
- * is called!
- */
-#define IncAlloc(p, sz, objv, nobj)                                     \
-    (ASSERT_EXPR((sz) >= 0),                                            \
-     (((inc_alloc_limit - global_htop) <= (sz)) ?                       \
-      erts_incremental_gc((p),(sz),(objv),(nobj)) : 0),                 \
-     ASSERT_EXPR(global_hend - global_htop > (sz)),                     \
-     global_htop += (sz), global_htop - (sz))
-
-
-/************************************************************************
- * INC_STORAGE, a dynamic circular storage for objects (INC_Object).    *
- * Use INC_STORE to add objects to the storage. The storage can then    *
- * be used either as a queue, using INC_STORAGE_GET to retreive         *
- * values, or as a stack, using INC_STORAGE_POP. It is OK to mix calls  *
- * to GET and POP if that is desired.                                   *
- * An iterator can be declared to traverse the storage without removing *
- * any elements, and INC_STORAGE_STEP will then return each element in  *
- * turn, oldest first.                                                  *
- ***********************************************************************/
-
-/* Declare a new storage; must be in the beginning of a block. Give
- * the storage a name that is used in all later calls to the storage. 
- * If this is an external declaration of the storage, pass the keyword
- * external as the first argument, otherwise leave it empty.
- */
-#define INC_STORAGE_DECLARATION(ext,name)                               \
-    ext INC_Storage *name##head;                                        \
-    ext INC_Storage *name##tail;                                        \
-    ext INC_Object *name##free;                                         \
-    ext INC_Object *name##last_free;                                    \
-    ext int name##size;
-
-
-/* Initialize the storage. Note that memory allocation is involved -
- * don't forget to erase the storage when you are done.
- */
-#define INC_STORAGE_INIT(name) do {                                     \
-    name##head = (INC_Storage*)erts_alloc(ERTS_ALC_T_OBJECT_STACK,      \
-                                          sizeof(INC_Storage));         \
-    name##head->next = name##head;                                      \
-    name##head->prev = name##head;                                      \
-    name##tail = name##head;                                            \
-    name##free = name##head->data;                                      \
-    name##last_free = name##free + INC_STORAGE_SIZE - 1;                \
-    name##size = 0;                                                     \
-} while(0)
-
-
-/*
-#define INC_STORAGE_SWAP(s1,s2) do {                                    \
-    INC_Storage *tmphead = s1##head;                                    \
-    INC_Storage *tmptail = s1##tail;                                    \
-    INC_Object *tmpfree = s1##free;                                     \
-    INC_Object *tmplast = s1##last_free;                                \
-    int tmpsize = s1##size;                                             \
-    s1##head = s2##head;                                                \
-    s1##tail = s2##tail;                                                \
-    s1##free = s2##free;                                                \
-    s1##last_free = s2##last_free;                                      \
-    s1##size = s2##size;                                                \
-    s2##head = tmphead;                                                 \
-    s2##tail = tmptail;                                                 \
-    s2##free = tmpfree;                                                 \
-    s2##last_free = tmplast;                                            \
-    s2##size = tmpsize;                                                 \
-} while(0)
-*/
-
-
-/* Return and remove the youngest element - treat the storage as a
- * stack. Always check that there are elements in the queue before
- * using INC_STORAGE_POP!
- */
-#define INC_STORAGE_POP(name) (ASSERT_EXPR(name##size != 0),            \
-    name##size--,                                                       \
-    (--name##free != name##head->data - 1) ?                            \
-    name##free : (name##head = name##head->prev,                        \
-                  name##free = name##head->data + INC_STORAGE_SIZE - 1))
-
-
-/* Return and remove the oldest element - treat the storage as a
- * queue. Always check that there are elements in the queue before
- * using INC_STORAGE_GET!
- */
-#define INC_STORAGE_GET(name) (ASSERT_EXPR(name##size != 0),            \
-    name##size--,                                                       \
-    (++name##last_free != name##tail->data + INC_STORAGE_SIZE) ?        \
-     name##last_free : (name##tail = name##tail->next,                  \
-                        name##last_free = name##tail->data))
-
-
-/* Advance the head to the next free location. If the storage is full,
- * a new storage is allocated and linked into the list.
- */
-#define INC_STORAGE_NEXT(name) do {                                     \
-    if (name##free == name##last_free) {                                \
-        name##tail = (INC_Storage*)erts_alloc(ERTS_ALC_T_OBJECT_STACK,  \
-                                              sizeof(INC_Storage));     \
-        memcpy(name##tail->data,name##head->data,                       \
-               INC_STORAGE_SIZE * sizeof(INC_Object));                  \
-        name##tail->next = name##head->next;                            \
-        name##head->next = name##tail;                                  \
-        name##tail->prev = name##tail->next->prev;                      \
-        name##tail->next->prev = name##tail;                            \
-        name##last_free = ((void*)name##tail +                          \
-                         ((void*)name##last_free - (void*)name##head)); \
-    }                                                                   \
-    name##free++;                                                       \
-    name##size++;                                                       \
-    if (name##free == name##head->data + INC_STORAGE_SIZE) {            \
-        name##head = name##head->next;                                  \
-        name##free = name##head->data;                                  \
-    }                                                                   \
-} while(0)
-
-
-/* The head of this storage is the next free location. This is where
- * the next element will be stored.
- */
-#define INC_STORAGE_HEAD(name) (name##free)
-
-
-/* Return the top - the youngest element in the storage. */
-/* #define INC_STORAGE_TOP(name) (name##free - 1 with some magic..) */
-
-
-/* True if the storage is empty, false otherwise */
-#define INC_STORAGE_EMPTY(name) (name##size == 0)
-
-
-/* Store a new element in the head of the storage and advance the head
- * to the next free location.
- */
-#define INC_STORE(name,ptr,sz) do {                                      \
-    INC_STORAGE_HEAD(name)->this = ptr;                                  \
-    INC_STORAGE_HEAD(name)->size = sz;                                   \
-    INC_STORAGE_NEXT(name);                                              \
-} while(0)
-
-
-/* An iterator. Use it together with INC_STORAGE_STEP to browse throuh
- * the storage. Please note that it is not possible to remove an entry
- * in the middle of the storage, use GET or POP to remove enties.
- */
-#define INC_STORAGE_ITERATOR(name)                                      \
-    INC_Storage *name##iterator_head = name##tail;                      \
-    INC_Object *name##iterator_current = name##last_free;               \
-    int name##iterator_left = name##size;
-
-
-/* Return the next element in the storage (sorted by age, oldest
- * first) or NULL if the storage is empty or the last element has been
- * returned already.
- */
-#define INC_STORAGE_STEP(name) (name##iterator_left == 0 ? NULL :       \
-    (name##iterator_left--,                                             \
-     (++name##iterator_current != name##iterator_head->data +           \
-      INC_STORAGE_SIZE) ? name##iterator_current :                      \
-     (name##iterator_head = name##iterator_head->next,                  \
-      name##iterator_current = name##iterator_head->data)))
-
-
-/* Erase the storage. */
-#define INC_STORAGE_ERASE(name)do {                             \
-    name##head->prev->next = NULL;                              \
-    while (name##head != NULL) {                                \
-        name##tail = name##head;                                \
-        name##head = name##head->next;                          \
-        erts_free(ERTS_ALC_T_OBJECT_STACK,(void*)name##tail);   \
-    }                                                           \
-    name##tail = NULL;                                          \
-    name##free = NULL;                                          \
-    name##last_free = NULL;                                     \
-    name##size = 0;                                             \
-} while(0)
-
-/*
- * Structures used by the non-moving memory manager
- */
-
-typedef struct
-{
-  Eterm *this;
-  unsigned long size;
-} INC_Object;
-
-typedef struct inc_storage {
-  struct inc_storage *next;
-  struct inc_storage *prev;
-  INC_Object data[INC_STORAGE_SIZE];
-} INC_Storage;
-
-typedef struct inc_mem_block
-{
-  unsigned long size;
-  struct inc_mem_block *prev;
-  struct inc_mem_block *next;
-} INC_MemBlock;
-
-typedef struct inc_page
-{
-  struct inc_page *next;
-  Eterm start[1]; /* Has to be last in struct, this is where the data start */
-} INC_Page;
-
-
-/*
- * Heap pointers for the non-moving memory area.
- */
-extern INC_Page *inc_used_mem;
-extern INC_MemBlock *inc_free_list;
-extern unsigned char *blackmap;
-
-extern Eterm **fwdptrs;
-extern Eterm *inc_fromspc;
-extern Eterm *inc_fromend;
-extern Process *inc_active_proc;
-extern Process *inc_active_last;
-extern Eterm *inc_alloc_limit;
-extern int   inc_words_to_go;
-
-INC_STORAGE_DECLARATION(extern,gray);
-INC_STORAGE_DECLARATION(extern,root);
-
-void erts_init_incgc(void);
-void erts_cleanup_incgc(void);
-void erts_incremental_gc(Process *p, int sz, Eterm* objv, int nobj);
-Eterm *erts_inc_alloc(int need);
-
-#else
-#  define INC_STORE(lst,ptr,sz)
-#  define INC_MARK_FORWARD(ptr)
-#  define INC_IS_FORWARDED(ptr)
-#  define INC_FORWARD_VALUE(ptr)
-#endif /* INCREMENTAL */
-
-#endif /* _ERL_NMGC_H_ */
diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h
index 87dbfc2a04..329a2204cc 100644
--- a/erts/emulator/beam/erl_node_container_utils.h
+++ b/erts/emulator/beam/erl_node_container_utils.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -176,7 +176,7 @@ extern int erts_use_r9_pids_ports;
  * 32-bit CPU.
  */
 
-#define ERTS_MAX_PROCESSES ((1L << 27)-1)
+#define ERTS_MAX_PROCESSES ((SWORD_CONSTANT(1) << 27)-1)
 #if (ERTS_MAX_PROCESSES > MAX_SMALL)
 # error "The maximum number of processes must fit in a SMALL."
 #endif
@@ -251,23 +251,36 @@ extern int erts_use_r9_pids_ports;
  * Refs                                                                    *
 \*                                                                         */
 
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 
 #define internal_ref_no_of_numbers(x)					\
   (internal_ref_data((x))[0])
+#define internal_thing_ref_no_of_numbers(thing)			        \
+  (internal_thing_ref_data(thing)[0])
 #define internal_ref_numbers(x)						\
   (&internal_ref_data((x))[1])
+#define internal_thing_ref_numbers(thing)				\
+  (&internal_thing_ref_data(thing)[1])
 #define external_ref_no_of_numbers(x)					\
   (external_ref_data((x))[0])
+#define external_thing_ref_no_of_numbers(thing)			        \
+  (external_thing_ref_data(thing)[0])
 #define external_ref_numbers(x)						\
   (&external_ref_data((x))[1])
+#define external_thing_ref_numbers(thing)				\
+  (&external_thing_ref_data(thing)[1])
+
 
 #else
 
 #define internal_ref_no_of_numbers(x)	(internal_ref_data_words((x)))
+#define internal_thing_ref_no_of_numbers(t) (internal_thing_ref_data_words(t))
 #define internal_ref_numbers(x)		(internal_ref_data((x)))
+#define internal_thing_ref_numbers(t)   (internal_thing_ref_data(t))
 #define external_ref_no_of_numbers(x)	(external_ref_data_words((x)))
+#define external_thing_ref_no_of_numbers(t) (external_thing_ref_data_words((t)))
 #define external_ref_numbers(x)		(external_ref_data((x)))
+#define external_thing_ref_numbers(t)   (external_thing_ref_data((t)))
 
 #endif
 
@@ -311,6 +324,8 @@ extern int erts_use_r9_pids_ports;
 					 : external_ref_channel_no((x)))
 #define is_ref(x)			(is_internal_ref((x))		\
 					 || is_external_ref((x)))
+#define is_ref_rel(x,Base)		(is_internal_ref_rel((x),Base)	\
+					 || is_external_ref_rel((x),Base))
 #define is_not_ref(x)			(!is_ref(x))
 
 #endif
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 42b28d987c..c7fd379367 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -26,6 +26,8 @@
 #include "dist.h"
 #include "big.h"
 #include "error.h"
+#include "erl_thr_progress.h"
+#include "dtrace-wrapper.h"
 
 Hash erts_dist_table;
 Hash erts_node_table;
@@ -41,6 +43,8 @@ Sint erts_no_of_not_connected_dist_entries;
 
 DistEntry *erts_this_dist_entry;
 ErlNode *erts_this_node;
+char erts_this_node_sysname_BUFFER[256],
+    *erts_this_node_sysname = "uninitialized yet";
 
 static Uint node_entries;
 static Uint dist_entries;
@@ -80,6 +84,8 @@ dist_table_alloc(void *dep_tmpl)
     Eterm chnl_nr;
     Eterm sysname;
     DistEntry *dep;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
 
     if(((DistEntry *) dep_tmpl) == erts_this_dist_entry)
 	return dep_tmpl;
@@ -92,7 +98,7 @@ dist_table_alloc(void *dep_tmpl)
 
     dep->prev				= NULL;
     erts_refc_init(&dep->refc, -1);
-    erts_smp_rwmtx_init_x(&dep->rwmtx, "dist_entry", chnl_nr);
+    erts_smp_rwmtx_init_opt_x(&dep->rwmtx, &rwmtx_opt, "dist_entry", chnl_nr);
     dep->sysname			= sysname;
     dep->cid				= NIL;
     dep->connection_id			= 0;
@@ -105,7 +111,7 @@ dist_table_alloc(void *dep_tmpl)
     dep->nlinks				= NULL;
     dep->monitors			= NULL;
 
-    erts_smp_spinlock_init_x(&dep->qlock, "dist_entry_out_queue", chnl_nr);
+    erts_smp_mtx_init_x(&dep->qlock, "dist_entry_out_queue", chnl_nr);
     dep->qflgs				= 0;
     dep->qsize				= 0;
     dep->out_queue.first		= NULL;
@@ -116,7 +122,7 @@ dist_table_alloc(void *dep_tmpl)
     dep->finalized_out_queue.first	= NULL;
     dep->finalized_out_queue.last	= NULL;
 
-    erts_smp_atomic_init(&dep->dist_cmd_scheduled, 0);
+    erts_smp_atomic_init_nob(&dep->dist_cmd_scheduled, 0);
     erts_port_task_handle_init(&dep->dist_cmd);
     dep->send				= NULL;
     dep->cache				= NULL;
@@ -170,7 +176,7 @@ dist_table_free(void *vdep)
     ASSERT(!dep->cache);
     erts_smp_rwmtx_destroy(&dep->rwmtx);
     erts_smp_mtx_destroy(&dep->lnk_mtx);
-    erts_smp_spinlock_destroy(&dep->qlock);
+    erts_smp_mtx_destroy(&dep->qlock);
 
 #ifdef DEBUG
     sys_memset(vdep, 0x77, sizeof(DistEntry));
@@ -233,7 +239,7 @@ erts_sysname_to_connected_dist_entry(Eterm sysname)
     erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
     res_dep = (DistEntry *) hash_get(&erts_dist_table, (void *) &de);
     if (res_dep) {
-	long refc = erts_refc_inctest(&res_dep->refc, 1);
+	erts_aint_t refc = erts_refc_inctest(&res_dep->refc, 1);
 	if (refc < 2) /* Pending delete */
 	    erts_refc_inc(&res_dep->refc, 1);
     }
@@ -255,7 +261,7 @@ DistEntry *erts_find_or_insert_dist_entry(Eterm sysname)
 {
     DistEntry *res;
     DistEntry de;
-    long refc;
+    erts_aint_t refc;
     res = erts_find_dist_entry(sysname);
     if (res)
 	return res;
@@ -277,7 +283,7 @@ DistEntry *erts_find_dist_entry(Eterm sysname)
     erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx);
     res = hash_get(&erts_dist_table, (void *) &de);
     if (res) {
-	long refc = erts_refc_inctest(&res->refc, 1);
+	erts_aint_t refc = erts_refc_inctest(&res->refc, 1);
 	if (refc < 2) /* Pending delete */
 	    erts_refc_inc(&res->refc, 1);
     }
@@ -580,11 +586,23 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint creation)
     ErlNode ne;
     ne.sysname = sysname;
     ne.creation = creation;
+
+    erts_smp_rwmtx_rlock(&erts_node_table_rwmtx);
+    res = hash_get(&erts_node_table, (void *) &ne);
+    if (res && res != erts_this_node) {
+	erts_aint_t refc = erts_refc_inctest(&res->refc, 0);
+	if (refc < 2) /* New or pending delete */
+	    erts_refc_inc(&res->refc, 1);
+    }
+    erts_smp_rwmtx_runlock(&erts_node_table_rwmtx);
+    if (res)
+	return res;
+
     erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx);
     res = hash_put(&erts_node_table, (void *) &ne);
     ASSERT(res);
     if (res != erts_this_node) {
-	long refc = erts_refc_inctest(&res->refc, 0);
+	erts_aint_t refc = erts_refc_inctest(&res->refc, 0);
 	if (refc < 2) /* New or pending delete */
 	    erts_refc_inc(&res->refc, 1);
     }
@@ -687,6 +705,9 @@ erts_set_this_node(Eterm sysname, Uint creation)
     (void) hash_erase(&erts_node_table, (void *) erts_this_node);
     erts_this_node->sysname = sysname;
     erts_this_node->creation = creation;
+    erts_this_node_sysname = erts_this_node_sysname_BUFFER;
+    erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname),
+                  "%T", sysname);
     (void) hash_put(&erts_node_table, (void *) erts_this_node);
 
     erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx);
@@ -696,8 +717,12 @@ erts_set_this_node(Eterm sysname, Uint creation)
 
 void erts_init_node_tables(void)
 {
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     HashFunctions f;
 
+    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+
     f.hash  = (H_FUN)			dist_table_hash;
     f.cmp   = (HCMP_FUN)		dist_table_cmp;
     f.alloc = (HALLOC_FUN)		dist_table_alloc;
@@ -719,9 +744,10 @@ void erts_init_node_tables(void)
     erts_this_dist_entry->prev				= NULL;
     erts_refc_init(&erts_this_dist_entry->refc, 1); /* erts_this_node */
 
-    erts_smp_rwmtx_init_x(&erts_this_dist_entry->rwmtx,
-			  "dist_entry",
-			  make_small(ERST_INTERNAL_CHANNEL_NO));
+    erts_smp_rwmtx_init_opt_x(&erts_this_dist_entry->rwmtx,
+			      &rwmtx_opt,
+			      "dist_entry",
+			      make_small(ERST_INTERNAL_CHANNEL_NO));
     erts_this_dist_entry->sysname			= am_Noname;
     erts_this_dist_entry->cid				= NIL;
     erts_this_dist_entry->connection_id			= 0;
@@ -736,9 +762,9 @@ void erts_init_node_tables(void)
     erts_this_dist_entry->nlinks			= NULL;
     erts_this_dist_entry->monitors			= NULL;
 
-    erts_smp_spinlock_init_x(&erts_this_dist_entry->qlock,
-			     "dist_entry_out_queue",
-			     make_small(ERST_INTERNAL_CHANNEL_NO));
+    erts_smp_mtx_init_x(&erts_this_dist_entry->qlock,
+			"dist_entry_out_queue",
+			make_small(ERST_INTERNAL_CHANNEL_NO));
     erts_this_dist_entry->qflgs				= 0;
     erts_this_dist_entry->qsize				= 0;
     erts_this_dist_entry->out_queue.first		= NULL;
@@ -748,7 +774,7 @@ void erts_init_node_tables(void)
 
     erts_this_dist_entry->finalized_out_queue.first	= NULL;
     erts_this_dist_entry->finalized_out_queue.last	= NULL;
-    erts_smp_atomic_init(&erts_this_dist_entry->dist_cmd_scheduled, 0);
+    erts_smp_atomic_init_nob(&erts_this_dist_entry->dist_cmd_scheduled, 0);
     erts_port_task_handle_init(&erts_this_dist_entry->dist_cmd);
     erts_this_dist_entry->send				= NULL;
     erts_this_dist_entry->cache				= NULL;
@@ -769,11 +795,14 @@ void erts_init_node_tables(void)
     erts_this_node->sysname			= am_Noname;
     erts_this_node->creation			= 0;
     erts_this_node->dist_entry			= erts_this_dist_entry;
+    erts_this_node_sysname = erts_this_node_sysname_BUFFER;
+    erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname),
+                  "%T", erts_this_node->sysname);
 
     (void) hash_put(&erts_node_table, (void *) erts_this_node);
 
-    erts_smp_rwmtx_init(&erts_node_table_rwmtx, "node_table");
-    erts_smp_rwmtx_init(&erts_dist_table_rwmtx, "dist_table");
+    erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table");
+    erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table");
 
     references_atoms_need_init = 1;
 }
@@ -820,9 +849,6 @@ static Eterm AM_dist_references;
 static Eterm AM_node_references;
 static Eterm AM_system;
 static Eterm AM_timer;
-#ifdef HYBRID
-static Eterm AM_processes;
-#endif
 
 static void setup_reference_table(void);
 static Eterm reference_table_term(Uint **hpp, Uint *szp);
@@ -888,7 +914,7 @@ erts_get_node_and_dist_references(struct process *proc)
 #endif
 
     erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
-    erts_smp_block_system(0);
+    erts_smp_thr_progress_block();
     /* No need to lock any thing since we are alone... */
 
     if (references_atoms_need_init) {
@@ -907,9 +933,6 @@ erts_get_node_and_dist_references(struct process *proc)
 	INIT_AM(node_references);
 	INIT_AM(timer);
 	INIT_AM(system);
-#ifdef HYBRID
-	INIT_AM(processes);
-#endif
 	references_atoms_need_init = 0;
     }
 
@@ -932,7 +955,7 @@ erts_get_node_and_dist_references(struct process *proc)
 
     delete_reference_table();
 
-    erts_smp_release_system();
+    erts_smp_thr_progress_unblock();
     erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
     return res;
 }
@@ -1087,49 +1110,62 @@ insert_offheap2(ErlOffHeap *oh, void *arg)
 static void
 insert_offheap(ErlOffHeap *oh, int type, Eterm id)
 {
-    if(oh->externals) {
-	ExternalThing *etp = oh->externals;
-	while (etp) {
-	    insert_node(etp->node, type, id);
-	    etp = etp->next;
-	}
-    }
+    union erl_off_heap_ptr u;
+    struct insert_offheap2_arg a;
+    a.type = BIN_REF;
 
-    if(oh->mso) {
-	ProcBin *pb;
-	struct insert_offheap2_arg a;
-	a.type = BIN_REF;
-	for(pb = oh->mso; pb; pb = pb->next) {
-	    if(IsMatchProgBinary(pb->val)) {
+    for (u.hdr = oh->first; u.hdr; u.hdr = u.hdr->next) {
+	switch (thing_subtag(u.hdr->thing_word)) {
+	case REFC_BINARY_SUBTAG:
+	    if(IsMatchProgBinary(u.pb->val)) {
 		InsertedBin *ib;
 		int insert_bin = 1;
 		for (ib = inserted_bins; ib; ib = ib->next)
-		    if(ib->bin_val == pb->val) {
+		    if(ib->bin_val == u.pb->val) {
 			insert_bin = 0;
 			break;
 		    }
 		if (insert_bin) {
-		    Uint id_heap[BIG_UINT_HEAP_SIZE];
+#if HALFWORD_HEAP
+		    UWord val = (UWord) u.pb->val;
+		    DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE*2); /* extra place allocated */
+#else
+		    DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE);
+#endif
 		    Uint *hp = &id_heap[0];
 		    InsertedBin *nib;
-		    a.id = erts_bld_uint(&hp, NULL, (Uint) pb->val);
-		    erts_match_prog_foreach_offheap(pb->val,
+#if HALFWORD_HEAP
+		    int actual_need = BIG_UWORD_HEAP_SIZE(val);
+		    ASSERT(actual_need <= (BIG_UINT_HEAP_SIZE*2));
+		    UseTmpHeapNoproc(actual_need);
+		    a.id = erts_bld_uword(&hp, NULL, (UWord) val);
+#else
+		    UseTmpHeapNoproc(BIG_UINT_HEAP_SIZE);
+		    a.id = erts_bld_uint(&hp, NULL, (Uint) u.pb->val);
+#endif
+		    erts_match_prog_foreach_offheap(u.pb->val,
 						    insert_offheap2,
 						    (void *) &a);
 		    nib = erts_alloc(ERTS_ALC_T_NC_TMP, sizeof(InsertedBin));
-		    nib->bin_val = pb->val;
+		    nib->bin_val = u.pb->val;
 		    nib->next = inserted_bins;
 		    inserted_bins = nib;
+#if HALFWORD_HEAP
+		    UnUseTmpHeapNoproc(actual_need);
+#else
+		    UnUseTmpHeapNoproc(BIG_UINT_HEAP_SIZE);
+#endif
 		}
-	    }
+	    }		
+	    break;
+	case FUN_SUBTAG:
+	    break; /* No need to */
+	default:
+	    ASSERT(is_external_header(u.hdr->thing_word));
+	    insert_node(u.ext->node, type, id);
+	    break;
 	}
     }
-
-#if 0
-    if(oh->funs) {
-	/* No need to */
-    }
-#endif
 }
 
 static void doit_insert_monitor(ErtsMonitor *monitor, void *p)
@@ -1190,12 +1226,15 @@ static void
 insert_bif_timer(Eterm receiver, Eterm msg, ErlHeapFragment *bp, void *arg)
 {
     if (bp) {
-	Eterm heap[3];
+	DeclareTmpHeapNoproc(heap,3);
+
+	UseTmpHeapNoproc(3);
 	insert_offheap(&bp->off_heap,
 		       TIMER_REF,
 		       (is_internal_pid(receiver)
 			? receiver
 			: TUPLE2(&heap[0], AM_process, receiver)));
+	UnUseTmpHeapNoproc(3);
     }
 }
 
@@ -1230,7 +1269,7 @@ setup_reference_table(void)
     DistEntry *dep;
     HashInfo hi;
     int i;
-    Eterm heap[3];
+    DeclareTmpHeapNoproc(heap,3);
 
     inserted_bins = NULL;
 
@@ -1251,21 +1290,18 @@ setup_reference_table(void)
     /* Go through the hole system, and build a table of all references
        to ErlNode and DistEntry structures */
 
+    UseTmpHeapNoproc(3);
     insert_node(erts_this_node,
 		SYSTEM_REF,
 		TUPLE2(&heap[0], AM_system, am_undefined));
 
-#ifdef HYBRID
-    /* Insert Heap */
-    insert_offheap(&erts_global_offheap,
-		   HEAP_REF,
-		   TUPLE2(&heap[0], AM_processes, am_undefined));
-#endif
+    UnUseTmpHeapNoproc(3);
 
     /* Insert all processes */
     for (i = 0; i < erts_max_processes; i++)
 	if (process_tab[i]) {
 	    ErlMessage *msg;
+
 	    /* Insert Heap */
 	    insert_offheap(&(process_tab[i]->off_heap),
 			   HEAP_REF,
@@ -1352,21 +1388,22 @@ setup_reference_table(void)
 
     { /* Add binaries stored elsewhere ... */
 	ErlOffHeap oh;
-	ProcBin pb[2] = {{0},{0}};
-	ProcBin *mso = NULL;
+	ProcBin pb[2];
 	int i = 0;
 	Binary *default_match_spec;
 	Binary *default_meta_match_spec;
 
-	/* Only the ProcBin members val and next will be inspected
+	oh.first = NULL;
+	/* Only the ProcBin members thing_word, val and next will be inspected
 	   (by insert_offheap()) */
 #undef  ADD_BINARY
-#define ADD_BINARY(Bin)					\
-	if ((Bin)) {					\
-	    pb[i].val = (Bin);				\
-	    pb[i].next = mso;				\
-	    mso = &pb[i];				\
-	    i++;					\
+#define ADD_BINARY(Bin)				 	     \
+	if ((Bin)) {					     \
+	    pb[i].thing_word = REFC_BINARY_SUBTAG;           \
+	    pb[i].val = (Bin);				     \
+	    pb[i].next = oh.first;		             \
+	    oh.first = (struct erl_off_heap_header*) &pb[i]; \
+	    i++;				             \
 	}
 
 	erts_get_default_trace_pattern(NULL,
@@ -1378,11 +1415,6 @@ setup_reference_table(void)
 	ADD_BINARY(default_match_spec);
 	ADD_BINARY(default_meta_match_spec);
 
-	oh.mso = mso;
-	oh.externals = NULL;
-#ifndef HYBRID /* FIND ME! */
-	oh.funs = NULL;
-#endif
 	insert_offheap(&oh, BIN_REF, AM_match_spec);
 #undef  ADD_BINARY
     }
diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h
index c48dac6219..4a015bdef9 100644
--- a/erts/emulator/beam/erl_node_tables.h
+++ b/erts/emulator/beam/erl_node_tables.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -61,7 +61,7 @@
 #define ERTS_DE_QFLGS_ALL			(ERTS_DE_QFLG_BUSY \
 						 | ERTS_DE_QFLG_EXIT)
 
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713f713f713UL)
 #else
 #define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713)
@@ -131,7 +131,7 @@ typedef struct dist_entry_ {
     ErtsLink *nlinks;           /* Link tree with subtrees */
     ErtsMonitor *monitors;      /* Monitor tree */
 
-    erts_smp_spinlock_t qlock;  /* Protects qflgs and out_queue */
+    erts_smp_mtx_t qlock;       /* Protects qflgs and out_queue */
     Uint32 qflgs;
     Sint qsize;
     ErtsDistOutputQueue out_queue;
@@ -169,6 +169,7 @@ extern Sint erts_no_of_not_connected_dist_entries;
 
 extern DistEntry *erts_this_dist_entry;
 extern ErlNode *erts_this_node;
+extern char *erts_this_node_sysname; /* must match erl_node_tables.c */
 
 DistEntry *erts_channel_no_to_dist_entry(Uint);
 DistEntry *erts_sysname_to_connected_dist_entry(Eterm);
diff --git a/erts/emulator/beam/erl_obsolete.c b/erts/emulator/beam/erl_obsolete.c
deleted file mode 100644
index 9c5a7c7ff9..0000000000
--- a/erts/emulator/beam/erl_obsolete.c
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-
-#include "sys.h"
-#include "erl_driver.h"
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
- *                                                                           *
- * ------------------------- OBSOLETE! DO NOT USE! ------------------------- *
- *                                                                           *
-\*                                                                           */
-
-/* cut from ../obsolete/driver.h (since it doesn't mix well with other
- * headers from the emulator).
- */
-#ifdef __WIN32__
-#ifdef CONST
-#  undef CONST
-#endif
-#endif
-
-#if ((defined(__STDC__) || defined(SABER)) && !defined(NO_PROTOTYPE)) || defined(__cplusplus) || defined(USE_PROTOTYPE)
-#   define _USING_PROTOTYPES_ 1
-#   define _ANSI_ARGS_(x)	x
-#   define CONST const
-#else
-#   define _ANSI_ARGS_(x)	()
-#   define CONST
-#endif
-
-typedef void* erl_mutex_t;
-typedef void* erl_cond_t;
-typedef void* erl_thread_t;
-
-EXTERN erl_mutex_t erts_mutex_create _ANSI_ARGS_((void));
-EXTERN int erts_mutex_destroy _ANSI_ARGS_((erl_mutex_t));
-EXTERN int erts_mutex_lock _ANSI_ARGS_((erl_mutex_t));
-EXTERN int erts_mutex_unlock _ANSI_ARGS_((erl_mutex_t));
-
-EXTERN erl_cond_t erts_cond_create _ANSI_ARGS_((void));
-EXTERN int erts_cond_destroy _ANSI_ARGS_((erl_cond_t));
-EXTERN int erts_cond_signal _ANSI_ARGS_((erl_cond_t));
-EXTERN int erts_cond_broadcast _ANSI_ARGS_((erl_cond_t));
-EXTERN int erts_cond_wait _ANSI_ARGS_((erl_cond_t, erl_mutex_t));
-EXTERN int erts_cond_timedwait _ANSI_ARGS_((erl_cond_t, erl_mutex_t, long));
-
-EXTERN int erts_thread_create _ANSI_ARGS_((erl_thread_t*,
-					 void* (*func)(void*),
-					 void* arg,
-					 int detached));
-EXTERN erl_thread_t erts_thread_self _ANSI_ARGS_((void));
-EXTERN void erts_thread_exit _ANSI_ARGS_((void*));
-EXTERN int  erts_thread_join _ANSI_ARGS_((erl_thread_t, void**));
-EXTERN int  erts_thread_kill _ANSI_ARGS_((erl_thread_t));
-
-/*
- * These functions implement the thread interface in ../obsolete/driver.h.
- * Do *not* use this interface! Within the emulator, use the erl_threads.h,
- * erl_smp.h, or ethread.h interface. From a driver use the thread interface
- * in erl_driver.h.
- */
-
-erl_mutex_t
-erts_mutex_create(void)
-{
-    return (erl_mutex_t) erl_drv_mutex_create(NULL);
-}
-
-int
-erts_mutex_destroy(erl_mutex_t mtx)
-{
-    erl_drv_mutex_destroy((ErlDrvMutex *) mtx);
-    return 0;
-}
-
-int
-erts_mutex_lock(erl_mutex_t mtx)
-{
-    erl_drv_mutex_lock((ErlDrvMutex *) mtx);
-    return 0;
-}
-
-int
-erts_mutex_unlock(erl_mutex_t mtx)
-{
-    erl_drv_mutex_unlock((ErlDrvMutex *) mtx);
-    return 0;
-}
-
-erl_cond_t
-erts_cond_create(void)
-{
-    return (erl_cond_t) erl_drv_cond_create(NULL);
-}
-
-int
-erts_cond_destroy(erl_cond_t cnd)
-{
-    erl_drv_cond_destroy((ErlDrvCond *) cnd);
-    return 0;
-}
-
-
-int
-erts_cond_signal(erl_cond_t cnd)
-{
-    erl_drv_cond_signal((ErlDrvCond *) cnd);
-    return 0;
-}
-
-int
-erts_cond_broadcast(erl_cond_t cnd)
-{
-    erl_drv_cond_broadcast((ErlDrvCond *) cnd);
-    return 0;
-}
-
-
-int
-erts_cond_wait(erl_cond_t cnd, erl_mutex_t mtx)
-{
-    erl_drv_cond_wait((ErlDrvCond *) cnd, (ErlDrvMutex *) mtx);
-    return 0;
-}
-
-int
-erts_cond_timedwait(erl_cond_t cnd, erl_mutex_t mtx, long ms)
-{
-    return ENOTSUP;
-}
-
-int
-erts_thread_create(erl_thread_t *tid,
-		   void* (*func)(void*),
-		   void* arg,
-		   int detached)
-{
-    if (detached)
-	return ENOTSUP;
-    return erl_drv_thread_create(NULL, (ErlDrvTid *) tid, func, arg, NULL);
-}
-
-erl_thread_t
-erts_thread_self(void)
-{
-    return (erl_thread_t) erl_drv_thread_self();
-}
-
-void
-erts_thread_exit(void *res)
-{
-    erl_drv_thread_exit(res);
-}
-
-int
-erts_thread_join(erl_thread_t tid, void **respp)
-{
-    return erl_drv_thread_join((ErlDrvTid) tid, respp);
-}
-
-int
-erts_thread_kill(erl_thread_t tid)
-{
-    return ENOTSUP;
-}
-
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 0b6bb0d8e9..0f1a0d441a 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2006-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -32,6 +32,7 @@
 #include "global.h"
 #include "erl_port_task.h"
 #include "dist.h"
+#include "dtrace-wrapper.h"
 
 #if defined(DEBUG) && 0
 #define HARD_DEBUG
@@ -61,6 +62,20 @@ do {					\
     (P)->sched.next = NULL;		\
 } while (0)
 
+#ifdef USE_VM_PROBES
+#define DTRACE_DRIVER(PROBE_NAME, PP)                              \
+    if (DTRACE_ENABLED(driver_ready_input)) {                      \
+        DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE);         \
+        DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE);            \
+                                                                   \
+        dtrace_pid_str(PP->connected, process_str);                \
+        dtrace_port_str(PP, port_str);                             \
+        DTRACE3(PROBE_NAME, process_str, port_str, PP->name);      \
+    }
+#else
+#define  DTRACE_DRIVER(PROBE_NAME, PP) do {} while(0)
+#endif
+
 erts_smp_atomic_t erts_port_task_outstanding_io_tasks;
 
 struct ErtsPortTaskQueue_ {
@@ -121,7 +136,7 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(port_taskq,
 static ERTS_INLINE ErtsPortTask *
 handle2task(ErtsPortTaskHandle *pthp)
 {
-    return (ErtsPortTask *) erts_smp_atomic_read(pthp);
+    return (ErtsPortTask *) erts_smp_atomic_read_nob(pthp);
 }
 
 static ERTS_INLINE void
@@ -129,7 +144,7 @@ reset_handle(ErtsPortTask *ptp)
 {
     if (ptp->handle) {
 	ASSERT(ptp == handle2task(ptp->handle));
-	erts_smp_atomic_set(ptp->handle, (long) NULL);
+	erts_smp_atomic_set_nob(ptp->handle, (erts_aint_t) NULL);
     }
 }
 
@@ -138,7 +153,7 @@ set_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp)
 {
     ptp->handle = pthp;
     if (pthp) {
-	erts_smp_atomic_set(pthp, (long) ptp);
+	erts_smp_atomic_set_nob(pthp, (erts_aint_t) ptp);
 	ASSERT(ptp == handle2task(ptp->handle));
     }
 }
@@ -479,8 +494,8 @@ erts_port_task_abort(Eterm id, ErtsPortTaskHandle *pthp)
     case ERTS_PORT_TASK_INPUT:
     case ERTS_PORT_TASK_OUTPUT:
     case ERTS_PORT_TASK_EVENT:
-	ASSERT(erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) > 0);
-	erts_smp_atomic_dec(&erts_port_task_outstanding_io_tasks);
+	ASSERT(erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) > 0);
+	erts_smp_atomic_dec_relb(&erts_port_task_outstanding_io_tasks);
 	break;
     default:
 	break;
@@ -568,7 +583,7 @@ erts_port_task_schedule(Eterm id,
 	ErtsRunQueue *xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL);
 	if (xrunq) {
 	    /* Port emigrated ... */
-	    erts_smp_atomic_set(&pp->run_queue, (long) xrunq);
+	    erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq);
 	    erts_smp_runq_unlock(runq);
 	    runq = xrunq;
 	}
@@ -594,7 +609,7 @@ erts_port_task_schedule(Eterm id,
     case ERTS_PORT_TASK_INPUT:
     case ERTS_PORT_TASK_OUTPUT:
     case ERTS_PORT_TASK_EVENT:
-	erts_smp_atomic_inc(&erts_port_task_outstanding_io_tasks);
+	erts_smp_atomic_inc_relb(&erts_port_task_outstanding_io_tasks);
 	/* Fall through... */
     default:
 	enqueue_task(pp->sched.taskq, ptp);
@@ -625,6 +640,7 @@ erts_port_task_schedule(Eterm id,
 
     if (!enq_port) {
 	ERTS_PT_CHK_PRES_PORTQ(runq, pp);
+	erts_smp_runq_unlock(runq);
     }
     else {
 	enqueue_port(runq, pp);
@@ -634,9 +650,10 @@ erts_port_task_schedule(Eterm id,
 	    profile_runnable_port(pp, am_active);
 	}
 
+	erts_smp_runq_unlock(runq);
+
 	erts_smp_notify_inc_runq(runq);
     }
-    erts_smp_runq_unlock(runq);
     return 0;
 }
 
@@ -656,13 +673,11 @@ erts_port_task_free_port(Port *pp)
 	   when scheduled out... */
 	ErtsPortTask *ptp = port_task_alloc();
 	erts_smp_port_state_lock(pp);
-	ASSERT(erts_smp_atomic_read(&erts_ports_alive) > 0);
-	erts_smp_atomic_dec(&erts_ports_alive);
 	pp->status &= ~ERTS_PORT_SFLG_CLOSING;
 	pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED;
 	erts_may_save_closed_port(pp);
 	erts_smp_port_state_unlock(pp);
-	ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&pp->refc) > 1);
+	ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 1);
 	ptp->type = ERTS_PORT_TASK_FREE;
 	ptp->event = (ErlDrvEvent) -1;
 	ptp->event_data = NULL;
@@ -679,15 +694,14 @@ erts_port_task_free_port(Port *pp)
 	    port_is_dequeued = 1;
 	}
 	erts_smp_port_state_lock(pp);
-	erts_smp_atomic_dec(&erts_ports_alive);
 	pp->status &= ~ERTS_PORT_SFLG_CLOSING;
 	pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED;
 	erts_may_save_closed_port(pp);
 	erts_smp_port_state_unlock(pp);
 #ifdef ERTS_SMP
-	erts_smp_atomic_dec(&pp->refc); /* Not alive */
+	erts_smp_atomic_dec_nob(&pp->refc); /* Not alive */
 #endif
-	ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&pp->refc) > 0); /* Lock */
+	ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */
 	handle_remaining_tasks(runq, pp); /* May release runq lock */
 	ASSERT(!pp->sched.exe_taskq && (!ptqp || !ptqp->first));
 	pp->sched.taskq = NULL;
@@ -712,22 +726,6 @@ typedef struct {
     int *resp;
 } ErtsPortTaskExeBlockData;
 
-static void
-prepare_for_block(void *vd)
-{
-    ErtsPortTaskExeBlockData *d = (ErtsPortTaskExeBlockData *) vd;
-    erts_smp_runq_unlock(d->runq);
-}
-
-static void
-resume_after_block(void *vd)
-{
-    ErtsPortTaskExeBlockData *d = (ErtsPortTaskExeBlockData *) vd;
-    erts_smp_runq_lock(d->runq);
-    if (d->resp)
-	*d->resp = erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != (long) 0;
-}
-
 /*
  * Run all scheduled tasks for the first port in run queue. If
  * new tasks appear while running reschedule port (free task is
@@ -746,17 +744,11 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
     ErtsPortTask *ptp;
     int res = 0;
     int reds = ERTS_PORT_REDS_EXECUTE;
-    long io_tasks_executed = 0;
+    erts_aint_t io_tasks_executed = 0;
     int fpe_was_unmasked;
-    ErtsPortTaskExeBlockData blk_data = {runq, NULL};
 
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
 
-    erts_smp_activity_begin(ERTS_ACTIVITY_IO,
-			    prepare_for_block,
-			    resume_after_block,
-			    (void *) &blk_data);
-
     ERTS_PT_CHK_PORTQ(runq);
 
     pp = pop_port(runq);
@@ -832,8 +824,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	    ASSERT(!ptqp->first
 		   && (!pp->sched.taskq || !pp->sched.taskq->first));
 #ifdef ERTS_SMP
-	    erts_smp_atomic_dec(&pp->refc); /* Not alive */
-	    ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&pp->refc) > 0); /* Lock */
+	    erts_smp_atomic_dec_nob(&pp->refc); /* Not alive */
+	    ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */
 #else
 	    erts_port_status_bor_set(pp, ERTS_PORT_SFLG_FREE);	    
 #endif
@@ -846,12 +838,15 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	    goto tasks_done;
 	case ERTS_PORT_TASK_TIMEOUT:
 	    reds += ERTS_PORT_REDS_TIMEOUT;
-	    if (!(pp->status & ERTS_PORT_SFLGS_DEAD))
+	    if (!(pp->status & ERTS_PORT_SFLGS_DEAD)) {
+                DTRACE_DRIVER(driver_timeout, pp);
 		(*pp->drv_ptr->timeout)((ErlDrvData) pp->drv_data);
+            }
 	    break;
 	case ERTS_PORT_TASK_INPUT:
 	    reds += ERTS_PORT_REDS_INPUT;
 	    ASSERT((pp->status & ERTS_PORT_SFLGS_DEAD) == 0);
+            DTRACE_DRIVER(driver_ready_input, pp);
 	    /* NOTE some windows drivers use ->ready_input for input and output */
 	    (*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data, ptp->event);
 	    io_tasks_executed++;
@@ -859,12 +854,14 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	case ERTS_PORT_TASK_OUTPUT:
 	    reds += ERTS_PORT_REDS_OUTPUT;
 	    ASSERT((pp->status & ERTS_PORT_SFLGS_DEAD) == 0);
+            DTRACE_DRIVER(driver_ready_output, pp);
 	    (*pp->drv_ptr->ready_output)((ErlDrvData) pp->drv_data, ptp->event);
 	    io_tasks_executed++;
 	    break;
 	case ERTS_PORT_TASK_EVENT:
 	    reds += ERTS_PORT_REDS_EVENT;
 	    ASSERT((pp->status & ERTS_PORT_SFLGS_DEAD) == 0);
+            DTRACE_DRIVER(driver_event, pp);
 	    (*pp->drv_ptr->event)((ErlDrvData) pp->drv_data, ptp->event, ptp->event_data);
 	    io_tasks_executed++;
 	    break;
@@ -906,14 +903,16 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
     erts_unblock_fpe(fpe_was_unmasked);
 
     if (io_tasks_executed) {
-	ASSERT(erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) >= io_tasks_executed);
-	erts_smp_atomic_add(&erts_port_task_outstanding_io_tasks, -1*io_tasks_executed);
+	ASSERT(erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks)
+	       >= io_tasks_executed);
+	erts_smp_atomic_add_relb(&erts_port_task_outstanding_io_tasks,
+				 -1*io_tasks_executed);
     }
 
     *curr_port_pp = NULL;
 
 #ifdef ERTS_SMP
-    ASSERT(runq == (ErtsRunQueue *) erts_smp_atomic_read(&pp->run_queue));
+    ASSERT(runq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue));
 #endif
 
     if (!pp->sched.taskq) {
@@ -940,18 +939,19 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
 	}
 	else {
 	    /* Port emigrated ... */
-	    erts_smp_atomic_set(&pp->run_queue, (long) xrunq);
+	    erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq);
 	    enqueue_port(xrunq, pp);
 	    ASSERT(pp->sched.exe_taskq);
 	    pp->sched.exe_taskq = NULL;
-	    erts_smp_notify_inc_runq(xrunq);
 	    erts_smp_runq_unlock(xrunq);
+	    erts_smp_notify_inc_runq(xrunq);
 	}
 #endif
 	port_was_enqueued = 1;
     }
 
-    res = erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != (long) 0;
+    res = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks)
+	   != (erts_aint_t) 0);
 
     ERTS_PT_CHK_PRES_PORTQ(runq, pp);
 
@@ -969,26 +969,21 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp)
     erts_port_release(pp);
 #else
     {
-	long refc = erts_smp_atomic_dectest(&pp->refc);
+	erts_aint_t refc;
+	erts_smp_mtx_unlock(pp->lock);
+	refc = erts_smp_atomic_dec_read_nob(&pp->refc);
 	ASSERT(refc >= 0);
-	if (refc > 0)
-	    erts_smp_mtx_unlock(pp->lock);
-	else {
+	if (refc == 0) {
 	    erts_smp_runq_unlock(runq);
 	    erts_port_cleanup(pp); /* Might aquire runq lock */
 	    erts_smp_runq_lock(runq);
-	    res = erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != (long) 0;
+	    res = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks)
+		   != (erts_aint_t) 0);
 	}
     }
 #endif
 
  done:
-    blk_data.resp = &res;
-    erts_smp_activity_end(ERTS_ACTIVITY_IO,
-			  prepare_for_block,
-			  resume_after_block,
-			  (void *) &blk_data);
-
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
 
     ERTS_PORT_REDUCTIONS_EXECUTED(runq, reds);
@@ -1070,8 +1065,6 @@ erts_port_migrate(Port *prt, int *prt_locked,
     ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked);
     ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked);
 
-    ASSERT(!erts_common_run_queue);
-
     if (!*from_locked || !*to_locked) {
 	if (from_rq < to_rq) {
 	    if (!*to_locked) {
@@ -1105,14 +1098,13 @@ erts_port_migrate(Port *prt, int *prt_locked,
     /* Refuse to migrate to a suspended run queue */
     if (to_rq->flags & ERTS_RUNQ_FLG_SUSPENDED)
 	return ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED;
-    if (from_rq != (ErtsRunQueue *) erts_smp_atomic_read(&prt->run_queue))
+    if (from_rq != (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue))
 	return ERTS_MIGRATE_FAILED_RUNQ_CHANGED;
     if (!ERTS_PORT_IS_IN_RUNQ(from_rq, prt))
 	return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ;
     dequeue_port(from_rq, prt);
-    erts_smp_atomic_set(&prt->run_queue, (long) to_rq);
+    erts_smp_atomic_set_nob(&prt->run_queue, (erts_aint_t) to_rq);
     enqueue_port(to_rq, prt);
-    erts_smp_notify_inc_runq(to_rq);
     return ERTS_MIGRATE_SUCCESS;
 }
 
@@ -1124,7 +1116,8 @@ erts_port_migrate(Port *prt, int *prt_locked,
 void
 erts_port_task_init(void)
 {
-    erts_smp_atomic_init(&erts_port_task_outstanding_io_tasks, (long) 0);
+    erts_smp_atomic_init_nob(&erts_port_task_outstanding_io_tasks,
+			     (erts_aint_t) 0);
     init_port_task_alloc();
     init_port_taskq_alloc();
 }
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index f12d02da0c..d7104e1143 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -79,13 +79,13 @@ ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void);
 ERTS_GLB_INLINE void
 erts_port_task_handle_init(ErtsPortTaskHandle *pthp)
 {
-    erts_smp_atomic_init(pthp, (long) NULL);
+    erts_smp_atomic_init_nob(pthp, (erts_aint_t) NULL);
 }
 
 ERTS_GLB_INLINE int
 erts_port_task_is_scheduled(ErtsPortTaskHandle *pthp)
 {
-    return ((void *) erts_smp_atomic_read(pthp)) != NULL;
+    return ((void *) erts_smp_atomic_read_nob(pthp)) != NULL;
 }
 
 ERTS_GLB_INLINE void
@@ -102,7 +102,8 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp)
 ERTS_GLB_INLINE int
 erts_port_task_have_outstanding_io_tasks(void)
 {
-    return erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != 0;
+    return (erts_smp_atomic_read_acqb(&erts_port_task_outstanding_io_tasks)
+	    != 0);
 }
 
 #endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c
index 7fe3f3bca5..34da9cab84 100644
--- a/erts/emulator/beam/erl_printf_term.c
+++ b/erts/emulator/beam/erl_printf_term.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -114,13 +114,13 @@ do {									\
 /* return 0 if list is not a non-empty flat list of printable characters */
 
 static int
-is_printable_string(Eterm list)
+is_printable_string(Eterm list, Eterm* base)
 {
     int len = 0;
     int c;
 
     while(is_list(list)) {
-	Eterm* consp = list_val(list);
+	Eterm* consp = list_val_rel(list, base);
 	Eterm hd = CAR(consp);
 
 	if (!is_byte(hd))
@@ -216,240 +216,289 @@ static int print_atom_name(fmtfn_t fn, void* arg, Eterm atom, long *dcount)
 }
 
 
+#define PRT_BAR ((Eterm) 0)
+#define PRT_COMMA ((Eterm) 1)
+#define PRT_CLOSE_LIST ((Eterm) 2)
+#define PRT_CLOSE_TUPLE ((Eterm) 3)
+#define PRT_TERM ((Eterm) 4)
+#define PRT_ONE_CONS ((Eterm) 5)
+#define PRT_PATCH_FUN_SIZE ((Eterm) 6)
+#define PRT_LAST_ARRAY_ELEMENT ((Eterm) 7) /* Note! Must be last... */
 
 static int
-print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount)
+print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount,
+	   Eterm* obj_base) /* ignored if !HALFWORD_HEAP */
 {
+    DECLARE_WSTACK(s);
     int res;
     int i;
+    Eterm val;
     Uint32 *ref_num;
+    union {
+	UWord word;
+	Eterm* ptr;
+    }popped;
     Eterm* nobj;
+    Wterm wobj;
 
     res = 0;
 
-    if ((*dcount)-- <= 0)
-	return res;
+    goto L_jump_start;
+
+ L_outer_loop:
+    while (!WSTACK_ISEMPTY(s)) {
+	switch (val = WSTACK_POP(s)) {
+	case PRT_COMMA:
+	    PRINT_CHAR(res, fn, arg, ',');
+	    goto L_outer_loop;
+	case PRT_BAR:
+	    PRINT_CHAR(res, fn, arg, '|');
+	    goto L_outer_loop;
+	case PRT_CLOSE_LIST:
+	    PRINT_CHAR(res, fn, arg, ']');
+	    goto L_outer_loop;
+	case PRT_CLOSE_TUPLE:
+	    PRINT_CHAR(res, fn, arg, '}');
+	    goto L_outer_loop;
+	default:
+	    popped.word = WSTACK_POP(s);
 
-#ifdef HYBRID___NOT_ACTIVE
-    /* Color coded output based on memory location */
-    if(ptr_val(obj) >= global_heap && ptr_val(obj) < global_hend)
-        PRINT_STRING(res, fn, arg, "\033[32m");
-#ifdef INCREMENTAL
-    else if(ptr_val(obj) >= inc_fromspc && ptr_val(obj) < inc_fromend)
-        PRINT_STRING(res, fn, arg, "\033[33m");
-#endif
-    else if(IS_CONST(obj))
-        PRINT_STRING(res, fn, arg, "\033[34m");
-    else
-        PRINT_STRING(res, fn, arg, "\033[31m");
-#endif
+	    switch (val) {
+	    case PRT_TERM:
+		obj = (Eterm) popped.word;
+		break;
+	    case PRT_ONE_CONS:
+		obj = (Eterm) popped.word;
+	    L_print_one_cons:
+		{
+		    Eterm* cons = list_val_rel(obj, obj_base);
+		    Eterm tl;
+		    
+		    obj = CAR(cons);
+		    tl = CDR(cons);
+		    if (is_not_nil(tl)) {
+			if (is_list(tl)) {
+			    WSTACK_PUSH(s, tl);
+			    WSTACK_PUSH(s, PRT_ONE_CONS);
+			    WSTACK_PUSH(s, PRT_COMMA);
+			} else {
+			    WSTACK_PUSH(s, tl);
+			    WSTACK_PUSH(s, PRT_TERM);
+			    WSTACK_PUSH(s, PRT_BAR);
+			}
+		    }
+		}
+		break;
+	    case PRT_LAST_ARRAY_ELEMENT:
+		obj = *popped.ptr;
+		break;
+	    default:		/* PRT_LAST_ARRAY_ELEMENT+1 and upwards */
+		obj = *popped.ptr;
+	        WSTACK_PUSH(s, (UWord) (popped.ptr + 1));
+		WSTACK_PUSH(s, val-1);
+		WSTACK_PUSH(s, PRT_COMMA);
+		break;
+	    }
+	    break;
+	}
 
-    if (is_CP(obj)) {
-	PRINT_STRING(res, fn, arg, "<cp/header:");
-	PRINT_POINTER(res, fn, arg, obj);
-	PRINT_CHAR(res, fn, arg, '>');
-	return res;
-    }
+    L_jump_start:
 
-    switch (tag_val_def(obj)) {
-    case NIL_DEF:
-	PRINT_STRING(res, fn, arg, "[]");
-	break;
-    case ATOM_DEF: {
-	int tres = print_atom_name(fn, arg, obj, dcount);
-	if (tres < 0)
-	    return tres;
-	res += tres;
-	if (*dcount <= 0)
-	    return res;
-	break;
-    }
-    case SMALL_DEF:
-	PRINT_SLONG(res, fn, arg, 'd', 0, 1, (signed long) signed_val(obj));
-	break;
-    case BIG_DEF: {
-	int print_res;
-	char def_buf[64];
-	char *buf, *big_str;
-	Uint sz = (Uint) big_decimal_estimate(obj);
-	sz++;
-	if (sz <= 64)
-	    buf = &def_buf[0];
-	else
-	    buf = erts_alloc(ERTS_ALC_T_TMP, sz);
-	big_str = erts_big_to_string(obj, buf, sz);
-	print_res = erts_printf_string(fn, arg, big_str);
-	if (buf != &def_buf[0])
-	    erts_free(ERTS_ALC_T_TMP, (void *) buf);
-	if (print_res < 0)
-	    return print_res;
-	res += print_res;
-	break;
-    }
-    case REF_DEF:
-    case EXTERNAL_REF_DEF:
-	PRINT_STRING(res, fn, arg, "#Ref<");
-	PRINT_ULONG(res, fn, arg, 'u', 0, 1,
-		    (unsigned long) ref_channel_no(obj));
-	ref_num = ref_numbers(obj);
-	for (i = ref_no_of_numbers(obj)-1; i >= 0; i--) {
-	    PRINT_CHAR(res, fn, arg, '.');
-	    PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) ref_num[i]);
+	if ((*dcount)-- <= 0)
+	    goto L_done;
+
+	if (is_CP(obj)) {
+	    PRINT_STRING(res, fn, arg, "<cp/header:");
+	    PRINT_POINTER(res, fn, arg, cp_val(obj));
+	    PRINT_CHAR(res, fn, arg, '>');
+	    goto L_done;
 	}
-	PRINT_CHAR(res, fn, arg, '>');
-	break;
-    case PID_DEF:
-    case EXTERNAL_PID_DEF:
-	PRINT_CHAR(res, fn, arg, '<');
-	PRINT_ULONG(res, fn, arg, 'u', 0, 1,
-		    (unsigned long) pid_channel_no(obj));
-	PRINT_CHAR(res, fn, arg, '.');
-	PRINT_ULONG(res, fn, arg, 'u', 0, 1,
-		    (unsigned long) pid_number(obj));
-	PRINT_CHAR(res, fn, arg, '.');
-	PRINT_ULONG(res, fn, arg, 'u', 0, 1,
-		    (unsigned long) pid_serial(obj));
-	PRINT_CHAR(res, fn, arg, '>');
-	break;
-    case PORT_DEF:
-    case EXTERNAL_PORT_DEF:
-	PRINT_STRING(res, fn, arg, "#Port<");
-	PRINT_ULONG(res, fn, arg, 'u', 0, 1,
-		    (unsigned long) port_channel_no(obj));
-	PRINT_CHAR(res, fn, arg, '.');
-	PRINT_ULONG(res, fn, arg, 'u', 0, 1,
-		    (unsigned long) port_number(obj));
-	PRINT_CHAR(res, fn, arg, '>');
-	break;
-    case LIST_DEF:
-	if (is_printable_string(obj)) {
-	    int c;
-	    PRINT_CHAR(res, fn, arg, '"');
-	    nobj = list_val(obj);
-	    while (1) {
-		if ((*dcount)-- <= 0)
-		    return res;
-		c = signed_val(*nobj++);
-		if (c == '\n')
-		    PRINT_STRING(res, fn, arg, "\\n");
-		else {
-		    if (c == '"')
-			PRINT_CHAR(res, fn, arg, '\\');
-		    PRINT_CHAR(res, fn, arg, (char) c);
-		}
-		if (is_not_list(*nobj))
-		    break;
-		nobj = list_val(*nobj);
-	    }
-	    PRINT_CHAR(res, fn, arg, '"');
-	} else {
-	    PRINT_CHAR(res, fn, arg, '[');
-	    nobj = list_val(obj);
-	    while (1) {
-		int tres = print_term(fn, arg, *nobj++, dcount);
-		if (tres < 0)
-		    return tres;
-		res += tres;
-		if (*dcount <= 0)
-		    return res;
-		if (is_not_list(*nobj))
-		    break;
-		PRINT_CHAR(res, fn, arg, ',');
-		nobj = list_val(*nobj);
-	    }
-	    if (is_not_nil(*nobj)) {
-		int tres;
-		PRINT_CHAR(res, fn, arg, '|');
-		tres = print_term(fn, arg, *nobj, dcount);
-		if (tres < 0)
-		    return tres;
-		res += tres;
-		if (*dcount <= 0)
-		    return res;
+#if HALFWORD_HEAP
+	wobj = is_immed(obj) ? (Wterm)obj : rterm2wterm(obj, obj_base);
+#else
+	wobj = (Wterm)obj;
+#endif
+	switch (tag_val_def(wobj)) {
+	case NIL_DEF:
+	    PRINT_STRING(res, fn, arg, "[]");
+	    break;
+	case ATOM_DEF: {
+	    int tres = print_atom_name(fn, arg, obj, dcount);
+	    if (tres < 0) {
+		res = tres;
+		goto L_done;
 	    }
-	    PRINT_CHAR(res, fn, arg, ']');
-	}
-	break;
-    case TUPLE_DEF:
-	nobj = tuple_val(obj);	/* pointer to arity */
-	i = arityval(*nobj);	/* arity */
-	PRINT_CHAR(res, fn, arg, '{');
-	while (i--) {
-	    int tres = print_term(fn, arg, *++nobj, dcount);
-	    if (tres < 0)
-		return tres;
 	    res += tres;
 	    if (*dcount <= 0)
-		return res;
-	    if (i >= 1)
-		PRINT_CHAR(res, fn, arg, ',');
-	}
-	PRINT_CHAR(res, fn, arg, '}');
-	break;
-    case FLOAT_DEF: {
-	    FloatDef ff;
-	    GET_DOUBLE(obj, ff);
-	    PRINT_DOUBLE(res, fn, arg, 'e', 6, 0, ff.fd);
+		goto L_done;
+	    break;
 	}
-	break;
-    case BINARY_DEF:
-	{
-	    ProcBin* pb = (ProcBin *) binary_val(obj);
-	    if (pb->size == 1)
-		PRINT_STRING(res, fn, arg, "<<1 byte>>");
-	    else {
-		PRINT_STRING(res, fn, arg, "<<");
-		PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) pb->size);
-		PRINT_STRING(res, fn, arg, " bytes>>");
+	case SMALL_DEF:
+	    PRINT_SLONG(res, fn, arg, 'd', 0, 1, (signed long) signed_val(obj));
+	    break;
+	case BIG_DEF: {
+	    int print_res;
+	    char def_buf[64];
+	    char *buf, *big_str;
+	    Uint sz = (Uint) big_decimal_estimate(wobj);
+	    sz++;
+	    if (sz <= 64)
+		buf = &def_buf[0];
+	    else
+		buf = erts_alloc(ERTS_ALC_T_TMP, sz);
+	    big_str = erts_big_to_string(wobj, buf, sz);
+	    print_res = erts_printf_string(fn, arg, big_str);
+	    if (buf != &def_buf[0])
+		erts_free(ERTS_ALC_T_TMP, (void *) buf);
+	    if (print_res < 0) {
+		res = print_res;
+		goto L_done;
 	    }
+	    res += print_res;
+	    break;
 	}
-	break;
-    case EXPORT_DEF:
-	{
-	    Export* ep = (Export *) (export_val(obj))[1];
-	    Atom* module = atom_tab(atom_val(ep->code[0]));
-	    Atom* name = atom_tab(atom_val(ep->code[1]));
-
-	    PRINT_STRING(res, fn, arg, "#Fun<");
-	    PRINT_BUF(res, fn, arg, module->name, module->len);
+	case REF_DEF:
+	case EXTERNAL_REF_DEF:
+	    PRINT_STRING(res, fn, arg, "#Ref<");
+	    PRINT_ULONG(res, fn, arg, 'u', 0, 1,
+			(unsigned long) ref_channel_no(wobj));
+	    ref_num = ref_numbers(wobj);
+	    for (i = ref_no_of_numbers(wobj)-1; i >= 0; i--) {
+		PRINT_CHAR(res, fn, arg, '.');
+		PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) ref_num[i]);
+	    }
+	    PRINT_CHAR(res, fn, arg, '>');
+	    break;
+	case PID_DEF:
+	case EXTERNAL_PID_DEF:
+	    PRINT_CHAR(res, fn, arg, '<');
+	    PRINT_ULONG(res, fn, arg, 'u', 0, 1,
+			(unsigned long) pid_channel_no(wobj));
 	    PRINT_CHAR(res, fn, arg, '.');
-	    PRINT_BUF(res, fn, arg, name->name, name->len);
+	    PRINT_ULONG(res, fn, arg, 'u', 0, 1,
+			(unsigned long) pid_number(wobj));
 	    PRINT_CHAR(res, fn, arg, '.');
-	    PRINT_SLONG(res, fn, arg, 'd', 0, 1,
-			(signed long) ep->code[2]);
+	    PRINT_ULONG(res, fn, arg, 'u', 0, 1,
+			(unsigned long) pid_serial(wobj));
 	    PRINT_CHAR(res, fn, arg, '>');
-	}
-	break;
-    case FUN_DEF:
-	{
-	    ErlFunThing *funp = (ErlFunThing *) fun_val(obj);
-	    Atom *ap = atom_tab(atom_val(funp->fe->module));
-
-	    PRINT_STRING(res, fn, arg, "#Fun<");
-	    PRINT_BUF(res, fn, arg, ap->name, ap->len);
-	    PRINT_CHAR(res, fn, arg, '.');
-	    PRINT_SLONG(res, fn, arg, 'd', 0, 1,
-			(signed long) funp->fe->old_index);
+	    break;
+	case PORT_DEF:
+	case EXTERNAL_PORT_DEF:
+	    PRINT_STRING(res, fn, arg, "#Port<");
+	    PRINT_ULONG(res, fn, arg, 'u', 0, 1,
+			(unsigned long) port_channel_no(wobj));
 	    PRINT_CHAR(res, fn, arg, '.');
-	    PRINT_SLONG(res, fn, arg, 'd', 0, 1,
-			(signed long) funp->fe->old_uniq);
+	    PRINT_ULONG(res, fn, arg, 'u', 0, 1,
+			(unsigned long) port_number(wobj));
+	    PRINT_CHAR(res, fn, arg, '>');
+	    break;
+	case LIST_DEF:
+	    if (is_printable_string(obj, obj_base)) {
+		int c;
+		PRINT_CHAR(res, fn, arg, '"');
+		nobj = list_val_rel(obj, obj_base);
+		while (1) {
+		    if ((*dcount)-- <= 0)
+			goto L_done;
+		    c = signed_val(*nobj++);
+		    if (c == '\n')
+			PRINT_STRING(res, fn, arg, "\\n");
+		    else {
+			if (c == '"')
+			    PRINT_CHAR(res, fn, arg, '\\');
+			PRINT_CHAR(res, fn, arg, (char) c);
+		    }
+		    if (is_not_list(*nobj))
+			break;
+		    nobj = list_val_rel(*nobj, obj_base);
+		}
+		PRINT_CHAR(res, fn, arg, '"');
+	    } else {
+		PRINT_CHAR(res, fn, arg, '[');
+		WSTACK_PUSH(s,PRT_CLOSE_LIST);
+		goto L_print_one_cons;
+	    }
+	    break;
+	case TUPLE_DEF:
+	    nobj = tuple_val(wobj);	/* pointer to arity */
+	    i = arityval(*nobj);	/* arity */
+	    PRINT_CHAR(res, fn, arg, '{');
+	    WSTACK_PUSH(s,PRT_CLOSE_TUPLE);
+	    ++nobj;
+	    if (i > 0) {
+		WSTACK_PUSH(s, (UWord) nobj);
+		WSTACK_PUSH(s, PRT_LAST_ARRAY_ELEMENT+i-1);
+	    }
+	    break;
+	case FLOAT_DEF: {
+	    FloatDef ff;
+	    GET_DOUBLE(wobj, ff);
+	    PRINT_DOUBLE(res, fn, arg, 'e', 6, 0, ff.fd);
+	}
+	    break;
+	case BINARY_DEF:
+	    {
+		ProcBin* pb = (ProcBin *) binary_val(wobj);
+		if (pb->size == 1)
+		    PRINT_STRING(res, fn, arg, "<<1 byte>>");
+		else {
+		    PRINT_STRING(res, fn, arg, "<<");
+		    PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) pb->size);
+		    PRINT_STRING(res, fn, arg, " bytes>>");
+		}
+	    }
+	    break;
+	case EXPORT_DEF:
+	    {
+		Export* ep = *((Export **) (export_val(wobj) + 1));
+		Atom* module = atom_tab(atom_val(ep->code[0]));
+		Atom* name = atom_tab(atom_val(ep->code[1]));
+
+		PRINT_STRING(res, fn, arg, "#Fun<");
+		PRINT_BUF(res, fn, arg, module->name, module->len);
+		PRINT_CHAR(res, fn, arg, '.');
+		PRINT_BUF(res, fn, arg, name->name, name->len);
+		PRINT_CHAR(res, fn, arg, '.');
+		PRINT_SLONG(res, fn, arg, 'd', 0, 1,
+			    (signed long) ep->code[2]);
+		PRINT_CHAR(res, fn, arg, '>');
+	    }
+	    break;
+	case FUN_DEF:
+	    {
+		ErlFunThing *funp = (ErlFunThing *) fun_val(wobj);
+		Atom *ap = atom_tab(atom_val(funp->fe->module));
+
+		PRINT_STRING(res, fn, arg, "#Fun<");
+		PRINT_BUF(res, fn, arg, ap->name, ap->len);
+		PRINT_CHAR(res, fn, arg, '.');
+		PRINT_SLONG(res, fn, arg, 'd', 0, 1,
+			    (signed long) funp->fe->old_index);
+		PRINT_CHAR(res, fn, arg, '.');
+		PRINT_SLONG(res, fn, arg, 'd', 0, 1,
+			    (signed long) funp->fe->old_uniq);
+		PRINT_CHAR(res, fn, arg, '>');
+	    }
+	    break;
+	default:
+	    PRINT_STRING(res, fn, arg, "<unknown:");
+	    PRINT_POINTER(res, fn, arg, wobj);
 	    PRINT_CHAR(res, fn, arg, '>');
+	    break;
 	}
-	break;
-    default:
-	PRINT_STRING(res, fn, arg, "<unknown:");
-	PRINT_POINTER(res, fn, arg, obj);
-	PRINT_CHAR(res, fn, arg, '>');
-	break;
     }
 
+ L_done:
+    
+    DESTROY_WSTACK(s);
     return res;
 }
 
 int
-erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision)
+erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision,
+		 unsigned long* term_base)
 {
-    int res = print_term(fn, arg, (Uint) term, &precision);
+    int res = print_term(fn, arg, (Eterm)term, &precision, (Eterm*)term_base);
     if (res < 0)
 	return res;
     if (precision <= 0)
diff --git a/erts/emulator/beam/erl_printf_term.h b/erts/emulator/beam/erl_printf_term.h
index 4f76028396..a48a3de34c 100644
--- a/erts/emulator/beam/erl_printf_term.h
+++ b/erts/emulator/beam/erl_printf_term.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -21,6 +21,6 @@
 #define ERL_PRINTF_TERM_H__
 
 #include "erl_printf_format.h"
-int erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision);
-
+int erts_printf_term(fmtfn_t fn, void* arg, unsigned long term, long precision,
+		     unsigned long* term_base);
 #endif
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 996806fc75..0fa2def5af 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -24,12 +24,10 @@
 #endif
 
 #include <stddef.h> /* offsetof() */
-#include <ctype.h>
 #include "sys.h"
 #include "erl_vm.h"
 #include "global.h"
 #include "erl_process.h"
-#include "erl_nmgc.h"
 #include "error.h"
 #include "bif.h"
 #include "erl_db.h"
@@ -38,6 +36,12 @@
 #include "erl_instrument.h"
 #include "erl_threads.h"
 #include "erl_binary.h"
+#include "beam_bp.h"
+#include "erl_cpu_topology.h"
+#include "erl_thr_progress.h"
+#include "erl_thr_queue.h"
+#include "erl_async.h"
+#include "dtrace-wrapper.h"
 
 #define ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED (2000*CONTEXT_REDS)
 #define ERTS_RUNQ_CALL_CHECK_BALANCE_REDS \
@@ -45,13 +49,23 @@
 
 #define ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST (CONTEXT_REDS/10)
 
-#define ERTS_SCHED_SLEEP_SPINCOUNT 10000
+#define ERTS_SCHED_SPIN_UNTIL_YIELD 100
 
-#define ERTS_WAKEUP_OTHER_LIMIT (100*CONTEXT_REDS/2)
-#define ERTS_WAKEUP_OTHER_DEC 10
-#define ERTS_WAKEUP_OTHER_FIXED_INC (CONTEXT_REDS/10)
+#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_LONG 40
+#define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_VERY_LONG 1000
+#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_LONG 20
+#define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_LONG 1000
+#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM 10
+#define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_MEDIUM 1000
+#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_SHORT 10
+#define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_SHORT 0
+#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_SHORT 5
+#define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_VERY_SHORT 0
+#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE 0
+#define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_NONE 0
 
-#define ERTS_MAX_CPU_TOPOLOGY_ID ((int) 0xffff)
+#define ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT 1000
+#define ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT 0
 
 #if 0 || defined(DEBUG)
 #define ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA
@@ -91,9 +105,12 @@ do {								\
 #define ERTS_EMPTY_RUNQ(RQ) \
   ((RQ)->len == 0 && (RQ)->misc.start == NULL)
 
-extern Eterm beam_apply[];
-extern Eterm beam_exit[];
-extern Eterm beam_continue_exit[];
+#define ERTS_EMPTY_RUNQ_PORTS(RQ) \
+  ((RQ)->ports.info.len == 0 && (RQ)->misc.start == NULL)
+
+extern BeamInstr beam_apply[];
+extern BeamInstr beam_exit[];
+extern BeamInstr beam_continue_exit[];
 
 static Sint p_last;
 static Sint p_next;
@@ -101,6 +118,7 @@ static Sint p_serial;
 static Uint p_serial_mask;
 static Uint p_serial_shift;
 
+int erts_sched_compact_load;
 Uint erts_no_schedulers;
 Uint erts_max_processes = ERTS_DEFAULT_MAX_PROCESSES;
 Uint erts_process_tab_index_mask;
@@ -111,23 +129,49 @@ int erts_sched_thread_suggested_stack_size = -1;
 ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];
 #endif
 
-#ifdef ERTS_SMP
+static struct {
+    int aux_work;
+    int tse;
+    int sys_schedule;
+} sched_busy_wait;
 
+#ifdef ERTS_SMP
 int erts_disable_proc_not_running_opt;
 
-#define ERTS_SCHED_CHANGING_ONLINE 1
-#define ERTS_SCHED_CHANGING_MULTI_SCHED 2
+static ErtsAuxWorkData *aux_thread_aux_work_data;
+
+#define ERTS_SCHDLR_SSPND_CHNG_WAITER		(((erts_aint32_t) 1) << 0)
+#define ERTS_SCHDLR_SSPND_CHNG_MSB		(((erts_aint32_t) 1) << 1)
+#define ERTS_SCHDLR_SSPND_CHNG_ONLN		(((erts_aint32_t) 1) << 2)
+
+#ifndef DEBUG
+
+#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \
+  erts_smp_atomic32_set_nob(&schdlr_sspnd.changing, (VAL))
+
+#else
+
+#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL)			\
+do {									\
+    erts_aint32_t old_val__;						\
+    old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.changing,     	\
+					   (VAL));			\
+    ASSERT(old_val__ == (OLD_VAL));					\
+} while (0)
+
+#endif
+
 
 static struct {
     erts_smp_mtx_t mtx;
     erts_smp_cnd_t cnd;
-    int changing;
     int online;
     int curr_online;
     int wait_curr_online;
-    erts_smp_atomic_t active;
+    erts_smp_atomic32_t changing;
+    erts_smp_atomic32_t active;
     struct {
-	erts_smp_atomic_t ongoing;
+	int ongoing;
 	long wait_active;
 	ErtsProcList *procs;
     } msb; /* Multi Scheduling Block */
@@ -135,11 +179,10 @@ static struct {
 
 static struct {
     erts_smp_mtx_t update_mtx;
-    erts_smp_atomic_t active_runqs;
+    erts_smp_atomic32_t no_runqs;
     int last_active_runqs;
-    erts_smp_atomic_t used_runqs;
     int forced_check_balance;
-    erts_smp_atomic_t checking_balance;
+    erts_smp_atomic32_t checking_balance;
     int halftime;
     int full_reds_history_index;
     struct {
@@ -159,59 +202,19 @@ do {							\
 
 #endif
 
-/*
- * Cpu topology hierarchy.
- */
-#define ERTS_TOPOLOGY_NODE		0
-#define ERTS_TOPOLOGY_PROCESSOR		1
-#define ERTS_TOPOLOGY_PROCESSOR_NODE	2
-#define ERTS_TOPOLOGY_CORE		3
-#define ERTS_TOPOLOGY_THREAD		4
-#define ERTS_TOPOLOGY_LOGICAL		5
-
-#define ERTS_TOPOLOGY_MAX_DEPTH		6
-
-typedef struct {
-    int bind_id;
-    int bound_id;
-} ErtsCpuBindData;
-
-static ErtsCpuBindData *scheduler2cpu_map;
-erts_smp_rwmtx_t erts_cpu_bind_rwmtx;
-
-typedef enum {
-    ERTS_CPU_BIND_SPREAD,
-    ERTS_CPU_BIND_PROCESSOR_SPREAD,
-    ERTS_CPU_BIND_THREAD_SPREAD,
-    ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD,
-    ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD,
-    ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD,
-    ERTS_CPU_BIND_NO_SPREAD,
-    ERTS_CPU_BIND_NONE
-} ErtsCpuBindOrder;
-
-ErtsCpuBindOrder cpu_bind_order;
-
-static erts_cpu_topology_t *user_cpudata;
-static int user_cpudata_size;
-static erts_cpu_topology_t *system_cpudata;
-static int system_cpudata_size;
-
 erts_sched_stat_t erts_sched_stat;
 
-ErtsRunQueue *erts_common_run_queue;
-
 #ifdef USE_THREADS
 static erts_tsd_key_t sched_data_key;
 #endif
 
 static erts_smp_mtx_t proc_tab_mtx;
 
-static erts_smp_atomic_t function_calls;
+static erts_smp_atomic32_t function_calls;
 
 #ifdef ERTS_SMP
-static erts_smp_atomic_t doing_sys_schedule;
-static erts_smp_atomic_t no_empty_run_queues;
+static erts_smp_atomic32_t doing_sys_schedule;
+static erts_smp_atomic32_t no_empty_run_queues;
 #else /* !ERTS_SMP */
 ErtsSchedulerData *erts_scheduler_data;
 #endif
@@ -219,23 +222,20 @@ ErtsSchedulerData *erts_scheduler_data;
 ErtsAlignedRunQueue *erts_aligned_run_queues;
 Uint erts_no_run_queues;
 
-typedef union {
-    ErtsSchedulerData esd;
-    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))];
-} ErtsAlignedSchedulerData;
-
 ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
 
-#ifndef BM_COUNTERS
-static int processes_busy;
-#endif
+typedef union {
+    ErtsSchedulerSleepInfo ssi;
+    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerSleepInfo))];
+} ErtsAlignedSchedulerSleepInfo;
+
+static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info;
 
 Process**  process_tab;
 static Uint last_reductions;
 static Uint last_exact_reductions;
 Uint erts_default_process_flags;
 Eterm erts_system_monitor;
-Eterm erts_system_monitor_msg_queue_len;
 Eterm erts_system_monitor_long_gc;
 Eterm erts_system_monitor_large_heap;
 struct erts_system_monitor_flags_t erts_system_monitor_flags;
@@ -244,12 +244,10 @@ struct erts_system_monitor_flags_t erts_system_monitor_flags;
 Eterm erts_system_profile;
 struct erts_system_profile_flags_t erts_system_profile_flags;
 
-#ifdef HYBRID
-Uint erts_num_active_procs;
-Process** erts_active_procs;
+#if ERTS_MAX_PROCESSES > 0x7fffffff
+#error "Need to store process_count in another type"
 #endif
-
-static erts_smp_atomic_t process_count;
+static erts_smp_atomic32_t process_count;
 
 typedef struct ErtsTermProcElement_ ErtsTermProcElement;
 struct ErtsTermProcElement_ {
@@ -283,8 +281,10 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist,
 				 200,
 				 ERTS_ALC_T_PROC_LIST)
 
-#define ERTS_RUNQ_IX(IX)	(&erts_aligned_run_queues[(IX)].runq)
-#define ERTS_SCHEDULER_IX(IX)	(&erts_aligned_scheduler_data[(IX)].esd)
+#define ERTS_SCHED_SLEEP_INFO_IX(IX)					\
+    (ASSERT_EXPR(-1 <= ((int) (IX))					\
+		 && ((int) (IX)) < ((int) erts_no_schedulers)),		\
+     &aligned_sched_sleep_info[(IX)].ssi)
 
 #define ERTS_FOREACH_RUNQ(RQVAR, DO)					\
 do {									\
@@ -334,22 +334,62 @@ do {									\
 static void init_processes_bif(void);
 static void save_terminating_process(Process *p);
 static void exec_misc_ops(ErtsRunQueue *);
-static void print_function_from_pc(int to, void *to_arg, Eterm* x);
+static void print_function_from_pc(int to, void *to_arg, BeamInstr* x);
 static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp,
 			      int yreg);
-#ifdef ERTS_SMP
-static void handle_pending_exiters(ErtsProcList *);
 
-static void cpu_bind_order_sort(erts_cpu_topology_t *cpudata,
-				int size,
-				ErtsCpuBindOrder bind_order,
-				int mk_seq);
-static void signal_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size);
+static void aux_work_timeout(void *unused);
+static void aux_work_timeout_early_init(int no_schedulers);
+static void aux_work_timeout_late_init(void);
+static void setup_aux_work_timer(void);
+
+#if defined(DEBUG) || 0
+#define ERTS_DBG_CHK_AUX_WORK_VAL(V) dbg_chk_aux_work_val((V))
+static void
+dbg_chk_aux_work_val(erts_aint32_t value)
+{
+    erts_aint32_t valid = 0;
+
+    valid |= ERTS_SSI_AUX_WORK_SET_TMO;
+    valid |= ERTS_SSI_AUX_WORK_MISC;
+    valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM;
+    valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC;
+#if ERTS_USE_ASYNC_READY_Q
+    valid |= ERTS_SSI_AUX_WORK_ASYNC_READY;
+    valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
+#endif
+#ifdef ERTS_SMP
+    valid |= ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP;
+    valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR;
+    valid |= ERTS_SSI_AUX_WORK_DD;
+    valid |= ERTS_SSI_AUX_WORK_DD_THR_PRGR;
+#endif
+#if HAVE_ERTS_MSEG
+    valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK;
+#endif
+#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
+    valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN;
+#endif
+#ifdef ERTS_SSI_AUX_WORK_REAP_PORTS
+    valid |= ERTS_SSI_AUX_WORK_REAP_PORTS;
+#endif
 
+    if (~valid & value)
+	erl_exit(ERTS_ABORT_EXIT,
+		 "Invalid aux_work value found: 0x%x\n",
+		 ~valid & value);
+}
+#define ERTS_DBG_CHK_SSI_AUX_WORK(SSI) \
+  ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&(SSI)->aux_work))
+#else
+#define ERTS_DBG_CHK_AUX_WORK_VAL(V)
+#define ERTS_DBG_CHK_SSI_AUX_WORK(SSI)
 #endif
 
-static void early_cpu_bind_init(void);
-static void late_cpu_bind_init(void);
+#ifdef ERTS_SMP
+static void handle_pending_exiters(ErtsProcList *);
+
+#endif
 
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
 int
@@ -388,7 +428,12 @@ erts_pre_init_process(void)
      erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].get_locks
 	 = ERTS_PSD_DIST_ENTRY_GET_LOCKS;
      erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].set_locks
-	 = ERTS_PSD_DIST_ENTRY_GET_LOCKS;
+	 = ERTS_PSD_DIST_ENTRY_SET_LOCKS;
+
+     erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].get_locks
+	 = ERTS_PSD_CALL_TIME_BP_GET_LOCKS;
+     erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks
+	 = ERTS_PSD_CALL_TIME_BP_SET_LOCKS;
 
      /* Check that we have locks for all entries */
      for (ix = 0; ix < ERTS_PSD_SIZE; ix++) {
@@ -401,18 +446,18 @@ erts_pre_init_process(void)
 
 /* initialize the scheduler */
 void
-erts_init_process(void)
+erts_init_process(int ncpu)
 {
     Uint proc_bits = ERTS_PROC_BITS;
 
 #ifdef ERTS_SMP
     erts_disable_proc_not_running_opt = 0;
-    erts_init_proc_lock();
+    erts_init_proc_lock(ncpu);
 #endif
 
     init_proclist_alloc();
 
-    erts_smp_atomic_init(&process_count, 0);
+    erts_smp_atomic32_init_nob(&process_count, 0);
 
     if (erts_use_r9_pids_ports) {
 	proc_bits = ERTS_R9_PROC_BITS;
@@ -422,12 +467,6 @@ erts_init_process(void)
     process_tab = (Process**) erts_alloc(ERTS_ALC_T_PROC_TABLE,
 					 erts_max_processes*sizeof(Process*));
     sys_memzero(process_tab, erts_max_processes * sizeof(Process*));
-#ifdef HYBRID
-    erts_active_procs = (Process**)
-        erts_alloc(ERTS_ALC_T_ACTIVE_PROCS,
-                   erts_max_processes * sizeof(Process*));
-    erts_num_active_procs = 0;
-#endif
 
     erts_smp_mtx_init(&proc_tab_mtx, "proc_tab");
     p_last = -1;
@@ -437,9 +476,6 @@ erts_init_process(void)
     p_serial_shift = erts_fit_in_bits(erts_max_processes - 1);
     p_serial_mask = ((~(~((Uint) 0) << proc_bits)) >> p_serial_shift);
     erts_process_tab_index_mask = ~(~((Uint) 0) << p_serial_shift);
-#ifndef BM_COUNTERS
-    processes_busy = 0;
-#endif
     last_reductions = 0;
     last_exact_reductions = 0;
     erts_default_process_flags = 0;
@@ -486,6 +522,213 @@ erts_late_init_process(void)
 
 }
 
+static void
+init_sched_wall_time(ErtsSchedWallTime *swtp)
+{
+    swtp->enabled = 0;
+    swtp->start = 0;
+    swtp->working.total = 0;
+    swtp->working.start = 0;
+    swtp->working.currently = 0;
+}
+
+static ERTS_INLINE Uint64
+sched_wall_time_ts(void)
+{
+#ifdef HAVE_GETHRTIME
+    return (Uint64) sys_gethrtime();
+#else
+    Uint64 res;
+    SysTimeval tv;
+    sys_gettimeofday(&tv);
+    res = (Uint64) tv.tv_sec*1000000;
+    res += (Uint64) tv.tv_usec;
+    return res;
+#endif
+}
+
+static ERTS_INLINE void
+sched_wall_time_change(ErtsSchedulerData *esdp, int working)
+{
+    if (esdp->sched_wall_time.enabled) {
+	Uint64 ts = sched_wall_time_ts();
+	if (working) {
+#ifdef DEBUG
+	    ASSERT(!esdp->sched_wall_time.working.currently);
+	    esdp->sched_wall_time.working.currently = 1;
+#endif
+	    ts -= esdp->sched_wall_time.start;
+	    esdp->sched_wall_time.working.start = ts;
+	}
+	else {
+#ifdef DEBUG
+	    ASSERT(esdp->sched_wall_time.working.currently);
+	    esdp->sched_wall_time.working.currently = 0;
+#endif
+	    ts -= esdp->sched_wall_time.start;
+	    ts -= esdp->sched_wall_time.working.start;
+	    esdp->sched_wall_time.working.total += ts;
+	}
+    }
+}
+
+typedef struct {
+    int set;
+    int enable;
+    Process *proc;
+    Eterm ref;
+    Eterm ref_heap[REF_THING_SIZE];
+    Uint req_sched;
+    erts_smp_atomic32_t refc;
+} ErtsSchedWallTimeReq;
+
+#if !HALFWORD_HEAP
+ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(swtreq,
+				 ErtsSchedWallTimeReq,
+				 5,
+				 ERTS_ALC_T_SCHED_WTIME_REQ)
+#else
+static ERTS_INLINE ErtsSchedWallTimeReq *
+swtreq_alloc(void)
+{
+    return erts_alloc(ERTS_ALC_T_SCHED_WTIME_REQ,
+		      sizeof(ErtsSchedWallTimeReq));
+}
+
+static ERTS_INLINE void
+swtreq_free(ErtsSchedWallTimeReq *ptr)
+{
+    erts_free(ERTS_ALC_T_SCHED_WTIME_REQ, ptr);
+}
+#endif
+
+static void
+reply_sched_wall_time(void *vswtrp)
+{
+    Uint64 working = 0, total = 0;
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    ErtsSchedWallTimeReq *swtrp = (ErtsSchedWallTimeReq *) vswtrp;
+    ErtsProcLocks rp_locks = (swtrp->req_sched == esdp->no
+			      ? ERTS_PROC_LOCK_MAIN
+			      : 0);
+    Process *rp = swtrp->proc;
+    Eterm ref_copy = NIL, msg;
+    Eterm *hp = NULL;
+    Eterm **hpp;
+    Uint sz, *szp;
+    ErlOffHeap *ohp = NULL;
+    ErlHeapFragment *bp = NULL;
+
+    ASSERT(esdp);
+    
+    if (swtrp->set) {
+	if (!swtrp->enable && esdp->sched_wall_time.enabled)
+	    esdp->sched_wall_time.enabled = 0;
+	else if (swtrp->enable && !esdp->sched_wall_time.enabled) {
+	    Uint64 ts = sched_wall_time_ts();
+	    esdp->sched_wall_time.enabled = 1;
+	    esdp->sched_wall_time.start = ts;
+	    esdp->sched_wall_time.working.total = 0;
+	    esdp->sched_wall_time.working.start = 0;
+	    esdp->sched_wall_time.working.currently = 1;
+	}
+    }
+
+    if (esdp->sched_wall_time.enabled) {
+	Uint64 ts = sched_wall_time_ts();
+	ASSERT(esdp->sched_wall_time.working.currently);
+	ts -= esdp->sched_wall_time.start;
+	total = ts;
+	ts -= esdp->sched_wall_time.working.start;
+	working = esdp->sched_wall_time.working.total + ts;
+    }
+
+    sz = 0;
+    hpp = NULL;
+    szp = &sz;
+
+    while (1) {
+	if (hpp)
+	    ref_copy = STORE_NC(hpp, ohp, swtrp->ref);
+	else
+	    *szp += REF_THING_SIZE;
+
+	if (swtrp->set)
+	    msg = ref_copy;
+	else {
+	    msg = (!esdp->sched_wall_time.enabled
+		   ? am_notsup
+		   : erts_bld_tuple(hpp, szp, 3,
+				    make_small(esdp->no),
+				    erts_bld_uint64(hpp, szp, working),
+				    erts_bld_uint64(hpp, szp, total)));
+
+	    msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg);
+	}
+	if (hpp)
+	    break;
+
+	hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks);
+	szp = NULL;
+	hpp = &hp;
+    }
+
+    erts_queue_message(rp, &rp_locks, bp, msg, NIL
+#ifdef USE_VM_PROBES
+			   , NIL
+#endif
+		       );
+
+    if (swtrp->req_sched == esdp->no)
+	rp_locks &= ~ERTS_PROC_LOCK_MAIN;
+ 
+    if (rp_locks)
+	erts_smp_proc_unlock(rp, rp_locks);
+
+    erts_smp_proc_dec_refc(rp);
+
+    if (erts_smp_atomic32_dec_read_nob(&swtrp->refc) == 0)
+	swtreq_free(vswtrp);
+}
+
+Eterm
+erts_sched_wall_time_request(Process *c_p, int set, int enable)
+{
+    ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p);
+    Eterm ref;
+    ErtsSchedWallTimeReq *swtrp;
+    Eterm *hp;
+
+    if (!set && !esdp->sched_wall_time.enabled)
+	return THE_NON_VALUE;
+
+    swtrp = swtreq_alloc();
+    ref = erts_make_ref(c_p);
+    hp = &swtrp->ref_heap[0];
+
+    swtrp->set = set;
+    swtrp->enable = enable;
+    swtrp->proc = c_p;
+    swtrp->ref = STORE_NC(&hp, NULL, ref);
+    swtrp->req_sched = esdp->no;
+    erts_smp_atomic32_init_nob(&swtrp->refc,
+			       (erts_aint32_t) erts_no_schedulers);
+
+    erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers);
+
+#ifdef ERTS_SMP
+    if (erts_no_schedulers > 1)
+	erts_schedule_multi_misc_aux_work(1,
+					  erts_no_schedulers,
+					  reply_sched_wall_time,
+					  (void *) swtrp);
+#endif
+
+    reply_sched_wall_time((void *) swtrp);
+
+    return ref;
+}
+
 static ERTS_INLINE ErtsProcList *
 proclist_create(Process *p)
 {
@@ -572,21 +815,921 @@ erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data)
 
 #ifdef ERTS_SMP
 
+void
+erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags)
+{
+    switch (flags & ERTS_SSI_FLGS_SLEEP_TYPE) {
+    case ERTS_SSI_FLG_POLL_SLEEPING:
+	erts_sys_schedule_interrupt(1);
+	break;
+    case ERTS_SSI_FLG_POLL_SLEEPING|ERTS_SSI_FLG_TSE_SLEEPING:
+	/*
+	 * Thread progress blocking while poll sleeping; need
+	 * to signal on both...
+	 */
+	erts_sys_schedule_interrupt(1);
+	/* fall through */
+    case ERTS_SSI_FLG_TSE_SLEEPING:
+	erts_tse_set(ssi->event);
+	break;
+    case 0:
+	break;
+    default:
+	erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n",
+		 __FILE__, __LINE__);
+	break;
+    }
+}
+
+#endif
+
+static ERTS_INLINE void
+set_aux_work_flags_wakeup_nob(ErtsSchedulerSleepInfo *ssi,
+			      erts_aint32_t flgs)
+{
+    erts_aint32_t old_flgs;
+
+    ERTS_DBG_CHK_SSI_AUX_WORK(ssi);
+
+    old_flgs = erts_atomic32_read_nob(&ssi->aux_work);
+    if ((old_flgs & flgs) == 0) {
+
+	old_flgs = erts_atomic32_read_bor_nob(&ssi->aux_work, flgs);
+
+	if ((old_flgs & flgs) == 0) {
+#ifdef ERTS_SMP
+	    erts_sched_poke(ssi);
+#else
+	    erts_sys_schedule_interrupt(1);
+#endif
+	}
+    }
+}
+
+static ERTS_INLINE void
+set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi,
+			       erts_aint32_t flgs)
+{
+    erts_aint32_t old_flgs;
+
+    ERTS_DBG_CHK_SSI_AUX_WORK(ssi);
+
+    old_flgs = erts_atomic32_read_bor_relb(&ssi->aux_work, flgs);
+
+    if ((old_flgs & flgs) == 0) {
+#ifdef ERTS_SMP
+	erts_sched_poke(ssi);
+#else
+	erts_sys_schedule_interrupt(1);
+#endif
+    }
+}
+
+static ERTS_INLINE erts_aint32_t
+set_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs)
+{
+    return erts_atomic32_read_bor_nob(&ssi->aux_work, flgs);
+}
+
+static ERTS_INLINE erts_aint32_t
+unset_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs)
+{
+    return erts_atomic32_read_band_nob(&ssi->aux_work, ~flgs);
+}
+
+#ifdef ERTS_SMP
+
+static ERTS_INLINE void
+haw_thr_prgr_current_reset(ErtsAuxWorkData *awdp)
+{
+    awdp->current_thr_prgr = ERTS_THR_PRGR_INVALID;
+}
+
+static ERTS_INLINE ErtsThrPrgrVal
+haw_thr_prgr_current(ErtsAuxWorkData *awdp)
+{
+    ErtsThrPrgrVal current = awdp->current_thr_prgr;
+    if (current == ERTS_THR_PRGR_INVALID) {
+	current = erts_thr_progress_current();
+	awdp->current_thr_prgr = current;
+    }
+    return current;
+}
+
+static ERTS_INLINE void
+haw_thr_prgr_current_check_progress(ErtsAuxWorkData *awdp)
+{
+    ErtsThrPrgrVal current = awdp->current_thr_prgr;
+    if (current != ERTS_THR_PRGR_INVALID
+	&& !erts_thr_progress_equal(current, erts_thr_progress_current())) {
+	/*
+	 * We have used a previouly read current value that isn't the
+	 * latest; need to poke ourselfs in order to guarantee no loss
+	 * of wakeups.
+	 */
+	erts_sched_poke(awdp->ssi);
+    }
+}
+
+static ERTS_INLINE erts_aint32_t
+handle_delayed_aux_work_wakeup(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
+{
+    int jix, max_jix;
+    unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP);
+
+    ERTS_THR_MEMORY_BARRIER;
+
+    max_jix = awdp->delayed_wakeup.jix;
+    awdp->delayed_wakeup.jix = -1;
+    for (jix = 0; jix <= max_jix; jix++) {
+	int sched = awdp->delayed_wakeup.job[jix].sched;
+	erts_aint32_t aux_work = awdp->delayed_wakeup.job[jix].aux_work;
+
+	ASSERT(awdp->delayed_wakeup.sched2jix[sched] == jix);
+	awdp->delayed_wakeup.sched2jix[sched] = -1;
+	set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(sched-1),
+				      aux_work);
+    }
+    return aux_work & ~ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP;
+}
+
+static ERTS_INLINE void
+schedule_aux_work_wakeup(ErtsAuxWorkData *awdp, int sched, erts_aint32_t aux_work)
+{
+    int jix = awdp->delayed_wakeup.sched2jix[sched];
+    if (jix >= 0) {
+	ASSERT(awdp->delayed_wakeup.job[jix].sched == sched);
+	awdp->delayed_wakeup.job[jix].aux_work |= aux_work;
+    }
+    else {
+	jix = ++awdp->delayed_wakeup.jix;
+	awdp->delayed_wakeup.sched2jix[sched] = jix;
+	awdp->delayed_wakeup.job[jix].sched = sched;
+	awdp->delayed_wakeup.job[jix].aux_work = aux_work;
+    }
+    set_aux_work_flags_wakeup_nob(awdp->ssi, ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP);
+}
+
+#endif
+
+typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t;
+struct erts_misc_aux_work_t_ {
+    void (*func)(void *);
+    void *arg;
+};
+
+ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_aux_work,
+				 erts_misc_aux_work_t,
+				 200,
+				 ERTS_ALC_T_MISC_AUX_WORK)
+
+typedef union {
+    ErtsThrQ_t q;
+    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsThrQ_t))];
+} erts_algnd_misc_aux_work_q_t;
+
+static erts_algnd_misc_aux_work_q_t *misc_aux_work_queues;
+
 static void
-prepare_for_block(void *vrq)
+notify_aux_work(void *vssi)
 {
-    erts_smp_runq_unlock((ErtsRunQueue *) vrq);
+    set_aux_work_flags_wakeup_nob((ErtsSchedulerSleepInfo *) vssi,
+				  ERTS_SSI_AUX_WORK_MISC);
 }
 
 static void
-resume_after_block(void *vrq)
+init_misc_aux_work(void)
 {
-    erts_smp_runq_lock((ErtsRunQueue *) vrq);
+    int ix;
+    ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT;
+    qinit.notify = notify_aux_work;
+
+    init_misc_aux_work_alloc();
+
+    misc_aux_work_queues = 
+	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_MISC_AUX_WORK_Q,
+					   sizeof(erts_algnd_misc_aux_work_q_t)
+					   * (erts_no_schedulers+1));
+
+#ifdef ERTS_SMP
+    ix = 0; /* aux_thread + schedulers */
+#else
+    ix = 1; /* scheduler only */
+#endif
+
+    for (; ix <= erts_no_schedulers; ix++) {
+	qinit.arg = (void *) ERTS_SCHED_SLEEP_INFO_IX(ix-1);
+	erts_thr_q_initialize(&misc_aux_work_queues[ix].q, &qinit);
+    }
+}
+
+static erts_aint32_t
+misc_aux_work_clean(ErtsThrQ_t *q,
+		    ErtsAuxWorkData *awdp,
+		    erts_aint32_t aux_work)
+{
+    switch (erts_thr_q_clean(q)) {
+    case ERTS_THR_Q_DIRTY:
+	set_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC);
+	return aux_work | ERTS_SSI_AUX_WORK_MISC;
+    case ERTS_THR_Q_NEED_THR_PRGR:
+#ifdef ERTS_SMP
+	set_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR);
+	erts_thr_progress_wakeup(awdp->esdp,
+				 erts_thr_q_need_thr_progress(q));
+#endif
+    case ERTS_THR_Q_CLEAN:
+	break;
+    }
+    return aux_work;
+}
+
+static ERTS_INLINE erts_aint32_t
+handle_misc_aux_work(ErtsAuxWorkData *awdp,
+		     erts_aint32_t aux_work)
+{
+    ErtsThrQ_t *q = &misc_aux_work_queues[awdp->sched_id].q;
+
+    unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC);
+    while (1) {
+	erts_misc_aux_work_t *mawp = erts_thr_q_dequeue(q);
+	if (!mawp)
+	    break;
+	mawp->func(mawp->arg);
+	misc_aux_work_free(mawp);
+    }
+
+    return misc_aux_work_clean(q, awdp, aux_work & ~ERTS_SSI_AUX_WORK_MISC);
+}
+
+#ifdef ERTS_SMP
+
+static ERTS_INLINE erts_aint32_t
+handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp,
+			      erts_aint32_t aux_work)
+{
+    if (!erts_thr_progress_has_reached_this(haw_thr_prgr_current(awdp),
+					    awdp->misc.thr_prgr))
+	return aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR;
+
+    unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR);
+
+    return misc_aux_work_clean(&misc_aux_work_queues[awdp->sched_id].q,
+			       awdp,
+			       aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR);
 }
 
 #endif
 
 static ERTS_INLINE void
+schedule_misc_aux_work(int sched_id,
+		       void (*func)(void *),
+		       void *arg)
+{
+    ErtsThrQ_t *q;
+    erts_misc_aux_work_t *mawp;
+
+#ifdef ERTS_SMP
+    ASSERT(0 <= sched_id && sched_id <= erts_no_schedulers);
+#else
+    ASSERT(sched_id == 1);
+#endif
+
+    q = &misc_aux_work_queues[sched_id].q;
+    mawp = misc_aux_work_alloc();
+    mawp->func = func;
+    mawp->arg = arg;
+    erts_thr_q_enqueue(q, mawp);
+}
+
+void
+erts_schedule_misc_aux_work(int sched_id,
+			    void (*func)(void *),
+			    void *arg)
+{
+    schedule_misc_aux_work(sched_id, func, arg);
+}
+
+void
+erts_schedule_multi_misc_aux_work(int ignore_self,
+				  int max_sched,
+				  void (*func)(void *),
+				  void *arg)
+{
+    int id, self = 0;
+
+    if (ignore_self) {
+	ErtsSchedulerData *esdp = erts_get_scheduler_data();
+	if (esdp)
+	    self = (int) esdp->no;
+    }
+
+    ASSERT(0 < max_sched && max_sched <= erts_no_schedulers);
+
+    for (id = 1; id <= max_sched; id++) {
+	if (id == self)
+	    continue;
+	schedule_misc_aux_work(id, func, arg);
+   }
+}
+
+#if ERTS_USE_ASYNC_READY_Q
+
+void
+erts_notify_check_async_ready_queue(void *vno)
+{
+    int ix = ((int) (SWord) vno) -1;
+    set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(ix),
+				  ERTS_SSI_AUX_WORK_ASYNC_READY);
+}
+
+static ERTS_INLINE erts_aint32_t
+handle_async_ready(ErtsAuxWorkData *awdp,
+		   erts_aint32_t aux_work)
+{
+    ErtsSchedulerSleepInfo *ssi = awdp->ssi;
+    unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY);
+    if (erts_check_async_ready(awdp->async_ready.queue)) {
+	if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY)
+	    & ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN) {
+	    unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN);
+	    aux_work &= ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
+	}
+	return aux_work;
+    }
+#ifdef ERTS_SMP
+    awdp->async_ready.need_thr_prgr = 0;
+#endif
+    set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN);
+    return ((aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY)
+	    | ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN);
+}
+
+static ERTS_INLINE erts_aint32_t
+handle_async_ready_clean(ErtsAuxWorkData *awdp,
+			 erts_aint32_t aux_work)
+{
+    void *thr_prgr_p;
+
+#ifdef ERTS_SMP
+    if (awdp->async_ready.need_thr_prgr
+	&& !erts_thr_progress_has_reached_this(haw_thr_prgr_current(awdp),
+					       awdp->async_ready.thr_prgr)) {
+	return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
+    }
+
+    awdp->async_ready.need_thr_prgr = 0;
+    thr_prgr_p = (void *) &awdp->async_ready.thr_prgr;
+#else
+    thr_prgr_p = NULL;
+#endif
+
+    switch (erts_async_ready_clean(awdp->async_ready.queue, thr_prgr_p)) {
+    case ERTS_ASYNC_READY_CLEAN:
+	unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN);
+	return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
+#ifdef ERTS_SMP
+    case ERTS_ASYNC_READY_NEED_THR_PRGR:
+	erts_thr_progress_wakeup(awdp->esdp,
+				 awdp->async_ready.thr_prgr);
+	awdp->async_ready.need_thr_prgr = 1;
+	return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
+#endif
+    default:
+	return aux_work;
+    }
+}
+
+#endif
+
+static ERTS_INLINE erts_aint32_t
+handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
+{
+    ErtsSchedulerSleepInfo *ssi = awdp->ssi;
+    erts_aint32_t res;
+
+    unset_aux_work_flags(ssi, (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
+			       | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC));
+    aux_work &= ~(ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
+		  | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC);
+    res = erts_alloc_fix_alloc_shrink(awdp->sched_id, aux_work);
+    if (res) {
+	set_aux_work_flags(ssi, res);
+	aux_work |= res;
+    }
+
+    return aux_work;
+}
+
+#ifdef ERTS_SMP
+
+void
+erts_alloc_notify_delayed_dealloc(int ix)
+{
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    if (esdp)
+	schedule_aux_work_wakeup(&esdp->aux_work_data,
+				 ix,
+				 ERTS_SSI_AUX_WORK_DD);
+    else
+	set_aux_work_flags_wakeup_relb(ERTS_SCHED_SLEEP_INFO_IX(ix-1),
+				       ERTS_SSI_AUX_WORK_DD);
+}
+
+static ERTS_INLINE erts_aint32_t
+handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
+{
+    ErtsSchedulerSleepInfo *ssi = awdp->ssi;
+    int need_thr_progress = 0;
+    ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
+    int more_work = 0;
+
+    unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD);
+    erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp,
+						&need_thr_progress,
+						&wakeup,
+						&more_work);
+    if (more_work) {
+	if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD)
+	    & ERTS_SSI_AUX_WORK_DD_THR_PRGR) {
+	    unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR);
+	    aux_work &= ~ERTS_SSI_AUX_WORK_DD_THR_PRGR;
+	}
+	return aux_work;
+    }
+
+    if (need_thr_progress) {
+	if (wakeup == ERTS_THR_PRGR_INVALID)
+	    wakeup = erts_thr_progress_later(awdp->esdp);
+	awdp->dd.thr_prgr = wakeup;
+	set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR);
+	awdp->dd.thr_prgr = wakeup;
+	erts_thr_progress_wakeup(awdp->esdp, wakeup);
+    }
+    else if (awdp->dd.completed_callback) {
+	awdp->dd.completed_callback(awdp->dd.completed_arg);
+	awdp->dd.completed_callback = NULL;
+	awdp->dd.completed_arg = NULL;
+    }
+    return aux_work & ~ERTS_SSI_AUX_WORK_DD;
+}
+
+static ERTS_INLINE erts_aint32_t
+handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
+{
+    ErtsSchedulerSleepInfo *ssi;
+    int need_thr_progress;
+    int more_work;
+    ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
+    ErtsThrPrgrVal current = haw_thr_prgr_current(awdp);
+
+    if (!erts_thr_progress_has_reached_this(current, awdp->dd.thr_prgr))
+	return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR;
+
+    ssi = awdp->ssi;
+    need_thr_progress = 0;
+    more_work = 0;
+
+    erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp,
+						&need_thr_progress,
+						&wakeup,
+						&more_work);
+    if (more_work) {
+	set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD);
+	unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR);
+	return ((aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR)
+		| ERTS_SSI_AUX_WORK_DD);
+    }
+
+    if (need_thr_progress) {
+	if (wakeup == ERTS_THR_PRGR_INVALID)
+	    wakeup = erts_thr_progress_later(awdp->esdp);
+	awdp->dd.thr_prgr = wakeup;
+	erts_thr_progress_wakeup(awdp->esdp, wakeup);
+    }
+    else {
+	unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR);
+	if (awdp->dd.completed_callback) {
+	    awdp->dd.completed_callback(awdp->dd.completed_arg);
+	    awdp->dd.completed_callback = NULL;
+	    awdp->dd.completed_arg = NULL;
+	}
+    }
+
+    return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR;
+}
+
+static erts_atomic32_t completed_dealloc_count;
+
+static void
+completed_dealloc(void *vproc)
+{
+    if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == 0) {
+	erts_resume((Process *) vproc, (ErtsProcLocks) 0);
+	erts_smp_proc_dec_refc((Process *) vproc);
+    }
+}
+
+static void
+setup_completed_dealloc(void *vproc)
+{
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    ErtsAuxWorkData *awdp = (esdp
+			     ? &esdp->aux_work_data
+			     : aux_thread_aux_work_data);
+    erts_alloc_fix_alloc_shrink(awdp->sched_id, 0);
+    set_aux_work_flags_wakeup_nob(awdp->ssi, ERTS_SSI_AUX_WORK_DD);
+    awdp->dd.completed_callback = completed_dealloc;
+    awdp->dd.completed_arg = vproc;
+}
+
+static void
+prep_setup_completed_dealloc(void *vproc)
+{
+    erts_aint32_t count = (erts_aint32_t) (erts_no_schedulers+1);
+    if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == count) {
+	/* scheduler threads */
+	erts_schedule_multi_misc_aux_work(0,
+					  erts_no_schedulers,
+					  setup_completed_dealloc,
+					  vproc);
+	/* aux_thread */
+	erts_schedule_misc_aux_work(0,
+				    setup_completed_dealloc,
+				    vproc);
+    }
+}
+
+#endif /* ERTS_SMP */
+
+int
+erts_debug_wait_deallocations(Process *c_p)
+{
+#ifndef ERTS_SMP
+    erts_alloc_fix_alloc_shrink(1, 0);
+    return 1;
+#else
+    /* Only one process at a time can do this */
+    erts_aint32_t count = (erts_aint32_t) (2*(erts_no_schedulers+1));
+    if (0 == erts_atomic32_cmpxchg_mb(&completed_dealloc_count,
+				      count,
+				      0)) {
+	erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
+	erts_smp_proc_inc_refc(c_p);
+	/* scheduler threads */
+	erts_schedule_multi_misc_aux_work(0,
+					  erts_no_schedulers,
+					  prep_setup_completed_dealloc,
+					  (void *) c_p);
+	/* aux_thread */
+	erts_schedule_misc_aux_work(0,
+				    prep_setup_completed_dealloc,
+				    (void *) c_p);
+	return 1;
+    }
+    return 0;
+#endif
+}
+
+
+#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
+void
+erts_smp_notify_check_children_needed(void)
+{
+    int i;
+    for (i = 0; i < erts_no_schedulers; i++)
+	set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(i),
+				      ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
+}
+
+static ERTS_INLINE erts_aint32_t
+handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
+{
+    unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
+    erts_check_children();
+    return aux_work & ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN;
+}
+
+#endif
+
+static void
+notify_reap_ports_relb(void)
+{
+    int i;
+    for (i = 0; i < erts_no_schedulers; i++) {
+	set_aux_work_flags_wakeup_relb(ERTS_SCHED_SLEEP_INFO_IX(i),
+				       ERTS_SSI_AUX_WORK_REAP_PORTS);
+    }
+}
+
+erts_smp_atomic32_t erts_halt_progress;
+int erts_halt_code;
+
+static ERTS_INLINE erts_aint32_t
+handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
+{
+    unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_REAP_PORTS);
+    awdp->esdp->run_queue->halt_in_progress = 1;
+    if (erts_smp_atomic32_dec_read_acqb(&erts_halt_progress) == 0) {
+	int i;
+	erts_smp_atomic32_set_nob(&erts_halt_progress, 1);
+	for (i = 0; i < erts_max_ports; i++) {
+	    Port *prt = &erts_port[i];
+	    erts_smp_port_state_lock(prt);
+	    if ((prt->status & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
+				| ERTS_PORT_SFLG_HALT))) {
+		erts_smp_port_state_unlock(prt);
+		continue;
+	    }
+	    /* We need to set the halt flag - get the port lock */
+#ifdef ERTS_SMP
+	    erts_smp_atomic_inc_nob(&prt->refc);
+#endif
+	    erts_smp_port_state_unlock(prt);
+#ifdef ERTS_SMP
+	    erts_smp_mtx_lock(prt->lock);
+#endif
+	    if ((prt->status & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP
+				| ERTS_PORT_SFLG_HALT))) {
+		erts_port_release(prt);
+		continue;
+	    }
+	    erts_port_status_bor_set(prt, ERTS_PORT_SFLG_HALT);
+	    erts_smp_atomic32_inc_nob(&erts_halt_progress);
+	    if (prt->status & (ERTS_PORT_SFLG_EXITING
+			       | ERTS_PORT_SFLG_CLOSING)) {
+		erts_port_release(prt);
+		continue;
+	    }
+	    erts_do_exit_port(prt, prt->id, am_killed);
+	    erts_port_release(prt);
+	}
+	if (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0) {
+	    erl_exit_flush_async(erts_halt_code, "");
+	}
+    }
+    return aux_work & ~ERTS_SSI_AUX_WORK_REAP_PORTS;
+}
+
+#if HAVE_ERTS_MSEG
+
+static ERTS_INLINE erts_aint32_t
+handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
+{
+    unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK);
+    erts_mseg_cache_check();
+    return aux_work & ~ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK;
+}
+
+#endif
+
+static ERTS_INLINE erts_aint32_t
+handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
+{
+    unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_SET_TMO);
+    setup_aux_work_timer();
+    return aux_work & ~ERTS_SSI_AUX_WORK_SET_TMO;
+}
+
+static erts_aint32_t
+handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
+{
+#undef HANDLE_AUX_WORK
+#define HANDLE_AUX_WORK(FLG, HNDLR) \
+    ignore |= FLG; \
+    if (aux_work & FLG) { \
+	aux_work = HNDLR(awdp, aux_work); \
+	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \
+	if (!(aux_work & ~ignore)) { \
+	    ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \
+	    return aux_work; \
+	} \
+    }
+
+    erts_aint32_t aux_work = orig_aux_work;
+    erts_aint32_t ignore = 0;
+
+#ifdef ERTS_SMP
+    haw_thr_prgr_current_reset(awdp);
+#endif
+
+    ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
+    ASSERT(aux_work);
+
+    /*
+     * Handlers are *only* allowed to modify flags in return value
+     * and ssi flags that are explicity handled by the handler.
+     * Handlers are, e.g., not allowed to read the ssi flag field and
+     * then unconditionally return that value.
+     *
+     * Flag field returned should only contain flags for work that
+     * can continue immediately.
+     */
+
+    /*
+     * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative
+     * eachother. Most frequent first.
+     */
+#ifdef ERTS_SMP
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP,
+		    handle_delayed_aux_work_wakeup);
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD,
+		    handle_delayed_dealloc);
+    /* DD must be before DD_THR_PRGR */
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD_THR_PRGR,
+		    handle_delayed_dealloc_thr_prgr);
+#endif
+
+    HANDLE_AUX_WORK((ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
+		     | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
+		    handle_fix_alloc);
+
+#if ERTS_USE_ASYNC_READY_Q
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY,
+		    handle_async_ready);
+    /* ASYNC_READY must be before ASYNC_READY_CLEAN */
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN,
+		    handle_async_ready_clean);
+#endif
+
+#ifdef ERTS_SMP
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC_THR_PRGR,
+		    handle_misc_aux_work_thr_prgr);
+#endif
+    /* MISC_THR_PRGR must be before MISC */
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC,
+		    handle_misc_aux_work);
+
+#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CHECK_CHILDREN,
+		    handle_check_children);
+#endif
+
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_SET_TMO,
+		    handle_setup_aux_work_timer);
+
+#if HAVE_ERTS_MSEG
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK,
+		    handle_mseg_cache_check);
+#endif
+
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_REAP_PORTS,
+		    handle_reap_ports);
+
+    ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
+
+#ifdef ERTS_SMP
+    if (waiting && !aux_work)
+	haw_thr_prgr_current_check_progress(awdp);
+#endif
+
+    return aux_work;
+
+#undef HANDLE_AUX_WORK
+
+}
+
+typedef struct {
+    union {
+	ErlTimer data;
+	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErlTimer))];
+    } timer;
+
+    int initialized;
+    erts_atomic32_t refc;
+    erts_atomic32_t type[1];
+} ErtsAuxWorkTmo;
+
+static ErtsAuxWorkTmo *aux_work_tmo;
+
+static void
+aux_work_timeout_early_init(int no_schedulers)
+{
+    int i;
+    UWord p;
+
+    /*
+     * This is done really early. Our own allocators have
+     * not been started yet.
+     */
+
+    p = (UWord) malloc((sizeof(ErtsAuxWorkTmo)
+			+ sizeof(erts_atomic32_t)*(no_schedulers+1))
+		       + ERTS_CACHE_LINE_SIZE-1);
+    if (p & ERTS_CACHE_LINE_MASK)
+	p = (p & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE;
+    ASSERT((p & ERTS_CACHE_LINE_MASK) == 0);
+
+    aux_work_tmo = (ErtsAuxWorkTmo *) p;
+    aux_work_tmo->initialized = 0;
+    erts_atomic32_init_nob(&aux_work_tmo->refc, 0);
+    for (i = 0; i <= no_schedulers; i++)
+	erts_atomic32_init_nob(&aux_work_tmo->type[i], 0);
+}
+
+void
+aux_work_timeout_late_init(void)
+{
+    aux_work_tmo->initialized = 1;
+    if (erts_atomic32_read_nob(&aux_work_tmo->refc)) {
+	aux_work_tmo->timer.data.active = 0;
+	erts_set_timer(&aux_work_tmo->timer.data,
+		       aux_work_timeout,
+		       NULL,
+		       NULL,
+		       1000);
+    }
+}
+
+static void
+aux_work_timeout(void *unused)
+{
+    erts_aint32_t refc;
+    int i;
+#ifdef ERTS_SMP
+    i = 0;
+#else
+    i = 1;
+#endif
+
+    for (; i <= erts_no_schedulers; i++) {
+	erts_aint32_t type;
+	type = erts_atomic32_read_acqb(&aux_work_tmo->type[i]);
+	if (type)
+	    set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(i-1),
+					  type);
+    }
+
+    refc = erts_atomic32_read_nob(&aux_work_tmo->refc);
+    ASSERT(refc >= 1);
+    if (refc != 1
+	|| 1 != erts_atomic32_cmpxchg_relb(&aux_work_tmo->refc, 0, 1)) {
+	/* Setup next timeout... */
+	aux_work_tmo->timer.data.active = 0;
+	erts_set_timer(&aux_work_tmo->timer.data,
+		       aux_work_timeout,
+		       NULL,
+		       NULL,
+		       1000);
+    }
+}
+
+static void
+setup_aux_work_timer(void)
+{
+#ifndef ERTS_SMP
+    if (!erts_get_scheduler_data())
+	set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(0),
+				      ERTS_SSI_AUX_WORK_SET_TMO);
+    else
+#endif
+    {
+	aux_work_tmo->timer.data.active = 0;
+	erts_set_timer(&aux_work_tmo->timer.data,
+		       aux_work_timeout,
+		       NULL,
+		       NULL,
+		       1000);
+    }
+}
+
+erts_aint32_t
+erts_set_aux_work_timeout(int ix, erts_aint32_t type, int enable)
+{
+    erts_aint32_t old, refc;
+
+#ifndef ERTS_SMP
+    ix = 1;
+#endif
+
+    ERTS_DBG_CHK_AUX_WORK_VAL(type);
+    ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&aux_work_tmo->type[ix]));
+//    erts_fprintf(stderr, "t(%d, 0x%x, %d)\n", ix, type, enable);
+
+    if (!enable) {
+	old = erts_atomic32_read_band_mb(&aux_work_tmo->type[ix], ~type);
+	ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&aux_work_tmo->type[ix]));
+	if (old != 0 && (old & ~type) == 0)
+	    erts_atomic32_dec_relb(&aux_work_tmo->refc);
+	return old;
+    }
+
+    old = erts_atomic32_read_bor_mb(&aux_work_tmo->type[ix], type);
+    ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&aux_work_tmo->type[ix]));
+    if (old == 0 && type != 0) {
+	refc = erts_atomic32_inc_read_acqb(&aux_work_tmo->refc);
+	if (refc == 1) {
+	    erts_atomic32_inc_acqb(&aux_work_tmo->refc);
+	    if (aux_work_tmo->initialized) 
+		setup_aux_work_timer();
+	}
+    }
+    return old;
+}
+
+
+
+static ERTS_INLINE void
 sched_waiting_sys(Uint no, ErtsRunQueue *rq)
 {
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
@@ -614,8 +1757,6 @@ sched_active_sys(Uint no, ErtsRunQueue *rq)
 Uint
 erts_active_schedulers(void)
 {
-    /* RRRRRRRRR */
-
     Uint as = erts_no_schedulers;
 
     ERTS_ATOMIC_FOREACH_RUNQ(rq, as -= abs(rq->waiting));
@@ -627,6 +1768,46 @@ erts_active_schedulers(void)
 #ifdef ERTS_SMP
 
 static ERTS_INLINE void
+clear_sys_scheduling(void)
+{
+    erts_smp_atomic32_set_mb(&doing_sys_schedule, 0);
+}
+
+static ERTS_INLINE int
+try_set_sys_scheduling(void)
+{
+    return 0 == erts_smp_atomic32_cmpxchg_acqb(&doing_sys_schedule, 1, 0);
+}
+
+#endif
+
+static ERTS_INLINE int
+prepare_for_sys_schedule(void)
+{
+#ifdef ERTS_SMP
+    while (!erts_port_task_have_outstanding_io_tasks()
+	   && try_set_sys_scheduling()) {
+	if (!erts_port_task_have_outstanding_io_tasks())
+	    return 1;
+	clear_sys_scheduling();
+    }
+    return 0;
+#else
+    return !erts_port_task_have_outstanding_io_tasks();
+#endif
+}
+
+#ifdef ERTS_SMP
+
+static ERTS_INLINE void
+sched_change_waiting_sys_to_waiting(Uint no, ErtsRunQueue *rq)
+{
+    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    ASSERT(rq->waiting < 0);
+    rq->waiting *= -1;
+}
+
+static ERTS_INLINE void
 sched_waiting(Uint no, ErtsRunQueue *rq)
 {
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
@@ -656,287 +1837,663 @@ sched_active(Uint no, ErtsRunQueue *rq)
 static int ERTS_INLINE
 ongoing_multi_scheduling_block(void)
 {
-    return erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing) != 0;
+    ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&schdlr_sspnd.mtx));
+    return schdlr_sspnd.msb.ongoing;
 }
 
 static ERTS_INLINE void
 empty_runq(ErtsRunQueue *rq)
 {
-    long oifls = erts_smp_atomic_band(&rq->info_flags, ~ERTS_RUNQ_IFLG_NONEMPTY);
+    erts_aint32_t oifls = erts_smp_atomic32_read_band_nob(&rq->info_flags,
+							  ~ERTS_RUNQ_IFLG_NONEMPTY);
     if (oifls & ERTS_RUNQ_IFLG_NONEMPTY) {
 #ifdef DEBUG
-	long empty = erts_smp_atomic_read(&no_empty_run_queues);
-	ASSERT(0 <= empty && empty < erts_no_run_queues);
+	erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues);
+	/*
+	 * For a short period of time no_empty_run_queues may have
+	 * been increased twice for a specific run queue.
+	 */
+	ASSERT(0 <= empty && empty < 2*erts_no_run_queues);
 #endif
-	erts_smp_atomic_inc(&no_empty_run_queues);
+	erts_smp_atomic32_inc_relb(&no_empty_run_queues);
     }
 }
 
 static ERTS_INLINE void
 non_empty_runq(ErtsRunQueue *rq)
 {
-    long oifls = erts_smp_atomic_bor(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY);
+    erts_aint32_t oifls = erts_smp_atomic32_read_bor_nob(&rq->info_flags,
+							 ERTS_RUNQ_IFLG_NONEMPTY);
     if (!(oifls & ERTS_RUNQ_IFLG_NONEMPTY)) {
 #ifdef DEBUG
-	long empty = erts_smp_atomic_read(&no_empty_run_queues);
-	ASSERT(0 < empty && empty <= erts_no_run_queues);
+	erts_aint32_t empty = erts_smp_atomic32_read_nob(&no_empty_run_queues);
+	/*
+	 * For a short period of time no_empty_run_queues may have
+	 * been increased twice for a specific run queue.
+	 */
+	ASSERT(0 < empty && empty <= 2*erts_no_run_queues);
 #endif
-	erts_smp_atomic_dec(&no_empty_run_queues);
+	erts_smp_atomic32_dec_relb(&no_empty_run_queues);
     }
 }
 
-static ERTS_INLINE int
-sched_spin_wake(ErtsRunQueue *rq)
+static erts_aint32_t
+sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi)
 {
-#if ERTS_SCHED_SLEEP_SPINCOUNT == 0
-    return 0;
-#else
-    long val;
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    erts_aint32_t oflgs;
+    erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING
+			   | ERTS_SSI_FLG_WAITING);
+    erts_aint32_t xflgs = 0;
 
-    val = erts_smp_atomic_read(&rq->spin_waiter);
-    ASSERT(val >= 0);
-    if (val != 0) {
-	erts_smp_atomic_inc(&rq->spin_wake);
-	return 1;
+    do {
+	oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
+	if (oflgs == xflgs)
+	    return nflgs;
+	xflgs = oflgs;
+    } while (!(oflgs & ERTS_SSI_FLG_SUSPENDED));
+    return oflgs;
+}
+
+static erts_aint32_t
+sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi)
+{
+    erts_aint32_t oflgs;
+    erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING
+			   | ERTS_SSI_FLG_WAITING);
+    erts_aint32_t xflgs = ERTS_SSI_FLG_WAITING;
+
+    do {
+	oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
+	if (oflgs == xflgs)
+	    return nflgs;
+	xflgs = oflgs;
+	nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED;
+    } while (oflgs & ERTS_SSI_FLG_WAITING);
+    return oflgs;
+}
+
+static erts_aint32_t
+sched_spin_wait(ErtsSchedulerSleepInfo *ssi, int spincount)
+{
+    int until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD;
+    int sc = spincount;
+    erts_aint32_t flgs;
+
+    do {
+	flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
+	if ((flgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING))
+	    != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) {
+	    break;
+	}
+	ERTS_SPIN_BODY;
+	if (--until_yield == 0) {
+	    until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD;
+	    erts_thr_yield();
+	}
+    } while (--sc > 0);
+    return flgs;
+}
+
+static erts_aint32_t
+sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type)
+{
+    erts_aint32_t oflgs;
+    erts_aint32_t nflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING|sleep_type;
+    erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING;
+
+    if (sleep_type == ERTS_SSI_FLG_TSE_SLEEPING)
+	erts_tse_reset(ssi->event);
+    else {
+	ASSERT(sleep_type == ERTS_SSI_FLG_POLL_SLEEPING);
+	erts_sys_schedule_interrupt(0);
+    }
+
+    while (1) {
+	oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
+	if (oflgs == xflgs)
+	    return nflgs;
+	if ((oflgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING))
+	    != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) {
+	    return oflgs;
+	}
+	xflgs = oflgs;
+	nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED;
     }
-    return 0;
-#endif
 }
 
-static ERTS_INLINE int
-sched_spin_wake_all(ErtsRunQueue *rq)
+#define ERTS_SCHED_WAIT_WOKEN(FLGS)				\
+  (((FLGS) & (ERTS_SSI_FLG_WAITING|ERTS_SSI_FLG_SUSPENDED))	\
+   != ERTS_SSI_FLG_WAITING)
+
+
+static void
+thr_prgr_wakeup(void *vssi)
 {
-#if ERTS_SCHED_SLEEP_SPINCOUNT == 0
-    return 0;
-#else
-    long val;
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    erts_sched_poke((ErtsSchedulerSleepInfo *) vssi);
+}
 
-    val = erts_smp_atomic_read(&rq->spin_waiter);
-    ASSERT(val >= 0);
-    if (val != 0)
-	erts_smp_atomic_add(&rq->spin_wake, val);
-    return val;
-#endif
+static void
+thr_prgr_prep_wait(void *vssi)
+{
+    ErtsSchedulerSleepInfo *ssi = (ErtsSchedulerSleepInfo *) vssi;
+    erts_smp_atomic32_read_bor_acqb(&ssi->flags,
+				    ERTS_SSI_FLG_SLEEPING);
 }
 
 static void
-sched_sys_wait(Uint no, ErtsRunQueue *rq)
+thr_prgr_wait(void *vssi)
 {
-    long dt;
-#if ERTS_SCHED_SLEEP_SPINCOUNT != 0
-    int val;
-    int spincount = ERTS_SCHED_SLEEP_SPINCOUNT;
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+    ErtsSchedulerSleepInfo *ssi = (ErtsSchedulerSleepInfo *) vssi;
+    erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING;
 
-#endif
+    erts_tse_reset(ssi->event);
+
+    while (1) {
+	erts_aint32_t aflgs, nflgs;
+	nflgs = xflgs | ERTS_SSI_FLG_TSE_SLEEPING;
+	aflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
+	if (aflgs == xflgs) {
+	    erts_tse_wait(ssi->event);
+	    break;
+	}
+	if ((aflgs & ERTS_SSI_FLG_SLEEPING) == 0)
+	    break;
+	xflgs = aflgs;
+    }
+}
 
-    sched_waiting_sys(no, rq);
+static void
+thr_prgr_fin_wait(void *vssi)
+{
+    ErtsSchedulerSleepInfo *ssi = (ErtsSchedulerSleepInfo *) vssi;
+    erts_smp_atomic32_read_band_nob(&ssi->flags,
+				    ~(ERTS_SSI_FLG_SLEEPING
+				      | ERTS_SSI_FLG_TSE_SLEEPING));
+}
 
-#if ERTS_SCHED_SLEEP_SPINCOUNT != 0
-    erts_smp_atomic_inc(&rq->spin_waiter);
-    erts_smp_runq_unlock(rq);
+static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp);
+
+static void *
+aux_thread(void *unused)
+{
+    ErtsAuxWorkData *awdp = aux_thread_aux_work_data;
+    ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(-1);
+    erts_aint32_t aux_work;
+    ErtsThrPrgrCallbacks callbacks;
+    int thr_prgr_active = 1;
 
-    erl_sys_schedule(1); /* Might give us something to do */
+    ssi->event = erts_tse_fetch();
 
-    dt = do_time_read_and_reset();
-    if (dt) bump_timer(dt);
+    callbacks.arg = (void *) ssi;
+    callbacks.wakeup = thr_prgr_wakeup;
+    callbacks.prepare_wait = thr_prgr_prep_wait;
+    callbacks.wait = thr_prgr_wait;
+    callbacks.finalize_wait = thr_prgr_fin_wait;
 
-    while (spincount-- > 0) {
-	val = erts_smp_atomic_read(&rq->spin_wake);
-	ASSERT(val >= 0);
-	if (val != 0) {
-	    erts_smp_runq_lock(rq);
-	    val = erts_smp_atomic_read(&rq->spin_wake);
-	    ASSERT(val >= 0);
-	    if (val != 0)
-		goto woken;
-	    if (spincount == 0)
-		goto sleep;
-	    erts_smp_runq_unlock(rq);
+    erts_thr_progress_register_managed_thread(NULL, &callbacks, 1);
+    init_aux_work_data(awdp, NULL, NULL);
+    awdp->ssi = ssi;
+
+    sched_prep_spin_wait(ssi);
+
+    while (1) {
+	erts_aint32_t flgs;
+
+	aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
+	if (aux_work) {
+	    if (!thr_prgr_active)
+		erts_thr_progress_active(NULL, thr_prgr_active = 1);
+	    aux_work = handle_aux_work(awdp, aux_work, 1);
+	    if (aux_work && erts_thr_progress_update(NULL))
+		erts_thr_progress_leader_update(NULL);
+	}
+
+	if (!aux_work) {
+	    if (thr_prgr_active)
+		erts_thr_progress_active(NULL, thr_prgr_active = 0);
+	    erts_thr_progress_prepare_wait(NULL);
+
+	    flgs = sched_spin_wait(ssi, 0);
+
+	    if (flgs & ERTS_SSI_FLG_SLEEPING) {
+		ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+		flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
+		if (flgs & ERTS_SSI_FLG_SLEEPING) {
+		    int res;
+		    ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
+		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+		    do {
+			res = erts_tse_wait(ssi->event);
+		    } while (res == EINTR);
+		}
+	    }
+	    erts_thr_progress_finalize_wait(NULL);
 	}
+
+	flgs = sched_prep_spin_wait(ssi);
     }
+    return NULL;
+}
 
-    erts_smp_runq_lock(rq);
-    val = erts_smp_atomic_read(&rq->spin_wake);
-    ASSERT(val >= 0);
-    if (val != 0) {
-    woken:
-	erts_smp_atomic_dec(&rq->spin_wake);
-	ASSERT(erts_smp_atomic_read(&rq->spin_wake) >= 0);
-	erts_smp_atomic_dec(&rq->spin_waiter);
-	ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0);
+#endif /* ERTS_SMP */
+
+static void
+scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
+{
+    int working = 1;
+    ErtsSchedulerSleepInfo *ssi = esdp->ssi;
+    int spincount;
+    erts_aint32_t aux_work = 0;
+#ifdef ERTS_SMP
+    int thr_prgr_active = 1;
+    erts_aint32_t flgs;
+
+    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+
+    flgs = sched_prep_spin_wait(ssi);
+    if (flgs & ERTS_SSI_FLG_SUSPENDED) {
+	/* Go suspend instead... */
+	return;
     }
-    else {
-    sleep:
-	erts_smp_atomic_dec(&rq->spin_waiter);
-	ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0);
+
+    /*
+     * If all schedulers are waiting, one of them *should*
+     * be waiting in erl_sys_schedule()
+     */
+
+    if (!prepare_for_sys_schedule()) {
+
+	sched_waiting(esdp->no, rq);
+
+	erts_smp_runq_unlock(rq);
+
+	spincount = sched_busy_wait.tse;
+
+    tse_wait:
+
+	if (thr_prgr_active != working)
+	    sched_wall_time_change(esdp, thr_prgr_active);
+
+	while (1) {
+
+	    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
+	    if (aux_work) {
+		if (!thr_prgr_active) {
+		    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+		    sched_wall_time_change(esdp, 1);
+		}
+		aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1);
+		if (aux_work && erts_thr_progress_update(esdp))
+		    erts_thr_progress_leader_update(esdp);
+	    }
+
+	    if (aux_work)
+		flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
+	    else {
+		if (thr_prgr_active) {
+		    erts_thr_progress_active(esdp, thr_prgr_active = 0);
+		    sched_wall_time_change(esdp, 0);
+		}
+		erts_thr_progress_prepare_wait(esdp);
+
+		flgs = sched_spin_wait(ssi, spincount);
+		if (flgs & ERTS_SSI_FLG_SLEEPING) {
+		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+		    flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
+		    if (flgs & ERTS_SSI_FLG_SLEEPING) {
+			int res;
+			ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
+			ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+			do {
+			    res = erts_tse_wait(ssi->event);
+			} while (res == EINTR);
+		    }
+		}
+		erts_thr_progress_finalize_wait(esdp);
+	    }
+
+	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
+		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
+		break;
+	    }
+
+	    flgs = sched_prep_cont_spin_wait(ssi);
+	    spincount = sched_busy_wait.aux_work;
+
+	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
+		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
+		break;
+	    }
+
+	}
+
+	if (flgs & ~ERTS_SSI_FLG_SUSPENDED)
+	    erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+
+	if (!thr_prgr_active) {
+	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+	    sched_wall_time_change(esdp, 1);
+	}
+
+	erts_smp_runq_lock(rq);
+	sched_active(esdp->no, rq);
+
+    }
+    else
+#endif
+    {
+	erts_aint_t dt;
+
+	erts_smp_atomic32_set_relb(&function_calls, 0);
+	*fcalls = 0;
+
+	sched_waiting_sys(esdp->no, rq);
+
+
+	erts_smp_runq_unlock(rq);
+
+	ASSERT(working);
+	sched_wall_time_change(esdp, working = 0);
+
+	spincount = sched_busy_wait.sys_schedule;
+	if (spincount == 0)
+	    goto sys_aux_work;
+
+	while (spincount-- > 0) {
+
+	sys_poll_aux_work:
+
+	    if (working)
+		sched_wall_time_change(esdp, working = 0);
+
+	    ASSERT(!erts_port_task_have_outstanding_io_tasks());
+
+	    erl_sys_schedule(1); /* Might give us something to do */
+
+	    dt = erts_do_time_read_and_reset();
+	    if (dt) erts_bump_timer(dt);
+
+	sys_aux_work:
+#ifndef ERTS_SMP
+	    erts_sys_schedule_interrupt(0);
+#endif
+
+	    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
+	    if (aux_work) {
+		if (!working)
+		    sched_wall_time_change(esdp, working = 1);
+#ifdef ERTS_SMP
+		if (!thr_prgr_active)
+		    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+#endif
+		aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1);
+#ifdef ERTS_SMP
+		if (aux_work && erts_thr_progress_update(esdp))
+		    erts_thr_progress_leader_update(esdp);
+#endif
+	    }
+
+#ifndef ERTS_SMP
+	    if (rq->len != 0 || rq->misc.start)
+		goto sys_woken;
+#else
+	    flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
+	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
+		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
+		goto sys_woken;
+	    }
+
+	    /*
+	     * If we got new I/O tasks we aren't allowed to
+	     * call erl_sys_schedule() until it is handled.
+	     */
+	    if (erts_port_task_have_outstanding_io_tasks()) {
+		clear_sys_scheduling();
+		/*
+		 * Got to check that we still got I/O tasks; otherwise
+		 * we have to continue checking for I/O...
+		 */
+		if (!prepare_for_sys_schedule()) {
+		    spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT;
+		    goto tse_wait;
+		}
+	    }
+#endif
+	}
+
+	erts_smp_runq_lock(rq);
+
+#ifdef ERTS_SMP
 	/*
 	 * If we got new I/O tasks we aren't allowed to
 	 * sleep in erl_sys_schedule().
 	 */
-	if (!erts_port_task_have_outstanding_io_tasks()) {
-#endif
+	if (erts_port_task_have_outstanding_io_tasks()) {
+	    clear_sys_scheduling();
 
-	    erts_sys_schedule_interrupt(0);
+	    /*
+	     * Got to check that we still got I/O tasks; otherwise
+	     * we have to wait in erl_sys_schedule() after all...
+	     */
+	    if (!prepare_for_sys_schedule()) {
+		/*
+		 * Not allowed to wait in erl_sys_schedule;
+		 * do tse wait instead...
+		 */
+		sched_change_waiting_sys_to_waiting(esdp->no, rq);
+		erts_smp_runq_unlock(rq);
+		spincount = 0;
+		goto tse_wait;
+	    }
+	}
+#endif
+	if (aux_work) {
 	    erts_smp_runq_unlock(rq);
+	    goto sys_poll_aux_work;
+	}
+#ifdef ERTS_SMP
+	flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING);
+	if (!(flgs & ERTS_SSI_FLG_SLEEPING)) {
+	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
+		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
+		goto sys_locked_woken;
+	    }
+	    erts_smp_runq_unlock(rq);
+	    flgs = sched_prep_cont_spin_wait(ssi);
+	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
+		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
+		goto sys_woken;
+	    }
+	    ASSERT(!erts_port_task_have_outstanding_io_tasks());
+	    goto sys_poll_aux_work;
+	}
 
-	    erl_sys_schedule(0);
+	ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING);
+	ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+#endif
 
-	    dt = do_time_read_and_reset();
-	    if (dt) bump_timer(dt);
+	erts_smp_runq_unlock(rq);
 
-	    erts_smp_runq_lock(rq);
+	if (working)
+	    sched_wall_time_change(esdp, working = 0);
+
+#ifdef ERTS_SMP
+	if (thr_prgr_active)
+	    erts_thr_progress_active(esdp, thr_prgr_active = 0);
+#endif
+
+	ASSERT(!erts_port_task_have_outstanding_io_tasks());
+
+	erl_sys_schedule(0);
+
+	dt = erts_do_time_read_and_reset();
+	if (dt) erts_bump_timer(dt);
 
-#if ERTS_SCHED_SLEEP_SPINCOUNT != 0
+#ifndef ERTS_SMP
+	if (rq->len == 0 && !rq->misc.start)
+	    goto sys_aux_work;
+    sys_woken:
+#else
+	flgs = sched_prep_cont_spin_wait(ssi);
+	if (flgs & ERTS_SSI_FLG_WAITING)
+	    goto sys_aux_work;
+
+    sys_woken:
+	if (!thr_prgr_active)
+	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+	erts_smp_runq_lock(rq);
+    sys_locked_woken:
+	if (!thr_prgr_active) {
+	    erts_smp_runq_unlock(rq);
+	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+	    erts_smp_runq_lock(rq);
 	}
-    }
+	clear_sys_scheduling();
+	if (flgs & ~ERTS_SSI_FLG_SUSPENDED)
+	    erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
 #endif
+	if (!working)
+	    sched_wall_time_change(esdp, working = 1);
+	sched_active_sys(esdp->no, rq);
+    }
+
+    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
+}
+
+#ifdef ERTS_SMP
 
-    sched_active_sys(no, rq);
+static ERTS_INLINE erts_aint32_t
+ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi)
+{
+    /* reset all flags but suspended */
+    erts_aint32_t oflgs;
+    erts_aint32_t nflgs = 0;
+    erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING;
+    while (1) {
+	oflgs = erts_smp_atomic32_cmpxchg_relb(&ssi->flags, nflgs, xflgs);
+	if (oflgs == xflgs)
+	    return oflgs;
+	nflgs = oflgs & ERTS_SSI_FLG_SUSPENDED;
+	xflgs = oflgs;
+    }
 }
 
 static void
-sched_cnd_wait(Uint no, ErtsRunQueue *rq)
+wake_scheduler(ErtsRunQueue *rq, int incq)
 {
-#if ERTS_SCHED_SLEEP_SPINCOUNT != 0
-    int val;
-    int spincount = ERTS_SCHED_SLEEP_SPINCOUNT;
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
-#endif
+    ErtsSchedulerSleepInfo *ssi;
+    erts_aint32_t flgs;
 
-    sched_waiting(no, rq);
-    erts_smp_activity_begin(ERTS_ACTIVITY_WAIT,
-			    prepare_for_block,
-			    resume_after_block,
-			    (void *) rq);
+    /*
+     * The unlocked run queue is not strictly necessary
+     * from a thread safety or deadlock prevention
+     * perspective. It will, however, cost us performance
+     * if it is locked during wakup of another scheduler,
+     * so all code *should* handle this without having
+     * the lock on the run queue.
+     */
+    ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq));
 
-#if ERTS_SCHED_SLEEP_SPINCOUNT == 0
-    erts_smp_cnd_wait(&rq->cnd, &rq->mtx);
-#else
-    erts_smp_atomic_inc(&rq->spin_waiter);
-    erts_smp_mtx_unlock(&rq->mtx);
-
-    while (spincount-- > 0) {
-	val = erts_smp_atomic_read(&rq->spin_wake);
-	ASSERT(val >= 0);
-	if (val != 0) {
-	    erts_smp_mtx_lock(&rq->mtx);
-	    val = erts_smp_atomic_read(&rq->spin_wake);
-	    ASSERT(val >= 0);
-	    if (val != 0)
-		goto woken;
-	    if (spincount == 0)
-		goto sleep;
-	    erts_smp_mtx_unlock(&rq->mtx);
-	}
-    }
-
-    erts_smp_mtx_lock(&rq->mtx);
-    val = erts_smp_atomic_read(&rq->spin_wake);
-    ASSERT(val >= 0);
-    if (val == 0) {
-    sleep:
-	erts_smp_atomic_dec(&rq->spin_waiter);
-	ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0);
-	erts_smp_cnd_wait(&rq->cnd, &rq->mtx);
-    }
-    else {
-    woken:
-	erts_smp_atomic_dec(&rq->spin_wake);
-	ASSERT(erts_smp_atomic_read(&rq->spin_wake) >= 0);
-	erts_smp_atomic_dec(&rq->spin_waiter);
-	ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0);
-    }
+    ssi = rq->scheduler->ssi;
+
+    flgs = ssi_flags_set_wake(ssi);
+    erts_sched_finish_poke(ssi, flgs);
+
+    if (incq && (flgs & ERTS_SSI_FLG_WAITING))
+	non_empty_runq(rq);
+}
+
+#define ERTS_NO_USED_RUNQS_SHIFT 16
+#define ERTS_NO_RUNQS_MASK 0xffff
+
+#if ERTS_MAX_NO_OF_SCHEDULERS > ERTS_NO_RUNQS_MASK
+#  error "Too large amount of schedulers allowed"
 #endif
 
-    erts_smp_activity_end(ERTS_ACTIVITY_WAIT,
-			  prepare_for_block,
-			  resume_after_block,
-			  (void *) rq);
+static ERTS_INLINE void
+init_no_runqs(int active, int used)
+{
+    erts_aint32_t no_runqs = (erts_aint32_t) (active & ERTS_NO_RUNQS_MASK);
+    no_runqs |= (erts_aint32_t) ((used & ERTS_NO_RUNQS_MASK) << ERTS_NO_USED_RUNQS_SHIFT);
+    erts_smp_atomic32_init_nob(&balance_info.no_runqs, no_runqs);
+}
 
-    sched_active(no, rq);
+static ERTS_INLINE void
+get_no_runqs(int *active, int *used)
+{
+    erts_aint32_t no_runqs = erts_smp_atomic32_read_nob(&balance_info.no_runqs);
+    if (active)
+	*active = (int) (no_runqs & ERTS_NO_RUNQS_MASK);
+    if (used)
+	*used = (int) ((no_runqs >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK);
 }
 
-static void
-wake_one_scheduler(void)
-{
-    ASSERT(erts_common_run_queue);
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(erts_common_run_queue));
-    if (erts_common_run_queue->waiting) {
-	if (!sched_spin_wake(erts_common_run_queue)) {
-	    if (erts_common_run_queue->waiting == -1) /* One scheduler waiting
-							 and doing so in
-							 sys_schedule */
-		erts_sys_schedule_interrupt(1);
-	    else
-		erts_smp_cnd_signal(&erts_common_run_queue->cnd);
-	}
+static ERTS_INLINE void
+set_no_used_runqs(int used)
+{
+    erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs);
+    while (1) {
+	erts_aint32_t act, new;
+	new = (used & ERTS_NO_RUNQS_MASK) << ERTS_NO_USED_RUNQS_SHIFT;
+	new |= exp & ERTS_NO_RUNQS_MASK;
+	act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp);
+	if (act == exp)
+	    break;
+	exp = act;
     }
 }
 
-static void
-wake_scheduler(ErtsRunQueue *rq, int incq)
+static ERTS_INLINE void
+set_no_active_runqs(int active)
 {
-    ASSERT(!erts_common_run_queue);
-    ASSERT(-1 <= rq->waiting && rq->waiting <= 1);
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
-    if (rq->waiting && !rq->woken) {
-	if (!sched_spin_wake(rq)) {
-	    if (rq->waiting < 0)
-		erts_sys_schedule_interrupt(1);
-	    else
-		erts_smp_cnd_signal(&rq->cnd);
-	}
-	rq->woken = 1;
-	if (incq)
-	    non_empty_runq(rq);
+    erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs);
+    while (1) {
+	erts_aint32_t act, new;
+	new = exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT);
+	new |= active & ERTS_NO_RUNQS_MASK;
+	act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp);
+	if (act == exp)
+	    break;
+	exp = act;
     }
 }
 
-static void
-wake_all_schedulers(void)
+static ERTS_INLINE int
+try_inc_no_active_runqs(int active)
 {
-    if (erts_common_run_queue) {
-	erts_smp_runq_lock(erts_common_run_queue);
-	if (erts_common_run_queue->waiting) {
-	    if (erts_common_run_queue->waiting < 0)
-		erts_sys_schedule_interrupt(1);
-	    sched_spin_wake_all(erts_common_run_queue);
-	    erts_smp_cnd_broadcast(&erts_common_run_queue->cnd);
-	}
-	erts_smp_runq_unlock(erts_common_run_queue);
-    }
-    else {
-	int ix;
-	for (ix = 0; ix < erts_no_run_queues; ix++) {
-	    ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
-	    erts_smp_runq_lock(rq);
-	    wake_scheduler(rq, 0);
-	    erts_smp_runq_unlock(rq);
-	}
+    erts_aint32_t exp = erts_smp_atomic32_read_nob(&balance_info.no_runqs);
+    if (((exp >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK) < active)
+	return 0;
+    if ((exp & ERTS_NO_RUNQS_MASK) + 1 == active) {
+	erts_aint32_t new, act;
+	new = exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT);
+	new |= active & ERTS_NO_RUNQS_MASK;
+	act = erts_smp_atomic32_cmpxchg_nob(&balance_info.no_runqs, new, exp);
+	if (act == exp)
+	    return 1;
     }
+    return 0;
 }
 
+
 static ERTS_INLINE int
 chk_wake_sched(ErtsRunQueue *crq, int ix, int activate)
 {
-    long iflgs;
+    erts_aint32_t iflgs;
     ErtsRunQueue *wrq;
     if (crq->ix == ix)
 	return 0;
     wrq = ERTS_RUNQ_IX(ix);
-    iflgs = erts_smp_atomic_read(&wrq->info_flags);
+    iflgs = erts_smp_atomic32_read_nob(&wrq->info_flags);
     if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) {
-	erts_smp_xrunq_lock(crq, wrq);
 	if (activate) {
-	    if (ix == erts_smp_atomic_cmpxchg(&balance_info.active_runqs, ix+1, ix)) {
+	    if (try_inc_no_active_runqs(ix+1)) {
+		erts_smp_xrunq_lock(crq, wrq);
 		wrq->flags &= ~ERTS_RUNQ_FLG_INACTIVE;
+		erts_smp_xrunq_unlock(crq, wrq);
 	    }
 	}
 	wake_scheduler(wrq, 0);
-	erts_smp_xrunq_unlock(crq, wrq);
 	return 1;
     }
     return 0;
@@ -947,8 +2504,9 @@ wake_scheduler_on_empty_runq(ErtsRunQueue *crq)
 {
     int ix = crq->ix;
     int stop_ix = ix;
-    int active_ix = erts_smp_atomic_read(&balance_info.active_runqs);
-    int balance_ix = erts_smp_atomic_read(&balance_info.used_runqs);
+    int active_ix, balance_ix;
+
+    get_no_runqs(&active_ix, &balance_ix);
 
     if (active_ix > balance_ix)
 	active_ix = balance_ix;
@@ -982,19 +2540,35 @@ static ERTS_INLINE void
 smp_notify_inc_runq(ErtsRunQueue *runq)
 {
 #ifdef ERTS_SMP
-    if (erts_common_run_queue)
-	wake_one_scheduler();
-    else
+    if (runq)
 	wake_scheduler(runq, 1);
 #endif
 }
 
 void
-erts_smp_notify_inc_runq__(ErtsRunQueue *runq)
+erts_smp_notify_inc_runq(ErtsRunQueue *runq)
 {
     smp_notify_inc_runq(runq);
 }
 
+void
+erts_sched_notify_check_cpu_bind(void)
+{
+#ifdef ERTS_SMP
+    int ix;
+    for (ix = 0; ix < erts_no_run_queues; ix++) {
+	ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
+	erts_smp_runq_lock(rq);
+	rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND;
+	erts_smp_runq_unlock(rq);
+	wake_scheduler(rq, 0);
+    }
+#else
+    erts_sched_check_cpu_bind(erts_get_scheduler_data());
+#endif
+}
+
+
 #ifdef ERTS_SMP
 
 ErtsRunQueue *
@@ -1136,20 +2710,24 @@ static void
 evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq)
 {
     Port *prt;
+    int notify_to_rq = 0;
     int prio;
     int prt_locked = 0;
     int rq_locked = 0;
     int evac_rq_locked = 1;
+    ErtsMigrateResult mres;
 
     erts_smp_runq_lock(evac_rq);
 
+    erts_smp_atomic32_read_bor_nob(&evac_rq->scheduler->ssi->flags,
+				   ERTS_SSI_FLG_SUSPENDED);
+
     evac_rq->flags &= ~ERTS_RUNQ_FLGS_IMMIGRATE_QMASK;
     evac_rq->flags |= (ERTS_RUNQ_FLGS_EMIGRATE_QMASK
 		       | ERTS_RUNQ_FLGS_EVACUATE_QMASK
 		       | ERTS_RUNQ_FLG_SUSPENDED);
 
-    erts_smp_atomic_bor(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED);
-
+    erts_smp_atomic32_read_bor_nob(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED);
     /*
      * Need to set up evacuation paths first since we
      * may release the run queue lock on evac_rq
@@ -1177,9 +2755,11 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq)
     /* Evacuate scheduled ports */
     prt = evac_rq->ports.start;
     while (prt) {
-	(void) erts_port_migrate(prt, &prt_locked,
+	mres = erts_port_migrate(prt, &prt_locked,
 				 evac_rq, &evac_rq_locked,
 				 rq, &rq_locked);
+	if (mres == ERTS_MIGRATE_SUCCESS)
+	    notify_to_rq = 1;
 	if (prt_locked)
 	    erts_smp_port_unlock(prt);
 	if (!evac_rq_locked) {
@@ -1208,9 +2788,11 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq)
 			goto end_of_proc;
 		}
 
-		(void) erts_proc_migrate(proc, &proc_locks,
+		mres = erts_proc_migrate(proc, &proc_locks,
 					 evac_rq, &evac_rq_locked,
 					 rq, &rq_locked);
+		if (mres == ERTS_MIGRATE_SUCCESS)
+		    notify_to_rq = 1;
 		if (proc_locks)
 		    erts_smp_proc_unlock(proc, proc_locks);
 		if (!evac_rq_locked) {
@@ -1242,10 +2824,13 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq)
     if (rq_locked)
 	erts_smp_runq_unlock(rq);
 
-    if (!evac_rq_locked)
-	erts_smp_runq_lock(evac_rq);
+    if (evac_rq_locked)
+	erts_smp_runq_unlock(evac_rq);
+
+    if (notify_to_rq)
+	smp_notify_inc_runq(rq);
+
     wake_scheduler(evac_rq, 0);
-    erts_smp_runq_unlock(evac_rq);
 }
 
 static int
@@ -1263,6 +2848,9 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq)
     ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp);
     ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked);
 
+    if (rq->halt_in_progress)
+	goto try_steal_port;
+
     /*
      * Check for a runnable process to steal...
      */
@@ -1349,6 +2937,8 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq)
 	vrq_locked = 1;
     }
 
+ try_steal_port:
+
     ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp);
     ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked);
 
@@ -1391,7 +2981,7 @@ static ERTS_INLINE int
 check_possible_steal_victim(ErtsRunQueue *rq, int *rq_lockedp, int vix)
 {
     ErtsRunQueue *vrq = ERTS_RUNQ_IX(vix);
-    long iflgs = erts_smp_atomic_read(&vrq->info_flags);
+    erts_aint32_t iflgs = erts_smp_atomic32_read_nob(&vrq->info_flags);
     if (iflgs & ERTS_RUNQ_IFLG_NONEMPTY)
 	return try_steal_task_from_victim(rq, rq_lockedp, vrq);
     else
@@ -1403,9 +2993,6 @@ static int
 try_steal_task(ErtsRunQueue *rq)
 {
     int res, rq_locked, vix, active_rqs, blnc_rqs;
-    
-    if (erts_common_run_queue)
-	return 0;
 
     /*
      * We are not allowed to steal jobs to this run queue
@@ -1421,8 +3008,7 @@ try_steal_task(ErtsRunQueue *rq)
 
     ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, rq_locked);
 
-    active_rqs = erts_smp_atomic_read(&balance_info.active_runqs);
-    blnc_rqs = erts_smp_atomic_read(&balance_info.used_runqs);
+    get_no_runqs(&active_rqs, &blnc_rqs);
 
     if (active_rqs > blnc_rqs)
 	active_rqs = blnc_rqs;
@@ -1433,7 +3019,7 @@ try_steal_task(ErtsRunQueue *rq)
 	if (active_rqs < blnc_rqs) {
 	    int no = blnc_rqs - active_rqs;
 	    int stop_ix = vix = active_rqs + rq->ix % no;
-	    while (erts_smp_atomic_read(&no_empty_run_queues) < blnc_rqs) {
+	    while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) {
 		res = check_possible_steal_victim(rq, &rq_locked, vix);
 		if (res)
 		    goto done;
@@ -1448,7 +3034,7 @@ try_steal_task(ErtsRunQueue *rq)
 	vix = rq->ix;
 
 	/* ... then try to steal a job from another active queue... */
-	while (erts_smp_atomic_read(&no_empty_run_queues) < blnc_rqs) {
+	while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) {
 	    vix++;
 	    if (vix >= active_rqs)
 		vix = 0;
@@ -1468,36 +3054,12 @@ try_steal_task(ErtsRunQueue *rq)
 	erts_smp_runq_lock(rq);
 
     if (!res)
-	res = !ERTS_EMPTY_RUNQ(rq);
+	res = rq->halt_in_progress ?
+	    !ERTS_EMPTY_RUNQ_PORTS(rq) : !ERTS_EMPTY_RUNQ(rq);
 
     return res;
 }
 
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
-void
-erts_smp_notify_check_children_needed(void)
-{
-    int i;
-    for (i = 0; i < erts_no_schedulers; i++) {
-	erts_smp_runq_lock(ERTS_SCHEDULER_IX(i)->run_queue);
-	ERTS_SCHEDULER_IX(i)->check_children = 1;
-	if (!erts_common_run_queue)
-	    wake_scheduler(ERTS_SCHEDULER_IX(i)->run_queue, 0);
-	erts_smp_runq_unlock(ERTS_SCHEDULER_IX(i)->run_queue);
-    }
-    if (ongoing_multi_scheduling_block()) {
-	/* Also blocked schedulers need to check children */
-	erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-	for (i = 0; i < erts_no_schedulers; i++)
-	    ERTS_SCHEDULER_IX(i)->blocked_check_children = 1;
-	erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
-	erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-    }
-    if (erts_common_run_queue)
-	wake_all_schedulers();
-}
-#endif
-
 /* Run queue balancing */
 
 typedef struct {
@@ -1561,20 +3123,23 @@ do {									\
 static void
 check_balance(ErtsRunQueue *c_rq)
 {
+#if ERTS_MAX_PROCESSES >= (1 << 27)
+#  error check_balance() assumes ERTS_MAX_PROCESS < (1 << 27)
+#endif
     ErtsRunQueueBalance avg = {0};
     Sint64 scheds_reds, full_scheds_reds;
     int forced, active, current_active, oowc, half_full_scheds, full_scheds,
 	mmax_len, blnc_no_rqs, qix, pix, freds_hist_ix;
 
-    if (erts_smp_atomic_xchg(&balance_info.checking_balance, 1)) {
+    if (erts_smp_atomic32_xchg_nob(&balance_info.checking_balance, 1)) {
 	c_rq->check_balance_reds = INT_MAX;
 	return;
     }
 
-    blnc_no_rqs = (int) erts_smp_atomic_read(&balance_info.used_runqs);
+    get_no_runqs(NULL, &blnc_no_rqs);
     if (blnc_no_rqs == 1) {
 	c_rq->check_balance_reds = INT_MAX;
-	erts_smp_atomic_set(&balance_info.checking_balance, 0);
+	erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0);
 	return;
     }
 
@@ -1582,7 +3147,7 @@ check_balance(ErtsRunQueue *c_rq)
 
     if (balance_info.halftime) {	
 	balance_info.halftime = 0;
-	erts_smp_atomic_set(&balance_info.checking_balance, 0);
+	erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0);
 	ERTS_FOREACH_RUNQ(rq,
 	{
 	    if (rq->waiting)
@@ -1610,12 +3175,13 @@ check_balance(ErtsRunQueue *c_rq)
     forced = balance_info.forced_check_balance;
     balance_info.forced_check_balance = 0;
 
-    blnc_no_rqs = (int) erts_smp_atomic_read(&balance_info.used_runqs);
+    get_no_runqs(&current_active, &blnc_no_rqs);
+
     if (blnc_no_rqs == 1) {
 	erts_smp_mtx_unlock(&balance_info.update_mtx);
 	erts_smp_runq_lock(c_rq);
 	c_rq->check_balance_reds = INT_MAX;
-	erts_smp_atomic_set(&balance_info.checking_balance, 0);
+	erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0);
 	return;
     }
 
@@ -1624,8 +3190,6 @@ check_balance(ErtsRunQueue *c_rq)
     if (balance_info.full_reds_history_index >= ERTS_FULL_REDS_HISTORY_SIZE)
 	balance_info.full_reds_history_index = 0;
 
-    current_active = erts_smp_atomic_read(&balance_info.active_runqs);
-
     /* Read balance information for all run queues */
     for (qix = 0; qix < blnc_no_rqs; qix++) {
 	ErtsRunQueue *rq = ERTS_RUNQ_IX(qix);
@@ -1684,12 +3248,14 @@ check_balance(ErtsRunQueue *c_rq)
 		    run_queue_info[qix].prio[pix].avail = 0;
 	    }
 	    else {
-		int xreds = 0;
-		int procreds = treds;
-		procreds -= run_queue_info[qix].prio[ERTS_PORT_PRIO_LEVEL].reds;
+		Sint64 xreds = 0;
+		Sint64 procreds = treds;
+		procreds -= 
+		    ((Sint64)
+		     run_queue_info[qix].prio[ERTS_PORT_PRIO_LEVEL].reds);
 
 		for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) {
-		    int av;
+		    Sint64 av;
 
 		    if (xreds == 0)
 			av = 100;
@@ -1700,9 +3266,10 @@ check_balance(ErtsRunQueue *c_rq)
 			if (av == 0)
 			    av = 1;
 		    }
-		    run_queue_info[qix].prio[pix].avail = av;
+		    run_queue_info[qix].prio[pix].avail = (int) av;
+		    ASSERT(run_queue_info[qix].prio[pix].avail >= 0);
 		    if (pix < PRIORITY_NORMAL) /* ie., max or high */
-			xreds += run_queue_info[qix].prio[pix].reds;
+			xreds += (Sint64) run_queue_info[qix].prio[pix].reds;
 		}
 		run_queue_info[qix].prio[ERTS_PORT_PRIO_LEVEL].avail = 100;
 	    }
@@ -1718,6 +3285,9 @@ check_balance(ErtsRunQueue *c_rq)
 	    mmax_len = run_queue_info[qix].max_len;
     }
 
+    if (!erts_sched_compact_load)
+	goto all_active;
+
     if (!forced && half_full_scheds != blnc_no_rqs) {
 	int min = 1;
 	if (min < half_full_scheds)
@@ -1807,7 +3377,8 @@ check_balance(ErtsRunQueue *c_rq)
 	    if (max_len != 0) {
 		int avail = avg.prio[pix].avail;
 		if (avail != 0) {
-		    max_len = ((100*max_len - 1) / avail) + 1;
+		    max_len = (int) ((100*((Sint64) max_len) - 1)
+				     / ((Sint64) avail)) + 1;
 		    avg.prio[pix].max_len = max_len;
 		    ASSERT(max_len >= 0);
 		}
@@ -1824,9 +3395,10 @@ check_balance(ErtsRunQueue *c_rq)
 		    || run_queue_info[qix].prio[pix].avail == 0)
 		    limit = 0;
 		else
-		    limit = (((avg.prio[pix].max_len
-			       * run_queue_info[qix].prio[pix].avail) - 1)
-			     / 100 + 1);
+		    limit = (int) (((((Sint64) avg.prio[pix].max_len)
+				     * ((Sint64) run_queue_info[qix].prio[pix].avail))
+				    - 1)
+				   / 100 + 1);
 		run_queue_info[qix].prio[pix].migration_limit = limit;
 	    }
 	}
@@ -1954,10 +3526,10 @@ erts_fprintf(stderr, "--------------------------------\n");
     }
 
     balance_info.last_active_runqs = active;
-    erts_smp_atomic_set(&balance_info.active_runqs, active);
+    set_no_active_runqs(active);
 
     balance_info.halftime = 1;
-    erts_smp_atomic_set(&balance_info.checking_balance, 0);
+    erts_smp_atomic32_set_nob(&balance_info.checking_balance, 0);
 
     /* Write migration paths and reset balance statistics in all queues */
     for (qix = 0; qix < blnc_no_rqs; qix++) {
@@ -2051,49 +3623,355 @@ erts_debug_nbalance(void)
 #endif
 }
 
-void
-erts_early_init_scheduling(void)
+/* Wakeup other schedulers */
+
+typedef enum {
+    ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH,
+    ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH,
+    ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM,
+    ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW,
+    ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW
+} ErtsSchedWakeupOtherThreshold;
+
+typedef enum {
+    ERTS_SCHED_WAKEUP_OTHER_TYPE_PROPOSAL,
+    ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY
+} ErtsSchedWakeupOtherType;
+
+/* First proposal */
+
+#define ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH (200*CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_HIGH (50*CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_MEDIUM (10*CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_LOW (CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW (CONTEXT_REDS/10)
+
+#define ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_HIGH 3
+#define ERTS_WAKEUP_OTHER_DEC_SHIFT_HIGH 1
+#define ERTS_WAKEUP_OTHER_DEC_SHIFT_MEDIUM 0
+#define ERTS_WAKEUP_OTHER_DEC_SHIFT_LOW -2
+#define ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_LOW -5 
+
+#define ERTS_WAKEUP_OTHER_DEC_SHIFT 2
+#define ERTS_WAKEUP_OTHER_FIXED_INC (CONTEXT_REDS/10)
+
+/* To be legacy */
+
+#define ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH_LEGACY (200*CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_HIGH_LEGACY (50*CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_MEDIUM_LEGACY (10*CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_LOW_LEGACY (CONTEXT_REDS)
+#define ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW_LEGACY (CONTEXT_REDS/10)
+
+#define ERTS_WAKEUP_OTHER_DEC_LEGACY 10
+#define ERTS_WAKEUP_OTHER_FIXED_INC_LEGACY (CONTEXT_REDS/10)
+
+#ifdef ERTS_SMP
+
+static struct {
+    ErtsSchedWakeupOtherThreshold threshold;
+    ErtsSchedWakeupOtherType type;
+    int limit;
+    int dec_shift;
+    int dec_mask;
+    void (*check)(ErtsRunQueue *rq);
+} wakeup_other;
+
+static void
+wakeup_other_check(ErtsRunQueue *rq)
+{
+    int wo_reds = rq->wakeup_other_reds;
+    if (wo_reds) {
+	int left_len = rq->len - 1;
+	if (left_len < 1) {
+	    int wo_reduce = wo_reds << wakeup_other.dec_shift;
+	    wo_reduce &= wakeup_other.dec_mask;
+	    rq->wakeup_other -= wo_reduce;
+	    if (rq->wakeup_other < 0)
+		rq->wakeup_other = 0;
+	}
+	else {
+	    rq->wakeup_other += (left_len*wo_reds
+				 + ERTS_WAKEUP_OTHER_FIXED_INC);
+	    if (rq->wakeup_other > wakeup_other.limit) {
+		int empty_rqs =
+		    erts_smp_atomic32_read_acqb(&no_empty_run_queues);
+		if (empty_rqs != 0)
+		    wake_scheduler_on_empty_runq(rq);
+		rq->wakeup_other = 0;
+	    }
+	}
+	rq->wakeup_other_reds = 0;
+    }
+}
+
+static void
+wakeup_other_set_limit(void)
 {
-    early_cpu_bind_init();
+    switch (wakeup_other.threshold) {
+    case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH:
+	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH;
+	wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_HIGH;
+	break;
+    case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH:
+	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH;
+	wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_HIGH;
+	break;
+    case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM:
+	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM;
+	wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_MEDIUM;
+	break;
+    case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW:
+	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_LOW;
+	wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_LOW;
+	break;
+    case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW:
+	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW;
+	wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_LOW;
+	break;
+    }
+    if (wakeup_other.dec_shift < 0)
+	wakeup_other.dec_mask = (1 << (sizeof(wakeup_other.dec_mask)*8
+				       + wakeup_other.dec_shift)) - 1;
+    else {
+	wakeup_other.dec_mask = 0;
+	wakeup_other.dec_mask = ~wakeup_other.dec_mask;
+    }
 }
 
+static void
+wakeup_other_check_legacy(ErtsRunQueue *rq)
+{
+    int wo_reds = rq->wakeup_other_reds;
+    if (wo_reds) {
+	if (rq->len < 2) {
+	    rq->wakeup_other -= ERTS_WAKEUP_OTHER_DEC_LEGACY*wo_reds;
+	    if (rq->wakeup_other < 0)
+		rq->wakeup_other = 0;
+	}
+	else if (rq->wakeup_other < wakeup_other.limit)
+	    rq->wakeup_other += rq->len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC_LEGACY;
+	else {
+	    if (erts_smp_atomic32_read_acqb(&no_empty_run_queues) != 0) {
+		wake_scheduler_on_empty_runq(rq);
+		rq->wakeup_other = 0;
+	    }
+	    rq->wakeup_other = 0;
+	}
+    }
+    rq->wakeup_other_reds = 0;
+}
+
+static void
+wakeup_other_set_limit_legacy(void)
+{
+    switch (wakeup_other.threshold) {
+    case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH:
+	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH_LEGACY;
+	break;
+    case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH:
+	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH_LEGACY;
+	break;
+    case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM:
+	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM_LEGACY;
+	break;
+    case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW:
+	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_LOW_LEGACY;
+	break;
+    case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW:
+	wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW_LEGACY;
+	break;
+    }
+}
+
+static void
+set_wakeup_other_data(void)
+{
+    switch (wakeup_other.type) {
+    case ERTS_SCHED_WAKEUP_OTHER_TYPE_PROPOSAL:
+	wakeup_other.check = wakeup_other_check;
+	wakeup_other_set_limit();
+	break;
+    case ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY:
+	wakeup_other.check = wakeup_other_check_legacy;
+	wakeup_other_set_limit_legacy();
+	break;
+    }
+}
+
+#endif
+
 void
-erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
+erts_early_init_scheduling(int no_schedulers)
 {
-    int ix, n;
+    aux_work_timeout_early_init(no_schedulers);
+#ifdef ERTS_SMP
+    wakeup_other.threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM;
+    wakeup_other.type = ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY;
+#endif
+    sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM;
+    sched_busy_wait.tse = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM
+			   * ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT);
+    sched_busy_wait.aux_work = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM
+				* ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_MEDIUM);
+}
 
-#ifndef ERTS_SMP
-    mrq = 0;
+int
+erts_sched_set_wakeup_other_thresold(char *str)
+{
+    ErtsSchedWakeupOtherThreshold threshold;
+    if (sys_strcmp(str, "very_high") == 0)
+	threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH;
+    else if (sys_strcmp(str, "high") == 0)
+	threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH;
+    else if (sys_strcmp(str, "medium") == 0)
+	threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM;
+    else if (sys_strcmp(str, "low") == 0)
+	threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW;
+    else if (sys_strcmp(str, "very_low") == 0)
+	threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW;
+    else
+	return EINVAL;
+#ifdef ERTS_SMP
+    wakeup_other.threshold = threshold;
+    set_wakeup_other_data();
+#endif
+    return 0;
+}
+
+int
+erts_sched_set_wakeup_other_type(char *str)
+{
+    ErtsSchedWakeupOtherType type;
+    if (sys_strcmp(str, "proposal") == 0)
+	type = ERTS_SCHED_WAKEUP_OTHER_TYPE_PROPOSAL;
+    else if (sys_strcmp(str, "default") == 0)
+	type = ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY;
+    else if (sys_strcmp(str, "legacy") == 0)
+	type = ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY;
+    else
+	return EINVAL;
+#ifdef ERTS_SMP
+    wakeup_other.type = type;
+#endif
+    return 0;
+}
+
+int
+erts_sched_set_busy_wait_threshold(char *str)
+{
+    int sys_sched;
+    int aux_work_fact;
+
+    if (sys_strcmp(str, "very_long") == 0) {
+	sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_LONG;
+	aux_work_fact = ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_VERY_LONG;
+    }
+    else if (sys_strcmp(str, "long") == 0) {
+	sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_LONG;
+	aux_work_fact = ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_LONG;
+    }
+    else if (sys_strcmp(str, "medium") == 0) {
+	sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM;
+	aux_work_fact = ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_MEDIUM;
+    }
+    else if (sys_strcmp(str, "short") == 0) {
+	sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_SHORT;
+	aux_work_fact = ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_SHORT;
+    }
+    else if (sys_strcmp(str, "very_short") == 0) {
+	sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_SHORT;
+	aux_work_fact = ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_VERY_SHORT;
+    }
+    else if (sys_strcmp(str, "none") == 0) {
+	sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE;
+	aux_work_fact = ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_NONE;
+    }
+    else {
+	return EINVAL;
+    }
+
+    sched_busy_wait.sys_schedule = sys_sched;
+    sched_busy_wait.tse = sys_sched*ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT;
+    sched_busy_wait.aux_work = sys_sched*aux_work_fact;
+
+    return 0;
+}
+static void
+init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
+{
+    awdp->sched_id = esdp ? (int) esdp->no : 0;
+    awdp->esdp = esdp;
+    awdp->ssi = esdp ? esdp->ssi : NULL;
+#ifdef ERTS_SMP
+    awdp->misc.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
+    awdp->dd.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
+    awdp->dd.completed_callback = NULL;
+    awdp->dd.completed_arg = NULL;
+#endif
+#ifdef ERTS_USE_ASYNC_READY_Q
+#ifdef ERTS_SMP
+    awdp->async_ready.need_thr_prgr = 0;
+    awdp->async_ready.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
+#endif
+    awdp->async_ready.queue = NULL;
+#endif
+#ifdef ERTS_SMP
+    if (!dawwp) {
+	awdp->delayed_wakeup.job = NULL;
+	awdp->delayed_wakeup.sched2jix = NULL;
+	awdp->delayed_wakeup.jix = -1;
+    }
+    else {
+	int i;
+	awdp->delayed_wakeup.job = (ErtsDelayedAuxWorkWakeupJob *) dawwp;
+	dawwp += sizeof(ErtsDelayedAuxWorkWakeupJob)*(erts_no_schedulers+1);
+	awdp->delayed_wakeup.sched2jix = (int *) dawwp;
+	awdp->delayed_wakeup.jix = -1;
+	for (i = 0; i <= erts_no_schedulers; i++)
+	    awdp->delayed_wakeup.sched2jix[i] = -1;
+    }
+#endif
+}
+
+void
+erts_init_scheduling(int no_schedulers, int no_schedulers_online)
+{
+    int ix, n, no_ssi;
+    char *daww_ptr;
+#ifdef ERTS_SMP
+    size_t daww_sz;
 #endif
 
     init_misc_op_list_alloc();
 
+#ifdef ERTS_SMP
+    set_wakeup_other_data();
+#endif
+
     ASSERT(no_schedulers_online <= no_schedulers);
     ASSERT(no_schedulers_online >= 1);
     ASSERT(no_schedulers >= 1);
 
     /* Create and initialize run queues */
 
-    n = (int) (mrq ? no_schedulers : 1);
-
-    erts_aligned_run_queues = erts_alloc(ERTS_ALC_T_RUNQS,
-					 (sizeof(ErtsAlignedRunQueue)*(n+1)));
-    if ((((Uint) erts_aligned_run_queues) & ERTS_CACHE_LINE_MASK) == 0)
-	erts_aligned_run_queues = ((ErtsAlignedRunQueue *)
-				   ((((Uint) erts_aligned_run_queues)
-				     & ~ERTS_CACHE_LINE_MASK)
-				    + ERTS_CACHE_LINE_SIZE));
+    n = no_schedulers;
 
+    erts_aligned_run_queues = 
+	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS,
+					   sizeof(ErtsAlignedRunQueue) * n);
 #ifdef ERTS_SMP
-    erts_smp_atomic_init(&no_empty_run_queues, 0);
+    erts_smp_atomic32_init_nob(&no_empty_run_queues, 0);
 #endif
 
+    erts_no_run_queues = n;
+
     for (ix = 0; ix < n; ix++) {
 	int pix, rix;
 	ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
 
 	rq->ix = ix;
-	erts_smp_atomic_init(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY);
+	erts_smp_atomic32_init_nob(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY);
 
 	/* make sure that the "extra" id correponds to the schedulers
 	 * id if the esdp->no <-> ix+1 mapping change.
@@ -2102,12 +3980,9 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
 	erts_smp_mtx_init_x(&rq->mtx, "run_queue", make_small(ix + 1));
 	erts_smp_cnd_init(&rq->cnd);
 
-	erts_smp_atomic_init(&rq->spin_waiter, 0);
-	erts_smp_atomic_init(&rq->spin_wake, 0);
-
 	rq->waiting = 0;
 	rq->woken = 0;
-	rq->flags = !mrq ? ERTS_RUNQ_FLG_SHARED_RUNQ : 0;
+	rq->flags = 0;
 	rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;
 	rq->full_reds_history_sum = 0;
 	for (rix = 0; rix < ERTS_FULL_REDS_HISTORY_SIZE; rix++) {
@@ -2119,6 +3994,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
 	rq->len = 0;
 	rq->wakeup_other = 0;
 	rq->wakeup_other_reds = 0;
+	rq->halt_in_progress = 0;
 
 	rq->procs.len = 0;
 	rq->procs.pending_exiters = NULL;
@@ -2153,9 +4029,6 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
 	rq->ports.end = NULL;
     }
 
-    erts_common_run_queue = !mrq ? ERTS_RUNQ_IX(0) : NULL;
-    erts_no_run_queues = n;
-
 #ifdef ERTS_SMP
 
     if (erts_no_run_queues != 1) {
@@ -2169,17 +4042,51 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
 
 #endif
 
+    n = (int) no_schedulers;
+    erts_no_schedulers = n;
+
+    /* Create and initialize scheduler sleep info */
+#ifdef ERTS_SMP
+    no_ssi = n+1;
+#else
+    no_ssi = 1;
+#endif
+    aligned_sched_sleep_info =
+	erts_alloc_permanent_cache_aligned(
+	    ERTS_ALC_T_SCHDLR_SLP_INFO,
+	    no_ssi*sizeof(ErtsAlignedSchedulerSleepInfo));
+    for (ix = 0; ix < no_ssi; ix++) {
+	ErtsSchedulerSleepInfo *ssi = &aligned_sched_sleep_info[ix].ssi;
+#ifdef ERTS_SMP
+#if 0 /* no need to initialize these... */
+	ssi->next = NULL;
+	ssi->prev = NULL;
+#endif
+	erts_smp_atomic32_init_nob(&ssi->flags, 0);
+	ssi->event = NULL; /* initialized in sched_thread_func */
+#endif
+	erts_atomic32_init_nob(&ssi->aux_work, 0);
+    }
+
+#ifdef ERTS_SMP
+    aligned_sched_sleep_info++;
+#endif
+
     /* Create and initialize scheduler specific data */
 
-    n = (int) no_schedulers;
-    erts_aligned_scheduler_data = erts_alloc(ERTS_ALC_T_SCHDLR_DATA,
-					     (sizeof(ErtsAlignedSchedulerData)
-					      *(n+1)));
-    if ((((Uint) erts_aligned_scheduler_data) & ERTS_CACHE_LINE_MASK) == 0)
-	erts_aligned_scheduler_data = ((ErtsAlignedSchedulerData *)
-				       ((((Uint) erts_aligned_scheduler_data)
-					 & ~ERTS_CACHE_LINE_MASK)
-					+ ERTS_CACHE_LINE_SIZE));
+#ifdef ERTS_SMP
+    daww_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE((sizeof(ErtsDelayedAuxWorkWakeupJob)
+					      + sizeof(int))*(n+1));
+    daww_ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
+						  daww_sz*n);
+#else
+    daww_ptr = NULL;
+#endif
+
+    erts_aligned_scheduler_data = 
+	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
+					   n*sizeof(ErtsAlignedSchedulerData));					   
+
     for (ix = 0; ix < n; ix++) {
 	ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix);
 #ifdef ERTS_SMP
@@ -2187,7 +4094,18 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
 	esdp->match_pseudo_process = NULL;
 	esdp->free_process = NULL;
 #endif
+	esdp->x_reg_array =
+	    erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER,
+					       ERTS_X_REGS_ALLOCATED *
+					       sizeof(Eterm));
+	esdp->f_reg_array =
+	    erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER,
+					       MAX_REG * sizeof(FloatDef));
+#if !HEAP_ON_C_STACK
+	esdp->num_tmp_heap_used = 0;
+#endif
 	esdp->no = (Uint) ix+1;
+	esdp->ssi = ERTS_SCHED_SLEEP_INFO_IX(ix);
 	esdp->current_process = NULL;
 	esdp->current_port = NULL;
 
@@ -2196,66 +4114,65 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
 
 	erts_init_atom_cache_map(&esdp->atom_cache_map);
 
-	if (erts_common_run_queue) {
-	    esdp->run_queue = erts_common_run_queue;
-	    esdp->run_queue->scheduler = NULL;
-	}
-	else {
-	    esdp->run_queue = ERTS_RUNQ_IX(ix);
-	    esdp->run_queue->scheduler = esdp;
-	}
+	esdp->run_queue = ERTS_RUNQ_IX(ix);
+	esdp->run_queue->scheduler = esdp;
 
+	init_aux_work_data(&esdp->aux_work_data, esdp, daww_ptr);
 #ifdef ERTS_SMP
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
-	esdp->check_children = 0;
-	esdp->blocked_check_children = 0;
-#endif
-	erts_smp_atomic_init(&esdp->suspended, 0);
-	erts_smp_atomic_init(&esdp->chk_cpu_bind, 0);
+	daww_ptr += daww_sz;
 #endif
+	init_sched_wall_time(&esdp->sched_wall_time);
     }
 
+    init_misc_aux_work();
+#if !HALFWORD_HEAP
+    init_swtreq_alloc();
+#endif
+
+
 #ifdef ERTS_SMP
+
+    erts_atomic32_init_nob(&completed_dealloc_count, 0); /* debug only */
+
+    aux_thread_aux_work_data =
+	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
+					   sizeof(ErtsAuxWorkData));
+
     erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd");
     erts_smp_cnd_init(&schdlr_sspnd.cnd);
 
-    schdlr_sspnd.changing = 0;
+    erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0);
     schdlr_sspnd.online = no_schedulers_online;
     schdlr_sspnd.curr_online = no_schedulers;
-    erts_smp_atomic_init(&schdlr_sspnd.msb.ongoing, 0);
-    erts_smp_atomic_init(&schdlr_sspnd.active, no_schedulers);
+    schdlr_sspnd.msb.ongoing = 0;
+    erts_smp_atomic32_init_nob(&schdlr_sspnd.active, no_schedulers);
     schdlr_sspnd.msb.procs = NULL;
-    erts_smp_atomic_set(&balance_info.used_runqs,
-			erts_common_run_queue ? 1 : no_schedulers_online);
-    erts_smp_atomic_init(&balance_info.active_runqs, no_schedulers);
+    init_no_runqs(no_schedulers, no_schedulers_online);
     balance_info.last_active_runqs = no_schedulers;
     erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update");
     balance_info.forced_check_balance = 0;
     balance_info.halftime = 1;
     balance_info.full_reds_history_index = 0;
-    erts_smp_atomic_init(&balance_info.checking_balance, 0);
+    erts_smp_atomic32_init_nob(&balance_info.checking_balance, 0);
     balance_info.prev_rise.active_runqs = 0;
     balance_info.prev_rise.max_len = 0;
     balance_info.prev_rise.reds = 0;
     balance_info.n = 0;
 
     if (no_schedulers_online < no_schedulers) {
-	if (erts_common_run_queue) {
-	    for (ix = no_schedulers_online; ix < no_schedulers; ix++)
-		erts_smp_atomic_set(&(ERTS_SCHEDULER_IX(ix)->suspended), 1);
-	}
-	else {
-	    for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++)
-		evacuate_run_queue(ERTS_RUNQ_IX(ix),
-				   ERTS_RUNQ_IX(ix % no_schedulers_online));
-	}
+	for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++)
+	    evacuate_run_queue(ERTS_RUNQ_IX(ix),
+			       ERTS_RUNQ_IX(ix % no_schedulers_online));
     }
 
     schdlr_sspnd.wait_curr_online = no_schedulers_online;
     schdlr_sspnd.curr_online *= 2; /* Boot strapping... */
-    schdlr_sspnd.changing = ERTS_SCHED_CHANGING_ONLINE;
+    ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
+				| ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
 
-    erts_smp_atomic_init(&doing_sys_schedule, 0);
+    erts_smp_atomic32_init_nob(&doing_sys_schedule, 0);
+
+    init_misc_aux_work();
 
 #else /* !ERTS_SMP */
     {
@@ -2269,20 +4186,30 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
     erts_no_schedulers = 1;
 #endif
 
-    erts_smp_atomic_init(&function_calls, 0);
+    erts_smp_atomic32_init_nob(&function_calls, 0);
 
     /* init port tasks */
     erts_port_task_init();
 
-    late_cpu_bind_init();
+    aux_work_timeout_late_init();
+
+#ifndef ERTS_SMP
+#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
+    erts_scheduler_data->verify_unused_temp_alloc
+	= erts_alloc_get_verify_unused_temp_alloc(
+	    &erts_scheduler_data->verify_unused_temp_alloc_data);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL);
+#endif
+#endif
+
+    erts_smp_atomic32_init_relb(&erts_halt_progress, -1);
+    erts_halt_code = 0;
 }
 
 ErtsRunQueue *
 erts_schedid2runq(Uint id)
 {
     int ix;
-    if (erts_common_run_queue)
-	return erts_common_run_queue;
     ix = (int) id - 1;
     ASSERT(0 <= ix && ix < erts_no_run_queues);
     return ERTS_RUNQ_IX(ix);
@@ -2374,7 +4301,6 @@ resume_process(Process *p)
 	return;
     switch(p->rstatus) {
     case P_RUNABLE:
-	*statusp = P_WAITING;  /* make erts_add_to_runq work */
 	erts_add_to_runq(p);
 	break;
     case P_WAITING:
@@ -2386,28 +4312,127 @@ resume_process(Process *p)
     p->rstatus = P_FREE;    
 }
 
+int
+erts_get_max_no_executing_schedulers(void)
+{
+#ifdef ERTS_SMP
+    if (erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))
+	return (int) erts_no_schedulers;
+    ERTS_THR_MEMORY_BARRIER;
+    return (int) erts_smp_atomic32_read_nob(&schdlr_sspnd.active);
+#else
+    return 1;
+#endif
+}
+
 #ifdef ERTS_SMP
 
 static void
-susp_sched_prep_block(void *unused)
+scheduler_ix_resume_wake(Uint ix)
+{
+    ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix);
+    erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING
+			   | ERTS_SSI_FLG_TSE_SLEEPING
+			   | ERTS_SSI_FLG_WAITING
+			   | ERTS_SSI_FLG_SUSPENDED);
+    erts_aint32_t oflgs;
+    do {
+	oflgs = erts_smp_atomic32_cmpxchg_relb(&ssi->flags, 0, xflgs);
+	if (oflgs == xflgs) {
+	    erts_sched_finish_poke(ssi, oflgs);
+	    break;
+	}
+	xflgs = oflgs;
+    } while (oflgs & ERTS_SSI_FLG_SUSPENDED);
+}
+
+static erts_aint32_t
+sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, erts_aint32_t xpct)
 {
-    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+    erts_aint32_t oflgs;
+    erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING
+			   | ERTS_SSI_FLG_WAITING
+			   | ERTS_SSI_FLG_SUSPENDED);
+    erts_aint32_t xflgs = xpct;
+
+    do {
+	oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
+	if (oflgs == xflgs)
+	    return nflgs;
+	xflgs = oflgs;
+    } while (oflgs & ERTS_SSI_FLG_SUSPENDED);
+
+    return oflgs;
 }
 
-static void
-susp_sched_resume_block(void *unused)
+static erts_aint32_t
+sched_spin_suspended(ErtsSchedulerSleepInfo *ssi, int spincount)
 {
-    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+    int until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD;
+    int sc = spincount;
+    erts_aint32_t flgs;
+
+    do {
+	flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
+	if ((flgs & (ERTS_SSI_FLG_SLEEPING
+		     | ERTS_SSI_FLG_WAITING
+		     | ERTS_SSI_FLG_SUSPENDED))
+	    != (ERTS_SSI_FLG_SLEEPING
+		| ERTS_SSI_FLG_WAITING
+		| ERTS_SSI_FLG_SUSPENDED)) {
+	    break;
+	}
+	ERTS_SPIN_BODY;
+	if (--until_yield == 0) {
+	    until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD;
+	    erts_thr_yield();
+	}
+    } while (--sc > 0);
+    return flgs;
+}
+
+static erts_aint32_t
+sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi)
+{
+    erts_aint32_t oflgs;
+    erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING
+			   | ERTS_SSI_FLG_TSE_SLEEPING
+			   | ERTS_SSI_FLG_WAITING
+			   | ERTS_SSI_FLG_SUSPENDED);
+    erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING
+			   | ERTS_SSI_FLG_WAITING
+			   | ERTS_SSI_FLG_SUSPENDED);
+
+    erts_tse_reset(ssi->event);
+
+    while (1) {
+	oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
+	if (oflgs == xflgs)
+	    return nflgs;
+	if ((oflgs & (ERTS_SSI_FLG_SLEEPING
+		      | ERTS_SSI_FLG_WAITING
+		      | ERTS_SSI_FLG_SUSPENDED))
+	    != (ERTS_SSI_FLG_SLEEPING
+		| ERTS_SSI_FLG_WAITING
+		| ERTS_SSI_FLG_SUSPENDED)) {
+	    return oflgs;
+	}
+	xflgs = oflgs;
+    }
 }
 
 static void
 suspend_scheduler(ErtsSchedulerData *esdp)
 {
+    erts_aint32_t flgs;
+    erts_aint32_t changing;
     long no = (long) esdp->no;
-    ErtsRunQueue *rq = esdp->run_queue;
+    ErtsSchedulerSleepInfo *ssi = esdp->ssi;
     long active_schedulers;
     int curr_online = 1;
     int wake = 0;
+    erts_aint32_t aux_work;
+    int thr_prgr_active = 1;
 
     /*
      * Schedulers may be suspended in two different ways:
@@ -2424,126 +4449,150 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
     erts_smp_runq_unlock(esdp->run_queue);
 
-    /* Unbind from cpu */
-    erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx);
-    if (scheduler2cpu_map[esdp->no].bound_id >= 0
-	&& erts_unbind_from_cpu(erts_cpuinfo) == 0) {
-	esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1;
-    }
-    erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx);
+    erts_sched_check_cpu_bind_prep_suspend(esdp);
 
     if (erts_system_profile_flags.scheduler)
     	profile_scheduler(make_small(esdp->no), am_inactive);
 
-    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+    sched_wall_time_change(esdp, 0);
 
-    active_schedulers = erts_smp_atomic_dectest(&schdlr_sspnd.active);
-    ASSERT(active_schedulers >= 1);
-    if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_MULTI_SCHED) {
-	if (active_schedulers == schdlr_sspnd.msb.wait_active)
-	    wake = 1;
-	if (active_schedulers == 1)
-	    schdlr_sspnd.changing = 0;
-    }
+    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 
-    while (1) {
+    flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED);
+    if (flgs & ERTS_SSI_FLG_SUSPENDED) {
 
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
-	int check_children;
-	erts_smp_runq_lock(esdp->run_queue);
-	check_children = esdp->check_children;
-	esdp->check_children = 0;
-	erts_smp_runq_unlock(esdp->run_queue);
-	if (check_children) {
-	    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-	    erts_check_children();
-	    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+	active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.active);
+	ASSERT(active_schedulers >= 1);
+	changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+	if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) {
+	    if (active_schedulers == schdlr_sspnd.msb.wait_active)
+		wake = 1;
+	    if (active_schedulers == 1) {
+		changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+							   ~ERTS_SCHDLR_SSPND_CHNG_MSB);
+		changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB;
+	    }
 	}
-#endif
 
-	if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE) {
-	    int changed = 0;
-	    if (no > schdlr_sspnd.online && curr_online) {
-		schdlr_sspnd.curr_online--;
-		curr_online = 0;
-		changed = 1;
-	    }
-	    else if (no <= schdlr_sspnd.online && !curr_online) {
-		schdlr_sspnd.curr_online++;
-		curr_online = 1;
-		changed = 1;
+	while (1) {
+	    if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) {
+		int changed = 0;
+		if (no > schdlr_sspnd.online && curr_online) {
+		    schdlr_sspnd.curr_online--;
+		    curr_online = 0;
+		    changed = 1;
+		}
+		else if (no <= schdlr_sspnd.online && !curr_online) {
+		    schdlr_sspnd.curr_online++;
+		    curr_online = 1;
+		    changed = 1;
+		}
+		if (changed
+		    && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online)
+		    wake = 1;
+		if (schdlr_sspnd.online == schdlr_sspnd.curr_online) {
+		    changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+							       ~ERTS_SCHDLR_SSPND_CHNG_ONLN);
+		    changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN;
+		}
 	    }
-	    if (changed
-		&& schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online)
-		wake = 1;
-	    if (schdlr_sspnd.online == schdlr_sspnd.curr_online)
-		schdlr_sspnd.changing = 0;
-	}
 
-	if (wake) {
-	    erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
-	    wake = 0;
-	}
+	    if (wake) {
+		erts_smp_cnd_signal(&schdlr_sspnd.cnd);
+		wake = 0;
+	    }
 
+	    if (curr_online && !ongoing_multi_scheduling_block()) {
+		flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
+		if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
+		    break;
+	    }
+	    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
 
-	if (!(rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ|ERTS_RUNQ_FLG_SUSPENDED)))
-	    break;
-	if ((rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ)
-	    && !erts_smp_atomic_read(&esdp->suspended))
-	    break;
+	    while (1) {
+		erts_aint32_t flgs;
 
-	erts_smp_activity_begin(ERTS_ACTIVITY_WAIT,
-				susp_sched_prep_block,
-				susp_sched_resume_block,
-				NULL);
-	while (1) {
+		aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
+		if (aux_work) {
+		    if (!thr_prgr_active) {
+			erts_thr_progress_active(esdp, thr_prgr_active = 1);
+			sched_wall_time_change(esdp, 1);
+		    }
+		    aux_work = handle_aux_work(&esdp->aux_work_data,
+					       aux_work,
+					       1);
+		    if (aux_work && erts_thr_progress_update(esdp))
+			erts_thr_progress_leader_update(esdp);
+		}
 
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
-	    if (esdp->blocked_check_children)
-		break;
-#endif
+		if (!aux_work) {
+		    if (thr_prgr_active) {
+			erts_thr_progress_active(esdp, thr_prgr_active = 0);
+			sched_wall_time_change(esdp, 0);
+		    }
+		    erts_thr_progress_prepare_wait(esdp);
+		    flgs = sched_spin_suspended(ssi,
+						ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT);
+		    if (flgs == (ERTS_SSI_FLG_SLEEPING
+				 | ERTS_SSI_FLG_WAITING
+				 | ERTS_SSI_FLG_SUSPENDED)) {
+			flgs = sched_set_suspended_sleeptype(ssi);
+			if (flgs == (ERTS_SSI_FLG_SLEEPING
+				     | ERTS_SSI_FLG_TSE_SLEEPING
+				     | ERTS_SSI_FLG_WAITING
+				     | ERTS_SSI_FLG_SUSPENDED)) {
+			    int res;
+
+			    do {
+				res = erts_tse_wait(ssi->event);
+			    } while (res == EINTR);
+			}
+		    }
+		    erts_thr_progress_finalize_wait(esdp);
+		}
 
-	    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
+		flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING
+						       | ERTS_SSI_FLG_SUSPENDED));
+		if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
+		    break;
+		changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+		if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)
+		    break;
+	    }
 
-	    if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE)
-		break;
+	    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+	    changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+	}
 
-	    if (!(rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ
-			       | ERTS_RUNQ_FLG_SUSPENDED)))
-		break;
-	    if ((rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ)
-		&& !erts_smp_atomic_read(&esdp->suspended))
-		break;
+	active_schedulers = erts_smp_atomic32_inc_read_nob(&schdlr_sspnd.active);
+	changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+	if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB)
+	    && schdlr_sspnd.online == active_schedulers) {
+	    erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+					    ~ERTS_SCHDLR_SSPND_CHNG_MSB);
 	}
 
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
-	esdp->blocked_check_children = 0;
-#endif
+	ASSERT(no <= schdlr_sspnd.online);
+	ASSERT(!ongoing_multi_scheduling_block());
 
-	erts_smp_activity_end(ERTS_ACTIVITY_WAIT,
-			      susp_sched_prep_block,
-			      susp_sched_resume_block,
-			      NULL);
     }
 
-    active_schedulers = erts_smp_atomic_inctest(&schdlr_sspnd.active);
-    if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_MULTI_SCHED
-	&& schdlr_sspnd.online == active_schedulers) {
-	schdlr_sspnd.changing = 0;
-    }
     erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
 
+    ASSERT(curr_online);
+
     if (erts_system_profile_flags.scheduler)
     	profile_scheduler(make_small(esdp->no), am_active);
 
+    if (!thr_prgr_active) {
+	erts_thr_progress_active(esdp, thr_prgr_active = 1);
+	sched_wall_time_change(esdp, 1);
+    }
+
     erts_smp_runq_lock(esdp->run_queue);
     non_empty_runq(esdp->run_queue);
 
-    /* Make sure we check if we should bind to a cpu or not... */
-    if (rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ)
-	erts_smp_atomic_set(&esdp->chk_cpu_bind, 1);
-    else
-	rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND;
+    erts_sched_check_cpu_bind_post_suspend(esdp);
 }
 
 #define ERTS_RUNQ_RESET_SUSPEND_INFO(RQ, DBG_ID)			\
@@ -2558,7 +4607,7 @@ do {									\
     (RQ)->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK				\
                     | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK);		\
     (RQ)->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;	\
-    erts_smp_atomic_band(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED);	\
+    erts_smp_atomic32_read_band_nob(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED);\
     for (pix__ = 0; pix__ < ERTS_NO_PROC_PRIO_LEVELS; pix__++) {	\
 	(RQ)->procs.prio_info[pix__].max_len = 0;			\
 	(RQ)->procs.prio_info[pix__].reds = 0;				\
@@ -2600,8 +4649,10 @@ erts_schedulers_state(Uint *total,
 		      int yield_allowed)
 {
     int res;
+    erts_aint32_t changing;
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-    if (yield_allowed && schdlr_sspnd.changing)
+    changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+    if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER))
 	res = ERTS_SCHDLR_SSPND_YIELD_RESTART;
     else {
 	*active = *online = schdlr_sspnd.online;
@@ -2620,17 +4671,23 @@ erts_set_schedulers_online(Process *p,
 			   Sint new_no,
 			   Sint *old_no)
 {
-    int ix, res, no, have_unlocked_plocks;
+    ErtsSchedulerData *esdp;
+    int ix, res, no, have_unlocked_plocks, end_wait;
+    erts_aint32_t changing;
 
     if (new_no < 1 || erts_no_schedulers < new_no)
 	return ERTS_SCHDLR_SSPND_EINVAL;
 
+    esdp = ERTS_PROC_GET_SCHDATA(p);
+    end_wait = 0;
+
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 
     have_unlocked_plocks = 0;
     no = (int) new_no;
 
-    if (schdlr_sspnd.changing) {
+    changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+    if (changing) {
 	res = ERTS_SCHDLR_SSPND_YIELD_RESTART;
     }
     else {
@@ -2639,17 +4696,15 @@ erts_set_schedulers_online(Process *p,
 	    res = ERTS_SCHDLR_SSPND_DONE;
 	}
 	else {
-	    schdlr_sspnd.changing = ERTS_SCHED_CHANGING_ONLINE;
+	    ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN
+					| ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
 	    schdlr_sspnd.online = no;
 	    if (no > online) {
 		int ix;
 		schdlr_sspnd.wait_curr_online = no;
-		if (ongoing_multi_scheduling_block())
-		    /* No schedulers to resume */;
-		else if (erts_common_run_queue) {
+		if (ongoing_multi_scheduling_block()) {
 		    for (ix = online; ix < no; ix++)
-			erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended,
-					    0);
+			erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix));
 		}
 		else {
 		    if (plocks) {
@@ -2663,6 +4718,7 @@ erts_set_schedulers_online(Process *p,
 			erts_smp_runq_lock(rq);
 			ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x5);
 			erts_smp_runq_unlock(rq);
+			scheduler_ix_resume_wake(ix);
 		    }
 		    /*
 		     * Spread evacuation paths among all online
@@ -2673,11 +4729,10 @@ erts_set_schedulers_online(Process *p,
 			ErtsRunQueue *to_rq = ERTS_RUNQ_IX(ix % no);
 			evacuate_run_queue(from_rq, to_rq);
 		    }
-		    erts_smp_atomic_set(&balance_info.used_runqs, no);
+		    set_no_used_runqs(no);
 		    erts_smp_mtx_unlock(&balance_info.update_mtx);
 		    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 		}
-		erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
 		res = ERTS_SCHDLR_SSPND_DONE;
 	    }
 	    else /* if (no < online) */ {
@@ -2694,13 +4749,9 @@ erts_set_schedulers_online(Process *p,
 		    schdlr_sspnd.wait_curr_online = no+1;
 		}
 
-		if (ongoing_multi_scheduling_block())
-		    erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
-		else if (erts_common_run_queue) {
+		if (ongoing_multi_scheduling_block()) {
 		    for (ix = no; ix < online; ix++)
-			erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended,
-					    1);
-		    wake_all_schedulers();
+			erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix));
 		}
 		else {
 		    if (plocks) {
@@ -2723,27 +4774,47 @@ erts_set_schedulers_online(Process *p,
 		    for (ix = erts_no_run_queues-1; ix >= no; ix--)
 			evacuate_run_queue(ERTS_RUNQ_IX(ix),
 					   ERTS_RUNQ_IX(ix % no));
-		    erts_smp_atomic_set(&balance_info.used_runqs, no);
+		    set_no_used_runqs(no);
 		    erts_smp_mtx_unlock(&balance_info.update_mtx);
 		    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-		    ERTS_FOREACH_OP_RUNQ(rq, wake_scheduler(rq, 0));
+		    for (ix = no; ix < online; ix++) {
+			ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
+			wake_scheduler(rq, 0);
+		    }
+		}
+	    }
+
+	    if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) {
+		erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+		if (plocks && !have_unlocked_plocks) {
+		    have_unlocked_plocks = 1;
+		    erts_smp_proc_unlock(p, plocks);
 		}
+		erts_thr_progress_active(esdp, 0);
+		erts_thr_progress_prepare_wait(esdp);
+		end_wait = 1;
+		erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 	    }
 
-	    erts_smp_activity_begin(ERTS_ACTIVITY_WAIT,
-				    susp_sched_prep_block,
-				    susp_sched_resume_block,
-				    NULL);
 	    while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online)
 		erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
-	    erts_smp_activity_end(ERTS_ACTIVITY_WAIT,
-				  susp_sched_prep_block,
-				  susp_sched_resume_block,
-				  NULL);
+
+	    ASSERT(res != ERTS_SCHDLR_SSPND_DONE
+		   ? (ERTS_SCHDLR_SSPND_CHNG_WAITER
+		      & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))
+		   : (ERTS_SCHDLR_SSPND_CHNG_WAITER
+		      == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)));
+	    erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+					    ~ERTS_SCHDLR_SSPND_CHNG_WAITER);
+
 	}
     }
 
     erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+    if (end_wait) {
+	erts_thr_progress_finalize_wait(esdp);
+	erts_thr_progress_active(esdp, 1);
+    }
     if (have_unlocked_plocks)
 	erts_smp_proc_lock(p, plocks);
 
@@ -2754,37 +4825,41 @@ ErtsSchedSuspendResult
 erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 {
     int ix, res, have_unlocked_plocks = 0;
+    erts_aint32_t changing;
     ErtsProcList *plp;
 
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-
-    if (schdlr_sspnd.changing) {
+    changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
+    if (changing) {
 	res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */
     }
     else if (on) { /* ------ BLOCK ------ */
-	if (erts_is_multi_scheduling_blocked()) {
+	if (schdlr_sspnd.msb.procs) {
 	    plp = proclist_create(p);
 	    plp->next = schdlr_sspnd.msb.procs;
 	    schdlr_sspnd.msb.procs = plp;
 	    p->flags |= F_HAVE_BLCKD_MSCHED;
-	    ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1);
+	    ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1);
 	    ASSERT(p->scheduler_data->no == 1);
 	    res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
 	}
 	else {
+	    int online = schdlr_sspnd.online;
 	    p->flags |= F_HAVE_BLCKD_MSCHED;
 	    if (plocks) {
 		have_unlocked_plocks = 1;
 		erts_smp_proc_unlock(p, plocks);
 	    }
-	    erts_smp_atomic_set(&schdlr_sspnd.msb.ongoing, 1);
-	    if (schdlr_sspnd.online == 1) {
+	    ASSERT(!ongoing_multi_scheduling_block());
+	    schdlr_sspnd.msb.ongoing = 1;
+	    if (online == 1) {
 		res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
-		ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1);
+		ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1);
 		ASSERT(p->scheduler_data->no == 1);
 	    }
 	    else {
-		schdlr_sspnd.changing = ERTS_SCHED_CHANGING_MULTI_SCHED;
+		ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB
+					    | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0);
 		if (p->scheduler_data->no == 1) {
 		    res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
 		    schdlr_sspnd.msb.wait_active = 1;
@@ -2797,42 +4872,65 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 		    res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED;
 		    schdlr_sspnd.msb.wait_active = 2;
 		}
-		if (erts_common_run_queue) {
-		    for (ix = 1; ix < schdlr_sspnd.online; ix++)
-			erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, 1);
-		    wake_all_schedulers();
+
+		erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+		erts_smp_mtx_lock(&balance_info.update_mtx);
+		set_no_used_runqs(1);
+		for (ix = 0; ix < online; ix++) {
+		    ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
+		    erts_smp_runq_lock(rq);
+		    ASSERT(!(rq->flags & ERTS_RUNQ_FLG_SUSPENDED));
+		    ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x7);
+		    erts_smp_runq_unlock(rq);
 		}
-		else {
+		/*
+		 * Evacuate all activities in all other run queues
+		 * into the first run queue. Note order is important,
+		 * online run queues has to be evacuated last.
+		 */
+		for (ix = erts_no_run_queues-1; ix >= 1; ix--)
+		    evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(0));
+		erts_smp_mtx_unlock(&balance_info.update_mtx);
+		erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+
+		if (erts_smp_atomic32_read_nob(&schdlr_sspnd.active)
+		    != schdlr_sspnd.msb.wait_active) {
+		    ErtsSchedulerData *esdp;
+
 		    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-		    erts_smp_mtx_lock(&balance_info.update_mtx);
-		    erts_smp_atomic_set(&balance_info.used_runqs, 1);
-		    for (ix = 0; ix < schdlr_sspnd.online; ix++) {
-			ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
-			erts_smp_runq_lock(rq);
-			ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x7);
-			erts_smp_runq_unlock(rq);
+
+		    if (plocks && !have_unlocked_plocks) {
+			have_unlocked_plocks = 1;
+			erts_smp_proc_unlock(p, plocks);
 		    }
-		    /*
-		     * Evacuate all activities in all other run queues
-		     * into the first run queue. Note order is important,
-		     * online run queues has to be evacuated last.
-		     */
-		    for (ix = erts_no_run_queues-1; ix >= 1; ix--)
-			evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(0));
-		    erts_smp_mtx_unlock(&balance_info.update_mtx);
+
+		    esdp = ERTS_PROC_GET_SCHDATA(p);
+
+		    erts_thr_progress_active(esdp, 0);
+		    erts_thr_progress_prepare_wait(esdp);
+
+		    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+
+		    while (erts_smp_atomic32_read_nob(&schdlr_sspnd.active)
+			   != schdlr_sspnd.msb.wait_active)
+			erts_smp_cnd_wait(&schdlr_sspnd.cnd,
+					  &schdlr_sspnd.mtx);
+		    
+		    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+		    
+		    erts_thr_progress_active(esdp, 1);
+		    erts_thr_progress_finalize_wait(esdp);
+
 		    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+
 		}
-		erts_smp_activity_begin(ERTS_ACTIVITY_WAIT,
-					susp_sched_prep_block,
-					susp_sched_resume_block,
-					NULL);
-		while (erts_smp_atomic_read(&schdlr_sspnd.active)
-		       != schdlr_sspnd.msb.wait_active)
-		    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
-		erts_smp_activity_end(ERTS_ACTIVITY_WAIT,
-				      susp_sched_prep_block,
-				      susp_sched_resume_block,
-				      NULL);
+		ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED
+		       ? (ERTS_SCHDLR_SSPND_CHNG_WAITER
+			  & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))
+		       : (ERTS_SCHDLR_SSPND_CHNG_WAITER
+			  == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)));
+		erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+						~ERTS_SCHDLR_SSPND_CHNG_WAITER);
 	    }
 	    plp = proclist_create(p);
 	    plp->next = schdlr_sspnd.msb.procs;
@@ -2876,7 +4974,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 	if (schdlr_sspnd.msb.procs)
 	    res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
 	else {
-	    schdlr_sspnd.changing = ERTS_SCHED_CHANGING_MULTI_SCHED;
+	    ERTS_SCHDLR_SSPND_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0);
 #ifdef DEBUG
 	    ERTS_FOREACH_RUNQ(rq,
 	    {
@@ -2899,17 +4997,11 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 	    });
 #endif
 	    p->flags &= ~F_HAVE_BLCKD_MSCHED;
-	    erts_smp_atomic_set(&schdlr_sspnd.msb.ongoing, 0);
+	    schdlr_sspnd.msb.ongoing = 0;
 	    if (schdlr_sspnd.online == 1) {
 		/* No schedulers to resume */
-		ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1);
-		schdlr_sspnd.changing = 0;
-	    }
-	    else if (erts_common_run_queue) {
-		for (ix = 1; ix < schdlr_sspnd.online; ix++)
-		    erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, 0);
-		wake_all_schedulers();
-		erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
+		ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1);
+		ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB);
 	    }
 	    else {
 		int online = schdlr_sspnd.online;
@@ -2926,6 +5018,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 		    erts_smp_runq_lock(rq);
 		    ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x4);
 		    erts_smp_runq_unlock(rq);
+		    scheduler_ix_resume_wake(ix);
 		}
 
 		/* Spread evacuation paths among all online run queues */
@@ -2933,7 +5026,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 		    evacuate_run_queue(ERTS_RUNQ_IX(ix),
 				       ERTS_RUNQ_IX(ix % online));
 
-		erts_smp_atomic_set(&balance_info.used_runqs, online);
+		set_no_used_runqs(online);
 		/* Make sure that we balance soon... */
 		balance_info.forced_check_balance = 1;
 		erts_smp_runq_lock(ERTS_RUNQ_IX(0));
@@ -2941,7 +5034,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 		erts_smp_runq_unlock(ERTS_RUNQ_IX(0));
 		erts_smp_mtx_unlock(&balance_info.update_mtx);
 		erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-		erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
 	    }
 	    res = ERTS_SCHDLR_SSPND_DONE;
 	}
@@ -2958,7 +5050,7 @@ void
 erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value)
 {
     if (return_value == am_blocked) {
-	long active = erts_smp_atomic_read(&schdlr_sspnd.active);
+	erts_aint32_t active = erts_smp_atomic32_read_nob(&schdlr_sspnd.active);
 	ASSERT(1 <= active && active <= 2);
 	ASSERT(ERTS_PROC_GET_SCHDATA(p)->no == 1);
     }
@@ -2968,8 +5060,11 @@ erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value)
 int
 erts_is_multi_scheduling_blocked(void)
 {
-    return (erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing)
-	    && erts_smp_atomic_read(&schdlr_sspnd.active) == 1);
+    int res;
+    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
+    res = schdlr_sspnd.msb.procs != NULL;
+    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
+    return res;
 }
 
 Eterm
@@ -2978,7 +5073,7 @@ erts_multi_scheduling_blockers(Process *p)
     Eterm res = NIL;
 
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-    if (erts_is_multi_scheduling_blocked()) {
+    if (schdlr_sspnd.msb.procs) {
 	Eterm *hp, *hp_end;
 	ErtsProcList *plp1, *plp2;
 	Uint max_size;
@@ -3010,60 +5105,92 @@ erts_multi_scheduling_blockers(Process *p)
 static void *
 sched_thread_func(void *vesdp)
 {
+    ErtsThrPrgrCallbacks callbacks;
+    ErtsSchedulerData *esdp = vesdp;
+    Uint no = esdp->no;
+#ifdef ERTS_SMP
+    ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch();
+    callbacks.arg = (void *) esdp->ssi;
+    callbacks.wakeup = thr_prgr_wakeup;
+    callbacks.prepare_wait = thr_prgr_prep_wait;
+    callbacks.wait = thr_prgr_wait;
+    callbacks.finalize_wait = thr_prgr_fin_wait;
+
+    erts_thr_progress_register_managed_thread(esdp, &callbacks, 0);
+    erts_alloc_register_scheduler(vesdp);
+#endif
 #ifdef ERTS_ENABLE_LOCK_CHECK
     {
 	char buf[31];
-	Uint no = ((ErtsSchedulerData *) vesdp)->no;
-	erts_snprintf(&buf[0], 31, "scheduler %bpu", no);
+	erts_snprintf(&buf[0], 31, "scheduler %beu", no);
 	erts_lc_set_thread_name(&buf[0]);
     }
 #endif
-    erts_alloc_reg_scheduler_id(((ErtsSchedulerData *) vesdp)->no);
     erts_tsd_set(sched_data_key, vesdp);
 #ifdef ERTS_SMP
+#if HAVE_ERTS_MSEG
+    erts_mseg_late_init();
+#endif
+#if ERTS_USE_ASYNC_READY_Q
+    esdp->aux_work_data.async_ready.queue = erts_get_async_ready_queue(no);
+#endif
+
+    erts_sched_init_check_cpu_bind(esdp);
+
     erts_proc_lock_prepare_proc_lock_waiter();
 #endif
-    erts_register_blockable_thread();
+
 #ifdef HIPE
     hipe_thread_signal_init();
 #endif
     erts_thread_init_float();
-    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 
-    ASSERT(schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE);
+    if (no == 1) {
+	erts_thr_progress_active(esdp, 0);
+	erts_thr_progress_prepare_wait(esdp);
+    }
 
-    schdlr_sspnd.curr_online--;
+    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 
-    if (((ErtsSchedulerData *) vesdp)->no != 1) {
-	if (schdlr_sspnd.online == schdlr_sspnd.curr_online) {
-	    schdlr_sspnd.changing = 0;
-	    erts_smp_cnd_broadcast(&schdlr_sspnd.cnd);
-	}
+    ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)
+	   & ERTS_SCHDLR_SSPND_CHNG_ONLN);
+
+    if (--schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) {
+	erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+					~ERTS_SCHDLR_SSPND_CHNG_ONLN);
+	if (no != 1)
+	    erts_smp_cnd_signal(&schdlr_sspnd.cnd);
     }
-    else if (schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online)
-	schdlr_sspnd.changing = 0;
-    else {
-	erts_smp_activity_begin(ERTS_ACTIVITY_WAIT,
-				susp_sched_prep_block,
-				susp_sched_resume_block,
-				NULL);
+
+    if (no == 1) {
 	while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online)
 	    erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx);
-	erts_smp_activity_end(ERTS_ACTIVITY_WAIT,
-			      susp_sched_prep_block,
-			      susp_sched_resume_block,
-			      NULL);
-	ASSERT(!schdlr_sspnd.changing);
+	ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER);
     }
     erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
 
+    if (no == 1) {
+	erts_thr_progress_finalize_wait(esdp);
+	erts_thr_progress_active(esdp, 1);
+    }
+
+#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
+    esdp->verify_unused_temp_alloc
+	= erts_alloc_get_verify_unused_temp_alloc(
+	    &esdp->verify_unused_temp_alloc_data);
+    ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL);
+#endif
+
     process_main();
     /* No schedulers should *ever* terminate */
-    erl_exit(ERTS_ABORT_EXIT, "Scheduler thread number %bpu terminated\n",
-	     ((ErtsSchedulerData *) vesdp)->no);
+    erl_exit(ERTS_ABORT_EXIT,
+	     "Scheduler thread number %beu terminated\n",
+	     no);
     return NULL;
 }
 
+static ethr_tid aux_tid;
+
 void
 erts_start_schedulers(void)
 {
@@ -3083,17 +5210,11 @@ erts_start_schedulers(void)
 	res = ENOTSUP;
     }
 
-    erts_block_system(0);
-
     while (actual < wanted) {
 	ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(actual);
 	actual++;
 	ASSERT(actual == esdp->no);
-#ifdef ERTS_ENABLE_LOCK_COUNT
-	res = erts_lcnt_thr_create(&esdp->tid,sched_thread_func,(void*)esdp,&opts);
-#else
 	res = ethr_thr_create(&esdp->tid,sched_thread_func,(void*)esdp,&opts);
-#endif
 	if (res != 0) {
 	    actual--;
 	    break;
@@ -3101,7 +5222,12 @@ erts_start_schedulers(void)
     }
     
     erts_no_schedulers = actual;
-    erts_release_system();
+
+    ERTS_THR_MEMORY_BARRIER;
+
+    res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts);
+    if (res != 0)
+	erl_exit(1, "Failed to create aux thread\n");
 
     if (actual < 1)
 	erl_exit(1,
@@ -3112,8 +5238,8 @@ erts_start_schedulers(void)
 	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
 	ASSERT(actual != wanted_no_schedulers);
 	erts_dsprintf(dsbufp,
-		      "Failed to create %bpu scheduler-threads (%s:%d); "
-		      "only %bpu scheduler-thread%s created.\n",
+		      "Failed to create %beu scheduler-threads (%s:%d); "
+		      "only %beu scheduler-thread%s created.\n",
 		      wanted_no_schedulers, erl_errno_id(res), res,
 		      actual, actual == 1 ? " was" : "s were");
 	erts_send_error_to_logger_nogl(dsbufp);
@@ -3122,1351 +5248,6 @@ erts_start_schedulers(void)
 
 #endif /* ERTS_SMP */
 
-static int
-int_cmp(const void *vx, const void *vy)
-{
-    return *((int *) vx) - *((int *) vy);
-}
-
-static int
-cpu_spread_order_cmp(const void *vx, const void *vy)
-{
-    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
-    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
-
-    if (x->thread != y->thread)
-	return x->thread - y->thread;
-    if (x->core != y->core)
-	return x->core - y->core;
-    if (x->processor_node != y->processor_node)
-	return x->processor_node - y->processor_node;
-    if (x->processor != y->processor)
-	return x->processor - y->processor;
-    if (x->node != y->node)
-	return x->node - y->node;
-    return 0;
-}
-
-static int
-cpu_processor_spread_order_cmp(const void *vx, const void *vy)
-{
-    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
-    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
-
-    if (x->thread != y->thread)
-	return x->thread - y->thread;
-    if (x->processor_node != y->processor_node)
-	return x->processor_node - y->processor_node;
-    if (x->core != y->core)
-	return x->core - y->core;
-    if (x->node != y->node)
-	return x->node - y->node;
-    if (x->processor != y->processor)
-	return x->processor - y->processor;
-    return 0;
-}
-
-static int
-cpu_thread_spread_order_cmp(const void *vx, const void *vy)
-{
-    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
-    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
-
-    if (x->thread != y->thread)
-	return x->thread - y->thread;
-    if (x->node != y->node)
-	return x->node - y->node;
-    if (x->processor != y->processor)
-	return x->processor - y->processor;
-    if (x->processor_node != y->processor_node)
-	return x->processor_node - y->processor_node;
-    if (x->core != y->core)
-	return x->core - y->core;
-    return 0;
-}
-
-static int
-cpu_thread_no_node_processor_spread_order_cmp(const void *vx, const void *vy)
-{
-    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
-    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
-
-    if (x->thread != y->thread)
-	return x->thread - y->thread;
-    if (x->node != y->node)
-	return x->node - y->node;
-    if (x->core != y->core)
-	return x->core - y->core;
-    if (x->processor != y->processor)
-	return x->processor - y->processor;
-    return 0;
-}
-
-static int
-cpu_no_node_processor_spread_order_cmp(const void *vx, const void *vy)
-{
-    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
-    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
-
-    if (x->node != y->node)
-	return x->node - y->node;
-    if (x->thread != y->thread)
-	return x->thread - y->thread;
-    if (x->core != y->core)
-	return x->core - y->core;
-    if (x->processor != y->processor)
-	return x->processor - y->processor;
-    return 0;
-}
-
-static int
-cpu_no_node_thread_spread_order_cmp(const void *vx, const void *vy)
-{
-    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
-    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
-
-    if (x->node != y->node)
-	return x->node - y->node;
-    if (x->thread != y->thread)
-	return x->thread - y->thread;
-    if (x->processor != y->processor)
-	return x->processor - y->processor;
-    if (x->core != y->core)
-	return x->core - y->core;
-    return 0;
-}
-
-static int
-cpu_no_spread_order_cmp(const void *vx, const void *vy)
-{
-    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
-    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
-
-    if (x->node != y->node)
-	return x->node - y->node;
-    if (x->processor != y->processor)
-	return x->processor - y->processor;
-    if (x->processor_node != y->processor_node)
-	return x->processor_node - y->processor_node;
-    if (x->core != y->core)
-	return x->core - y->core;
-    if (x->thread != y->thread)
-	return x->thread - y->thread;
-    return 0;
-}
-
-static ERTS_INLINE void
-make_cpudata_id_seq(erts_cpu_topology_t *cpudata, int size, int no_node)
-{
-    int ix;
-    int node = -1;
-    int processor = -1;
-    int processor_node = -1;
-    int processor_node_node = -1;
-    int core = -1;
-    int thread = -1;
-    int old_node = -1;
-    int old_processor = -1;
-    int old_processor_node = -1;
-    int old_core = -1;
-    int old_thread = -1;
-
-    for (ix = 0; ix < size; ix++) {
-	if (!no_node || cpudata[ix].node >= 0) {
-	    if (old_node == cpudata[ix].node)
-		cpudata[ix].node = node;
-	    else {
-		old_node = cpudata[ix].node;
-		old_processor = processor = -1;
-		if (!no_node)
-		    old_processor_node = processor_node = -1;
-		old_core = core = -1;
-		old_thread = thread = -1;
-		if (no_node || cpudata[ix].node >= 0)
-		    cpudata[ix].node = ++node;
-	    }
-	}
-	if (old_processor == cpudata[ix].processor)
-	    cpudata[ix].processor = processor;
-	else {
-	    old_processor = cpudata[ix].processor;
-	    if (!no_node)
-		processor_node_node = old_processor_node = processor_node = -1;
-	    old_core = core = -1;
-	    old_thread = thread = -1;
-	    cpudata[ix].processor = ++processor;
-	}
-	if (no_node && cpudata[ix].processor_node < 0)
-	    old_processor_node = -1;
-	else {
-	    if (old_processor_node == cpudata[ix].processor_node) {
-		if (no_node)
-		    cpudata[ix].node = cpudata[ix].processor_node = node;
-		else {
-		    if (processor_node_node >= 0)
-			cpudata[ix].node = processor_node_node;
-		    cpudata[ix].processor_node = processor_node;
-		}
-	    }
-	    else {
-		old_processor_node = cpudata[ix].processor_node;
-		old_core = core = -1;
-		old_thread = thread = -1;
-		if (no_node)
-		    cpudata[ix].node = cpudata[ix].processor_node = ++node;
-		else {
-		    cpudata[ix].node = processor_node_node = ++node;
-		    cpudata[ix].processor_node = ++processor_node;
-		}
-	    }
-	}
-	if (!no_node && cpudata[ix].processor_node < 0)
-	    cpudata[ix].processor_node = 0;
-	if (old_core == cpudata[ix].core)
-	    cpudata[ix].core = core;
-	else {
-	    old_core = cpudata[ix].core;
-	    old_thread = thread = -1;
-	    cpudata[ix].core = ++core;
-	}
-	if (old_thread == cpudata[ix].thread)
-	    cpudata[ix].thread = thread;
-	else
-	    old_thread = cpudata[ix].thread = ++thread;
-    }
-}
-
-static void
-cpu_bind_order_sort(erts_cpu_topology_t *cpudata,
-		    int size,
-		    ErtsCpuBindOrder bind_order,
-		    int mk_seq)
-{
-    if (size > 1) {
-	int no_node = 0;
-	int (*cmp_func)(const void *, const void *);
-	switch (bind_order) {
-	case ERTS_CPU_BIND_SPREAD:
-	    cmp_func = cpu_spread_order_cmp;
-	    break;
-	case ERTS_CPU_BIND_PROCESSOR_SPREAD:
-	    cmp_func = cpu_processor_spread_order_cmp;
-	    break;
-	case ERTS_CPU_BIND_THREAD_SPREAD:
-	    cmp_func = cpu_thread_spread_order_cmp;
-	    break;
-	case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD:
-	    no_node = 1;
-	    cmp_func = cpu_thread_no_node_processor_spread_order_cmp;
-	    break;
-	case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD:
-	    no_node = 1;
-	    cmp_func = cpu_no_node_processor_spread_order_cmp;
-	    break;
-	case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD:
-	    no_node = 1;
-	    cmp_func = cpu_no_node_thread_spread_order_cmp;
-	    break;
-	case ERTS_CPU_BIND_NO_SPREAD:
-	    cmp_func = cpu_no_spread_order_cmp;
-	    break;
-	default:
-	    cmp_func = NULL;
-	    erl_exit(ERTS_ABORT_EXIT,
-		     "Bad cpu bind type: %d\n",
-		     (int) cpu_bind_order);
-	    break;
-	}
-
-	if (mk_seq)
-	    make_cpudata_id_seq(cpudata, size, no_node);
-
-	qsort(cpudata, size, sizeof(erts_cpu_topology_t), cmp_func);
-    }
-}
-
-static int
-processor_order_cmp(const void *vx, const void *vy)
-{
-    erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx;
-    erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy;
-
-    if (x->processor != y->processor)
-	return x->processor - y->processor;
-    if (x->node != y->node)
-	return x->node - y->node;
-    if (x->processor_node != y->processor_node)
-	return x->processor_node - y->processor_node;
-    if (x->core != y->core)
-	return x->core - y->core;
-    if (x->thread != y->thread)
-	return x->thread - y->thread;
-    return 0;
-}
-
-static void
-check_cpu_bind(ErtsSchedulerData *esdp)
-{
-    int res;
-    int cpu_id;
-    erts_smp_runq_unlock(esdp->run_queue);
-    erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx);
-    cpu_id = scheduler2cpu_map[esdp->no].bind_id;
-    if (cpu_id >= 0 && cpu_id != scheduler2cpu_map[esdp->no].bound_id) {
-	res = erts_bind_to_cpu(erts_cpuinfo, cpu_id);
-	if (res == 0)
-	    esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = cpu_id;
-	else {
-	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	    erts_dsprintf(dsbufp, "Scheduler %d failed to bind to cpu %d: %s\n",
-			  (int) esdp->no, cpu_id, erl_errno_id(-res));
-	    erts_send_error_to_logger_nogl(dsbufp);
-	    if (scheduler2cpu_map[esdp->no].bound_id >= 0)
-		goto unbind;
-	}
-    }
-    else if (cpu_id < 0 && scheduler2cpu_map[esdp->no].bound_id >= 0) {
-    unbind:
-	/* Get rid of old binding */
-	res = erts_unbind_from_cpu(erts_cpuinfo);
-	if (res == 0)
-	    esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1;
-	else {
-	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	    erts_dsprintf(dsbufp, "Scheduler %d failed to unbind from cpu %d: %s\n",
-			  (int) esdp->no, cpu_id, erl_errno_id(-res));
-	    erts_send_error_to_logger_nogl(dsbufp);
-	}
-    }
-    erts_smp_runq_lock(esdp->run_queue);
-#ifdef ERTS_SMP
-    if (erts_common_run_queue)
-	erts_smp_atomic_set(&esdp->chk_cpu_bind, 0);
-    else {
-	esdp->run_queue->flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND;
-    }
-#endif
-    erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx);
-
-}
-
-static void
-signal_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size)
-{
-    int s_ix = 1;
-    int cpu_ix;
-
-    if (cpu_bind_order != ERTS_CPU_BIND_NONE) {
-
-	cpu_bind_order_sort(cpudata, size, cpu_bind_order, 1);
-
-	for (cpu_ix = 0; cpu_ix < size && cpu_ix < erts_no_schedulers; cpu_ix++)
-	    if (erts_is_cpu_available(erts_cpuinfo, cpudata[cpu_ix].logical))
-		scheduler2cpu_map[s_ix++].bind_id = cpudata[cpu_ix].logical;
-    }
-
-    if (s_ix <= erts_no_schedulers)
-	for (; s_ix <= erts_no_schedulers; s_ix++)
-	    scheduler2cpu_map[s_ix].bind_id = -1;
-
-#ifdef ERTS_SMP
-    if (erts_common_run_queue) {
-	for (s_ix = 0; s_ix < erts_no_schedulers; s_ix++)
-	    erts_smp_atomic_set(&ERTS_SCHEDULER_IX(s_ix)->chk_cpu_bind, 1);
-	wake_all_schedulers();
-    }
-    else {
-	ERTS_FOREACH_RUNQ(rq,
-	{
-	    rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND;
-	    wake_scheduler(rq, 0);
-	});
-    }
-#else
-    check_cpu_bind(erts_get_scheduler_data());
-#endif
-}
-
-int
-erts_init_scheduler_bind_type(char *how)
-{
-    if (erts_bind_to_cpu(erts_cpuinfo, -1) == -ENOTSUP)
-	return ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED;
-
-    if (!system_cpudata && !user_cpudata)
-	return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY;
-
-    if (sys_strcmp(how, "s") == 0)
-	cpu_bind_order = ERTS_CPU_BIND_SPREAD;
-    else if (sys_strcmp(how, "ps") == 0)
-	cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD;
-    else if (sys_strcmp(how, "ts") == 0)
-	cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD;
-    else if (sys_strcmp(how, "db") == 0
-	     || sys_strcmp(how, "tnnps") == 0)
-	cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD;
-    else if (sys_strcmp(how, "nnps") == 0)
-	cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD;
-    else if (sys_strcmp(how, "nnts") == 0)
-	cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD;
-    else if (sys_strcmp(how, "ns") == 0)
-	cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD;
-    else if (sys_strcmp(how, "u") == 0)
-	cpu_bind_order = ERTS_CPU_BIND_NONE;
-    else
-	return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE;
-
-    return ERTS_INIT_SCHED_BIND_TYPE_SUCCESS;
-}
-
-typedef struct {
-    int *id;
-    int used;
-    int size;
-} ErtsCpuTopIdSeq;
-
-typedef struct {
-    ErtsCpuTopIdSeq logical;
-    ErtsCpuTopIdSeq thread;
-    ErtsCpuTopIdSeq core;
-    ErtsCpuTopIdSeq processor_node;
-    ErtsCpuTopIdSeq processor;
-    ErtsCpuTopIdSeq node;
-} ErtsCpuTopEntry;
-
-static void
-init_cpu_top_entry(ErtsCpuTopEntry *cte)
-{
-    int size = 10;
-    cte->logical.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
-				 sizeof(int)*size);
-    cte->logical.size = size;
-    cte->thread.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
-				sizeof(int)*size);
-    cte->thread.size = size;
-    cte->core.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
-			      sizeof(int)*size);
-    cte->core.size = size;
-    cte->processor_node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
-					sizeof(int)*size);
-    cte->processor_node.size = size;
-    cte->processor.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
-				   sizeof(int)*size);
-    cte->processor.size = size;
-    cte->node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS,
-			      sizeof(int)*size);
-    cte->node.size = size;
-}
-
-static void
-destroy_cpu_top_entry(ErtsCpuTopEntry *cte)
-{
-    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->logical.id);
-    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->thread.id);
-    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->core.id);
-    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor_node.id);
-    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor.id);
-    erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->node.id);
-}
-
-static int
-get_cput_value_or_range(int *v, int *vr, char **str)
-{
-    long l;
-    char *c = *str;
-    errno = 0;
-    if (!isdigit((unsigned char)*c))
-	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID;
-    l = strtol(c, &c, 10);
-    if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l)
-	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID;
-    *v = (int) l;
-    if (*c == '-') {
-	c++;
-	if (!isdigit((unsigned char)*c))
-	    return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
-	l = strtol(c, &c, 10);
-	if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l)
-	    return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
-	*vr = (int) l;
-    }
-    *str = c;
-    return ERTS_INIT_CPU_TOPOLOGY_OK;
-}
-
-static int
-get_cput_id_seq(ErtsCpuTopIdSeq *idseq, char **str)
-{
-    int ix = 0;
-    int need_size = 0;
-    char *c = *str;
-
-    while (1) {
-	int res;
-	int val;
-	int nids;
-	int val_range = -1;
-	res = get_cput_value_or_range(&val, &val_range, &c);
-	if (res != ERTS_INIT_CPU_TOPOLOGY_OK)
-	    return res;
-	if (val_range < 0 || val_range == val)
-	    nids = 1;
-	else {
-	    if (val_range > val)
-		nids = val_range - val + 1;
-	    else
-		nids = val - val_range + 1;
-	}
-	need_size += nids;
-	if (need_size > idseq->size) {
-	    idseq->size = need_size + 10;
-	    idseq->id = erts_realloc(ERTS_ALC_T_TMP_CPU_IDS,
-				      idseq->id,
-				      sizeof(int)*idseq->size);
-	}
-	if (nids == 1)
-	    idseq->id[ix++] = val;
-	else if (val_range > val) {
-	    for (; val <= val_range; val++)
-		idseq->id[ix++] = val;
-	}
-	else {
-	    for (; val >= val_range; val--)
-		idseq->id[ix++] = val;
-	}
-	if (*c != ',')
-	    break;
-	c++;
-    }
-    *str = c;
-    idseq->used = ix;
-    return ERTS_INIT_CPU_TOPOLOGY_OK;
-}
-
-static int
-get_cput_entry(ErtsCpuTopEntry *cput, char **str)
-{
-    int h;
-    char *c = *str;
-
-    cput->logical.used = 0;
-    cput->thread.id[0] = 0;
-    cput->thread.used = 1;
-    cput->core.id[0] = 0;
-    cput->core.used = 1;
-    cput->processor_node.id[0] = -1;
-    cput->processor_node.used = 1;
-    cput->processor.id[0] = 0;
-    cput->processor.used = 1;
-    cput->node.id[0] = -1;
-    cput->node.used = 1;
-
-    h = ERTS_TOPOLOGY_MAX_DEPTH;
-    while (*c != ':' && *c != '\0') {
-	int res;
-	ErtsCpuTopIdSeq *idseqp;
-	switch (*c++) {
-	case 'L':
-	    if (h <= ERTS_TOPOLOGY_LOGICAL)
-		return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
-	    idseqp = &cput->logical;
-	    h = ERTS_TOPOLOGY_LOGICAL;
-	    break;
-	case 't':
-	case 'T':
-	    if (h <= ERTS_TOPOLOGY_THREAD)
-		return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
-	    idseqp = &cput->thread;
-	    h = ERTS_TOPOLOGY_THREAD;
-	    break;
-	case 'c':
-	case 'C':
-	    if (h <= ERTS_TOPOLOGY_CORE)
-		return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
-	    idseqp = &cput->core;
-	    h = ERTS_TOPOLOGY_CORE;
-	    break;
-	case 'p':
-	case 'P':
-	    if (h <= ERTS_TOPOLOGY_PROCESSOR)
-		return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
-	    idseqp = &cput->processor;
-	    h = ERTS_TOPOLOGY_PROCESSOR;
-	    break;
-	case 'n':
-	case 'N':
-	    if (h <= ERTS_TOPOLOGY_PROCESSOR) {
-	    do_node:
-		if (h <= ERTS_TOPOLOGY_NODE)
-		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
-		idseqp = &cput->node;
-		h = ERTS_TOPOLOGY_NODE;
-	    }
-	    else {
-		int p_node = 0;
-		char *p_chk = c;
-		while (*p_chk != '\0' && *p_chk != ':') {
-		    if (*p_chk == 'p' || *p_chk == 'P') {
-			p_node = 1;
-			break;
-		    }
-		    p_chk++;
-		}
-		if (!p_node)
-		    goto do_node;
-		if (h <= ERTS_TOPOLOGY_PROCESSOR_NODE)
-		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY;
-		idseqp = &cput->processor_node;
-		h = ERTS_TOPOLOGY_PROCESSOR_NODE;
-	    }
-	    break;
-	default:
-	    return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE;
-	}
-	res = get_cput_id_seq(idseqp, &c);
-	if (res != ERTS_INIT_CPU_TOPOLOGY_OK)
-		return res;
-    }
-
-    if (cput->logical.used < 1)
-	return ERTS_INIT_CPU_TOPOLOGY_MISSING_LID;
-
-    if (*c == ':') {
-	c++;
-    }
-
-    if (cput->thread.used != 1
-	&& cput->thread.used != cput->logical.used)
-	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
-    if (cput->core.used != 1
-	&& cput->core.used != cput->logical.used)
-	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
-    if (cput->processor_node.used != 1
-	&& cput->processor_node.used != cput->logical.used)
-	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
-    if (cput->processor.used != 1
-	&& cput->processor.used != cput->logical.used)
-	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
-    if (cput->node.used != 1
-	&& cput->node.used != cput->logical.used)
-	return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE;
-
-    *str = c;
-    return ERTS_INIT_CPU_TOPOLOGY_OK;
-}
-
-static int
-verify_topology(erts_cpu_topology_t *cpudata, int size)
-{
-    if (size > 0) {
-	int *logical;
-	int node, processor, no_nodes, i;
-
-	/* Verify logical ids */
-	logical = erts_alloc(ERTS_ALC_T_TMP, sizeof(int)*size);
-
-	for (i = 0; i < user_cpudata_size; i++)
-	    logical[i] = user_cpudata[i].logical;
-
-	qsort(logical, user_cpudata_size, sizeof(int), int_cmp);
-	for (i = 0; i < user_cpudata_size-1; i++) {
-	    if (logical[i] == logical[i+1]) {
-		erts_free(ERTS_ALC_T_TMP, logical);
-		return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS;
-	    }
-	}
-
-	erts_free(ERTS_ALC_T_TMP, logical);
-
-	qsort(cpudata, size, sizeof(erts_cpu_topology_t), processor_order_cmp);
-
-	/* Verify unique entities */
-
-	for (i = 1; i < user_cpudata_size; i++) {
-	    if (user_cpudata[i-1].processor == user_cpudata[i].processor
-		&& user_cpudata[i-1].node == user_cpudata[i].node
-		&& (user_cpudata[i-1].processor_node
-		    == user_cpudata[i].processor_node)
-		&& user_cpudata[i-1].core == user_cpudata[i].core
-		&& user_cpudata[i-1].thread == user_cpudata[i].thread) {
-		return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES;
-	    }
-	}
-
-	/* Verify numa nodes */
-	node = cpudata[0].node;
-	processor = cpudata[0].processor;
-	no_nodes = cpudata[0].node < 0 && cpudata[0].processor_node < 0;
-	for (i = 1; i < size; i++) {
-	    if (no_nodes) {
-		if (cpudata[i].node >= 0 || cpudata[i].processor_node >= 0)
-		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES;
-	    }
-	    else {
-		if (cpudata[i].processor == processor && cpudata[i].node != node)
-		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES;
-		node = cpudata[i].node;
-		processor = cpudata[i].processor;
-		if (node >= 0 && cpudata[i].processor_node >= 0)
-		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES;
-		if (node < 0 && cpudata[i].processor_node < 0)
-		    return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES;
-	    }
-	}
-    }
-
-    return ERTS_INIT_CPU_TOPOLOGY_OK;
-}
-
-int
-erts_init_cpu_topology(char *topology_str)
-{
-    ErtsCpuTopEntry cput;
-    int need_size;
-    char *c;
-    int ix;
-    int error = ERTS_INIT_CPU_TOPOLOGY_OK;
-
-    if (user_cpudata)
-	erts_free(ERTS_ALC_T_CPUDATA, user_cpudata);
-    user_cpudata_size = 10;
-
-    user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA,
-			      (sizeof(erts_cpu_topology_t)
-			       * user_cpudata_size));
-
-    init_cpu_top_entry(&cput);
-
-    ix = 0;
-    need_size = 0;
-
-    c = topology_str;
-    if (*c == '\0') {
-	error = ERTS_INIT_CPU_TOPOLOGY_MISSING;
-	goto fail;
-    }
-    do {
-	int r;
-	error = get_cput_entry(&cput, &c);
-	if (error != ERTS_INIT_CPU_TOPOLOGY_OK)
-	    goto fail;
-	need_size += cput.logical.used;
-	if (user_cpudata_size < need_size) {
-	    user_cpudata_size = need_size + 10;
-	    user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA,
-					user_cpudata,
-					(sizeof(erts_cpu_topology_t)
-					 * user_cpudata_size));
-	}
-
-	ASSERT(cput.thread.used == 1
-	       || cput.thread.used == cput.logical.used);
-	ASSERT(cput.core.used == 1
-	       || cput.core.used == cput.logical.used);
-	ASSERT(cput.processor_node.used == 1
-	       || cput.processor_node.used == cput.logical.used);
-	ASSERT(cput.processor.used == 1
-	       || cput.processor.used == cput.logical.used);
-	ASSERT(cput.node.used == 1
-	       || cput.node.used == cput.logical.used);
-
-	for (r = 0; r < cput.logical.used; r++) {
-	    user_cpudata[ix].logical = cput.logical.id[r];
-	    user_cpudata[ix].thread =
-		cput.thread.id[cput.thread.used == 1 ? 0 : r];
-	    user_cpudata[ix].core =
-		cput.core.id[cput.core.used == 1 ? 0 : r];
-	    user_cpudata[ix].processor_node =
-		cput.processor_node.id[cput.processor_node.used == 1 ? 0 : r];
-	    user_cpudata[ix].processor =
-		cput.processor.id[cput.processor.used == 1 ? 0 : r];
-	    user_cpudata[ix].node =
-		cput.node.id[cput.node.used == 1 ? 0 : r];
-	    ix++;
-	}
-    } while (*c != '\0');
-
-    if (user_cpudata_size != ix) {
-	user_cpudata_size = ix;
-	user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA,
-				    user_cpudata,
-				    (sizeof(erts_cpu_topology_t)
-				     * user_cpudata_size));
-    }
-
-    error = verify_topology(user_cpudata, user_cpudata_size);
-    if (error == ERTS_INIT_CPU_TOPOLOGY_OK) {
-	destroy_cpu_top_entry(&cput);
-	return ERTS_INIT_CPU_TOPOLOGY_OK;
-    }
-
- fail:
-    if (user_cpudata)
-	erts_free(ERTS_ALC_T_CPUDATA, user_cpudata);
-    user_cpudata_size = 0;
-    destroy_cpu_top_entry(&cput);
-    return error;
-}
-
-#define ERTS_GET_CPU_TOPOLOGY_ERROR		-1
-#define ERTS_GET_USED_CPU_TOPOLOGY		0
-#define ERTS_GET_DETECTED_CPU_TOPOLOGY		1
-#define ERTS_GET_DEFINED_CPU_TOPOLOGY		2
-
-static Eterm get_cpu_topology_term(Process *c_p, int type);
-
-Eterm
-erts_set_cpu_topology(Process *c_p, Eterm term)
-{
-    erts_cpu_topology_t *cpudata = NULL;
-    int cpudata_size = 0;
-    Eterm res;
-
-    erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx);
-    res = get_cpu_topology_term(c_p, ERTS_GET_USED_CPU_TOPOLOGY);
-    if (term == am_undefined) {
-	if (user_cpudata)
-	    erts_free(ERTS_ALC_T_CPUDATA, user_cpudata);
-	user_cpudata = NULL;
-	user_cpudata_size = 0;
-
-	if (cpu_bind_order != ERTS_CPU_BIND_NONE && system_cpudata) {
-	    cpudata_size = system_cpudata_size;
-	    cpudata = erts_alloc(ERTS_ALC_T_TMP,
-				 (sizeof(erts_cpu_topology_t)
-				  * cpudata_size));
-
-	    sys_memcpy((void *) cpudata,
-		       (void *) system_cpudata,
-		       sizeof(erts_cpu_topology_t)*cpudata_size);
-	}
-    }
-    else if (is_not_list(term)) {
-    error:
-	res = THE_NON_VALUE;
-	goto done;
-    }
-    else {
-	Eterm list = term;
-	int ix = 0;
-
-	cpudata_size = 100;
-	cpudata = erts_alloc(ERTS_ALC_T_TMP,
-			     (sizeof(erts_cpu_topology_t)
-			      * cpudata_size));
-
-	while (is_list(list)) {
-	    Eterm *lp = list_val(list);
-	    Eterm cpu = CAR(lp);
-	    Eterm* tp;
-	    Sint id;
-		
-	    if (is_not_tuple(cpu))
-		goto error;
-
-	    tp = tuple_val(cpu);
-
-	    if (arityval(tp[0]) != 7 || tp[1] != am_cpu)
-		goto error;
-
-	    if (ix >= cpudata_size) {
-		cpudata_size += 100;
-		cpudata = erts_realloc(ERTS_ALC_T_TMP,
-				       cpudata,
-				       (sizeof(erts_cpu_topology_t)
-					* cpudata_size));
-	    }
-
-	    id = signed_val(tp[2]);
-	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
-		goto error;
-	    cpudata[ix].node = (int) id;
-
-	    id = signed_val(tp[3]);
-	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
-		goto error;
-	    cpudata[ix].processor = (int) id;
-
-	    id = signed_val(tp[4]);
-	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
-		goto error;
-	    cpudata[ix].processor_node = (int) id;
-
-	    id = signed_val(tp[5]);
-	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
-		goto error;
-	    cpudata[ix].core = (int) id;
-
-	    id = signed_val(tp[6]);
-	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
-		goto error;
-	    cpudata[ix].thread = (int) id;
-
-	    id = signed_val(tp[7]);
-	    if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id)
-		goto error;
-	    cpudata[ix].logical = (int) id;
-
-	    list = CDR(lp);
-	    ix++;
-	}
-
-	if (is_not_nil(list))
-	    goto error;
-	
-	cpudata_size = ix;
-
-	if (ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(cpudata, cpudata_size))
-	    goto error;
-
-	if (user_cpudata_size != cpudata_size) {
-	    if (user_cpudata)
-		erts_free(ERTS_ALC_T_CPUDATA, user_cpudata);
-	    user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA,
-				      sizeof(erts_cpu_topology_t)*cpudata_size);
-	    user_cpudata_size = cpudata_size;
-	}
-
-	sys_memcpy((void *) user_cpudata,
-		   (void *) cpudata,
-		   sizeof(erts_cpu_topology_t)*cpudata_size);
-    }
-
-    signal_schedulers_bind_change(cpudata, cpudata_size);
-
- done:
-    erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx);
-
-    if (cpudata)
-	erts_free(ERTS_ALC_T_TMP, cpudata);
-
-    return res;
-}
-
-static Eterm
-bound_schedulers_term(ErtsCpuBindOrder order)
-{
-    switch (order) {
-    case ERTS_CPU_BIND_SPREAD: {
-	ERTS_DECL_AM(spread);
-	return AM_spread;
-    }
-    case ERTS_CPU_BIND_PROCESSOR_SPREAD: {
-	ERTS_DECL_AM(processor_spread);
-	return AM_processor_spread;
-    }
-    case ERTS_CPU_BIND_THREAD_SPREAD: {
-	ERTS_DECL_AM(thread_spread);
-	return AM_thread_spread;
-    }
-    case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD: {
-	ERTS_DECL_AM(thread_no_node_processor_spread);
-	return AM_thread_no_node_processor_spread;
-    }
-    case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD: {
-	ERTS_DECL_AM(no_node_processor_spread);
-	return AM_no_node_processor_spread;
-    }
-    case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD: {
-	ERTS_DECL_AM(no_node_thread_spread);
-	return AM_no_node_thread_spread;
-    }
-    case ERTS_CPU_BIND_NO_SPREAD: {
-	ERTS_DECL_AM(no_spread);
-	return AM_no_spread;
-    }
-    case ERTS_CPU_BIND_NONE: {
-	ERTS_DECL_AM(unbound);
-	return AM_unbound;
-    }
-    default:
-	ASSERT(0);
-	return THE_NON_VALUE;
-    }
-}
-
-Eterm
-erts_bound_schedulers_term(Process *c_p)
-{
-    ErtsCpuBindOrder order;
-    erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx);
-    order = cpu_bind_order;
-    erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx);
-    return bound_schedulers_term(order);
-}
-
-static void
-create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata, int *cpudata_size)
-{
-    if (user_cpudata) {
-	*cpudata_size = user_cpudata_size;
-	*cpudata = erts_alloc(ERTS_ALC_T_TMP,
-			      (sizeof(erts_cpu_topology_t)
-			       * (*cpudata_size)));
-	sys_memcpy((void *) *cpudata,
-		   (void *) user_cpudata,
-		   sizeof(erts_cpu_topology_t)*(*cpudata_size));
-    }
-    else if (system_cpudata) {
-	*cpudata_size = system_cpudata_size;
-	*cpudata = erts_alloc(ERTS_ALC_T_TMP,
-			      (sizeof(erts_cpu_topology_t)
-			       * (*cpudata_size)));
-	sys_memcpy((void *) *cpudata,
-		   (void *) system_cpudata,
-		   sizeof(erts_cpu_topology_t)*(*cpudata_size));
-    }
-    else {
-	*cpudata = NULL;
-	*cpudata_size = 0;
-    }
-}
-
-static void
-destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata)
-{
-    if (cpudata)
-	erts_free(ERTS_ALC_T_TMP, cpudata);
-}
-
-Eterm
-erts_bind_schedulers(Process *c_p, Eterm how)
-{
-    Eterm res;
-    erts_cpu_topology_t *cpudata;
-    int cpudata_size;
-    ErtsCpuBindOrder old_cpu_bind_order;
-
-    erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx);
-
-    if (erts_bind_to_cpu(erts_cpuinfo, -1) == -ENOTSUP) {
-	ERTS_BIF_PREP_ERROR(res, c_p, EXC_NOTSUP);
-    }
-    else {
-
-	old_cpu_bind_order = cpu_bind_order;
-
-	if (ERTS_IS_ATOM_STR("spread", how))
-	    cpu_bind_order = ERTS_CPU_BIND_SPREAD;
-	else if (ERTS_IS_ATOM_STR("processor_spread", how))
-	    cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD;
-	else if (ERTS_IS_ATOM_STR("thread_spread", how))
-	    cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD;
-	else if (ERTS_IS_ATOM_STR("default_bind", how)
-		 || ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how))
-	    cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD;
-	else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how))
-	    cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD;
-	else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how))
-	    cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD;
-	else if (ERTS_IS_ATOM_STR("no_spread", how))
-	    cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD;
-	else if (ERTS_IS_ATOM_STR("unbound", how))
-	    cpu_bind_order = ERTS_CPU_BIND_NONE;
-	else {
-	    cpu_bind_order = old_cpu_bind_order;
-	    ERTS_BIF_PREP_ERROR(res, c_p, BADARG);
-	    goto done;
-	}
-
-	create_tmp_cpu_topology_copy(&cpudata, &cpudata_size);
-
-	if (!cpudata) {
-	    cpu_bind_order = old_cpu_bind_order;
-	    ERTS_BIF_PREP_ERROR(res, c_p, BADARG);
-	    goto done;
-	}
-
-	signal_schedulers_bind_change(cpudata, cpudata_size);
-
-	destroy_tmp_cpu_topology_copy(cpudata);
-    
-	res = bound_schedulers_term(old_cpu_bind_order);
-    }
-
- done:
-
-    erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx);
-
-    return res;
-}
-
-Eterm
-erts_fake_scheduler_bindings(Process *p, Eterm how)
-{
-    ErtsCpuBindOrder fake_cpu_bind_order;
-    erts_cpu_topology_t *cpudata;
-    int cpudata_size;
-    Eterm res;
-
-    if (ERTS_IS_ATOM_STR("spread", how))
-	fake_cpu_bind_order = ERTS_CPU_BIND_SPREAD;
-    else if (ERTS_IS_ATOM_STR("processor_spread", how))
-	fake_cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD;
-    else if (ERTS_IS_ATOM_STR("thread_spread", how))
-	fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD;
-    else if (ERTS_IS_ATOM_STR("default_bind", how)
-	     || ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how))
-	fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD;
-    else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how))
-	fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD;
-    else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how))
-	fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD;
-    else if (ERTS_IS_ATOM_STR("no_spread", how))
-	fake_cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD;
-    else if (ERTS_IS_ATOM_STR("unbound", how))
-	fake_cpu_bind_order = ERTS_CPU_BIND_NONE;
-    else {
-	ERTS_BIF_PREP_ERROR(res, p, BADARG);
-	return res;
-    }
-
-    erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx);
-    create_tmp_cpu_topology_copy(&cpudata, &cpudata_size);
-    erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx);
-
-    if (!cpudata || fake_cpu_bind_order == ERTS_CPU_BIND_NONE)
-	ERTS_BIF_PREP_RET(res, am_false);
-    else {
-	int i;
-	Eterm *hp;
-	
-	cpu_bind_order_sort(cpudata, cpudata_size, fake_cpu_bind_order, 1);
-
-#ifdef ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA
-
-	erts_fprintf(stderr, "node:          ");
-	for (i = 0; i < cpudata_size; i++)
-	    erts_fprintf(stderr, " %2d", cpudata[i].node);
-	erts_fprintf(stderr, "\n");
-	erts_fprintf(stderr, "processor:     ");
-	for (i = 0; i < cpudata_size; i++)
-	    erts_fprintf(stderr, " %2d", cpudata[i].processor);
-	erts_fprintf(stderr, "\n");
-	if (fake_cpu_bind_order != ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD
-	    && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD
-	    && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD) {
-	    erts_fprintf(stderr, "processor_node:");
-	    for (i = 0; i < cpudata_size; i++)
-		erts_fprintf(stderr, " %2d", cpudata[i].processor_node);
-	    erts_fprintf(stderr, "\n");
-	}
-	erts_fprintf(stderr, "core:          ");
-	for (i = 0; i < cpudata_size; i++)
-	    erts_fprintf(stderr, " %2d", cpudata[i].core);
-	erts_fprintf(stderr, "\n");
-	erts_fprintf(stderr, "thread:        ");
-	for (i = 0; i < cpudata_size; i++)
-	    erts_fprintf(stderr, " %2d", cpudata[i].thread);
-	erts_fprintf(stderr, "\n");
-	erts_fprintf(stderr, "logical:       ");
-	for (i = 0; i < cpudata_size; i++)
-	    erts_fprintf(stderr, " %2d", cpudata[i].logical);
-	erts_fprintf(stderr, "\n");
-#endif
-
-	hp = HAlloc(p, cpudata_size+1);
-	ERTS_BIF_PREP_RET(res, make_tuple(hp));
-	*hp++ = make_arityval((Uint) cpudata_size);
-	for (i = 0; i < cpudata_size; i++)
-	    *hp++ = make_small((Uint) cpudata[i].logical);
-    }
-
-    destroy_tmp_cpu_topology_copy(cpudata);
-
-    return res;
-}
-
-Eterm
-erts_get_schedulers_binds(Process *c_p)
-{
-    int ix;
-    ERTS_DECL_AM(unbound);
-    Eterm *hp = HAlloc(c_p, erts_no_schedulers+1);
-    Eterm res = make_tuple(hp);
-
-    *(hp++) = make_arityval(erts_no_schedulers);
-    erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx);
-    for (ix = 1; ix <= erts_no_schedulers; ix++)
-	*(hp++) = (scheduler2cpu_map[ix].bound_id >= 0
-		   ? make_small(scheduler2cpu_map[ix].bound_id)
-		   : AM_unbound);
-    erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx);
-    return res;
-}
-
-static Eterm
-bld_topology_term(Eterm **hpp,
-		  Uint *hszp,
-		  erts_cpu_topology_t *cpudata,
-		  int size)
-{
-    Eterm res = NIL;
-    int i;
-
-    if (size == 0)
-	return am_undefined;
-
-    for (i = size-1; i >= 0; i--) {
-	res = erts_bld_cons(hpp,
-			    hszp,
-			    erts_bld_tuple(hpp,
-					   hszp,
-					   7,
-					   am_cpu,
-					   make_small(cpudata[i].node),
-					   make_small(cpudata[i].processor),
-					   make_small(cpudata[i].processor_node),
-					   make_small(cpudata[i].core),
-					   make_small(cpudata[i].thread),
-					   make_small(cpudata[i].logical)),
-			    res);
-    }
-    return res;
-}
-
-static Eterm
-get_cpu_topology_term(Process *c_p, int type)
-{
-#ifdef DEBUG
-    Eterm *hp_end;
-#endif
-    Eterm *hp;
-    Uint hsz;
-    Eterm res = THE_NON_VALUE;
-    erts_cpu_topology_t *cpudata = NULL;
-    int size = 0;
-
-    switch (type) {
-    case ERTS_GET_USED_CPU_TOPOLOGY:
-	if (user_cpudata)
-	    goto defined;
-	else
-	    goto detected;
-    case ERTS_GET_DETECTED_CPU_TOPOLOGY:
-    detected:
-	if (!system_cpudata)
-	    res = am_undefined;
-	else {
-	    size = system_cpudata_size;
-	    cpudata = erts_alloc(ERTS_ALC_T_TMP,
-				 (sizeof(erts_cpu_topology_t)
-				  * size));
-	    sys_memcpy((void *) cpudata,
-		       (void *) system_cpudata,
-		       sizeof(erts_cpu_topology_t)*size);
-	}
-	break;
-    case ERTS_GET_DEFINED_CPU_TOPOLOGY:
-    defined:
-	if (!user_cpudata)
-	    res = am_undefined;
-	else {
-	    size = user_cpudata_size;
-	    cpudata = user_cpudata;
-	}
-	break;
-    default:
-	erl_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type);
-	break;
-    }
-
-    if (res == am_undefined) {
-	ASSERT(!cpudata);
-	return res;
-    }
-
-    hsz = 0;
-
-    bld_topology_term(NULL, &hsz,
-		      cpudata, size);
-
-    hp = HAlloc(c_p, hsz);
-
-#ifdef DEBUG
-    hp_end = hp + hsz;
-#endif
-
-    res = bld_topology_term(&hp, NULL,
-			    cpudata, size);
-
-    ASSERT(hp_end == hp);
-
-    if (cpudata && cpudata != system_cpudata && cpudata != user_cpudata)
-	erts_free(ERTS_ALC_T_TMP, cpudata);
-
-    return res;
-}
-
-Eterm
-erts_get_cpu_topology_term(Process *c_p, Eterm which)
-{
-    Eterm res;
-    int type;
-    erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx);
-    if (ERTS_IS_ATOM_STR("used", which))
-	type = ERTS_GET_USED_CPU_TOPOLOGY;
-    else if (ERTS_IS_ATOM_STR("detected", which))
-	type = ERTS_GET_DETECTED_CPU_TOPOLOGY;
-    else if (ERTS_IS_ATOM_STR("defined", which))
-	type = ERTS_GET_DEFINED_CPU_TOPOLOGY;
-    else
-	type = ERTS_GET_CPU_TOPOLOGY_ERROR;
-    if (type == ERTS_GET_CPU_TOPOLOGY_ERROR)
-	res = THE_NON_VALUE;
-    else
-	res = get_cpu_topology_term(c_p, type);
-    erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx);
-    return res;
-}
-
-static void
-early_cpu_bind_init(void)
-{
-    user_cpudata = NULL;
-    user_cpudata_size = 0;
-
-    system_cpudata_size = erts_get_cpu_topology_size(erts_cpuinfo);
-    system_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA,
-				(sizeof(erts_cpu_topology_t)
-				 * system_cpudata_size));
-
-    cpu_bind_order = ERTS_CPU_BIND_NONE;
-
-    if (!erts_get_cpu_topology(erts_cpuinfo, system_cpudata)
-	|| ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(system_cpudata,
-							system_cpudata_size)) {
-	erts_free(ERTS_ALC_T_CPUDATA, system_cpudata);
-	system_cpudata = NULL;
-	system_cpudata_size = 0;
-    }
-}
-
-static void
-late_cpu_bind_init(void)
-{
-    int ix;
-
-    erts_smp_rwmtx_init(&erts_cpu_bind_rwmtx, "cpu_bind");
-
-    scheduler2cpu_map = erts_alloc(ERTS_ALC_T_CPUDATA,
-				   (sizeof(ErtsCpuBindData)
-				    * (erts_no_schedulers+1)));
-    for (ix = 1; ix <= erts_no_schedulers; ix++) {
-	scheduler2cpu_map[ix].bind_id = -1;
-	scheduler2cpu_map[ix].bound_id = -1;
-    }
-
-    if (cpu_bind_order != ERTS_CPU_BIND_NONE) {
-	erts_cpu_topology_t *cpudata;
-	int cpudata_size;
-	create_tmp_cpu_topology_copy(&cpudata, &cpudata_size);
-	ASSERT(cpudata);
-	signal_schedulers_bind_change(cpudata, cpudata_size);
-	destroy_tmp_cpu_topology_copy(cpudata);
-    }
-}
-
 #ifdef ERTS_SMP
 
 static void
@@ -4481,7 +5262,7 @@ add_pend_suspend(Process *suspendee,
 					 sizeof(ErtsPendingSuspend));
     psp->next = NULL;
 #ifdef DEBUG
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
     psp->end = (ErtsPendingSuspend *) 0xdeaddeaddeaddead;
 #else
     psp->end = (ErtsPendingSuspend *) 0xdeaddead;
@@ -4572,21 +5353,9 @@ handle_pend_sync_suspend(Process *suspendee,
     }
 }
 
-/*
- * Like erts_pid2proc() but:
- *
- * * At least ERTS_PROC_LOCK_MAIN have to be held on c_p.
- * * At least ERTS_PROC_LOCK_MAIN have to be taken on pid.
- * * It also waits for proc to be in a state != running and garbing.
- * * If ERTS_PROC_LOCK_BUSY is returned, the calling process has to
- *   yield (ERTS_BIF_YIELD[0-3]()). c_p might in this case have been
- *   suspended.
- */
-
-
-Process *
-erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
-			  Eterm pid, ErtsProcLocks pid_locks)
+static Process *
+pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
+		     Eterm pid, ErtsProcLocks pid_locks, int suspend)
 {
     Process *rp;
     int unlock_c_p_status;
@@ -4613,7 +5382,7 @@ erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 	c_p->suspendee = NIL;
 	ASSERT(c_p->flags & F_P2PNR_RESCHED);
 	c_p->flags &= ~F_P2PNR_RESCHED;
-	if (rp)
+	if (!suspend && rp)
 	    resume_process(rp);
     }
     else {
@@ -4677,6 +5446,8 @@ erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 	    }
 
 	    /* rp is not running and we got the locks we want... */
+	    if (suspend)
+		suspend_process(rp_rq, rp);
 	}
 	erts_smp_runqs_unlock(cp_rq, rp_rq);
     }
@@ -4689,6 +5460,35 @@ erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
     return rp;
 }
 
+
+/*
+ * Like erts_pid2proc() but:
+ *
+ * * At least ERTS_PROC_LOCK_MAIN have to be held on c_p.
+ * * At least ERTS_PROC_LOCK_MAIN have to be taken on pid.
+ * * It also waits for proc to be in a state != running and garbing.
+ * * If ERTS_PROC_LOCK_BUSY is returned, the calling process has to
+ *   yield (ERTS_BIF_YIELD[0-3]()). c_p might in this case have been
+ *   suspended.
+ */
+Process *
+erts_pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
+			  Eterm pid, ErtsProcLocks pid_locks)
+{
+    return pid2proc_not_running(c_p, c_p_locks, pid, pid_locks, 0);
+}
+
+/*
+ * Like erts_pid2proc_not_running(), but hands over the process
+ * in a suspended state unless (c_p is looked up).
+ */
+Process *
+erts_pid2proc_suspend(Process *c_p, ErtsProcLocks c_p_locks,
+		      Eterm pid, ErtsProcLocks pid_locks)
+{
+    return pid2proc_not_running(c_p, c_p_locks, pid, pid_locks, 1);
+}
+
 /*
  * erts_pid2proc_nropt() is normally the same as
  * erts_pid2proc_not_running(). However it is only
@@ -4802,6 +5602,21 @@ handle_pend_bif_async_suspend(Process *suspendee,
     }
 }
 
+#else
+
+/*
+ * Non-smp version of erts_pid2proc_suspend().
+ */
+Process *
+erts_pid2proc_suspend(Process *c_p, ErtsProcLocks c_p_locks,
+		      Eterm pid, ErtsProcLocks pid_locks)
+{
+    Process *rp = erts_pid2proc(c_p, c_p_locks, pid, pid_locks);
+    if (rp)
+	erts_suspend(rp, pid_locks, NULL);
+    return rp;
+}
+
 #endif /* ERTS_SMP */
 
 /*
@@ -4866,7 +5681,7 @@ suspend_process_2(BIF_ALIST_2)
 
     /* This is really a piece of cake without SMP support... */
     if (!smon->active) {
-	suspend_process(erts_common_run_queue, suspendee);
+	suspend_process(ERTS_RUNQ_IX(0), suspendee);
 	smon->active++;
 	res = am_true;
     }
@@ -5322,7 +6137,7 @@ dequeue_process(ErtsRunQueue *runq, Process *p)
 }
 
 /* schedule a process */
-static ERTS_INLINE void
+static ERTS_INLINE ErtsRunQueue *
 internal_add_to_runq(ErtsRunQueue *runq, Process *p)
 {
     Uint32 prev_status = p->status;
@@ -5333,12 +6148,12 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p)
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
 
     if (p->status_flags & ERTS_PROC_SFLG_INRUNQ)
-	return;
+	return NULL;
     else if (p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) {
-	ASSERT(p->status != P_SUSPENDED);
+	ASSERT(ERTS_PROC_IS_EXITING(p) || p->rcount == 0);
 	ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p);
 	p->status_flags |= ERTS_PROC_SFLG_PENDADD2SCHEDQ;
-	return;
+	return NULL;
     }
     ASSERT(!p->scheduler_data);
 #endif
@@ -5346,9 +6161,8 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p)
     ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p);
 #ifndef ERTS_SMP
     /* Never schedule a suspended process (ok in smp case) */
-    ASSERT(p->status != P_SUSPENDED);
+    ASSERT(ERTS_PROC_IS_EXITING(p) || p->rcount == 0);
     add_runq = runq;
-
 #else
     ASSERT(!p->bound_runq || p->bound_runq == p->run_queue);
     if (p->bound_runq) {
@@ -5377,20 +6191,23 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p)
     	profile_runnable_proc(p, am_active);
     }
 
-    smp_notify_inc_runq(add_runq);
-
     if (add_runq != runq)
 	erts_smp_runq_unlock(add_runq);
+
+    return add_runq;
 }
 
 
 void
 erts_add_to_runq(Process *p)
 {
+    ErtsRunQueue *notify_runq;
     ErtsRunQueue *runq = erts_get_runq_proc(p);
     erts_smp_runq_lock(runq);
-    internal_add_to_runq(runq, p);
+    notify_runq = internal_add_to_runq(runq, p);
     erts_smp_runq_unlock(runq);
+    smp_notify_inc_runq(notify_runq);
+
 }
 
 /* Possibly remove a scheduled process we need to suspend */
@@ -5434,8 +6251,6 @@ erts_proc_migrate(Process *p, ErtsProcLocks *plcks,
 		       || from_locked);
     ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked);
     ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked);
-
-    ASSERT(!erts_common_run_queue);
     
     /*
      * If we have the lock on the run queue to migrate to,
@@ -5529,8 +6344,6 @@ erts_proc_migrate(Process *p, ErtsProcLocks *plcks,
     p->run_queue = to_rq;
     enqueue_process(to_rq, p);
 
-    smp_notify_inc_runq(to_rq);
-
     return ERTS_MIGRATE_SUCCESS;
 }
 #endif /* ERTS_SMP */
@@ -5588,25 +6401,17 @@ erts_process_status(Process *c_p, ErtsProcLocks c_p_locks,
 	int i;
 	ErtsSchedulerData *esdp;
 
-	if (erts_common_run_queue)
-	    erts_smp_runq_lock(erts_common_run_queue);
-	    
 	for (i = 0; i < erts_no_schedulers; i++) {
 	    esdp = ERTS_SCHEDULER_IX(i);
-	    if (!erts_common_run_queue)
-		erts_smp_runq_lock(esdp->run_queue);
+	    erts_smp_runq_lock(esdp->run_queue);
 	    if (esdp->free_process && esdp->free_process->id == rpid) {
 		res = am_free;
-		if (!erts_common_run_queue)
-		    erts_smp_runq_unlock(esdp->run_queue);
+		erts_smp_runq_unlock(esdp->run_queue);
 		break;
 	    }
-	    if (!erts_common_run_queue)
-		erts_smp_runq_unlock(esdp->run_queue);
+	    erts_smp_runq_unlock(esdp->run_queue);
 	}
 
-	if (erts_common_run_queue)
-	    erts_smp_runq_unlock(erts_common_run_queue);
 #endif
 
     }
@@ -5727,30 +6532,6 @@ erts_set_process_priority(Process *p, Eterm new_value)
     return old_value;
 }
 
-#ifdef ERTS_SMP
-
-static ERTS_INLINE int
-prepare_for_sys_schedule(void)
-{
-    while (!erts_port_task_have_outstanding_io_tasks()
-	   && !erts_smp_atomic_xchg(&doing_sys_schedule, 1)) {
-	if (!erts_port_task_have_outstanding_io_tasks())
-	    return 1;
-	erts_smp_atomic_set(&doing_sys_schedule, 0);
-    }
-    return 0;
-}
-
-#else
-
-static ERTS_INLINE int
-prepare_for_sys_schedule(void)
-{
-    return !erts_port_task_have_outstanding_io_tasks();
-}
-
-#endif
-
 /* note that P_RUNNING is only set so that we don't try to remove
 ** running processes from the schedule queue if they exit - a running
 ** process not being in the schedule queue!! 
@@ -5780,14 +6561,23 @@ Process *schedule(Process *p, int calls)
 {
     ErtsRunQueue *rq;
     ErtsRunPrioQueue *rpq;
-    long dt;
+    erts_aint_t dt;
     ErtsSchedulerData *esdp;
     int context_reds;
-    long fcalls;
+    int fcalls;
     int input_reductions;
     int actual_reds;
     int reds;
 
+#ifdef USE_VM_PROBES
+    if (p != NULL && DTRACE_ENABLED(process_unscheduled)) {
+        DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE);
+
+        dtrace_proc_str(p, process_buf);
+        DTRACE1(process_unscheduled, process_buf);
+    }
+#endif
+
     if (ERTS_USE_MODIFIED_TIMING()) {
 	context_reds = ERTS_MODIFIED_TIMING_CONTEXT_REDS;
 	input_reductions = ERTS_MODIFIED_TIMING_INPUT_REDS;
@@ -5797,7 +6587,7 @@ Process *schedule(Process *p, int calls)
 	input_reductions = INPUT_REDUCTIONS;
     }
 
-    ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING);
+    ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
 
     /*
      * Clean up after the process being scheduled out.
@@ -5806,7 +6596,7 @@ Process *schedule(Process *p, int calls)
 	esdp = erts_get_scheduler_data();
 	rq = erts_get_runq_current(esdp);
 	ASSERT(esdp);
-	fcalls = erts_smp_atomic_read(&function_calls);
+	fcalls = (int) erts_smp_atomic32_read_acqb(&function_calls);
 	actual_reds = reds = 0;
 	erts_smp_runq_lock(rq);
     } else {
@@ -5824,7 +6614,7 @@ Process *schedule(Process *p, int calls)
 	    reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST;
 	esdp->virtual_reds = 0;
 
-	fcalls = erts_smp_atomic_addtest(&function_calls, reds);
+	fcalls = (int) erts_smp_atomic32_add_read_acqb(&function_calls, reds);
 	ASSERT(esdp && esdp == erts_get_scheduler_data());
 
 	rq = erts_get_runq_current(esdp);
@@ -5839,6 +6629,9 @@ Process *schedule(Process *p, int calls)
 	}
 
 	if (IS_TRACED(p)) {
+	    if (IS_TRACED_FL(p, F_TRACE_CALLS) &&  p->status != P_FREE) {
+		erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_OUT);
+	    }
 	    switch (p->status) {
 	    case P_EXITING:
 		if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT))
@@ -5868,7 +6661,7 @@ Process *schedule(Process *p, int calls)
 	    handle_pending_suspend(p,
 				   ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
 	    ASSERT(!(p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ)
-		   || p->status != P_SUSPENDED);
+		   || p->rcount == 0);
 	}
 #endif
 	erts_smp_runq_lock(rq);
@@ -5882,8 +6675,11 @@ Process *schedule(Process *p, int calls)
 	p->status_flags &= ~ERTS_PROC_SFLG_RUNNING;
 
 	if (p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) {
+	    ErtsRunQueue *notify_runq;
 	    p->status_flags &= ~ERTS_PROC_SFLG_PENDADD2SCHEDQ;
-	    internal_add_to_runq(rq, p);
+	    notify_runq = internal_add_to_runq(rq, p);
+	    if (notify_runq != rq)
+		smp_notify_inc_runq(notify_runq);
 	}
 #endif
 
@@ -5919,27 +6715,26 @@ Process *schedule(Process *p, int calls)
 
 	ERTS_SMP_CHK_NO_PROC_LOCKS;
 
-	dt = do_time_read_and_reset();
+	dt = erts_do_time_read_and_reset();
 	if (dt) {
 	    erts_smp_runq_unlock(rq);
-	    bump_timer(dt);
+	    erts_bump_timer(dt);
 	    erts_smp_runq_lock(rq);
 	}
 	BM_STOP_TIMER(system);
 
     }
 
-    ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING);
+    ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
+
  check_activities_to_run: {
 
 #ifdef ERTS_SMP
 
-	if (!(rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ)
-	    && rq->check_balance_reds <= 0) {
+	if (rq->check_balance_reds <= 0)
 	    check_balance(rq);
-	}
 
-	ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING);
+	ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
 	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 
 	if (rq->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK)
@@ -5947,46 +6742,51 @@ Process *schedule(Process *p, int calls)
 
  continue_check_activities_to_run:
 
-	if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ
-			 | ERTS_RUNQ_FLG_CHK_CPU_BIND
+	if (rq->flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND
 			 | ERTS_RUNQ_FLG_SUSPENDED)) {
-	    if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED)
-		|| erts_smp_atomic_read(&esdp->suspended)) {
+	    if (rq->flags & ERTS_RUNQ_FLG_SUSPENDED) {
+		ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags)
+		       & ERTS_SSI_FLG_SUSPENDED);
 		suspend_scheduler(esdp);
 	    }
-	    if ((rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND)
-		|| erts_smp_atomic_read(&esdp->chk_cpu_bind)) {
-		check_cpu_bind(esdp);
-	    }
+	    if (rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND)
+		erts_sched_check_cpu_bind(esdp);
 	}
 
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
-	if (esdp->check_children) {
-	    esdp->check_children = 0;
-	    erts_smp_runq_unlock(rq);
-	    erts_check_children();
-	    erts_smp_runq_lock(rq);
+	{
+	    erts_aint32_t aux_work;
+	    int leader_update = erts_thr_progress_update(esdp);
+	    aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work);
+	    if (aux_work | leader_update) {
+		erts_smp_runq_unlock(rq);
+		if (leader_update)
+		    erts_thr_progress_leader_update(esdp);
+		if (aux_work)
+		    handle_aux_work(&esdp->aux_work_data, aux_work, 0);
+		erts_smp_runq_lock(rq);
+	    }
 	}
-#endif
-
-	erts_smp_chk_system_block(prepare_for_block,
-				  resume_after_block,
-				  (void *) rq);
 
-	ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING);
+	ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
 	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 
-#endif
+#else /* ERTS_SMP */
+	{
+	    erts_aint32_t aux_work;
+	    aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work);
+	    if (aux_work)
+		handle_aux_work(&esdp->aux_work_data, aux_work, 0);
+	}
+#endif /* ERTS_SMP */
 
 	ASSERT(rq->len == rq->procs.len + rq->ports.info.len);
 
-#ifndef ERTS_SMP
+	if ((rq->len == 0 && !rq->misc.start)
+	    || (rq->halt_in_progress
+		&& rq->ports.info.len == 0 && !rq->misc.start)) {
 
-	if (rq->len == 0 && !rq->misc.start)
-	    goto do_sys_schedule;
+#ifdef ERTS_SMP
 
-#else /* ERTS_SMP */
-	if (rq->len == 0 && !rq->misc.start) {
 	    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 
 	    rq->wakeup_other = 0;
@@ -5994,13 +6794,11 @@ Process *schedule(Process *p, int calls)
 
 	    empty_runq(rq);
 
-	    if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ
-			     | ERTS_RUNQ_FLG_SUSPENDED)) {
-		if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED)
-		    || erts_smp_atomic_read(&esdp->suspended)) {
-		    non_empty_runq(rq);
-		    goto continue_check_activities_to_run;
-		}
+	    if (rq->flags & ERTS_RUNQ_FLG_SUSPENDED) {
+		ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags)
+		       & ERTS_SSI_FLG_SUSPENDED);
+		non_empty_runq(rq);
+		goto continue_check_activities_to_run;
 	    }
 	    else if (!(rq->flags & ERTS_RUNQ_FLG_INACTIVE)) {
 		/*
@@ -6014,57 +6812,38 @@ Process *schedule(Process *p, int calls)
 		}
 	    }
 
-	    if (prepare_for_sys_schedule()) {
-		erts_smp_atomic_set(&function_calls, 0);
-		fcalls = 0;
-		sched_sys_wait(esdp->no, rq);
-		erts_smp_atomic_set(&doing_sys_schedule, 0);
-	    }
-	    else {
-		/* If all schedulers are waiting, one of them *should*
-		   be waiting in erl_sys_schedule() */
-		sched_cnd_wait(esdp->no, rq);
-	    }
-
-	    non_empty_runq(rq);
+#endif
 
-	    goto check_activities_to_run;
-	}
-	else
-#endif /* ERTS_SMP */
-	if (fcalls > input_reductions && prepare_for_sys_schedule()) {
-	    int runnable;
+	    scheduler_wait(&fcalls, esdp, rq);
 
 #ifdef ERTS_SMP
-	    runnable = 1;
-#else
-	do_sys_schedule:
-	    runnable = rq->len != 0;
-	    if (!runnable)
-		sched_waiting_sys(esdp->no, rq);
+	    non_empty_runq(rq);
 #endif
 
+	    goto check_activities_to_run;
+	}
+	else if (fcalls > input_reductions && prepare_for_sys_schedule()) {
 	    /*
 	     * Schedule system-level activities.
 	     */
 
-	    erts_smp_atomic_set(&function_calls, 0);
+	    erts_smp_atomic32_set_relb(&function_calls, 0);
 	    fcalls = 0;
+
 	    ASSERT(!erts_port_task_have_outstanding_io_tasks());
-#ifdef ERTS_SMP
-	    /* erts_sys_schedule_interrupt(0); */
+
+#if 0 /* Not needed since we wont wait in sys schedule */
+	    erts_sys_schedule_interrupt(0);
 #endif
 	    erts_smp_runq_unlock(rq);
-	    erl_sys_schedule(runnable);
-	    dt = do_time_read_and_reset();
-	    if (dt) bump_timer(dt);
+	    erl_sys_schedule(1);
+	    dt = erts_do_time_read_and_reset();
+	    if (dt) erts_bump_timer(dt);
 #ifdef ERTS_SMP
 	    erts_smp_runq_lock(rq);
-	    erts_smp_atomic_set(&doing_sys_schedule, 0);
+	    clear_sys_scheduling();
 	    goto continue_check_activities_to_run;
 #else
-	    if (!runnable)
-		sched_active_sys(esdp->no, rq);
 	    goto check_activities_to_run;
 #endif
 	}
@@ -6073,30 +6852,7 @@ Process *schedule(Process *p, int calls)
 	    exec_misc_ops(rq);
 
 #ifdef ERTS_SMP
-	{
-	    int wo_reds = rq->wakeup_other_reds;
-	    if (wo_reds) {
-		if (rq->len < 2) {
-		    rq->wakeup_other -= ERTS_WAKEUP_OTHER_DEC*wo_reds;
-		    if (rq->wakeup_other < 0)
-			rq->wakeup_other = 0;
-		}
-		else if (rq->wakeup_other < ERTS_WAKEUP_OTHER_LIMIT)
-		    rq->wakeup_other += rq->len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC;
-		else {
-		    if (erts_common_run_queue) {
-			if (erts_common_run_queue->waiting)
-			    wake_one_scheduler();
-		    }
-		    else if (erts_smp_atomic_read(&no_empty_run_queues) != 0) {
-			wake_scheduler_on_empty_runq(rq);
-			rq->wakeup_other = 0;
-		    }
-		    rq->wakeup_other = 0;
-		}
-	    }
-	    rq->wakeup_other_reds = 0;
-	}
+	wakeup_other.check(rq);
 #endif
 
 	/*
@@ -6106,7 +6862,8 @@ Process *schedule(Process *p, int calls)
 	if (rq->ports.info.len) {
 	    int have_outstanding_io;
 	    have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port);
-	    if (have_outstanding_io && fcalls > 2*input_reductions) {
+	    if ((have_outstanding_io && fcalls > 2*input_reductions)
+		|| rq->halt_in_progress) {
 		/*
 		 * If we have performed more than 2*INPUT_REDUCTIONS since
 		 * last call to erl_sys_schedule() and we still haven't
@@ -6231,10 +6988,10 @@ Process *schedule(Process *p, int calls)
 	erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
 
 	if (erts_sched_stat.enabled) {
-	    Uint old = ERTS_PROC_SCHED_ID(p,
+	    UWord old = ERTS_PROC_SCHED_ID(p,
 					  (ERTS_PROC_LOCK_MAIN
 					   | ERTS_PROC_LOCK_STATUS),
-					  esdp->no);
+					  (UWord) esdp->no);
 	    int migrated = old && old != esdp->no;
 
 	    erts_smp_spin_lock(&erts_sched_stat.lock);
@@ -6259,7 +7016,6 @@ Process *schedule(Process *p, int calls)
 #endif
 	ASSERT(p->status != P_SUSPENDED); /* Never run a suspended process */
 
-        ACTIVATE(p);
 	reds = context_reds;
 
 	if (IS_TRACED(p)) {
@@ -6275,7 +7031,11 @@ Process *schedule(Process *p, int calls)
 		    trace_virtual_sched(p, am_in);
 		break;
 	    }
+	    if (IS_TRACED_FL(p, F_TRACE_CALLS)) {
+		erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_IN);
+	    }
 	}
+
 	if (p->status != P_EXITING)
 	    p->status = P_RUNNING;
 
@@ -6296,7 +7056,6 @@ Process *schedule(Process *p, int calls)
 	}
 	    
 	p->fcalls = reds;
-	ASSERT(IS_ACTIVE(p));
 	ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 	return p;
     }
@@ -6308,14 +7067,14 @@ erts_sched_stat_modify(int what)
     int ix;
     switch (what) {
     case ERTS_SCHED_STAT_MODIFY_ENABLE:
-	erts_smp_block_system(0);
+	erts_smp_thr_progress_block();
 	erts_sched_stat.enabled = 1;
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 	break;
     case ERTS_SCHED_STAT_MODIFY_DISABLE:
-	erts_smp_block_system(0);
+	erts_smp_thr_progress_block();
 	erts_sched_stat.enabled = 1;
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 	break;
     case ERTS_SCHED_STAT_MODIFY_CLEAR:
 	erts_smp_spin_lock(&erts_sched_stat.lock);
@@ -6375,7 +7134,8 @@ erts_sched_stat_term(Process *p, int total)
 void
 erts_schedule_misc_op(void (*func)(void *), void *arg)
 {
-    ErtsRunQueue *rq = erts_get_runq_current(NULL);
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    ErtsRunQueue *rq = esdp ? esdp->run_queue : ERTS_RUNQ_IX(0);
     ErtsMiscOpList *molp = misc_op_list_alloc();
 
     erts_smp_runq_lock(rq);
@@ -6397,8 +7157,8 @@ erts_schedule_misc_op(void (*func)(void *), void *arg)
     else
 	rq->misc.start = molp;
     rq->misc.end = molp;
-    smp_notify_inc_runq(rq);
     erts_smp_runq_unlock(rq);
+    smp_notify_inc_runq(rq);
 }
 
 static void
@@ -6469,7 +7229,7 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp)
      * Wait for other schedulers to schedule out their processes
      * and update 'reductions'.
      */
-    erts_smp_block_system(0);
+    erts_smp_thr_progress_block();
     for (reds = 0, ix = 0; ix < erts_no_run_queues; ix++)
 	reds += ERTS_RUNQ_IX(ix)->procs.reductions;
     if (redsp)
@@ -6477,7 +7237,7 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp)
     if (diffp)
 	*diffp = reds - last_exact_reductions;
     last_exact_reductions = reds;
-    erts_smp_release_system();
+    erts_smp_thr_progress_unblock();
     erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
 }
 
@@ -6532,7 +7292,7 @@ erts_test_next_pid(int set, Uint next)
 
 Uint erts_process_count(void)
 {
-    long res = erts_smp_atomic_read(&process_count);
+    erts_aint32_t res = erts_smp_atomic32_read_nob(&process_count);
     ASSERT(res >= 0);
     return (Uint) res;
 }
@@ -6581,7 +7341,7 @@ alloc_process(void)
     ASSERT(!process_tab[p_next]);
 
     process_tab[p_next] = p;
-    erts_smp_atomic_inc(&process_count);
+    erts_smp_atomic32_inc_nob(&process_count);
     p->id = make_internal_pid(p_serial << p_serial_shift | p_next);
     if (p->id == ERTS_INVALID_PID) {
 	/* Do not use the invalid pid; change serial */
@@ -6640,12 +7400,10 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 		   Eterm args,	/* Arguments for function (must be well-formed list). */
 		   ErlSpawnOpts* so) /* Options for spawn. */
 {
-    ErtsRunQueue *rq;
+    ErtsRunQueue *rq, *notify_runq;
     Process *p;
     Sint arity;			/* Number of arguments. */
-#ifndef HYBRID
     Uint arg_size;		/* Size of arguments. */
-#endif
     Uint sz;			/* Needed words on heap. */
     Uint heap_need;		/* Size needed on heap. */
     Eterm res = THE_NON_VALUE;
@@ -6654,17 +7412,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR);
 #endif
 
-#ifdef HYBRID
-    /*
-     * Copy the arguments to the global heap
-     * Since global GC might occur we want to do this before adding the
-     * new process to the process_tab.
-     */
-    BM_SWAP_TIMER(system,copy);
-    LAZY_COPY(parent,args);
-    BM_SWAP_TIMER(copy,system);
-    heap_need = 0;
-#endif /* HYBRID */
     /*
      * Check for errors.
      */
@@ -6682,15 +7429,15 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	goto error;
     }
 
+#ifdef BM_COUNTERS
     processes_busy++;
+#endif
     BM_COUNT(processes_spawned);
 
-#ifndef HYBRID
     BM_SWAP_TIMER(system,size);
     arg_size = size_object(args);
     BM_SWAP_TIMER(size,system);
     heap_need = arg_size;
-#endif
 
     p->flags = erts_default_process_flags;
 
@@ -6707,7 +7454,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	p->min_heap_size  = H_MIN_SIZE;
 	p->min_vheap_size = BIN_VH_MIN_SIZE;
 	p->prio           = PRIORITY_NORMAL;
-	p->max_gen_gcs    = (Uint16) erts_smp_atomic_read(&erts_max_gen_gcs);
+	p->max_gen_gcs    = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs);
     }
     p->skipped = 0;
     ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0));
@@ -6719,11 +7466,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     /*
      * Must initialize binary lists here before copying binaries to process.
      */
-    p->off_heap.mso = NULL;
-#ifndef HYBRID /* FIND ME! */
-    p->off_heap.funs = NULL;
-#endif
-    p->off_heap.externals = NULL;
+    p->off_heap.first = NULL;
     p->off_heap.overhead = 0;
 
     heap_need +=
@@ -6745,9 +7488,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*sz);
     p->old_hend = p->old_htop = p->old_heap = NULL;
     p->high_water = p->heap;
-#ifdef INCREMENTAL
-    p->scan_top = p->high_water;
-#endif
     p->gen_gcs = 0;
     p->stop = p->hend = p->heap + sz;
     p->htop = p->heap;
@@ -6757,13 +7497,14 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->bin_vheap_sz     = p->min_vheap_size;
     p->bin_old_vheap_sz = p->min_vheap_size;
     p->bin_old_vheap    = 0;
+    p->bin_vheap_mature = 0;
 
     /* No need to initialize p->fcalls. */
 
     p->current = p->initial+INITIAL_MOD;
 
-    p->i = (Eterm *) beam_apply;
-    p->cp = (Eterm *) beam_apply+1;
+    p->i = (BeamInstr *) beam_apply;
+    p->cp = (BeamInstr *) beam_apply+1;
 
     p->arg_reg = p->def_arg_reg;
     p->max_arg_reg = sizeof(p->def_arg_reg)/sizeof(p->def_arg_reg[0]);
@@ -6772,19 +7513,10 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     BM_STOP_TIMER(system);
     BM_MESSAGE(args,p,parent);
     BM_START_TIMER(system);
-#ifdef HYBRID
-    p->arg_reg[2] = args;
-#ifdef INCREMENTAL
-    p->active = 0;
-    if (ptr_val(args) >= inc_fromspc && ptr_val(args) < inc_fromend)
-        INC_ACTIVATE(p);
-#endif
-#else
     BM_SWAP_TIMER(system,copy);
     p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap);
     BM_MESSAGE_COPIED(arg_size);
     BM_SWAP_TIMER(copy,system);
-#endif
     p->arity = 3;
 
     p->fvalue = NIL;
@@ -6813,7 +7545,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	p->group_leader =
 	    IS_CONST(parent->group_leader)
 	    ? parent->group_leader
-	    : STORE_NC(&p->htop, &p->off_heap.externals, parent->group_leader);
+	    : STORE_NC(&p->htop, &p->off_heap, parent->group_leader);
     }
 
     erts_get_default_tracing(&p->trace_flags, &p->tracer_proc);
@@ -6836,14 +7568,11 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->seq_trace_lastcnt = 0;
     p->seq_trace_clock = 0;
     SEQ_TRACE_TOKEN(p) = NIL;
-    p->parent = parent->id == ERTS_INVALID_PID ? NIL : parent->id;
-
-#ifdef HYBRID
-    p->rrma  = NULL;
-    p->rrsrc = NULL;
-    p->nrr   = 0;
-    p->rrsz  = 0;
+#ifdef USE_VM_PROBES
+    DT_UTAG(p) = NIL;
+    DT_UTAG_FLAGS(p) = 0;
 #endif
+    p->parent = parent->id == ERTS_INVALID_PID ? NIL : parent->id;
 
     INIT_HOLE_CHECK(p);
 #ifdef DEBUG
@@ -6913,15 +7642,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	so->mref = mref;
     }
 
-#ifdef HYBRID
-    /*
-     * Add process to the array of active processes.
-     */
-    ACTIVATE(p);
-    p->active_index = erts_num_active_procs++;
-    erts_active_procs[p->active_index] = p;
-#endif
-
 #ifdef ERTS_SMP
     p->scheduler_data = NULL;
     p->is_exiting = 0;
@@ -6933,7 +7653,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->pending_exit.bp = NULL;
 #endif
 
-#if !defined(NO_FPE_SIGNALS)
+#if !defined(NO_FPE_SIGNALS) || defined(HIPE)
     p->fp_exception = 0;
 #endif
 
@@ -6957,15 +7677,27 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 #endif
 
     p->status = P_WAITING;
-    internal_add_to_runq(rq, p);
+    notify_runq = internal_add_to_runq(rq, p);
 
     erts_smp_runq_unlock(rq);
 
+    smp_notify_inc_runq(notify_runq);
+
     res = p->id;
     erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
 
     VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->id));
 
+#ifdef USE_VM_PROBES
+    if (DTRACE_ENABLED(process_spawn)) {
+        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);
+        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);
+
+        dtrace_fun_decode(p, mod, func, arity, process_name, mfa);
+        DTRACE2(process_spawn, process_name, mfa);
+    }
+#endif
+
  error:
 
     erts_smp_proc_unlock(parent, ERTS_PROC_LOCKS_ALL_MINOR);
@@ -7007,6 +7739,7 @@ void erts_init_empty_process(Process *p)
     p->bin_vheap_sz = BIN_VH_MIN_SIZE;
     p->bin_old_vheap_sz = BIN_VH_MIN_SIZE;
     p->bin_old_vheap = 0;
+    p->bin_vheap_mature = 0;
 #ifdef ERTS_SMP
     p->u.ptimer = NULL;
     p->bound_runq = NULL;
@@ -7014,18 +7747,11 @@ void erts_init_empty_process(Process *p)
     memset(&(p->u.tm), 0, sizeof(ErlTimer));
 #endif
     p->next = NULL;
-    p->off_heap.mso = NULL;
-#ifndef HYBRID /* FIND ME! */
-    p->off_heap.funs = NULL;
-#endif
-    p->off_heap.externals = NULL;
+    p->off_heap.first = NULL;
     p->off_heap.overhead = 0;
     p->reg = NULL;
     p->heap_sz = 0;
     p->high_water = NULL;
-#ifdef INCREMENTAL
-    p->scan_top = NULL;
-#endif
     p->old_hend = NULL;
     p->old_htop = NULL;
     p->old_heap = NULL;
@@ -7077,14 +7803,6 @@ void erts_init_empty_process(Process *p)
 #endif
 #endif
 
-    ACTIVATE(p);
-
-#ifdef HYBRID
-    p->rrma  = NULL;
-    p->rrsrc = NULL;
-    p->nrr   = 0;
-    p->rrsz  = 0;
-#endif
     INIT_HOLE_CHECK(p);
 #ifdef DEBUG
     p->last_old_htop = NULL;
@@ -7108,7 +7826,7 @@ void erts_init_empty_process(Process *p)
     p->run_queue = ERTS_RUNQ_IX(0);
 #endif
 
-#if !defined(NO_FPE_SIGNALS)
+#if !defined(NO_FPE_SIGNALS) || defined(HIPE)
     p->fp_exception = 0;
 #endif
 
@@ -7132,9 +7850,6 @@ erts_debug_verify_clean_empty_process(Process* p)
     ASSERT(p->reg == NULL);
     ASSERT(p->heap_sz == 0);
     ASSERT(p->high_water == NULL);
-#ifdef INCREMENTAL
-    ASSERT(p->scan_top == NULL);
-#endif
     ASSERT(p->old_hend == NULL);
     ASSERT(p->old_htop == NULL);
     ASSERT(p->old_heap == NULL);
@@ -7165,11 +7880,7 @@ erts_debug_verify_clean_empty_process(Process* p)
 
     /* Thing that erts_cleanup_empty_process() cleans up */
 
-    ASSERT(p->off_heap.mso == NULL);
-#ifndef HYBRID /* FIND ME! */
-    ASSERT(p->off_heap.funs == NULL);
-#endif
-    ASSERT(p->off_heap.externals == NULL);
+    ASSERT(p->off_heap.first == NULL);
     ASSERT(p->off_heap.overhead == 0);
 
     ASSERT(p->mbuf == NULL);
@@ -7180,25 +7891,16 @@ erts_debug_verify_clean_empty_process(Process* p)
 void
 erts_cleanup_empty_process(Process* p)
 {
-    ErlHeapFragment* mbufp;
-
     /* We only check fields that are known to be used... */
 
     erts_cleanup_offheap(&p->off_heap);
-    p->off_heap.mso = NULL;
-#ifndef HYBRID /* FIND ME! */
-    p->off_heap.funs = NULL;
-#endif
-    p->off_heap.externals = NULL;
+    p->off_heap.first = NULL;
     p->off_heap.overhead = 0;
 
-    mbufp = p->mbuf;
-    while (mbufp) {
-	ErlHeapFragment *next = mbufp->next;
-	free_message_buffer(mbufp);
-	mbufp = next;
+    if (p->mbuf != NULL) {
+	free_message_buffer(p->mbuf);
+	p->mbuf = NULL;
     }
-    p->mbuf = NULL;
 #if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP)
     erts_lcnt_proc_lock_destroy(p);
 #endif
@@ -7214,7 +7916,6 @@ static void
 delete_process(Process* p)
 {
     ErlMessage* mp;
-    ErlHeapFragment* bp;
 
     VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->id));
 
@@ -7230,7 +7931,7 @@ delete_process(Process* p)
      * The mso list should not be used anymore, but if it is, make sure that
      * we'll notice.
      */
-    p->off_heap.mso = (void *) 0x8DEFFACD;
+    p->off_heap.first = (void *) 0x8DEFFACD;
 
     if (p->arg_reg != p->def_arg_reg) {
 	erts_free(ERTS_ALC_T_ARG_REG, p->arg_reg);
@@ -7264,11 +7965,8 @@ delete_process(Process* p)
     /*
      * Free all pending message buffers.
      */
-    bp = p->mbuf;
-    while (bp != NULL) {
-	ErlHeapFragment* next_bp = bp->next;
-	free_message_buffer(bp);
-	bp = next_bp;
+    if (p->mbuf != NULL) {	
+	free_message_buffer(p->mbuf);
     }
 
     erts_erase_dicts(p);
@@ -7299,22 +7997,6 @@ delete_process(Process* p)
     ASSERT(!p->suspend_monitors);
 
     p->fvalue = NIL;
-
-#ifdef HYBRID
-    erts_active_procs[p->active_index] =
-        erts_active_procs[--erts_num_active_procs];
-    erts_active_procs[p->active_index]->active_index = p->active_index;
-#ifdef INCREMENTAL
-    if (INC_IS_ACTIVE(p))
-         INC_DEACTIVATE(p);
-#endif
-
-    if (p->rrma != NULL) {
-        erts_free(ERTS_ALC_T_ROOTSET,p->rrma);
-        erts_free(ERTS_ALC_T_ROOTSET,p->rrsrc);
-    }
-#endif
-
 }
 
 static ERTS_INLINE void
@@ -7348,7 +8030,7 @@ set_proc_exiting(Process *p, Eterm reason, ErlHeapFragment *bp)
     p->freason = EXTAG_EXIT;
     KILL_CATCHES(p);
     cancel_timer(p);
-    p->i = (Eterm *) beam_exit;
+    p->i = (BeamInstr *) beam_exit;
 }
 
 
@@ -7437,7 +8119,11 @@ static ERTS_INLINE void
 send_exit_message(Process *to, ErtsProcLocks *to_locksp,
 		  Eterm exit_term, Uint term_size, Eterm token)
 {
-    if (token == NIL) {
+    if (token == NIL 
+#ifdef USE_VM_PROBES
+	|| token == am_have_dt_utag
+#endif
+	) {
 	Eterm* hp;
 	Eterm mess;
 	ErlHeapFragment* bp;
@@ -7445,7 +8131,11 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp,
 
 	hp = erts_alloc_message_heap(term_size, &bp, &ohp, to, to_locksp);
 	mess = copy_struct(exit_term, term_size, &hp, ohp);
-	erts_queue_message(to, to_locksp, bp, mess, NIL);
+	erts_queue_message(to, to_locksp, bp, mess, NIL
+#ifdef USE_VM_PROBES
+			   , NIL
+#endif
+			   );
     } else {
 	ErlHeapFragment* bp;
 	Eterm* hp;
@@ -7461,7 +8151,11 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp,
 	/* the trace token must in this case be updated by the caller */
 	seq_trace_output(token, mess, SEQ_TRACE_SEND, to->id, NULL);
 	temp_token = copy_struct(token, sz_token, &hp, &bp->off_heap);
-	erts_queue_message(to, to_locksp, bp, mess, temp_token);
+	erts_queue_message(to, to_locksp, bp, mess, temp_token
+#ifdef USE_VM_PROBES
+			   , NIL
+#endif
+			   );
     }
 }
 
@@ -7554,9 +8248,26 @@ send_exit_signal(Process *c_p,		/* current process if and only
 
     ASSERT(reason != THE_NON_VALUE);
 
+#ifdef USE_VM_PROBES
+    if(DTRACE_ENABLED(process_exit_signal) && is_pid(from)) {
+        DTRACE_CHARBUF(sender_str, DTRACE_TERM_BUF_SIZE);
+        DTRACE_CHARBUF(receiver_str, DTRACE_TERM_BUF_SIZE);
+        DTRACE_CHARBUF(reason_buf, DTRACE_TERM_BUF_SIZE);
+
+        dtrace_pid_str(from, sender_str);
+        dtrace_proc_str(rp, receiver_str);
+        erts_snprintf(reason_buf, sizeof(reason_buf) - 1, "%T", reason);
+        DTRACE3(process_exit_signal, sender_str, receiver_str, reason_buf);
+    }
+#endif
+
     if (ERTS_PROC_IS_TRAPPING_EXITS(rp)
 	&& (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) {
-	if (is_not_nil(token) && token_update)
+	if (is_not_nil(token) 
+#ifdef USE_VM_PROBES
+	    && token != am_have_dt_utag
+#endif
+	    && token_update)
 	    seq_trace_update_send(token_update);
 	if (is_value(exit_tuple))
 	    send_exit_message(rp, rp_locks, exit_tuple, exit_tuple_sz, token);
@@ -7657,7 +8368,6 @@ send_exit_signal(Process *c_p,		/* current process if and only
 	    set_proc_exiting(rp,
 			     is_immed(rsn) ? rsn : copy_object(rsn, rp),
 			     NULL);
-	    ACTIVATE(rp);
 	    if (old_status != P_RUNABLE && old_status != P_RUNNING)
 		erts_add_to_runq(rp);
 	}
@@ -7778,13 +8488,14 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 	    erts_port_release(prt); 
 	} else if (is_internal_pid(mon->pid)) {/* local by name or pid */
 	    Eterm watched;
-	    Eterm lhp[3];
+	    DeclareTmpHeapNoproc(lhp,3);
 	    ErtsProcLocks rp_locks = (ERTS_PROC_LOCK_LINK
 				      | ERTS_PROC_LOCKS_MSG_SEND);
 	    rp = erts_pid2proc(NULL, 0, mon->pid, rp_locks);
 	    if (rp == NULL) {
 		goto done;
 	    }
+	    UseTmpHeapNoproc(3);
 	    rmon = erts_remove_monitor(&(rp->monitors),mon->ref);
 	    if (rmon) {
 		erts_destroy_monitor(rmon);
@@ -7795,6 +8506,7 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext)
 		erts_queue_monitor_message(rp, &rp_locks, mon->ref, am_process, 
 					   watched, pcontext->reason);
 	    }
+	    UnUseTmpHeapNoproc(3);
 	    /* else: demonitor while we exited, i.e. do nothing... */
 	    erts_smp_proc_unlock(rp, rp_locks);
 	} else { /* external by pid or name */
@@ -7937,15 +8649,6 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext)
 	    if (rlnk)
 		erts_destroy_link(rlnk);
 	    erts_deref_dist_entry(dep);
-	} else {
-#ifndef ERTS_SMP
-	    /* XXX Is this possible? Shouldn't this link
-	       previously have been removed if the node
-	       had previously been disconnected. */
-	    ASSERT(0);
-#endif
-	    /* This is possible when smp support has been enabled,
-	       and dist port and process exits simultaneously. */
 	}
 	break;
 	
@@ -7987,7 +8690,18 @@ erts_do_exit_process(Process* p, Eterm reason)
 
     p->arity = 0;		/* No live registers */
     p->fvalue = reason;
-    
+
+#ifdef USE_VM_PROBES
+    if (DTRACE_ENABLED(process_exit)) {
+        DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE);
+        DTRACE_CHARBUF(reason_buf, DTRACE_TERM_BUF_SIZE);
+
+        dtrace_proc_str(p, process_buf);
+        erts_snprintf(reason_buf, DTRACE_TERM_BUF_SIZE - 1, "%T", reason);
+        DTRACE2(process_exit, process_buf, reason_buf);
+    }
+#endif
+
 #ifdef ERTS_SMP
     ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
     /* By locking all locks (main lock is already locked) when going
@@ -8025,8 +8739,13 @@ erts_do_exit_process(Process* p, Eterm reason)
     ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p);
 #endif
 
-    if (IS_TRACED_FL(p,F_TRACE_PROCS))
-	trace_proc(p, p, am_exit, reason);
+    if (IS_TRACED(p)) {
+	if (IS_TRACED_FL(p, F_TRACE_CALLS))
+	    erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_EXITING);
+
+	if (IS_TRACED_FL(p,F_TRACE_PROCS))
+	    trace_proc(p, p, am_exit, reason);
+    }
 
     erts_trace_check_exiting(p->id);
 
@@ -8075,6 +8794,8 @@ continue_exit_process(Process *p
     Eterm reason = p->fvalue;
     DistEntry *dep;
     struct saved_calls *scb;
+    process_breakpoint_time_t *pbt;
+
 #ifdef DEBUG
     int yield_allowed = 1;
 #endif
@@ -8176,8 +8897,8 @@ continue_exit_process(Process *p
 	p->status_flags = 0;
 #endif
 	process_tab[pix] = NULL; /* Time of death! */
-	ASSERT(erts_smp_atomic_read(&process_count) > 0);
-	erts_smp_atomic_dec(&process_count);
+	ASSERT(erts_smp_atomic32_read_nob(&process_count) > 0);
+	erts_smp_atomic32_dec_nob(&process_count);
 
 #ifdef ERTS_SMP
 	erts_pix_unlock(pix_lock);
@@ -8214,9 +8935,12 @@ continue_exit_process(Process *p
 	   ? ERTS_PROC_SET_DIST_ENTRY(p, ERTS_PROC_LOCKS_ALL, NULL)
 	   : NULL);
     scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, ERTS_PROC_LOCKS_ALL, NULL);
+    pbt = ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCKS_ALL, NULL);
 
     erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
+#ifdef BM_COUNTERS
     processes_busy--;
+#endif
 
     if (dep) {
 	erts_do_net_exits(dep, reason);
@@ -8228,11 +8952,12 @@ continue_exit_process(Process *p
      * Pre-build the EXIT tuple if there are any links.
      */
     if (lnk) {
-	Eterm tmp_heap[4];
+	DeclareTmpHeap(tmp_heap,4,p);
 	Eterm exit_tuple;
 	Uint exit_tuple_sz;
 	Eterm* hp;
 
+	UseTmpHeap(4,p);
 	hp = &tmp_heap[0];
 
 	exit_tuple = TUPLE3(hp, am_EXIT, p->id, reason);
@@ -8243,16 +8968,21 @@ continue_exit_process(Process *p
 	    ExitLinkContext context = {p, reason, exit_tuple, exit_tuple_sz};
 	    erts_sweep_links(lnk, &doit_exit_link, &context);
 	}
+	UnUseTmpHeap(4,p);
     }
 
     {
 	ExitMonitorContext context = {reason, p};
-	erts_sweep_monitors(mon,&doit_exit_monitor,&context);
+	erts_sweep_monitors(mon,&doit_exit_monitor,&context); /* Allocates TmpHeap, but we
+								 have none here */
     }
 
     if (scb)
         erts_free(ERTS_ALC_T_CALLS_BUF, (void *) scb);
 
+    if (pbt)
+        erts_free(ERTS_ALC_T_BPD, (void *) pbt);
+
     delete_process(p);
 
     erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN);
@@ -8271,7 +9001,7 @@ continue_exit_process(Process *p
 
     ASSERT(p->status == P_EXITING);
 
-    p->i = (Eterm *) beam_continue_exit;
+    p->i = (BeamInstr *) beam_continue_exit;
 
     if (!(curr_locks & ERTS_PROC_LOCK_STATUS)) {
 	erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS);
@@ -8291,14 +9021,33 @@ continue_exit_process(Process *p
 static void
 timeout_proc(Process* p)
 {
-    p->i = (Eterm *) p->def_arg_reg[0];
+    BeamInstr** pi = (BeamInstr **) p->def_arg_reg;
+    p->i = *pi;
     p->flags |= F_TIMO;
     p->flags &= ~F_INSLPQUEUE;
 
-    if (p->status == P_WAITING)
-	erts_add_to_runq(p); 
-    if (p->status == P_SUSPENDED)
+    switch (p->status) {
+    case P_GARBING:
+	switch (p->gcstatus) {
+	case P_SUSPENDED:
+	    goto suspended;
+	case P_WAITING:
+	    goto waiting;
+	default:
+	    break;
+	}
+	break;
+    case P_WAITING:
+    waiting:
+	erts_add_to_runq(p);
+	break;
+    case P_SUSPENDED:
+    suspended:
 	p->rstatus = P_RUNABLE;   /* MUST set resume status to runnable */
+	break;
+    default:
+	break;
+    }
 }
 
 
@@ -8310,7 +9059,7 @@ cancel_timer(Process* p)
 #ifdef ERTS_SMP
     erts_cancel_smp_ptimer(p->u.ptimer);
 #else
-    erl_cancel_timer(&p->u.tm);
+    erts_cancel_timer(&p->u.tm);
 #endif
 }
 
@@ -8336,7 +9085,7 @@ set_timer(Process* p, Uint timeout)
 			   (ErlTimeoutProc) timeout_proc,
 			   timeout);
 #else
-    erl_set_timer(&p->u.tm,
+    erts_set_timer(&p->u.tm,
 		  (ErlTimeoutProc) timeout_proc,
 		  NULL,
 		  (void*) p,
@@ -8384,15 +9133,15 @@ erts_program_counter_info(int to, void *to_arg, Process *p)
 	     * only cause problems.
 	     */
 	    for (i = 0; i < p->arity; i++)
-		erts_print(to, to_arg, "   %T\n", p->arg_reg[i]);
+		erts_print(to, to_arg, "   %.*T\n", INT_MAX, p->arg_reg[i]);
 	}
     }
 }
 
 static void
-print_function_from_pc(int to, void *to_arg, Eterm* x)
+print_function_from_pc(int to, void *to_arg, BeamInstr* x)
 {
-    Eterm* addr = find_function_from_pc(x);
+    BeamInstr* addr = find_function_from_pc(x);
     if (addr == NULL) {
         if (x == beam_exit) {
             erts_print(to, to_arg, "<terminate process>");
@@ -8426,7 +9175,7 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg)
     }
 
     if (is_CP(x)) {
-        erts_print(to, to_arg, "Return addr %p (", (Eterm *) x);
+        erts_print(to, to_arg, "Return addr %p (", (Eterm *) EXPAND_POINTER(x));
         print_function_from_pc(to, to_arg, cp_val(x));
         erts_print(to, to_arg, ")\n");
         yreg = 0;
@@ -9255,8 +10004,8 @@ init_processes_bif(void)
     processes_trap_export.code[0] = am_erlang;
     processes_trap_export.code[1] = am_processes_trap;
     processes_trap_export.code[2] = 2;
-    processes_trap_export.code[3] = (Eterm) em_apply_bif;
-    processes_trap_export.code[4] = (Eterm) &processes_trap;
+    processes_trap_export.code[3] = (BeamInstr) em_apply_bif;
+    processes_trap_export.code[4] = (BeamInstr) &processes_trap;
 
 #if ERTS_PROCESSES_BIF_DEBUGLEVEL >= ERTS_PROCS_DBGLVL_CHK_TERM_PROC_LIST
     erts_get_emu_time(&debug_tv_start);
@@ -9268,6 +10017,22 @@ init_processes_bif(void)
  * Debug stuff
  */
 
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+int
+erts_dbg_check_halloc_lock(Process *p)
+{
+    if (ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p))
+	return 1;
+    if (p->id == ERTS_INVALID_PID)
+	return 1;
+    if (p->scheduler_data && p == p->scheduler_data->match_pseudo_process)
+	return 1;
+    if (erts_thr_progress_is_blocking())
+	return 1;
+    return 0;
+}
+#endif
+
 Eterm
 erts_debug_processes(Process *c_p)
 {
@@ -9502,3 +10267,30 @@ debug_processes_assert_error(char* expr, char* file, int line)
 /*                                                                           *\
  * End of the processes/0 BIF implementation.                                *
 \* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * A nice system halt closing all open port goes as follows:
+ * 1) This function schedules the aux work ERTS_SSI_AUX_WORK_REAP_PORTS
+ *    on all schedulers, then schedules itself out.
+ * 2) All shedulers detect this and set the flag halt_in_progress
+ *    on their run queue. The last scheduler sets all non-closed ports
+ *    ERTS_PORT_SFLG_HALT. Global atomic erts_halt_progress is used
+ *    as refcount to determine which is last.
+ * 3) While the run ques has flag halt_in_progress no processes
+ *    will be scheduled, only ports.
+ * 4) When the last port closes that scheduler calls erlang:halt/1.
+ *    The same global atomic is used as refcount.
+ *
+ * A BIF that calls this should make sure to schedule out to never come back:
+ *    erl_halt((int)(- code));
+ *    ERTS_BIF_YIELD1(bif_export[BIF_erlang_halt_1], BIF_P, NIL);
+ */
+void erl_halt(int code)
+{
+    if (-1 == erts_smp_atomic32_cmpxchg_acqb(&erts_halt_progress,
+					     erts_no_schedulers,
+					     -1)) {
+	erts_halt_code = code;
+	notify_reap_ports_relb();
+    }
+}
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index f58b6932b3..9e7a5a5c74 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -28,6 +28,12 @@
 #define ERTS_INCLUDE_SCHEDULER_INTERNALS
 #endif
 
+/* #define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC */
+
+#if !defined(ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC) && defined(DEBUG)
+#  define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
+#endif
+
 typedef struct process Process;
 
 #include "sys.h"
@@ -47,11 +53,18 @@ typedef struct process Process;
 #include "erl_time.h"
 #include "erl_atom_table.h"
 #include "external.h"
+#include "erl_mseg.h"
+#include "erl_async.h"
 
 #ifdef HIPE
 #include "hipe_process.h"
 #endif
 
+#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
+#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
+#include "erl_thr_progress.h"
+#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
+
 struct ErtsNodesMonitor_;
 struct port;
 
@@ -82,6 +95,7 @@ struct saved_calls {
 };
 
 extern Export exp_send, exp_receive, exp_timeout;
+extern int erts_sched_compact_load;
 extern Uint erts_no_schedulers;
 extern Uint erts_no_run_queues;
 extern int erts_sched_thread_suggested_stack_size;
@@ -130,12 +144,10 @@ extern int erts_sched_thread_suggested_stack_size;
   (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 1))
 #define ERTS_RUNQ_FLG_SUSPENDED \
   (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 2))
-#define ERTS_RUNQ_FLG_SHARED_RUNQ \
-  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 3))
 #define ERTS_RUNQ_FLG_CHK_CPU_BIND \
-  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 4))
+  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 3))
 #define ERTS_RUNQ_FLG_INACTIVE \
-  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5))
+  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 4))
 
 #define ERTS_RUNQ_FLGS_MIGRATION_QMASKS	\
   (ERTS_RUNQ_FLGS_EMIGRATE_QMASK	\
@@ -174,12 +186,12 @@ extern int erts_sched_thread_suggested_stack_size;
 #define ERTS_UNSET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
   ((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO)))
 
-#define ERTS_RUNQ_IFLG_SUSPENDED		(((long) 1) << 0)
-#define ERTS_RUNQ_IFLG_NONEMPTY			(((long) 1) << 1)
+#define ERTS_RUNQ_IFLG_SUSPENDED		(((erts_aint32_t) 1) << 0)
+#define ERTS_RUNQ_IFLG_NONEMPTY			(((erts_aint32_t) 1) << 1)
 
 
 #ifdef DEBUG
-#  ifdef ARCH_64
+#  if defined(ARCH_64) && !HALFWORD_HEAP
 #    define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \
        (*((char **) &(RQP)) = (char *) (0xdeadbeefdead0003 | ((N) << 4)))
 #  define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \
@@ -194,8 +206,8 @@ do { \
 #  define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \
 do { \
     ASSERT((RQP) != NULL); \
-    ASSERT(((((Uint) (RQP)) & ((Uint) 1))) == ((Uint) 0)); \
-    ASSERT((((Uint) (RQP)) & ~((Uint) 0xffff)) != ((Uint) 0xdead0000)); \
+    ASSERT(((((UWord) (RQP)) & ((UWord) 1))) == ((UWord) 0)); \
+    ASSERT((((UWord) (RQP)) & ~((UWord) 0xffff)) != ((UWord) 0xdead0000)); \
 } while (0)
 #  endif
 #else
@@ -219,6 +231,54 @@ typedef enum {
     ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED
 } ErtsMigrateResult;
 
+#define ERTS_SSI_FLG_SLEEPING		(((erts_aint32_t) 1) << 0)
+#define ERTS_SSI_FLG_POLL_SLEEPING 	(((erts_aint32_t) 1) << 1)
+#define ERTS_SSI_FLG_TSE_SLEEPING 	(((erts_aint32_t) 1) << 2)
+#define ERTS_SSI_FLG_WAITING		(((erts_aint32_t) 1) << 3)
+#define ERTS_SSI_FLG_SUSPENDED	 	(((erts_aint32_t) 1) << 4)
+
+#define ERTS_SSI_FLGS_SLEEP_TYPE			\
+ (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING)
+
+#define ERTS_SSI_FLGS_SLEEP				\
+ (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLGS_SLEEP_TYPE)
+
+#define ERTS_SSI_FLGS_ALL				\
+ (ERTS_SSI_FLGS_SLEEP					\
+  | ERTS_SSI_FLG_WAITING				\
+  | ERTS_SSI_FLG_SUSPENDED)
+
+/*
+ * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative
+ * eachother. Most frequent - lowest bit number.
+ */
+
+#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP	(((erts_aint32_t) 1) << 0)
+#define ERTS_SSI_AUX_WORK_DD			(((erts_aint32_t) 1) << 1)
+#define ERTS_SSI_AUX_WORK_DD_THR_PRGR		(((erts_aint32_t) 1) << 2)
+#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC	(((erts_aint32_t) 1) << 3)
+#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM	(((erts_aint32_t) 1) << 4)
+#define ERTS_SSI_AUX_WORK_ASYNC_READY		(((erts_aint32_t) 1) << 5)
+#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN	(((erts_aint32_t) 1) << 6)
+#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR		(((erts_aint32_t) 1) << 7)
+#define ERTS_SSI_AUX_WORK_MISC			(((erts_aint32_t) 1) << 8)
+#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN	(((erts_aint32_t) 1) << 9)
+#define ERTS_SSI_AUX_WORK_SET_TMO		(((erts_aint32_t) 1) << 10)
+#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK	(((erts_aint32_t) 1) << 11)
+#define ERTS_SSI_AUX_WORK_REAP_PORTS		(((erts_aint32_t) 1) << 12)
+
+typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;
+
+struct ErtsSchedulerSleepInfo_ {
+#ifdef ERTS_SMP
+    ErtsSchedulerSleepInfo *next;
+    ErtsSchedulerSleepInfo *prev;
+    erts_smp_atomic32_t flags;
+    erts_tse_t *event;
+#endif
+    erts_atomic32_t aux_work;
+};
+
 /* times to reschedule low prio process before running */
 #define RESCHEDULE_LOW        8
 
@@ -266,14 +326,11 @@ typedef struct {
 
 struct ErtsRunQueue_ {
     int ix;
-    erts_smp_atomic_t info_flags;
+    erts_smp_atomic32_t info_flags;
 
     erts_smp_mtx_t mtx;
     erts_smp_cnd_t cnd;
 
-    erts_smp_atomic_t spin_waiter;
-    erts_smp_atomic_t spin_wake;
-
     ErtsSchedulerData *scheduler;
     int waiting; /* < 0 in sys schedule; > 0 on cnd variable */
     int woken;
@@ -286,6 +343,7 @@ struct ErtsRunQueue_ {
     int len;
     int wakeup_other;
     int wakeup_other_reds;
+    int halt_in_progress;
 
     struct {
 	int len;
@@ -319,7 +377,6 @@ typedef union {
 } ErtsAlignedRunQueue;
 
 extern ErtsAlignedRunQueue *erts_aligned_run_queues;
-extern ErtsRunQueue *erts_common_run_queue;
 
 #define ERTS_PROC_REDUCTIONS_EXECUTED(RQ, PRIO, REDS, AREDS)	\
 do {								\
@@ -336,42 +393,109 @@ do {								\
     (RQ)->wakeup_other_reds += (REDS);				\
 } while (0)
 
-struct ErtsSchedulerData_ {
+typedef struct {
+    int enabled;
+    Uint64 start;
+    struct {
+	Uint64 total;
+	Uint64 start;
+	int currently;
+    } working;
+} ErtsSchedWallTime;
 
+typedef struct {
+    int sched;
+    erts_aint32_t aux_work;
+} ErtsDelayedAuxWorkWakeupJob;
+
+typedef struct {
+    int sched_id;
+    ErtsSchedulerData *esdp;
+    ErtsSchedulerSleepInfo *ssi;
 #ifdef ERTS_SMP
+    ErtsThrPrgrVal current_thr_prgr;
+#endif
+    struct {
+	int ix;
+#ifdef ERTS_SMP
+	ErtsThrPrgrVal thr_prgr;
+#endif
+    } misc;
+#ifdef ERTS_SMP
+    struct {
+	ErtsThrPrgrVal thr_prgr;
+	void (*completed_callback)(void *);
+	void (*completed_arg)(void *);
+    } dd;
+#endif
+#ifdef ERTS_USE_ASYNC_READY_Q
+    struct {
+#ifdef ERTS_SMP
+	int need_thr_prgr;
+	ErtsThrPrgrVal thr_prgr;
+#endif
+	void *queue;
+    } async_ready;
+#endif
+#ifdef ERTS_SMP
+    struct {
+	int *sched2jix;
+	int jix;
+	ErtsDelayedAuxWorkWakeupJob *job;
+    } delayed_wakeup;
+#endif
+} ErtsAuxWorkData;
+
+struct ErtsSchedulerData_ {
     /*
      * Keep X registers first (so we get as many low
      * numbered registers as possible in the same cache
      * line).
      */
-    Eterm save_reg[ERTS_X_REGS_ALLOCATED]; /* X registers */
-    FloatDef freg[MAX_REG];	/* Floating point registers. */
+    Eterm* x_reg_array;		/* X registers */
+    FloatDef* f_reg_array;	/* Floating point registers. */
+
+#ifdef ERTS_SMP
     ethr_tid tid;		/* Thread id */
     struct erl_bits_state erl_bits_state; /* erl_bits.c state */
     void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */
     Process *free_process;
+    ErtsThrPrgrData thr_progress_data;
 #endif
-
+#if !HEAP_ON_C_STACK
+    Eterm tmp_heap[TMP_HEAP_SIZE];
+    int num_tmp_heap_used;
+    Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE];
+    Eterm cmp_tmp_heap[CMP_TMP_HEAP_SIZE];
+    Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE];
+#endif
+    ErtsSchedulerSleepInfo *ssi;
     Process *current_process;
     Uint no;			/* Scheduler number */
     struct port *current_port;
     ErtsRunQueue *run_queue;
     int virtual_reds;
     int cpu_id;			/* >= 0 when bound */
-
+    ErtsAuxWorkData aux_work_data;
     ErtsAtomCacheMap atom_cache_map;
 
-#ifdef ERTS_SMP
-    /* NOTE: These fields are modified under held mutexes by other threads */
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
-    int check_children; /* run queue mutex */
-    int blocked_check_children; /* schdlr_sspnd mutex */
-#endif
-    erts_smp_atomic_t suspended; /* Only used when common run queue */
-    erts_smp_atomic_t chk_cpu_bind; /* Only used when common run queue */
+    ErtsSchedAllocData alloc_data;
+
+    ErtsSchedWallTime sched_wall_time;
+
+#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
+    erts_alloc_verify_func_t verify_unused_temp_alloc;
+    Allctr_t *verify_unused_temp_alloc_data;
 #endif
 };
 
+typedef union {
+    ErtsSchedulerData esd;
+    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))];
+} ErtsAlignedSchedulerData;
+
+extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
+
 #ifndef ERTS_SMP
 extern ErtsSchedulerData *erts_scheduler_data;
 #endif
@@ -386,8 +510,9 @@ extern ErtsSchedulerData *erts_scheduler_data;
 #define ERTS_PSD_SAVED_CALLS_BUF		1
 #define ERTS_PSD_SCHED_ID			2
 #define ERTS_PSD_DIST_ENTRY			3
+#define ERTS_PSD_CALL_TIME_BP			4
 
-#define ERTS_PSD_SIZE				4
+#define ERTS_PSD_SIZE				5
 
 typedef struct {
     void *data[ERTS_PSD_SIZE];
@@ -408,6 +533,9 @@ typedef struct {
 #define ERTS_PSD_DIST_ENTRY_GET_LOCKS ERTS_PROC_LOCK_MAIN
 #define ERTS_PSD_DIST_ENTRY_SET_LOCKS ERTS_PROC_LOCK_MAIN
 
+#define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN
+#define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN
+
 typedef struct {
     ErtsProcLocks get_locks;
     ErtsProcLocks set_locks;
@@ -479,6 +607,7 @@ struct ErtsPendingSuspend_ {
 
 #  define MIN_VHEAP_SIZE(p)   (p)->min_vheap_size
 #  define BIN_VHEAP_SZ(p)     (p)->bin_vheap_sz
+#  define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature
 #  define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz
 #  define BIN_OLD_VHEAP(p)    (p)->bin_old_vheap
 
@@ -498,7 +627,7 @@ struct process {
     Uint min_heap_size;         /* Minimum size of heap (in words). */
     Uint min_vheap_size;        /* Minimum size of virtual heap (in words). */
 
-#if !defined(NO_FPE_SIGNALS)
+#if !defined(NO_FPE_SIGNALS) || defined(HIPE)
     volatile unsigned long fp_exception;
 #endif
 
@@ -518,8 +647,8 @@ struct process {
     unsigned max_arg_reg;	/* Maximum number of argument registers available. */
     Eterm def_arg_reg[6];	/* Default array for argument registers. */
 
-    Eterm* cp;			/* Continuation pointer (for threaded code). */
-    Eterm* i;			/* Program counter for threaded code. */
+    BeamInstr* cp;		/* (untagged) Continuation pointer (for threaded code). */
+    BeamInstr* i;		/* Program counter for threaded code. */
     Sint catches;		/* Number of catches on stack */
     Sint fcalls;		/* 
 				 * Number of reductions left to execute.
@@ -566,11 +695,16 @@ struct process {
     Uint seq_trace_lastcnt;
     Eterm seq_trace_token;	/* Sequential trace token (tuple size 5 see below) */
 
-    Eterm initial[3];		/* Initial module(0), function(1), arity(2) */
-    Eterm* current;		/* Current Erlang function:
+#ifdef USE_VM_PROBES
+    Eterm dt_utag;              /* Place to store the dynamc trace user tag */
+    Uint dt_utag_flags;         /* flag field for the dt_utag */
+#endif       
+    BeamInstr initial[3];	/* Initial module(0), function(1), arity(2), often used instead
+				   of pointer to funcinfo instruction, hence the BeamInstr datatype */
+    BeamInstr* current;		/* Current Erlang function, part of the funcinfo:
 				 * module(0), function(1), arity(2)
 				 * (module and functions are tagged atoms;
-				 * arity an untagged integer).
+				 * arity an untagged integer). BeamInstr * because it references code
 				 */
     
     /*
@@ -595,9 +729,10 @@ struct process {
     Uint mbuf_sz;		/* Size of all message buffers */
     ErtsPSD *psd;		/* Rarely used process specific data */
 
-    Uint bin_vheap_sz;		/* Virtual heap block size for binaries */
-    Uint bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
-    Uint bin_old_vheap;		/* Virtual old heap size for binaries */
+    Uint64 bin_vheap_sz;	/* Virtual heap block size for binaries */
+    Uint64 bin_vheap_mature;	/* Virtual heap block size for binaries */
+    Uint64 bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
+    Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */
 
     union {
 #ifdef ERTS_SMP
@@ -626,24 +761,6 @@ struct process {
 #endif
 #endif
 
-#ifdef HYBRID
-    Eterm *rrma;                /* Remembered roots to Message Area */
-    Eterm **rrsrc;              /* The source of the root */
-    Uint nrr;                   /* Number of remembered roots */
-    Uint rrsz;                  /* Size of root array */
-#endif
-
-#ifdef HYBRID
-    Uint active;                /* Active since last major collection? */
-    Uint active_index;          /* Index in the active process array */
-#endif
- 
-#ifdef INCREMENTAL
-    Process *active_next; /* Active processes to scan for roots */
-    Process *active_prev; /* in collection of the message area  */
-    Eterm *scan_top;
-#endif
-
 #ifdef CHECK_FOR_HOLES
     Eterm* last_htop;		/* No need to scan the heap below this point. */
     ErlHeapFragment* last_mbuf;	/* No need to scan beyond this mbuf. */
@@ -753,22 +870,18 @@ ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp)
 {
     ErlHeapFragment* hf = MBUF(p);
 
-    ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->size));
+    ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->alloc_size));
 
     hf->used_size = hp - hf->mem;
 }	
 #endif /* inline */
 
-Eterm* erts_heap_alloc(Process* p, Uint need);
+Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra);
 #ifdef CHECK_FOR_HOLES
 Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz);
 #endif
 
 extern Process** process_tab;
-#ifdef HYBRID
-extern Uint erts_num_active_procs;
-extern Process** erts_active_procs;
-#endif
 extern Uint erts_max_processes;
 extern Uint erts_process_tab_index_mask;
 extern Uint erts_default_process_flags;
@@ -812,13 +925,14 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 #define F_INSLPQUEUE         (1 <<  1) /* Set if in timer queue */
 #define F_TIMO               (1 <<  2) /* Set if timeout */
 #define F_HEAP_GROW          (1 <<  3)
-#define F_NEED_FULLSWEEP     (1 <<  4) /* If process has old binaries & funs. */
+#define F_NEED_FULLSWEEP     (1 <<  4)
 #define F_USING_DB           (1 <<  5) /* If have created tables */
 #define F_DISTRIBUTION       (1 <<  6) /* Process used in distribution */
 #define F_USING_DDLL         (1 <<  7) /* Process has used the DDLL interface */
 #define F_HAVE_BLCKD_MSCHED  (1 <<  8) /* Process has blocked multi-scheduling */
 #define F_P2PNR_RESCHED      (1 <<  9) /* Process has been rescheduled via erts_pid2proc_not_running() */
 #define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
+#define F_HIBERNATE_SCHED    (1 << 11) /* Schedule out after hibernate op */
 
 /* process trace_flags */
 #define F_SENSITIVE          (1 << 0)
@@ -878,6 +992,14 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 #define SEQ_TRACE_PRINT    (1 << 2)
 #define SEQ_TRACE_TIMESTAMP (1 << 3)
 
+#ifdef USE_VM_PROBES
+#define DT_UTAG_PERMANENT (1 << 0)
+#define DT_UTAG_SPREADING (1 << 1)
+#define DT_UTAG(P) ((P)->dt_utag)
+#define DT_UTAG_FLAGS(P)  ((P)->dt_utag_flags) 
+#endif
+
+
 #ifdef ERTS_SMP
 /* Status flags ... */
 #define ERTS_PROC_SFLG_PENDADD2SCHEDQ	(((Uint32) 1) << 0)	/* Pending
@@ -946,39 +1068,35 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags;
 	    (p)->flags &= ~F_TIMO; \
     } while (0)
 
-
-#define ERTS_INIT_SCHED_BIND_TYPE_SUCCESS		0
-#define ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED		1
-#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY	2
-#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE	3
-
-int erts_init_scheduler_bind_type(char *how);
-
-#define ERTS_INIT_CPU_TOPOLOGY_OK			0
-#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID		1
-#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE		2
-#define ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY	3
-#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE		4
-#define ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES		5
-#define ERTS_INIT_CPU_TOPOLOGY_MISSING_LID		6
-#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS		7
-#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES	8
-#define ERTS_INIT_CPU_TOPOLOGY_MISSING			9
-
-int erts_init_cpu_topology(char *topology_str);
+#define ERTS_RUNQ_IX(IX)						\
+  (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_run_queues),			\
+   &erts_aligned_run_queues[(IX)].runq)
+#define ERTS_SCHEDULER_IX(IX)						\
+  (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers),			\
+   &erts_aligned_scheduler_data[(IX)].esd)
 
 void erts_pre_init_process(void);
 void erts_late_init_process(void);
-void erts_early_init_scheduling(void);
-void erts_init_scheduling(int, int, int);
+void erts_early_init_scheduling(int);
+void erts_init_scheduling(int, int);
+
+Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
 
 ErtsProcList *erts_proclist_create(Process *);
 void erts_proclist_destroy(ErtsProcList *);
 int erts_proclist_same(ErtsProcList *, Process *);
 
+int erts_sched_set_wakeup_other_thresold(char *str);
+int erts_sched_set_wakeup_other_type(char *str);
+int erts_sched_set_busy_wait_threshold(char *str);
+
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
+int erts_dbg_check_halloc_lock(Process *p);
+#endif
 #ifdef DEBUG
 void erts_dbg_multi_scheduling_return_trap(Process *, Eterm);
 #endif
+int erts_get_max_no_executing_schedulers(void);
 #ifdef ERTS_SMP
 ErtsSchedSuspendResult
 erts_schedulers_state(Uint *, Uint *, Uint *, int);
@@ -992,10 +1110,23 @@ erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int);
 int erts_is_multi_scheduling_blocked(void);
 Eterm erts_multi_scheduling_blockers(Process *);
 void erts_start_schedulers(void);
+void erts_alloc_notify_delayed_dealloc(int);
 void erts_smp_notify_check_children_needed(void);
 #endif
+#if ERTS_USE_ASYNC_READY_Q
+void erts_notify_check_async_ready_queue(void *);
+#endif
+void erts_schedule_misc_aux_work(int sched_id,
+				 void (*func)(void *),
+				 void *arg);
+void erts_schedule_multi_misc_aux_work(int ignore_self,
+				       int max_sched,
+				       void (*func)(void *),
+				       void *arg);
+erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int);
+void erts_sched_notify_check_cpu_bind(void);
 Uint erts_active_schedulers(void);
-void erts_init_process(void);
+void erts_init_process(int);
 Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm);
 Uint erts_run_queues_len(Uint *);
 void erts_add_to_runq(Process *);
@@ -1069,10 +1200,14 @@ void erts_handle_pending_exit(Process *, ErtsProcLocks);
 
 void erts_deep_process_dump(int, void *);
 
+Eterm erts_get_reader_groups_map(Process *c_p);
+Eterm erts_debug_reader_groups_map(Process *c_p, int groups);
+
 Sint erts_test_next_pid(int, Uint);
 Eterm erts_debug_processes(Process *c_p);
 Eterm erts_debug_processes_bif_info(Process *c_p);
 Uint erts_debug_nbalance(void);
+int erts_debug_wait_deallocations(Process *c_p);
 
 #ifdef ERTS_SMP
 #  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data)
@@ -1082,6 +1217,20 @@ Uint erts_debug_nbalance(void);
 #  define ERTS_PROC_GET_SCHDATA(PROC) (erts_scheduler_data)
 #endif
 
+#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
+#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P)					\
+do {										\
+    ErtsSchedulerData *esdp__ = ((P)						\
+				 ? ERTS_PROC_GET_SCHDATA((Process *) (P))	\
+				 : erts_get_scheduler_data());			\
+    if (esdp__)									\
+	esdp__->verify_unused_temp_alloc(					\
+	    esdp__->verify_unused_temp_alloc_data);				\
+} while (0)
+#else
+#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(ESDP)
+#endif
+
 #if defined(ERTS_SMP) || defined(USE_THREADS)
 ErtsSchedulerData *erts_get_scheduler_data(void);
 #else
@@ -1129,16 +1278,11 @@ erts_psd_get(Process *p, int ix)
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
     ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
     if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].get_locks)
-	ERTS_SMP_LC_ASSERT(locks
-			   || erts_is_system_blocked(0)
-			   || (ERTS_IS_CRASH_DUMPING
-			       && erts_is_system_blocked(ERTS_BS_FLG_ALLOW_GC)));
+	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
     else {
 	locks &= erts_psd_required_locks[ix].get_locks;
 	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks == locks
-			   || erts_is_system_blocked(0)
-			   || (ERTS_IS_CRASH_DUMPING
-			       && erts_is_system_blocked(ERTS_BS_FLG_ALLOW_GC)));
+			   || erts_thr_progress_is_blocking());
     }
 #endif
     ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
@@ -1155,16 +1299,11 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
     ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
     if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks)
-	ERTS_SMP_LC_ASSERT(locks
-			   || erts_is_system_blocked(0)
-			   || (ERTS_IS_CRASH_DUMPING
-			       && erts_is_system_blocked(ERTS_BS_FLG_ALLOW_GC)));
+	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
     else {
 	locks &= erts_psd_required_locks[ix].set_locks;
 	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks
-			   || erts_is_system_blocked(0)
-			   || (ERTS_IS_CRASH_DUMPING
-			       && erts_is_system_blocked(ERTS_BS_FLG_ALLOW_GC)));
+			   || erts_thr_progress_is_blocking());
     }
 #endif
     ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
@@ -1184,7 +1323,7 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
 #endif
 
 #define ERTS_PROC_SCHED_ID(P, L, ID) \
-  ((Uint) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID)))
+  ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID)))
 
 #define ERTS_PROC_GET_DIST_ENTRY(P) \
   ((DistEntry *) erts_psd_get((P), ERTS_PSD_DIST_ENTRY))
@@ -1196,6 +1335,12 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
 #define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \
   ((struct saved_calls *) erts_psd_set((P), (L), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB)))
 
+#define ERTS_PROC_GET_CALL_TIME(P) \
+  ((process_breakpoint_time_t *) erts_psd_get((P), ERTS_PSD_CALL_TIME_BP))
+#define ERTS_PROC_SET_CALL_TIME(P, L, PBT) \
+  ((process_breakpoint_time_t *) erts_psd_set((P), (L), ERTS_PSD_CALL_TIME_BP, (void *) (PBT)))
+
+
 ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
 ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p,
 						  ErtsProcLocks plocks,
@@ -1209,8 +1354,8 @@ erts_proc_get_error_handler(Process *p)
     if (!val)
 	return am_error_handler;
     else {
-	ASSERT(is_atom(((Eterm) val)));
-	return (Eterm) val;
+	ASSERT(is_atom(((Eterm) (UWord) val)));
+	return (Eterm) (UWord) val;
     }
 }
 
@@ -1220,13 +1365,13 @@ erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler)
     void *old_val;
     void *new_val;
     ASSERT(is_atom(handler));
-    new_val = handler == am_error_handler ? NULL : (void *) handler;
+    new_val = (handler == am_error_handler) ? NULL : (void *) (UWord) handler;
     old_val = erts_psd_set(p, plocks, ERTS_PSD_ERROR_HANDLER, new_val);
     if (!old_val)
 	return am_error_handler;
     else {
-	ASSERT(is_atom(((Eterm) old_val)));
-	return (Eterm) old_val;
+	ASSERT(is_atom(((Eterm) (UWord) old_val)));
+	return (Eterm) (UWord) old_val;
     }
 }
 
@@ -1247,10 +1392,14 @@ ERTS_GLB_INLINE Eterm erts_get_current_pid(void);
 ERTS_GLB_INLINE Uint erts_get_scheduler_id(void);
 ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p);
 ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp);
+#ifndef ERTS_ENABLE_LOCK_COUNT
 ERTS_GLB_INLINE void erts_smp_runq_lock(ErtsRunQueue *rq);
+#endif
 ERTS_GLB_INLINE int erts_smp_runq_trylock(ErtsRunQueue *rq);
 ERTS_GLB_INLINE void erts_smp_runq_unlock(ErtsRunQueue *rq);
+#ifndef ERTS_ENABLE_LOCK_COUNT
 ERTS_GLB_INLINE void erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
+#endif
 ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
 ERTS_GLB_INLINE void erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
 ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
@@ -1322,8 +1471,7 @@ erts_get_runq_proc(Process *p)
     ASSERT(p->run_queue);
     return p->run_queue;
 #else
-    ASSERT(erts_common_run_queue);
-    return erts_common_run_queue;
+    return ERTS_RUNQ_IX(0);
 #endif
 }
 
@@ -1336,11 +1484,16 @@ erts_get_runq_current(ErtsSchedulerData *esdp)
 	esdp = erts_get_scheduler_data();
     return esdp->run_queue;
 #else
-    ASSERT(erts_common_run_queue);
-    return erts_common_run_queue;
+    return ERTS_RUNQ_IX(0);
 #endif
 }
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+
+#define erts_smp_runq_lock(rq)	erts_smp_mtx_lock_x(&(rq)->mtx, __FILE__, __LINE__)
+
+#else
+
 ERTS_GLB_INLINE void
 erts_smp_runq_lock(ErtsRunQueue *rq)
 {
@@ -1349,6 +1502,8 @@ erts_smp_runq_lock(ErtsRunQueue *rq)
 #endif
 }
 
+#endif
+
 ERTS_GLB_INLINE int
 erts_smp_runq_trylock(ErtsRunQueue *rq)
 {
@@ -1367,6 +1522,31 @@ erts_smp_runq_unlock(ErtsRunQueue *rq)
 #endif
 }
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+
+#define erts_smp_xrunq_lock(rq, xrq)	erts_smp_xrunq_lock_x((rq), (xrq), __FILE__, __LINE__)
+
+ERTS_GLB_INLINE void
+erts_smp_xrunq_lock_x(ErtsRunQueue *rq, ErtsRunQueue *xrq, char* file, int line)
+{
+#ifdef ERTS_SMP
+    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&rq->mtx));
+    if (xrq != rq) {
+	if (erts_smp_mtx_trylock(&xrq->mtx) == EBUSY) {
+	    if (rq < xrq)
+		erts_smp_mtx_lock_x(&xrq->mtx, file, line);
+	    else {
+		erts_smp_mtx_unlock(&rq->mtx);
+		erts_smp_mtx_lock_x(&xrq->mtx, file, line);
+		erts_smp_mtx_lock_x(&rq->mtx, file, line);
+	    }
+	}
+    }
+#endif
+}
+
+#else
+
 ERTS_GLB_INLINE void
 erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
 {
@@ -1386,6 +1566,8 @@ erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
 #endif
 }
 
+#endif
+
 ERTS_GLB_INLINE void
 erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
 {
@@ -1440,6 +1622,10 @@ erts_get_atom_cache_map(Process *c_p)
 }
 #endif
 
+Process *erts_pid2proc_suspend(Process *,
+			       ErtsProcLocks,
+			       Eterm,
+			       ErtsProcLocks);
 #ifdef ERTS_SMP
 
 Process *erts_pid2proc_not_running(Process *,
@@ -1487,29 +1673,30 @@ extern int erts_disable_proc_not_running_opt;
 #define ERTS_MIN_PROCESSES		16
 #endif
 
-#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
-ERTS_GLB_INLINE void erts_smp_notify_inc_runq(ErtsRunQueue *runq);
-void erts_smp_notify_inc_runq__(ErtsRunQueue *runq);
-#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
+void erts_smp_notify_inc_runq(ErtsRunQueue *runq);
 
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+#ifdef ERTS_SMP
+void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t);
+ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi);
 
-#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
 ERTS_GLB_INLINE void
-erts_smp_notify_inc_runq(ErtsRunQueue *runq)
+erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
 {
-#ifdef ERTS_SMP
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq));
-    if (runq->waiting)
-	erts_smp_notify_inc_runq__(runq);
-#endif
+    erts_aint32_t flags;
+    ERTS_THR_MEMORY_BARRIER;
+    flags = erts_smp_atomic32_read_nob(&ssi->flags);
+    if (flags & ERTS_SSI_FLG_SLEEPING) {
+	flags = erts_smp_atomic32_read_band_nob(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP);
+	erts_sched_finish_poke(ssi, flags);
+    }
 }
 
-#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
-
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
+#endif /* #ifdef ERTS_SMP */
+
 #include "erl_process_lock.h"
 
 #undef ERTS_INCLUDE_SCHEDULER_INTERNALS
@@ -1517,4 +1704,6 @@ erts_smp_notify_inc_runq(ErtsRunQueue *runq)
 #endif
 
 
-
+void erl_halt(int code);
+extern erts_smp_atomic32_t erts_halt_progress;
+extern int erts_halt_code;
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index 1666509c72..3550f1396c 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -45,16 +45,16 @@ static void dump_dist_ext(int to, void *to_arg, ErtsDistExternal *edep);
 static void dump_element_nl(int to, void *to_arg, Eterm x);
 static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp,
 			      int yreg);
-static void print_function_from_pc(int to, void *to_arg, Eterm* x);
+static void print_function_from_pc(int to, void *to_arg, BeamInstr* x);
 static void heap_dump(int to, void *to_arg, Eterm x);
 static void dump_binaries(int to, void *to_arg, Binary* root);
 static void dump_externally(int to, void *to_arg, Eterm term);
 
 static Binary* all_binaries;
 
-extern Eterm beam_apply[];
-extern Eterm beam_exit[];
-extern Eterm beam_continue_exit[];
+extern BeamInstr beam_apply[];
+extern BeamInstr beam_exit[];
+extern BeamInstr beam_continue_exit[];
 
 
 void
@@ -194,7 +194,7 @@ dump_element(int to, void *to_arg, Eterm x)
 	} else if (is_pid(x)) {
 	    erts_print(to, to_arg, "P%T", x);
 	} else if (is_port(x)) {
-	    erts_print(to, to_arg, "p<%bpu.%bpu>",
+	    erts_print(to, to_arg, "p<%beu.%beu>",
 		       port_channel_no(x), port_number(x));
 	} else if (is_nil(x)) {
 	    erts_putc(to, to_arg, 'N');
@@ -223,7 +223,7 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg)
     }
 
     if (is_CP(x)) {
-        erts_print(to, to_arg, "SReturn addr 0x%X (", (Eterm *) x);
+        erts_print(to, to_arg, "SReturn addr 0x%X (", cp_val(x));
         print_function_from_pc(to, to_arg, cp_val(x));
         erts_print(to, to_arg, ")\n");
         yreg = 0;
@@ -239,9 +239,9 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg)
 }
 
 static void
-print_function_from_pc(int to, void *to_arg, Eterm* x)
+print_function_from_pc(int to, void *to_arg, BeamInstr* x)
 {
-    Eterm* addr = find_function_from_pc(x);
+    BeamInstr* addr = find_function_from_pc(x);
     if (addr == NULL) {
         if (x == beam_exit) {
             erts_print(to, to_arg, "<terminate process>");
@@ -261,139 +261,139 @@ print_function_from_pc(int to, void *to_arg, Eterm* x)
 static void
 heap_dump(int to, void *to_arg, Eterm x)
 {
+    DeclareTmpHeapNoproc(last,1);
+    Eterm* next = last;
     Eterm* ptr;
-    Eterm last = OUR_NIL;
-    Eterm* next = &last;
 
     if (is_immed(x) || is_CP(x)) {
 	return;
     }
-
- again:
-    if (x == OUR_NIL) {	/* We are done. */
-	return;
-    } if (is_CP(x)) {
-	next = (Eterm *) x;
-    } else if (is_list(x)) {
-	ptr = list_val(x);
-	if (ptr[0] != OUR_NIL) {
-	    erts_print(to, to_arg, ADDR_FMT ":l", ptr);
-	    dump_element(to, to_arg, ptr[0]);
-	    erts_putc(to, to_arg, '|');
-	    dump_element(to, to_arg, ptr[1]);
-	    erts_putc(to, to_arg, '\n');
-	    if (is_immed(ptr[1])) {
-		ptr[1] = make_small(0);
-	    }
-	    x = ptr[0];
-	    ptr[0] = (Eterm) next;
-	    next = ptr + 1;
-	    goto again;
-	}
-    } else if (is_boxed(x)) {
-	Eterm hdr;
-	
-	ptr = boxed_val(x);
-	hdr = *ptr;
-	if (hdr != OUR_NIL) {	/* If not visited */
-	    erts_print(to, to_arg, ADDR_FMT ":", ptr);
-	    if (is_arity_value(hdr)) {
-		Uint i;
-		Uint arity = arityval(hdr);
-
-		erts_print(to, to_arg, "t" WORD_FMT ":", arity);
-		for (i = 1; i <= arity; i++) {
-		    dump_element(to, to_arg, ptr[i]);
-		    if (is_immed(ptr[i])) {
-			ptr[i] = make_small(0);
-		    }
-		    if (i < arity) {
-			erts_putc(to, to_arg, ',');
-		    }
-		}
+    UseTmpHeapNoproc(1);
+    *last = OUR_NIL;
+
+    while (x != OUR_NIL) {
+	if (is_CP(x)) {
+	    next = (Eterm *) EXPAND_POINTER(x);
+	} else if (is_list(x)) {
+	    ptr = list_val(x);
+	    if (ptr[0] != OUR_NIL) {
+		erts_print(to, to_arg, ADDR_FMT ":l", ptr);
+		dump_element(to, to_arg, ptr[0]);
+		erts_putc(to, to_arg, '|');
+		dump_element(to, to_arg, ptr[1]);
 		erts_putc(to, to_arg, '\n');
-		if (arity == 0) {
-		    ptr[0] = OUR_NIL;
-		} else {
-		    x = ptr[arity];
-		    ptr[0] = (Eterm) next;
-		    next = ptr + arity - 1;
-		    goto again;
+		if (is_immed(ptr[1])) {
+		    ptr[1] = make_small(0);
 		}
-	    } else if (hdr == HEADER_FLONUM) {
-		FloatDef f;
-		char sbuf[31];
-		int i;
-
-		GET_DOUBLE_DATA((ptr+1), f);
-		i = sys_double_to_chars(f.fd, (char*) sbuf);
-		sys_memset(sbuf+i, 0, 31-i);
-		erts_print(to, to_arg, "F%X:%s\n", i, sbuf);
-		*ptr = OUR_NIL;
-	    } else if (_is_bignum_header(hdr)) {
-		erts_print(to, to_arg, "B%T\n", x);
-		*ptr = OUR_NIL;
-	    } else if (is_binary_header(hdr)) {
-		Uint tag = thing_subtag(hdr);
-		Uint size = binary_size(x);
-		Uint i;
-
-		if (tag == HEAP_BINARY_SUBTAG) {
-		    byte* p;
-
-		    erts_print(to, to_arg, "Yh%X:", size);
-		    p = binary_bytes(x);
-		    for (i = 0; i < size; i++) {
-			erts_print(to, to_arg, "%02X", p[i]);
+		x = ptr[0];
+		ptr[0] = (Eterm) COMPRESS_POINTER(next);
+		next = ptr + 1;
+		continue;
+	    }
+	} else if (is_boxed(x)) {
+	    Eterm hdr;
+
+	    ptr = boxed_val(x);
+	    hdr = *ptr;
+	    if (hdr != OUR_NIL) {	/* If not visited */
+		erts_print(to, to_arg, ADDR_FMT ":", ptr);
+	        if (is_arity_value(hdr)) {
+		    Uint i;
+		    Uint arity = arityval(hdr);
+
+		    erts_print(to, to_arg, "t" WORD_FMT ":", arity);
+		    for (i = 1; i <= arity; i++) {
+			dump_element(to, to_arg, ptr[i]);
+			if (is_immed(ptr[i])) {
+			    ptr[i] = make_small(0);
+			}
+			if (i < arity) {
+			    erts_putc(to, to_arg, ',');
+			}
 		    }
-		} else if (tag == REFC_BINARY_SUBTAG) {
-		    ProcBin* pb = (ProcBin *) binary_val(x);
-		    Binary* val = pb->val;
-
-		    if (erts_smp_atomic_xchg(&val->refc, 0) != 0) {
-			val->flags = (Uint) all_binaries;
-			all_binaries = val;
+		    erts_putc(to, to_arg, '\n');
+		    if (arity == 0) {
+			ptr[0] = OUR_NIL;
+		    } else {
+			x = ptr[arity];
+			ptr[0] = (Eterm) COMPRESS_POINTER(next);
+			next = ptr + arity - 1;
+			continue;
 		    }
-		    erts_print(to, to_arg, "Yc%X:%X:%X", val,
-			       pb->bytes - (byte *)val->orig_bytes,
-			       size);
-		} else if (tag == SUB_BINARY_SUBTAG) {
-		    ErlSubBin* Sb = (ErlSubBin *) binary_val(x);
-		    Eterm* real_bin = binary_val(Sb->orig);
-		    void* val;
-
-		    if (thing_subtag(*real_bin) == REFC_BINARY_SUBTAG) {
-			ProcBin* pb = (ProcBin *) real_bin;
-			val = pb->val;
-		    } else {	/* Heap binary */
-			val = real_bin;
+		} else if (hdr == HEADER_FLONUM) {
+		    FloatDef f;
+		    char sbuf[31];
+		    int i;
+
+		    GET_DOUBLE_DATA((ptr+1), f);
+		    i = sys_double_to_chars(f.fd, (char*) sbuf);
+		    sys_memset(sbuf+i, 0, 31-i);
+		    erts_print(to, to_arg, "F%X:%s\n", i, sbuf);
+		    *ptr = OUR_NIL;
+		} else if (_is_bignum_header(hdr)) {
+		    erts_print(to, to_arg, "B%T\n", x);
+		    *ptr = OUR_NIL;
+		} else if (is_binary_header(hdr)) {
+		    Uint tag = thing_subtag(hdr);
+		    Uint size = binary_size(x);
+		    Uint i;
+
+		    if (tag == HEAP_BINARY_SUBTAG) {
+			byte* p;
+
+			erts_print(to, to_arg, "Yh%X:", size);
+			p = binary_bytes(x);
+			for (i = 0; i < size; i++) {
+			    erts_print(to, to_arg, "%02X", p[i]);
+			}
+		    } else if (tag == REFC_BINARY_SUBTAG) {
+			ProcBin* pb = (ProcBin *) binary_val(x);
+			Binary* val = pb->val;
+
+			if (erts_smp_atomic_xchg_nob(&val->refc, 0) != 0) {
+			    val->flags = (UWord) all_binaries;
+			    all_binaries = val;
+			}
+			erts_print(to, to_arg, "Yc%X:%X:%X", val,
+				   pb->bytes - (byte *)val->orig_bytes,
+				   size);
+		    } else if (tag == SUB_BINARY_SUBTAG) {
+			ErlSubBin* Sb = (ErlSubBin *) binary_val(x);
+			Eterm* real_bin = binary_val(Sb->orig);
+			void* val;
+
+			if (thing_subtag(*real_bin) == REFC_BINARY_SUBTAG) {
+			    ProcBin* pb = (ProcBin *) real_bin;
+			    val = pb->val;
+			} else {	/* Heap binary */
+			    val = real_bin;
+			}
+			erts_print(to, to_arg, "Ys%X:%X:%X", val, Sb->offs, size);
 		    }
-		    erts_print(to, to_arg, "Ys%X:%X:%X", val, Sb->offs, size);
+		    erts_putc(to, to_arg, '\n');
+		    *ptr = OUR_NIL;
+		} else if (is_external_pid_header(hdr)) {
+		    erts_print(to, to_arg, "P%T\n", x);
+		    *ptr = OUR_NIL;
+		} else if (is_external_port_header(hdr)) {
+		    erts_print(to, to_arg, "p<%beu.%beu>\n",
+			       port_channel_no(x), port_number(x));
+		    *ptr = OUR_NIL;
+		} else {
+		    /*
+		     * All other we dump in the external term format.
+		     */
+			dump_externally(to, to_arg, x);
+		    erts_putc(to, to_arg, '\n');
+		    *ptr = OUR_NIL;
 		}
-		erts_putc(to, to_arg, '\n');
-		*ptr = OUR_NIL;
-	    } else if (is_external_pid_header(hdr)) {
-		erts_print(to, to_arg, "P%T\n", x);
-		*ptr = OUR_NIL;
-	    } else if (is_external_port_header(hdr)) {
-		erts_print(to, to_arg, "p<%bpu.%bpu>\n",
-			   port_channel_no(x), port_number(x));
-		*ptr = OUR_NIL;
-	    } else {
-		/*
-		 * All other we dump in the external term format.
-		 */
-		dump_externally(to, to_arg, x);
-		erts_putc(to, to_arg, '\n');
-		*ptr = OUR_NIL;
 	    }
 	}
+	x = *next;
+	*next = OUR_NIL;
+	next--;
     }
-
-    x = *next;
-    *next = OUR_NIL;
-    next--;
-    goto again;
+    UnUseTmpHeapNoproc(1);
 }
 
 static void
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index 52440fb635..34d591df40 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2007-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2007-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -71,9 +71,12 @@ const Process erts_proc_lock_busy;
 
 #ifdef ERTS_SMP
 
-/*#define ERTS_PROC_LOCK_SPIN_ON_GATE*/
-#define ERTS_PROC_LOCK_SPIN_COUNT_MAX  16000
+#define ERTS_PROC_LOCK_SPIN_COUNT_MAX  2000
+#define ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC 32
 #define ERTS_PROC_LOCK_SPIN_COUNT_BASE 1000
+#define ERTS_PROC_LOCK_AUX_SPIN_COUNT 50
+
+#define ERTS_PROC_LOCK_SPIN_UNTIL_YIELD 25
 
 #ifdef ERTS_PROC_LOCK_DEBUG
 #define ERTS_PROC_LOCK_HARD_DEBUG
@@ -83,32 +86,9 @@ const Process erts_proc_lock_busy;
 static void check_queue(erts_proc_lock_t *lck);
 #endif
 
-
-typedef struct erts_proc_lock_waiter_t_ erts_proc_lock_waiter_t;
-struct erts_proc_lock_waiter_t_ {
-    erts_proc_lock_waiter_t *next;
-    erts_proc_lock_waiter_t *prev;
-    ErtsProcLocks wait_locks;
-    erts_smp_gate_t gate;
-    erts_proc_lock_queues_t *queues;
-};
-
-struct erts_proc_lock_queues_t_ {
-    erts_proc_lock_queues_t *next;
-    erts_proc_lock_waiter_t *queue[ERTS_PROC_LOCK_MAX_BIT+1];
-};
-
-struct erts_proc_lock_thr_spec_data_t_ {
-    erts_proc_lock_queues_t *qs;
-    erts_proc_lock_waiter_t *wtr;
-};
-
-static erts_proc_lock_queues_t zeroqs = {0};
-
-static erts_smp_spinlock_t wtr_lock;
-static erts_proc_lock_waiter_t *waiter_free_list;
-static erts_proc_lock_queues_t *queue_free_list;
-static erts_tsd_key_t waiter_key;
+#if SIZEOF_INT < 4
+#error "The size of the 'uflgs' field of the erts_tse_t type is too small"
+#endif
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
 static struct {
@@ -122,165 +102,122 @@ static struct {
 erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS];
 
 static int proc_lock_spin_count;
-static int proc_lock_trans_spin_cost;
+static int aux_thr_proc_lock_spin_count;
 
-static void cleanup_waiter(void);
+static void cleanup_tse(void);
 
 void
-erts_init_proc_lock(void)
+erts_init_proc_lock(int cpus)
 {
     int i;
-    int cpus;
-    erts_smp_spinlock_init(&wtr_lock, "proc_lck_wtr_alloc");
     for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) {
-#if ERTS_PROC_LOCK_MUTEX_IMPL
 #ifdef ERTS_ENABLE_LOCK_COUNT
-	erts_smp_mtx_init_x(&erts_pix_locks[i].u.mtx, "pix_lock", make_small(i));
+	erts_mtx_init_x(&erts_pix_locks[i].u.mtx,
+			"pix_lock", make_small(i));
 #else
-	erts_smp_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock");
-#endif
-#else
-#ifdef ERTS_ENABLE_LOCK_COUNT
-	erts_smp_spinlock_init_x(&erts_pix_locks[i].u.spnlck, "pix_lock", make_small(i));
-#else
-	erts_smp_spinlock_init(&erts_pix_locks[i].u.spnlck, "pix_lock");
-#endif
+	erts_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock");
 #endif
     }
-    waiter_free_list = NULL;
-    queue_free_list = NULL;
-    erts_tsd_key_create(&waiter_key);
-    erts_thr_install_exit_handler(cleanup_waiter);
+    erts_thr_install_exit_handler(cleanup_tse);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     lc_id.proc_lock_main	= erts_lc_get_lock_order_id("proc_main");
     lc_id.proc_lock_link	= erts_lc_get_lock_order_id("proc_link");
     lc_id.proc_lock_msgq	= erts_lc_get_lock_order_id("proc_msgq");
     lc_id.proc_lock_status	= erts_lc_get_lock_order_id("proc_status");
 #endif
-    cpus = erts_get_cpu_configured(erts_cpuinfo);
-    if (cpus > 1)
-	proc_lock_spin_count = (ERTS_PROC_LOCK_SPIN_COUNT_BASE
-				* ((int) erts_no_schedulers));
-    else if (cpus == 1)
-	proc_lock_spin_count = 0;
-    else /* No of cpus unknown. Assume multi proc, but be conservative. */
+    if (cpus > 1) {
 	proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE;
-    if (proc_lock_spin_count > ERTS_PROC_LOCK_SPIN_COUNT_MAX)
-	proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_MAX;
-    proc_lock_trans_spin_cost = proc_lock_spin_count/20;
-}
-
-static ERTS_INLINE erts_proc_lock_waiter_t *
-alloc_wtr(void)
-{
-    erts_proc_lock_waiter_t *wtr;
-    erts_smp_spin_lock(&wtr_lock);
-    wtr = waiter_free_list;
-    if (wtr) {
-	waiter_free_list = wtr->next;
-	ERTS_LC_ASSERT(queue_free_list);
-	wtr->queues = queue_free_list;
-	queue_free_list = wtr->queues->next;
-	erts_smp_spin_unlock(&wtr_lock);
+	proc_lock_spin_count += (ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC
+				 * ((int) erts_no_schedulers));
+	aux_thr_proc_lock_spin_count = ERTS_PROC_LOCK_AUX_SPIN_COUNT;
     }
-    else {
-	erts_smp_spin_unlock(&wtr_lock);
-	wtr = erts_alloc(ERTS_ALC_T_PROC_LCK_WTR,
-			 sizeof(erts_proc_lock_waiter_t));
-	erts_smp_gate_init(&wtr->gate);
-	wtr->wait_locks = (ErtsProcLocks) 0;
-	wtr->queues = erts_alloc(ERTS_ALC_T_PROC_LCK_QS,
-				 sizeof(erts_proc_lock_queues_t));
-	sys_memcpy((void *) wtr->queues,
-		   (void *) &zeroqs,
-		   sizeof(erts_proc_lock_queues_t));
+    else if (cpus == 1) {
+	proc_lock_spin_count = 0;
+	aux_thr_proc_lock_spin_count = 0;
     }
-    return wtr;
+    else { /* No of cpus unknown. Assume multi proc, but be conservative. */
+	proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE/2;
+	aux_thr_proc_lock_spin_count = ERTS_PROC_LOCK_AUX_SPIN_COUNT/2;
+    }
+    if (proc_lock_spin_count > ERTS_PROC_LOCK_SPIN_COUNT_MAX)
+	proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_MAX;
 }
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
-static void
-check_unused_waiter(erts_proc_lock_waiter_t *wtr)
-{
-    int i;
-    ERTS_LC_ASSERT(wtr->wait_locks == 0);
-    for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++)
-	ERTS_LC_ASSERT(!wtr->queues->queue[i]);
-}
-#define CHECK_UNUSED_WAITER(W) check_unused_waiter((W))
+#define CHECK_UNUSED_TSE(W) ERTS_LC_ASSERT((W)->uflgs == 0)
 #else
-#define CHECK_UNUSED_WAITER(W)
+#define CHECK_UNUSED_TSE(W)
 #endif
 
+static ERTS_INLINE erts_tse_t *
+tse_fetch(erts_pix_lock_t *pix_lock)
+{
+    erts_tse_t *tse = erts_tse_fetch();
+    tse->uflgs = 0;
+    return tse;
+}
 
 static ERTS_INLINE void
-free_wtr(erts_proc_lock_waiter_t *wtr)
+tse_return(erts_tse_t *tse)
 {
-    CHECK_UNUSED_WAITER(wtr);
-    erts_smp_spin_lock(&wtr_lock);
-    wtr->next = waiter_free_list;
-    waiter_free_list = wtr;
-    wtr->queues->next = queue_free_list;
-    queue_free_list = wtr->queues;
-    erts_smp_spin_unlock(&wtr_lock);
+    CHECK_UNUSED_TSE(tse);
+    erts_tse_return(tse);
 }
 
 void
 erts_proc_lock_prepare_proc_lock_waiter(void)
 {
-    erts_tsd_set(waiter_key, (void *) alloc_wtr());
+    tse_return(tse_fetch(NULL));
 }
 
 
 static void
-cleanup_waiter(void)
+cleanup_tse(void)
 {
-    erts_proc_lock_waiter_t *wtr = erts_tsd_get(waiter_key);
-    if (wtr)
-	free_wtr(wtr);
+    erts_tse_t *tse = erts_tse_fetch();
+    if (tse)
+	erts_tse_return(tse);
 }
 
 
 /*
  * Waiters are queued in a circular double linked list;
- * where qs->queue[lock_ix] is the first waiter in queue, and
- * qs->queue[lock_ix]->prev is the last waiter in queue.
+ * where lck->queue[lock_ix] is the first waiter in queue, and
+ * lck->queue[lock_ix]->prev is the last waiter in queue.
  */
 
 static ERTS_INLINE void
-enqueue_waiter(erts_proc_lock_queues_t *qs,
-	       int ix,
-	       erts_proc_lock_waiter_t *wtr)
+enqueue_waiter(erts_proc_lock_t *lck, int ix, erts_tse_t *wtr)
 {
-    if (!qs->queue[ix]) {
-	qs->queue[ix] = wtr;
+    if (!lck->queue[ix]) {
+	lck->queue[ix] = wtr;
 	wtr->next = wtr;
 	wtr->prev = wtr;
     }
     else {
-	ERTS_LC_ASSERT(qs->queue[ix]->next && qs->queue[ix]->prev);
-	wtr->next = qs->queue[ix];
-	wtr->prev = qs->queue[ix]->prev;
+	ERTS_LC_ASSERT(lck->queue[ix]->next && lck->queue[ix]->prev);
+	wtr->next = lck->queue[ix];
+	wtr->prev = lck->queue[ix]->prev;
 	wtr->prev->next = wtr;
-	qs->queue[ix]->prev = wtr;
+	lck->queue[ix]->prev = wtr;
     }
 }
 
-static erts_proc_lock_waiter_t *
-dequeue_waiter(erts_proc_lock_queues_t *qs, int ix)
+static erts_tse_t *
+dequeue_waiter(erts_proc_lock_t *lck, int ix)
 {
-    erts_proc_lock_waiter_t *wtr = qs->queue[ix];
-    ERTS_LC_ASSERT(qs->queue[ix]);
+    erts_tse_t *wtr = lck->queue[ix];
+    ERTS_LC_ASSERT(lck->queue[ix]);
     if (wtr->next == wtr) {
-	ERTS_LC_ASSERT(qs->queue[ix]->prev == wtr);
-	qs->queue[ix] = NULL;
+	ERTS_LC_ASSERT(lck->queue[ix]->prev == wtr);
+	lck->queue[ix] = NULL;
     }
     else {
 	ERTS_LC_ASSERT(wtr->next != wtr);
 	ERTS_LC_ASSERT(wtr->prev != wtr);
 	wtr->next->prev = wtr->prev;
 	wtr->prev->next = wtr->next;
-	qs->queue[ix] = wtr->next;
+	lck->queue[ix] = wtr->next;
     }
     return wtr;
 }
@@ -295,29 +232,28 @@ dequeue_waiter(erts_proc_lock_queues_t *qs, int ix)
  *       lock.
  */
 static ERTS_INLINE void
-try_aquire(erts_proc_lock_t *lck, erts_proc_lock_waiter_t *wtr)
+try_aquire(erts_proc_lock_t *lck, erts_tse_t *wtr)
 {
     ErtsProcLocks got_locks = (ErtsProcLocks) 0;
-    ErtsProcLocks locks = wtr->wait_locks;
+    ErtsProcLocks locks = wtr->uflgs;
     int lock_no;
 
-    ERTS_LC_ASSERT(lck->queues);
     ERTS_LC_ASSERT(got_locks != locks);
 
     for (lock_no = 0; lock_no <= ERTS_PROC_LOCK_MAX_BIT; lock_no++) {
 	ErtsProcLocks lock = ((ErtsProcLocks) 1) << lock_no;
 	if (locks & lock) {
 	    ErtsProcLocks wflg, old_lflgs;
-	    if (lck->queues->queue[lock_no]) {
+	    if (lck->queue[lock_no]) {
 		/* Others already waiting */
 	    enqueue:
 		ERTS_LC_ASSERT(ERTS_PROC_LOCK_FLGS_READ_(lck)
 			       & (lock << ERTS_PROC_LOCK_WAITER_SHIFT));
-		enqueue_waiter(lck->queues, lock_no, wtr);
+		enqueue_waiter(lck, lock_no, wtr);
 		break;
 	    }
 	    wflg = lock << ERTS_PROC_LOCK_WAITER_SHIFT;
-	    old_lflgs = ERTS_PROC_LOCK_FLGS_BOR_(lck, wflg | lock);
+	    old_lflgs = ERTS_PROC_LOCK_FLGS_BOR_ACQB_(lck, wflg | lock);
 	    if (old_lflgs & lock) {
 		/* Didn't get the lock */
 		goto enqueue;
@@ -334,7 +270,7 @@ try_aquire(erts_proc_lock_t *lck, erts_proc_lock_waiter_t *wtr)
 	}
     }
 
-    wtr->wait_locks &= ~got_locks;
+    wtr->uflgs &= ~got_locks;
 }
 
 /*
@@ -350,8 +286,8 @@ transfer_locks(Process *p,
 	       int unlock)
 {
     int transferred = 0;
-    erts_proc_lock_waiter_t *wake = NULL;
-    erts_proc_lock_waiter_t *wtr;
+    erts_tse_t *wake = NULL;
+    erts_tse_t *wtr;
     ErtsProcLocks unset_waiter = 0;
     ErtsProcLocks tlocks = trnsfr_lcks;
     int lock_no;
@@ -365,7 +301,6 @@ transfer_locks(Process *p,
     for (lock_no = 0; tlocks && lock_no <= ERTS_PROC_LOCK_MAX_BIT; lock_no++) {
 	ErtsProcLocks lock = ((ErtsProcLocks) 1) << lock_no;
 	if (tlocks & lock) {
-	    erts_proc_lock_queues_t *qs = p->lock.queues;
 	    /* Transfer lock */
 #ifdef ERTS_ENABLE_LOCK_CHECK
 	    tlocks &= ~lock;
@@ -373,15 +308,15 @@ transfer_locks(Process *p,
 	    ERTS_LC_ASSERT(ERTS_PROC_LOCK_FLGS_READ_(&p->lock)
 			   & (lock << ERTS_PROC_LOCK_WAITER_SHIFT));
 	    transferred++;
-	    wtr = dequeue_waiter(qs, lock_no);
+	    wtr = dequeue_waiter(&p->lock, lock_no);
 	    ERTS_LC_ASSERT(wtr);
-	    if (!qs->queue[lock_no])
+	    if (!p->lock.queue[lock_no])
 		unset_waiter |= lock;
-	    ERTS_LC_ASSERT(wtr->wait_locks & lock);
-	    wtr->wait_locks &= ~lock;
-	    if (wtr->wait_locks)
+	    ERTS_LC_ASSERT(wtr->uflgs & lock);
+	    wtr->uflgs &= ~lock;
+	    if (wtr->uflgs)
 		try_aquire(&p->lock, wtr);
-	    if (!wtr->wait_locks) {
+	    if (!wtr->uflgs) {
 		/*
 		 * The other thread got all locks it needs;
 		 * need to wake it up.
@@ -412,9 +347,10 @@ transfer_locks(Process *p,
 	erts_pix_unlock(pix_lock);
     
 	do {
-	    erts_proc_lock_waiter_t *tmp = wake;
+	    erts_tse_t *tmp = wake;
 	    wake = wake->next;
-	    erts_smp_gate_let_through(&tmp->gate, 1);
+	    erts_atomic32_set_nob(&tmp->uaflgs, 0);
+	    erts_tse_set(tmp);
 	} while (wake);
 
 	if (!unlock)
@@ -462,26 +398,15 @@ wait_for_locks(Process *p,
                ErtsProcLocks olflgs)
 {
     erts_pix_lock_t *pix_lock = pixlck ? pixlck : ERTS_PID2PIXLOCK(p->id);
-    int tsd;
-    erts_proc_lock_waiter_t *wtr;
+    erts_tse_t *wtr;
 
     /* Acquire a waiter object on which this thread can wait. */
-    wtr = erts_tsd_get(waiter_key);
-    if (wtr)
-	tsd = 1;
-    else {
-#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL
-	erts_pix_unlock(pix_lock);
-#endif
-	wtr = alloc_wtr();
-	tsd = 0;
-#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL
-	erts_pix_lock(pix_lock);
-#endif
-    }
+    wtr = tse_fetch(pix_lock);
     
     /* Record which locks this waiter needs. */
-    wtr->wait_locks = need_locks;
+    wtr->uflgs = need_locks;
+
+    ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
 
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
     erts_pix_lock(pix_lock);
@@ -489,16 +414,6 @@ wait_for_locks(Process *p,
 
     ERTS_LC_ASSERT(erts_lc_pix_lock_is_locked(pix_lock));
 
-    /* Provide the process with waiter queues, if it doesn't have one. */
-    if (!p->lock.queues) {
-	wtr->queues->next = NULL;
-	p->lock.queues = wtr->queues;
-    }
-    else {
-	wtr->queues->next = p->lock.queues->next;
-	p->lock.queues->next = wtr->queues;
-    }
-
 #ifdef ERTS_PROC_LOCK_HARD_DEBUG
     check_queue(&p->lock);
 #endif
@@ -506,46 +421,45 @@ wait_for_locks(Process *p,
     /* Try to aquire locks one at a time in lock order and set wait flag */
     try_aquire(&p->lock, wtr);
 
+    ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
+
 #ifdef ERTS_PROC_LOCK_HARD_DEBUG
     check_queue(&p->lock);
 #endif
 
-    if (wtr->wait_locks) { /* We didn't get them all; need to wait... */
-	/* Got to wait for locks... */
+    if (wtr->uflgs == 0)
 	erts_pix_unlock(pix_lock);
+    else {
+	/* We didn't get them all; need to wait... */
 
-	/*
-	 * Wait for needed locks. When we return all needed locks have
-	 * have been acquired by other threads and transfered to us.
-	 */
-#ifdef ERTS_PROC_LOCK_SPIN_ON_GATE
-	erts_smp_gate_swait(&wtr->gate, proc_lock_spin_count);
-#else
-	erts_smp_gate_wait(&wtr->gate);
-#endif
+	ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
 
-	erts_pix_lock(pix_lock);
-    }
+	erts_atomic32_set_nob(&wtr->uaflgs, 1);
+	erts_pix_unlock(pix_lock);
 
-    /* Recover some queues to store in the waiter. */
-    ERTS_LC_ASSERT(p->lock.queues);
-    if (p->lock.queues->next) {
-	wtr->queues = p->lock.queues->next;
-	p->lock.queues->next = wtr->queues->next;
-    }
-    else {
-	wtr->queues = p->lock.queues;
-	p->lock.queues = NULL;
-    }
+	while (1) {
+	    int res;
+	    erts_tse_reset(wtr);
 
-    erts_pix_unlock(pix_lock);
+	    if (erts_atomic32_read_nob(&wtr->uaflgs) == 0)
+		break;
+
+	    /*
+	     * Wait for needed locks. When we are woken all needed locks have
+	     * have been acquired by other threads and transfered to us.
+	     * However, we need to be prepared for spurious wakeups.
+	     */
+	    do {
+		res = erts_tse_wait(wtr); /* might return EINTR */
+	    } while (res != 0);
+	}
+
+	ASSERT(wtr->uflgs == 0);
+    }
 
     ERTS_LC_ASSERT(locks == (ERTS_PROC_LOCK_FLGS_READ_(&p->lock) & locks));
 
-    if (tsd)
-	CHECK_UNUSED_WAITER(wtr);
-    else
-	free_wtr(wtr);
+    tse_return(wtr);
 }
 
 /*
@@ -563,52 +477,57 @@ erts_proc_lock_failed(Process *p,
 		      ErtsProcLocks locks,
 		      ErtsProcLocks old_lflgs)
 {
-#ifdef ERTS_PROC_LOCK_SPIN_ON_GATE
-    int spin_count = 0;
-#else
-    int spin_count = proc_lock_spin_count;
-#endif
-
+    int until_yield = ERTS_PROC_LOCK_SPIN_UNTIL_YIELD;
+    int thr_spin_count;
+    int spin_count;
     ErtsProcLocks need_locks = locks;
     ErtsProcLocks olflgs = old_lflgs;
 
-    while (need_locks != 0)
-    {
-        ErtsProcLocks can_grab = in_order_locks(olflgs, need_locks);
+    if (erts_thr_get_main_status())
+	thr_spin_count = proc_lock_spin_count;
+    else
+	thr_spin_count = aux_thr_proc_lock_spin_count;
 
-        if (can_grab == 0)
-        {
+    spin_count = thr_spin_count;
+
+    while (need_locks != 0) {
+        ErtsProcLocks can_grab;
+
+	can_grab = in_order_locks(olflgs, need_locks);
+
+        if (can_grab == 0) {
             /* Someone already has the lowest-numbered lock we want. */
 
-            if (spin_count-- <= 0)
-            {
+            if (spin_count-- <= 0) {
                 /* Too many retries, give up and sleep for the lock. */
                 wait_for_locks(p, pixlck, locks, need_locks, olflgs);
                 return;
             }
 
+	    ERTS_SPIN_BODY;
+
+	    if (--until_yield == 0) {
+		until_yield = ERTS_PROC_LOCK_SPIN_UNTIL_YIELD;
+		erts_thr_yield();
+	    }
+
             olflgs = ERTS_PROC_LOCK_FLGS_READ_(&p->lock);
         }
-        else
-        {
+        else {
             /* Try to grab all of the grabbable locks at once with cmpxchg. */
             ErtsProcLocks grabbed = olflgs | can_grab;
             ErtsProcLocks nflgs =
-                ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock, grabbed, olflgs);
+                ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(&p->lock, grabbed, olflgs);
 
-            if (nflgs == olflgs)
-            {
+            if (nflgs == olflgs) {
                 /* Success! We grabbed the 'can_grab' locks. */
                 olflgs = grabbed;
                 need_locks &= ~can_grab;
 
-#ifndef ERTS_PROC_LOCK_SPIN_ON_GATE
                 /* Since we made progress, reset the spin count. */
-                spin_count = proc_lock_spin_count;
-#endif
+                spin_count = thr_spin_count;
             }
-            else
-            {
+            else {
                 /* Compare-and-exchange failed, try again. */
                 olflgs = nflgs;
             }
@@ -659,7 +578,9 @@ proc_safelock(Process *a_proc,
 	      ErtsProcLocks b_need_locks)
 {
     Process *p1, *p2;
+#ifdef ERTS_ENABLE_LOCK_CHECK
     Eterm pid1, pid2;
+#endif
     erts_pix_lock_t *pix_lck1, *pix_lck2;
     ErtsProcLocks need_locks1, have_locks1, need_locks2, have_locks2;
     ErtsProcLocks unlock_mask;
@@ -674,24 +595,32 @@ proc_safelock(Process *a_proc,
     if (a_proc) {
 	if (a_proc->id < b_proc->id) {
 	    p1 = a_proc;
+#ifdef ERTS_ENABLE_LOCK_CHECK
 	    pid1 = a_proc->id;
+#endif
 	    pix_lck1 = a_pix_lck;
 	    need_locks1 = a_need_locks;
 	    have_locks1 = a_have_locks;
 	    p2 = b_proc;
+#ifdef ERTS_ENABLE_LOCK_CHECK
 	    pid2 = b_proc->id;
+#endif
 	    pix_lck2 = b_pix_lck;
 	    need_locks2 = b_need_locks;
 	    have_locks2 = b_have_locks;
 	}
 	else if (a_proc->id > b_proc->id) {
 	    p1 = b_proc;
+#ifdef ERTS_ENABLE_LOCK_CHECK
 	    pid1 = b_proc->id;
+#endif
 	    pix_lck1 = b_pix_lck;
 	    need_locks1 = b_need_locks;
 	    have_locks1 = b_have_locks;
 	    p2 = a_proc;
+#ifdef ERTS_ENABLE_LOCK_CHECK
 	    pid2 = a_proc->id;
+#endif
 	    pix_lck2 = a_pix_lck;
 	    need_locks2 = a_need_locks;
 	    have_locks2 = a_have_locks;
@@ -700,12 +629,16 @@ proc_safelock(Process *a_proc,
 	    ERTS_LC_ASSERT(a_proc == b_proc);
 	    ERTS_LC_ASSERT(a_proc->id == b_proc->id);
 	    p1 = a_proc;
+#ifdef ERTS_ENABLE_LOCK_CHECK
 	    pid1 = a_proc->id;
+#endif
 	    pix_lck1 = a_pix_lck;
 	    need_locks1 = a_need_locks | b_need_locks;
 	    have_locks1 = a_have_locks | b_have_locks;
 	    p2 = NULL;
+#ifdef ERTS_ENABLE_LOCK_CHECK
 	    pid2 = 0;
+#endif
 	    pix_lck2 = NULL;
 	    need_locks2 = 0;
 	    have_locks2 = 0;
@@ -713,12 +646,16 @@ proc_safelock(Process *a_proc,
     }
     else {
 	p1 = b_proc;
+#ifdef ERTS_ENABLE_LOCK_CHECK
 	pid1 = b_proc->id;
+#endif
 	pix_lck1 = b_pix_lck;
 	need_locks1 = b_need_locks;
 	have_locks1 = b_have_locks;
 	p2 = NULL;
+#ifdef ERTS_ENABLE_LOCK_CHECK
 	pid2 = 0;
+#endif
 	pix_lck2 = NULL;
 	need_locks2 = 0;
 	have_locks2 = 0;
@@ -943,13 +880,16 @@ erts_pid2proc_safelock(Process *c_p,
 void
 erts_proc_lock_init(Process *p)
 {
+    int i;
     /* We always start with all locks locked */
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
-    erts_smp_atomic_init(&p->lock.flags, (long) ERTS_PROC_LOCKS_ALL);
+    erts_smp_atomic32_init_nob(&p->lock.flags,
+			       (erts_aint32_t) ERTS_PROC_LOCKS_ALL);
 #else
     p->lock.flags = ERTS_PROC_LOCKS_ALL;
 #endif
-    p->lock.queues = NULL;
+    for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++)
+	p->lock.queue[i] = NULL;
     p->lock.refc = 1;
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_proc_lock_init(p);
@@ -961,11 +901,8 @@ erts_proc_lock_init(Process *p)
     erts_proc_lc_trylock(p, ERTS_PROC_LOCKS_ALL, 1);
 #endif
 #ifdef ERTS_PROC_LOCK_DEBUG
-    {
-	int i;
-	for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++)
-	    erts_smp_atomic_init(&p->lock.locked[i], (long) 1);
-    }
+    for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++)
+	erts_smp_atomic32_init_nob(&p->lock.locked[i], (erts_aint32_t) 1);
 #endif
 }
 
@@ -973,7 +910,7 @@ erts_proc_lock_init(Process *p)
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
 void erts_lcnt_proc_lock_init(Process *p) {
-    
+    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) {
     if (p->id != ERTS_INVALID_PID) {
 	erts_lcnt_init_lock_x(&(p->lock.lcnt_main),   "proc_main",   ERTS_LCNT_LT_PROCLOCK, p->id);
 	erts_lcnt_init_lock_x(&(p->lock.lcnt_msgq),   "proc_msgq",   ERTS_LCNT_LT_PROCLOCK, p->id);
@@ -985,6 +922,12 @@ void erts_lcnt_proc_lock_init(Process *p) {
 	erts_lcnt_init_lock(&(p->lock.lcnt_link),   "proc_link",   ERTS_LCNT_LT_PROCLOCK);
 	erts_lcnt_init_lock(&(p->lock.lcnt_status), "proc_status", ERTS_LCNT_LT_PROCLOCK);
     }
+    } else {
+	sys_memzero(&(p->lock.lcnt_main), sizeof(p->lock.lcnt_main));
+	sys_memzero(&(p->lock.lcnt_msgq), sizeof(p->lock.lcnt_msgq));
+	sys_memzero(&(p->lock.lcnt_link), sizeof(p->lock.lcnt_link));
+	sys_memzero(&(p->lock.lcnt_status), sizeof(p->lock.lcnt_status));
+    }
 }
 	
 
@@ -1079,6 +1022,26 @@ void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res
     }
 }
 
+
+void erts_lcnt_enable_proc_lock_count(int enable) {
+    int i;
+
+    for (i = 0; i < erts_max_processes; ++i) {
+	Process* p = process_tab[i];
+	if (p) {
+	    if (enable) {
+		if (!ERTS_LCNT_LOCK_TYPE(&(p->lock.lcnt_main))) {
+		    erts_lcnt_proc_lock_init(p);
+		}
+	    } else {
+		if (ERTS_LCNT_LOCK_TYPE(&(p->lock.lcnt_main))) {
+		    erts_lcnt_proc_lock_destroy(p);
+		}
+	    }
+	}
+    }
+}
+
 #endif /* ifdef ERTS_ENABLE_LOCK_COUNT */
 
 
@@ -1407,22 +1370,22 @@ check_queue(erts_proc_lock_t *lck)
 	wtr = (((ErtsProcLocks) 1) << lock_no) << ERTS_PROC_LOCK_WAITER_SHIFT;
 	if (lflgs & wtr) {
 	    int n;
-	    erts_proc_lock_waiter_t *wtr;
-	    ERTS_LC_ASSERT(lck->queues && lck->queues->queue[lock_no]);
-	    wtr = lck->queues->queue[lock_no];
+	    erts_tse_t *wtr;
+	    ERTS_LC_ASSERT(lck->queue[lock_no]);
+	    wtr = lck->queue[lock_no];
 	    n = 0;
 	    do {
 		wtr = wtr->next;
 		n++;
-	    } while (wtr != lck->queues->queue[lock_no]);
+	    } while (wtr != lck->queue[lock_no]);
 	    do {
 		wtr = wtr->prev;
 		n--;
-	    } while (wtr != lck->queues->queue[lock_no]);
+	    } while (wtr != lck->queue[lock_no]);
 	    ERTS_LC_ASSERT(n == 0);
 	}
 	else {
-	    ERTS_LC_ASSERT(!lck->queues || !lck->queues->queue[lock_no]);
+	    ERTS_LC_ASSERT(!lck->queue[lock_no]);
 	}
     }
 }
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index d71e5a0a6e..290084d8ca 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2007-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -41,10 +41,10 @@
 #define ERTS_PROC_LOCK_SPINLOCK_IMPL 0
 #define ERTS_PROC_LOCK_MUTEX_IMPL 0
 
-#if defined(ETHR_HAVE_OPTIMIZED_ATOMIC_OPS)
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
 #  undef ERTS_PROC_LOCK_ATOMIC_IMPL
 #  define ERTS_PROC_LOCK_ATOMIC_IMPL 1
-#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCK)
+#elif defined(ETHR_HAVE_NATIVE_SPINLOCKS)
 #  undef ERTS_PROC_LOCK_SPINLOCK_IMPL
 #  define ERTS_PROC_LOCK_SPINLOCK_IMPL 1
 #else
@@ -54,20 +54,18 @@
 
 #define ERTS_PROC_LOCK_MAX_BIT 3
 
-typedef Uint32 ErtsProcLocks;
-
-typedef struct erts_proc_lock_queues_t_ erts_proc_lock_queues_t;
+typedef erts_aint32_t ErtsProcLocks;
 
 typedef struct erts_proc_lock_t_ {
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
-    erts_smp_atomic_t flags;
+    erts_smp_atomic32_t flags;
 #else
     ErtsProcLocks flags;
 #endif
-    erts_proc_lock_queues_t *queues;
-    long refc;
+    erts_tse_t *queue[ERTS_PROC_LOCK_MAX_BIT+1];
+    Sint32 refc;
 #ifdef ERTS_PROC_LOCK_DEBUG
-    erts_smp_atomic_t locked[ERTS_PROC_LOCK_MAX_BIT+1];
+    erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1];
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_t lcnt_main;
@@ -215,6 +213,8 @@ void erts_lcnt_proc_lock_unaquire(erts_proc_lock_t *lock, ErtsProcLocks locks);
 void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks);
 void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res);
 
+void erts_lcnt_enable_proc_lock_count(int enable);
+
 #endif /* ERTS_ENABLE_LOCK_COUNT*/
 
 
@@ -255,12 +255,8 @@ void erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks);
 
 typedef struct {
     union {
-#if ERTS_PROC_LOCK_MUTEX_IMPL
-	erts_smp_mtx_t mtx;
-#else
-	erts_smp_spinlock_t spnlck;
-#endif
-	char buf[64]; /* Try to get locks in different cache lines */
+	erts_mtx_t mtx;
+	char buf[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_mtx_t))];
     } u;
 } erts_pix_lock_t;
 
@@ -274,14 +270,21 @@ typedef struct {
 #if ERTS_PROC_LOCK_ATOMIC_IMPL
 
 #define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) \
-  ((ErtsProcLocks) erts_smp_atomic_band(&(L)->flags, (long) (MSK)))
-#define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) \
-  ((ErtsProcLocks) erts_smp_atomic_bor(&(L)->flags, (long) (MSK)))
-#define ERTS_PROC_LOCK_FLGS_CMPXCHG_(L, NEW, EXPECTED) \
-  ((ErtsProcLocks) erts_smp_atomic_cmpxchg(&(L)->flags, \
-                                           (long) (NEW), (long) (EXPECTED)))
+  ((ErtsProcLocks) erts_smp_atomic32_read_band_nob(&(L)->flags, \
+						   (erts_aint32_t) (MSK)))
+#define ERTS_PROC_LOCK_FLGS_BOR_ACQB_(L, MSK) \
+  ((ErtsProcLocks) erts_smp_atomic32_read_bor_acqb(&(L)->flags, \
+						   (erts_aint32_t) (MSK)))
+#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \
+  ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_acqb(&(L)->flags, \
+						  (erts_aint32_t) (NEW), \
+						  (erts_aint32_t) (EXPECTED)))
+#define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \
+  ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_relb(&(L)->flags, \
+						  (erts_aint32_t) (NEW), \
+						  (erts_aint32_t) (EXPECTED)))
 #define ERTS_PROC_LOCK_FLGS_READ_(L) \
-  ((ErtsProcLocks) erts_smp_atomic_read(&(L)->flags))
+  ((ErtsProcLocks) erts_smp_atomic32_read_nob(&(L)->flags))
 
 #else /* no opt atomic ops */
 
@@ -289,6 +292,9 @@ ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_band(erts_proc_lock_t *,
 							ErtsProcLocks);
 ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_bor(erts_proc_lock_t *,
 						       ErtsProcLocks);
+ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *,
+							   ErtsProcLocks,
+							   ErtsProcLocks);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
@@ -321,8 +327,10 @@ erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new,
 #endif
 
 #define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) erts_proc_lock_flags_band((L), (MSK))
-#define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) erts_proc_lock_flags_bor((L), (MSK))
-#define ERTS_PROC_LOCK_FLGS_CMPXCHG_(L, NEW, EXPECTED) \
+#define ERTS_PROC_LOCK_FLGS_BOR_ACQB_(L, MSK) erts_proc_lock_flags_bor((L), (MSK))
+#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \
+  erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED))
+#define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \
   erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED))
 #define ERTS_PROC_LOCK_FLGS_READ_(L) ((L)->flags)
 
@@ -330,7 +338,7 @@ erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new,
 
 extern erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS];
 
-void erts_init_proc_lock(void);
+void erts_init_proc_lock(int cpus);
 void erts_proc_lock_prepare_proc_lock_waiter(void);
 void erts_proc_lock_failed(Process *,
 			   erts_pix_lock_t *,
@@ -348,9 +356,9 @@ ERTS_GLB_INLINE ErtsProcLocks erts_smp_proc_raw_trylock__(Process *p,
 							  ErtsProcLocks locks);
 #ifdef ERTS_ENABLE_LOCK_COUNT
 ERTS_GLB_INLINE void erts_smp_proc_lock_x__(Process *,
-					  erts_pix_lock_t *,
-					  ErtsProcLocks,
-					  char *file, unsigned int line);
+					    erts_pix_lock_t *,
+					    ErtsProcLocks,
+					    char *file, unsigned int line);
 #else
 ERTS_GLB_INLINE void erts_smp_proc_lock__(Process *,
 					  erts_pix_lock_t *,
@@ -372,30 +380,18 @@ ERTS_GLB_INLINE void erts_proc_lock_op_debug(Process *, ErtsProcLocks, int);
 ERTS_GLB_INLINE void erts_pix_lock(erts_pix_lock_t *pixlck)
 {
     ERTS_LC_ASSERT(pixlck);
-#if ERTS_PROC_LOCK_MUTEX_IMPL
-    erts_smp_mtx_lock(&pixlck->u.mtx);
-#else
-    erts_smp_spin_lock(&pixlck->u.spnlck);
-#endif
+    erts_mtx_lock(&pixlck->u.mtx);
 }
 
 ERTS_GLB_INLINE void erts_pix_unlock(erts_pix_lock_t *pixlck)
 {
     ERTS_LC_ASSERT(pixlck);
-#if ERTS_PROC_LOCK_MUTEX_IMPL
-    erts_smp_mtx_unlock(&pixlck->u.mtx);
-#else
-    erts_smp_spin_unlock(&pixlck->u.spnlck);
-#endif
+    erts_mtx_unlock(&pixlck->u.mtx);
 }
 
 ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *pixlck)
 {
-#if ERTS_PROC_LOCK_MUTEX_IMPL
-    return erts_smp_lc_mtx_is_locked(&pixlck->u.mtx);
-#else
-    return erts_smp_lc_spinlock_is_locked(&pixlck->u.spnlck);
-#endif
+    return erts_lc_mtx_is_locked(&pixlck->u.mtx);
 }
 
 /*
@@ -417,9 +413,9 @@ erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks)
     ErtsProcLocks expct_lflgs = 0;
 
     while (1) {
-        ErtsProcLocks lflgs = ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock,
-							   expct_lflgs | locks,
-							   expct_lflgs);
+        ErtsProcLocks lflgs = ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(&p->lock,
+								expct_lflgs | locks,
+								expct_lflgs);
         if (ERTS_LIKELY(lflgs == expct_lflgs)) {
             /* We successfully grabbed all locks. */
             return 0;
@@ -535,7 +531,7 @@ erts_smp_proc_unlock__(Process *p,
 
         if (want_lflgs != old_lflgs) {
             ErtsProcLocks new_lflgs =
-                ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock, want_lflgs, old_lflgs);
+                ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(&p->lock, want_lflgs, old_lflgs);
 
             if (new_lflgs != old_lflgs) {
                 /* cmpxchg failed, try again. */
@@ -627,13 +623,13 @@ erts_proc_lock_op_debug(Process *p, ErtsProcLocks locks, int locked)
     for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) {
 	ErtsProcLocks lock = ((ErtsProcLocks) 1) << i;
 	if (locks & lock) {
-	    long lock_count;
+	    erts_aint32_t lock_count;
 	    if (locked) {
-		lock_count = erts_smp_atomic_inctest(&p->lock.locked[i]);
+		lock_count = erts_smp_atomic32_inc_read_nob(&p->lock.locked[i]);
 		ERTS_LC_ASSERT(lock_count == 1);
 	    }
 	    else {
-		lock_count = erts_smp_atomic_dectest(&p->lock.locked[i]);
+		lock_count = erts_smp_atomic32_dec_read_nob(&p->lock.locked[i]);
 		ERTS_LC_ASSERT(lock_count == 0);
 	    }
 	}
@@ -655,7 +651,7 @@ ERTS_GLB_INLINE int erts_smp_proc_trylock(Process *, ErtsProcLocks);
 
 ERTS_GLB_INLINE void erts_smp_proc_inc_refc(Process *);
 ERTS_GLB_INLINE void erts_smp_proc_dec_refc(Process *);
-
+ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *, Sint32);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
@@ -741,6 +737,21 @@ ERTS_GLB_INLINE void erts_smp_proc_dec_refc(Process *p)
 #endif
 }
 
+ERTS_GLB_INLINE void erts_smp_proc_add_refc(Process *p, Sint32 refc)
+{
+#ifdef ERTS_SMP
+    Process *fp;
+    erts_pix_lock_t *pixlck = ERTS_PID2PIXLOCK(p->id);
+    erts_pix_lock(pixlck);
+    ERTS_LC_ASSERT(p->lock.refc > 0);
+    p->lock.refc += refc;
+    fp = p->lock.refc == 0 ? p : NULL; 
+    erts_pix_unlock(pixlck);
+    if (fp)
+	erts_free_proc(fp);
+#endif
+}
+
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 #ifdef ERTS_SMP
@@ -947,8 +958,6 @@ erts_pid2proc_opt(Process *c_p,
 		if (flags & ERTS_P2P_FLG_TRY_LOCK)
 		    proc = ERTS_PROC_LOCK_BUSY;
 		else {
-		    if (flags & ERTS_P2P_FLG_SMP_INC_REFC)
-			proc->lock.refc++;
 #ifdef ERTS_ENABLE_LOCK_COUNT
 		    erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks);
 #endif
@@ -958,6 +967,8 @@ erts_pid2proc_opt(Process *c_p,
 					   pid_need_locks,
 					   pix_lock,
 					   flags);
+		    if (proc && (flags & ERTS_P2P_FLG_SMP_INC_REFC))
+			proc->lock.refc++;
 		}
 	    }
         }
diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
new file mode 100644
index 0000000000..a490aec734
--- /dev/null
+++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
@@ -0,0 +1,327 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Scheduler specific pre-allocators. Each scheduler
+ *              thread allocates memory in its own private chunk of
+ *              memory. Memory blocks deallocated by remote
+ *              schedulers (or other threads) are passed back to
+ *              the chunk owner via a lock-free data structure.
+ *
+ * Author: 	Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#ifdef ERTS_SMP
+
+#include "erl_process.h"
+#include "erl_thr_progress.h"
+
+erts_sspa_data_t *
+erts_sspa_create(size_t blk_sz, int pa_size)
+{
+    erts_sspa_data_t *data;
+    size_t tot_size;
+    size_t chunk_mem_size;
+    char *p;
+    char *chunk_start;
+    int cix;
+    int no_blocks = pa_size;
+    int no_blocks_per_chunk;
+
+    if (erts_no_schedulers == 1)
+	no_blocks_per_chunk = no_blocks;
+    else {
+	int extra = (no_blocks - 1)/4 + 1;
+	if (extra == 0)
+	    extra = 1;
+	no_blocks_per_chunk = no_blocks;
+	no_blocks_per_chunk += extra*erts_no_schedulers;
+	no_blocks_per_chunk /= erts_no_schedulers;
+    }
+    no_blocks = no_blocks_per_chunk * erts_no_schedulers;
+    chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_chunk_header_t));
+    chunk_mem_size += blk_sz * no_blocks_per_chunk;
+    chunk_mem_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(chunk_mem_size);
+    tot_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t));
+    tot_size += chunk_mem_size*erts_no_schedulers;
+
+    p = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_PRE_ALLOC_DATA, tot_size);
+    data = (erts_sspa_data_t *) p;
+    p += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_data_t));
+    chunk_start = p;
+
+    data->chunks_mem_size = chunk_mem_size;
+    data->start = chunk_start;
+    data->end = chunk_start + chunk_mem_size*erts_no_schedulers;
+
+    /* Initialize all chunks */
+    for (cix = 0; cix < erts_no_schedulers; cix++) {
+	erts_sspa_chunk_t *chnk = erts_sspa_cix2chunk(data, cix);
+	erts_sspa_chunk_header_t *chdr = &chnk->aligned.header;
+	erts_sspa_blk_t *blk;
+	int i;
+
+	erts_atomic_init_nob(&chdr->tail.data.last, (erts_aint_t) &chdr->tail.data.marker);
+	erts_atomic_init_nob(&chdr->tail.data.marker.next_atmc, ERTS_AINT_NULL);
+	erts_atomic_init_nob(&chdr->tail.data.um_refc[0], 0);
+	erts_atomic_init_nob(&chdr->tail.data.um_refc[1], 0);
+	erts_atomic32_init_nob(&chdr->tail.data.um_refc_ix, 0);
+
+	chdr->head.no_thr_progress_check = 0;
+	chdr->head.used_marker = 1;
+	chdr->head.first = &chdr->tail.data.marker;
+	chdr->head.unref_end = &chdr->tail.data.marker;
+	chdr->head.next.thr_progress = erts_thr_progress_current();
+	chdr->head.next.thr_progress_reached = 1;
+	chdr->head.next.um_refc_ix = 1;
+	chdr->head.next.unref_end = &chdr->tail.data.marker;
+
+	p = &chnk->data[0];
+	chdr->local.first = (erts_sspa_blk_t *) p;
+	blk = (erts_sspa_blk_t *) p;
+	for (i = 0; i < no_blocks_per_chunk; i++) {
+	    blk = (erts_sspa_blk_t *) p;
+	    p += blk_sz;
+	    blk->next_ptr = (erts_sspa_blk_t *) p;
+	}
+
+	blk->next_ptr = NULL;
+	chdr->local.last = blk;
+	chdr->local.cnt = no_blocks_per_chunk;
+	chdr->local.lim = no_blocks_per_chunk / 3;
+
+	ERTS_SSPA_DBG_CHK_LCL(chdr);
+    }
+
+    return data;
+}
+
+static ERTS_INLINE void
+enqueue_remote_managed_thread(erts_sspa_chunk_header_t *chdr,
+			      erts_sspa_blk_t *this,
+			      int cinit)
+{
+    erts_aint_t itmp;
+    erts_sspa_blk_t *enq;
+
+    erts_atomic_init_nob(&this->next_atmc, ERTS_AINT_NULL);
+    /* Enqueue at end of list... */
+
+    enq = (erts_sspa_blk_t *) erts_atomic_read_nob(&chdr->tail.data.last);
+    itmp = erts_atomic_cmpxchg_relb(&enq->next_atmc,
+				    (erts_aint_t) this,
+				    ERTS_AINT_NULL);
+    if (itmp == ERTS_AINT_NULL) {
+	/* We are required to move last pointer */
+#ifdef DEBUG
+	ASSERT(ERTS_AINT_NULL == erts_atomic_read_nob(&this->next_atmc));
+	ASSERT(((erts_aint_t) enq)
+	       == erts_atomic_xchg_relb(&chdr->tail.data.last,
+				      (erts_aint_t) this));
+#else
+	erts_atomic_set_relb(&chdr->tail.data.last, (erts_aint_t) this);
+#endif
+    }
+    else {
+	/*
+	 * We *need* to insert element somewhere in between the
+	 * last element we read earlier and the actual last element.
+	 */
+	int i = cinit;
+
+	while (1) {
+	    erts_aint_t itmp2;
+	    erts_atomic_set_nob(&this->next_atmc, itmp);
+	    itmp2 = erts_atomic_cmpxchg_relb(&enq->next_atmc,
+					     (erts_aint_t) this,
+					     itmp);
+	    if (itmp == itmp2)
+		break; /* inserted this */
+	    if ((i & 1) == 0)
+		itmp = itmp2;
+	    else {
+		enq = (erts_sspa_blk_t *) itmp;
+		itmp = erts_atomic_read_acqb(&enq->next_atmc);
+		ASSERT(itmp != ERTS_AINT_NULL);
+	    }
+	    i++;
+	}
+    }
+}
+
+static ERTS_INLINE erts_aint_t
+check_insert_marker(erts_sspa_chunk_header_t *chdr, erts_aint_t ilast)
+{
+    if (!chdr->head.used_marker
+	&& chdr->head.unref_end == (erts_sspa_blk_t *) ilast) {
+	erts_aint_t itmp;
+	erts_sspa_blk_t *last = (erts_sspa_blk_t *) ilast;
+
+	erts_atomic_init_nob(&chdr->tail.data.marker.next_atmc, ERTS_AINT_NULL);
+	itmp = erts_atomic_cmpxchg_relb(&last->next_atmc,
+					(erts_aint_t) &chdr->tail.data.marker,
+					ERTS_AINT_NULL);
+	if (itmp == ERTS_AINT_NULL) {
+	    ilast = (erts_aint_t) &chdr->tail.data.marker;
+	    chdr->head.used_marker = !0;
+	    erts_atomic_set_relb(&chdr->tail.data.last, ilast);
+	}
+    }
+    return ilast;
+}
+
+void
+erts_sspa_remote_free(erts_sspa_chunk_header_t *chdr,
+		      erts_sspa_blk_t *blk,
+		      int cinit)
+{
+    int um_refc_ix = 0;
+    int managed_thread = erts_thr_progress_is_managed_thread();
+    if (!managed_thread) {
+	um_refc_ix = erts_atomic32_read_acqb(&chdr->tail.data.um_refc_ix);
+	while (1) {
+	    int tmp_um_refc_ix;
+	    erts_atomic_inc_acqb(&chdr->tail.data.um_refc[um_refc_ix]);
+	    tmp_um_refc_ix = erts_atomic32_read_acqb(&chdr->tail.data.um_refc_ix);
+	    if (tmp_um_refc_ix == um_refc_ix)
+		break;
+	    erts_atomic_dec_relb(&chdr->tail.data.um_refc[um_refc_ix]);
+	    um_refc_ix = tmp_um_refc_ix;
+	}
+    }
+
+    enqueue_remote_managed_thread(chdr, blk, cinit);
+
+    if (!managed_thread)
+	erts_atomic_dec_relb(&chdr->tail.data.um_refc[um_refc_ix]);
+}
+
+static ERTS_INLINE void
+fetch_remote(erts_sspa_chunk_header_t *chdr, int max)
+{
+    int new_local = 0;
+
+    if (chdr->head.no_thr_progress_check < ERTS_SSPA_FORCE_THR_CHECK_PROGRESS)
+	chdr->head.no_thr_progress_check++;
+    else {
+	erts_aint_t ilast;
+
+	chdr->head.no_thr_progress_check = 0;
+
+	ilast = erts_atomic_read_nob(&chdr->tail.data.last);
+	if (((erts_sspa_blk_t *) ilast) == &chdr->tail.data.marker
+	    && chdr->head.first == &chdr->tail.data.marker)
+	    return;
+
+	if (chdr->head.next.thr_progress_reached
+	    || erts_thr_progress_has_reached(chdr->head.next.thr_progress)) {
+	    int um_refc_ix;
+	    chdr->head.next.thr_progress_reached = 1;
+	    um_refc_ix = chdr->head.next.um_refc_ix;
+	    if (erts_atomic_read_nob(&chdr->tail.data.um_refc[um_refc_ix]) == 0) {
+
+		ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+
+		/* Move unreferenced end pointer forward... */
+
+		chdr->head.unref_end = chdr->head.next.unref_end;
+
+		ilast = check_insert_marker(chdr, ilast);
+
+		if (chdr->head.unref_end != (erts_sspa_blk_t *) ilast) {
+		    chdr->head.next.unref_end = (erts_sspa_blk_t *) ilast;
+		    chdr->head.next.thr_progress = erts_thr_progress_later(NULL);
+		    erts_atomic32_set_relb(&chdr->tail.data.um_refc_ix,
+					   um_refc_ix);
+		    chdr->head.next.um_refc_ix = um_refc_ix == 0 ? 1 : 0;
+		    chdr->head.next.thr_progress_reached = 0;
+		}
+	    }
+	}
+    }
+
+    if (new_local < max && chdr->head.first != chdr->head.unref_end) {
+	erts_sspa_blk_t *first, *this, *next, *last;
+	first = chdr->head.first;
+	if (first == &chdr->tail.data.marker) {
+	    chdr->head.used_marker = 0;
+	    first = ((erts_sspa_blk_t *)
+		     erts_atomic_read_nob(&first->next_atmc));
+	    chdr->head.first = first;
+	}
+	if (first != chdr->head.unref_end) {
+
+	    ERTS_SSPA_DBG_CHK_LCL(chdr);
+
+	    this = last = first;
+	    do {
+		next = (erts_sspa_blk_t *) erts_atomic_read_nob(&this->next_atmc);
+		if (this == &chdr->tail.data.marker)
+		    chdr->head.used_marker = 0;
+		else {
+		    last->next_ptr = this;
+		    last = this;
+		    new_local++;
+		}
+		this = next;
+	    } while (new_local < max && this != chdr->head.unref_end);
+	    chdr->head.first = this;
+	    if (!chdr->local.last)
+		chdr->local.first = first;
+	    else
+		chdr->local.last->next_ptr = first;
+	    chdr->local.last = last;
+	    last->next_ptr = NULL;
+	    chdr->local.cnt += new_local;
+
+	    ERTS_SSPA_DBG_CHK_LCL(chdr);
+	}
+    }
+
+}
+
+erts_sspa_blk_t *
+erts_sspa_process_remote_frees(erts_sspa_chunk_header_t *chdr,
+			       erts_sspa_blk_t *old_res)
+{
+    erts_sspa_blk_t *res = old_res;
+
+    fetch_remote(chdr, ERTS_SSPA_MAX_GET_NEW_LOCAL);
+
+    if (!res && chdr->local.first) {
+
+	ERTS_SSPA_DBG_CHK_LCL(chdr);
+
+	res = chdr->local.first;
+	chdr->local.first = res->next_ptr;
+	chdr->local.cnt--;
+	if (!chdr->local.first)
+	    chdr->local.last = NULL;
+
+	ERTS_SSPA_DBG_CHK_LCL(chdr);
+    }
+
+    return res;
+}
+
+#endif /* ERTS_SMP */
diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.h b/erts/emulator/beam/erl_sched_spec_pre_alloc.h
new file mode 100644
index 0000000000..bccb1aba7a
--- /dev/null
+++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.h
@@ -0,0 +1,240 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Scheduler specific pre-allocators. Each scheduler
+ *              thread allocates memory in its own private chunk of
+ *              memory. Memory blocks deallocated by remote
+ *              schedulers (or other threads) are passed back to
+ *              the chunk owner via a lock-free data structure.
+ *
+ * Author: 	Rickard Green
+ */
+
+#ifndef ERTS_SCHED_SPEC_PRE_ALLOC_H__
+#define ERTS_SCHED_SPEC_PRE_ALLOC_H__
+
+#ifdef ERTS_SMP
+
+#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
+#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
+#include "erl_thr_progress.h"
+#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
+
+#ifdef DEBUG
+#define ERTS_SPPA_DBG_CHK_IN_CHNK(A, C, P)				\
+do {									\
+    ASSERT((void *) (C) < (void *) (P));				\
+    ASSERT((void *) (P)							\
+	   < (void *) (((char *) (C)) + (A)->chunks_mem_size));		\
+} while (0)
+#else
+#define ERTS_SPPA_DBG_CHK_IN_CHNK(A, C, P)
+#endif
+
+#ifdef DEBUG
+extern Uint erts_no_schedulers;
+#endif
+
+#define ERTS_SSPA_FORCE_THR_CHECK_PROGRESS 10
+#define ERTS_SSPA_MAX_GET_NEW_LOCAL 5
+
+typedef struct {
+    char *start;
+    char *end;
+    int chunks_mem_size;
+} erts_sspa_data_t;
+
+typedef union erts_sspa_blk_t_ erts_sspa_blk_t;
+union erts_sspa_blk_t_ {
+    erts_atomic_t next_atmc;
+    erts_sspa_blk_t *next_ptr;
+};
+
+typedef struct {
+    erts_sspa_blk_t *first;
+    erts_sspa_blk_t *last;
+    int cnt;
+    int lim;
+} erts_sspa_local_freelist_t;
+
+typedef struct {
+    erts_sspa_blk_t marker;
+    erts_atomic_t last;
+    erts_atomic_t um_refc[2];
+    erts_atomic32_t um_refc_ix;
+} erts_sspa_tail_t;
+
+typedef struct {
+    /*
+     * This structure needs to be cache line aligned for best
+     * performance.
+     */
+    union {
+	/* Modified by threads returning memory to this chunk */
+	erts_sspa_tail_t data;
+	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_tail_t))];
+    } tail;
+    /*
+     * Everything below this point is *only* accessed by the
+     * thread owning this chunk.
+     */
+    struct {
+	int no_thr_progress_check;
+	int used_marker;
+	erts_sspa_blk_t *first;
+	erts_sspa_blk_t *unref_end;
+	struct {
+	    ErtsThrPrgrVal thr_progress;
+	    int thr_progress_reached;
+	    int um_refc_ix;
+	    erts_sspa_blk_t *unref_end;
+	} next;
+    } head;
+    erts_sspa_local_freelist_t local;
+} erts_sspa_chunk_header_t;
+
+typedef struct {
+    union {
+	erts_sspa_chunk_header_t header;
+	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(
+		sizeof(erts_sspa_chunk_header_t))];
+    } aligned;
+    char data[1];
+} erts_sspa_chunk_t;
+
+#ifdef DEBUG
+ERTS_GLB_INLINE void
+check_local_list(erts_sspa_chunk_header_t *chdr);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE void
+check_local_list(erts_sspa_chunk_header_t *chdr)
+{
+    erts_sspa_blk_t *blk;
+    int n = 0;
+    for (blk = chdr->local.first; blk; blk = blk->next_ptr)
+	n++;
+    ASSERT(n == chdr->local.cnt);
+}
+#endif
+#define ERTS_SSPA_DBG_CHK_LCL(CHDR) check_local_list((CHDR))
+#else
+#define ERTS_SSPA_DBG_CHK_LCL(CHDR)
+#endif
+
+erts_sspa_data_t *erts_sspa_create(size_t blk_sz,
+				   int pa_size);
+void erts_sspa_remote_free(erts_sspa_chunk_header_t *chdr,
+			   erts_sspa_blk_t *blk,
+			   int cinit);
+erts_sspa_blk_t *erts_sspa_process_remote_frees(erts_sspa_chunk_header_t *chdr,
+						erts_sspa_blk_t *old_res);
+
+ERTS_GLB_INLINE erts_sspa_chunk_t *erts_sspa_cix2chunk(erts_sspa_data_t *data,
+						       int cix);
+ERTS_GLB_INLINE int erts_sspa_ptr2cix(erts_sspa_data_t *data, void *ptr);
+ERTS_GLB_INLINE char *erts_sspa_alloc(erts_sspa_data_t *data, int cix);
+ERTS_GLB_INLINE int erts_sspa_free(erts_sspa_data_t *data, int cix, char *blk);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE erts_sspa_chunk_t *
+erts_sspa_cix2chunk(erts_sspa_data_t *data, int cix)
+{
+    ASSERT(0 <= cix && cix < erts_no_schedulers);
+    return (erts_sspa_chunk_t *) (data->start + cix*data->chunks_mem_size);
+}
+
+ERTS_GLB_INLINE int
+erts_sspa_ptr2cix(erts_sspa_data_t *data, void *ptr)
+{
+    int cix;
+    size_t diff;
+    if ((char *) ptr < data->start || data->end <= (char *) ptr)
+	return -1;
+    diff = ((char *) ptr) - data->start;
+    cix = (int) diff / data->chunks_mem_size;
+    ASSERT(0 <= cix && cix < erts_no_schedulers);
+    return cix;
+}
+
+ERTS_GLB_INLINE char *
+erts_sspa_alloc(erts_sspa_data_t *data, int cix)
+{
+    erts_sspa_chunk_t *chnk;
+    erts_sspa_chunk_header_t *chdr;
+    erts_sspa_blk_t *res;
+
+    chnk = erts_sspa_cix2chunk(data, cix);
+    chdr = &chnk->aligned.header;
+    res = chdr->local.first;
+    ERTS_SSPA_DBG_CHK_LCL(chdr);
+    if (res) {
+	ERTS_SSPA_DBG_CHK_LCL(chdr);
+	chdr->local.first = res->next_ptr;
+	chdr->local.cnt--;
+	if (!chdr->local.first)
+	    chdr->local.last = NULL;
+	ERTS_SSPA_DBG_CHK_LCL(chdr);
+    }
+    if (chdr->local.cnt <= chdr->local.lim)
+	return (char *) erts_sspa_process_remote_frees(chdr, res);
+    else if (chdr->head.no_thr_progress_check < ERTS_SSPA_FORCE_THR_CHECK_PROGRESS)
+	chdr->head.no_thr_progress_check++;
+    ASSERT(res);
+    return (char *) res;
+}
+
+ERTS_GLB_INLINE int
+erts_sspa_free(erts_sspa_data_t *data, int cix, char *cblk)
+{
+    erts_sspa_chunk_t *chnk;
+    erts_sspa_chunk_header_t *chdr;
+    erts_sspa_blk_t *blk = (erts_sspa_blk_t *) cblk;
+    int chnk_cix = erts_sspa_ptr2cix(data, blk);
+
+    if (chnk_cix < 0)
+	return 0;
+
+    chnk = erts_sspa_cix2chunk(data, chnk_cix);
+    chdr = &chnk->aligned.header;
+    if (chnk_cix != cix) {
+	/* Remote chunk */
+	erts_sspa_remote_free(chdr, blk, chnk_cix - cix);
+    }
+    else {
+	/* Local chunk */
+	ERTS_SSPA_DBG_CHK_LCL(chdr);
+	blk->next_ptr = chdr->local.first;
+	chdr->local.first = blk;
+	if (!chdr->local.last)
+	    chdr->local.last = blk;
+	chdr->local.cnt++;
+	ERTS_SSPA_DBG_CHK_LCL(chdr);
+    }
+
+    return 1;
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+#endif /* ERTS_SMP */
+
+#endif /* ERTS_SCHED_SPEC_PRE_ALLOC_H__ */
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index 03d2a586e3..a32e9d9d7c 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /*
@@ -43,27 +43,55 @@ typedef erts_thr_init_data_t erts_smp_thr_init_data_t;
 typedef erts_tid_t erts_smp_tid_t;
 typedef erts_mtx_t erts_smp_mtx_t;
 typedef erts_cnd_t erts_smp_cnd_t;
+#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER ERTS_RWMTX_OPT_DEFAULT_INITER
+#define ERTS_SMP_RWMTX_TYPE_NORMAL ERTS_RWMTX_TYPE_NORMAL
+#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ ERTS_RWMTX_TYPE_FREQUENT_READ
+#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \
+  ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ
+#define ERTS_SMP_RWMTX_LONG_LIVED ERTS_RWMTX_LONG_LIVED
+#define ERTS_SMP_RWMTX_SHORT_LIVED ERTS_RWMTX_SHORT_LIVED
+#define ERTS_SMP_RWMTX_UNKNOWN_LIVED ERTS_RWMTX_UNKNOWN_LIVED
+typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t;
 typedef erts_rwmtx_t erts_smp_rwmtx_t;
 typedef erts_tsd_key_t erts_smp_tsd_key_t;
-typedef erts_gate_t erts_smp_gate_t;
-typedef ethr_atomic_t erts_smp_atomic_t;
+#define erts_smp_dw_atomic_t erts_dw_atomic_t
+#define erts_smp_atomic_t erts_atomic_t
+#define erts_smp_atomic32_t erts_atomic32_t
 typedef erts_spinlock_t erts_smp_spinlock_t;
 typedef erts_rwlock_t erts_smp_rwlock_t;
-typedef erts_thr_timeval_t erts_smp_thr_timeval_t;
 void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */
 
+#define ERTS_SMP_MEMORY_BARRIER ERTS_THR_MEMORY_BARRIER
+#define ERTS_SMP_WRITE_MEMORY_BARRIER ERTS_THR_WRITE_MEMORY_BARRIER
+#define ERTS_SMP_READ_MEMORY_BARRIER ERTS_THR_READ_MEMORY_BARRIER
+#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
+
 #else /* #ifdef ERTS_SMP */
 
-#define ERTS_SMP_THR_OPTS_DEFAULT_INITER 0
+#define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0}
 typedef int erts_smp_thr_opts_t;
 typedef int erts_smp_thr_init_data_t;
 typedef int erts_smp_tid_t;
 typedef int erts_smp_mtx_t;
 typedef int erts_smp_cnd_t;
+#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER {0}
+#define ERTS_SMP_RWMTX_TYPE_NORMAL 0
+#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ 0
+#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0
+#define ERTS_SMP_RWMTX_LONG_LIVED 0
+#define ERTS_SMP_RWMTX_SHORT_LIVED 0
+#define ERTS_SMP_RWMTX_UNKNOWN_LIVED 0
+typedef struct {
+    char type;
+    char lived;
+    int main_spincount;
+    int aux_spincount;
+} erts_smp_rwmtx_opt_t;
 typedef int erts_smp_rwmtx_t;
 typedef int erts_smp_tsd_key_t;
-typedef int erts_smp_gate_t;
-typedef long erts_smp_atomic_t;
+#define erts_smp_dw_atomic_t erts_no_dw_atomic_t
+#define erts_smp_atomic_t erts_no_atomic_t
+#define erts_smp_atomic32_t erts_no_atomic32_t
 #if __GNUC__ > 2
 typedef struct { } erts_smp_spinlock_t;
 typedef struct { } erts_smp_rwlock_t;
@@ -72,10 +100,10 @@ typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t;
 typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t;
 #endif
 
-typedef struct {
-    long tv_sec;
-    long tv_nsec;
-} erts_smp_thr_timeval_t;
+#define ERTS_SMP_MEMORY_BARRIER
+#define ERTS_SMP_WRITE_MEMORY_BARRIER
+#define ERTS_SMP_READ_MEMORY_BARRIER
+#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER
 
 #endif /* #ifdef ERTS_SMP */
 
@@ -103,8 +131,6 @@ ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx,
 ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name);
 ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name);
 ERTS_GLB_INLINE void erts_smp_mtx_destroy(erts_smp_mtx_t *mtx);
-ERTS_GLB_INLINE void erts_smp_mtx_set_forksafe(erts_smp_mtx_t *mtx);
-ERTS_GLB_INLINE void erts_smp_mtx_unset_forksafe(erts_smp_mtx_t *mtx);
 ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
 ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, int line);
@@ -119,9 +145,17 @@ ERTS_GLB_INLINE void erts_smp_cnd_wait(erts_smp_cnd_t *cnd,
 					   erts_smp_mtx_t *mtx);
 ERTS_GLB_INLINE void erts_smp_cnd_signal(erts_smp_cnd_t *cnd);
 ERTS_GLB_INLINE void erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd);
+ERTS_GLB_INLINE void erts_smp_rwmtx_set_reader_group(int no);
+ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
+					       erts_smp_rwmtx_opt_t *opt,
+					       char *name,
+					       Eterm extra);
 ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx,
 					   char *name,
 					   Eterm extra);
+ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
+					     erts_smp_rwmtx_opt_t *opt,
+					     char *name);
 ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx,
 					 char *name);
 ERTS_GLB_INLINE void erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx);
@@ -138,23 +172,6 @@ ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx);
 ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx);
-ERTS_GLB_INLINE void erts_smp_atomic_init(erts_smp_atomic_t *var, long i);
-ERTS_GLB_INLINE void erts_smp_atomic_set(erts_smp_atomic_t *var, long i);
-ERTS_GLB_INLINE long erts_smp_atomic_read(erts_smp_atomic_t *var);
-ERTS_GLB_INLINE long erts_smp_atomic_inctest(erts_smp_atomic_t *incp);
-ERTS_GLB_INLINE long erts_smp_atomic_dectest(erts_smp_atomic_t *decp);
-ERTS_GLB_INLINE void erts_smp_atomic_inc(erts_smp_atomic_t *incp);
-ERTS_GLB_INLINE void erts_smp_atomic_dec(erts_smp_atomic_t *decp);
-ERTS_GLB_INLINE long erts_smp_atomic_addtest(erts_smp_atomic_t *addp,
-					     long i);
-ERTS_GLB_INLINE void erts_smp_atomic_add(erts_smp_atomic_t *addp, long i);
-ERTS_GLB_INLINE long erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp,
-					  long new);
-ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp,
-					     long new,
-					     long expected);
-ERTS_GLB_INLINE long erts_smp_atomic_bor(erts_smp_atomic_t *var, long mask);
-ERTS_GLB_INLINE long erts_smp_atomic_band(erts_smp_atomic_t *var, long mask);
 ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock,
 					      char *name,
 					      Eterm extra);
@@ -185,17 +202,10 @@ ERTS_GLB_INLINE void erts_smp_write_lock(erts_smp_rwlock_t *lock);
 ERTS_GLB_INLINE void erts_smp_write_unlock(erts_smp_rwlock_t *lock);
 ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock);
 ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock);
-ERTS_GLB_INLINE void erts_smp_thr_time_now(erts_smp_thr_timeval_t *time);
 ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp);
 ERTS_GLB_INLINE void erts_smp_tsd_key_delete(erts_smp_tsd_key_t key);
 ERTS_GLB_INLINE void erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value);
 ERTS_GLB_INLINE void * erts_smp_tsd_get(erts_smp_tsd_key_t key);
-ERTS_GLB_INLINE void erts_smp_gate_init(erts_smp_gate_t *gp);
-ERTS_GLB_INLINE void erts_smp_gate_destroy(erts_smp_gate_t *gp);
-ERTS_GLB_INLINE void erts_smp_gate_close(erts_smp_gate_t *gp);
-ERTS_GLB_INLINE void erts_smp_gate_let_through(erts_smp_gate_t *gp, unsigned no);
-ERTS_GLB_INLINE void erts_smp_gate_wait(erts_smp_gate_t *gp);
-ERTS_GLB_INLINE void erts_smp_gate_swait(erts_smp_gate_t *gp, int spincount);
 
 #ifdef ERTS_THR_HAVE_SIG_FUNCS
 #define ERTS_SMP_THR_HAVE_SIG_FUNCS 1
@@ -205,6 +215,490 @@ ERTS_GLB_INLINE void erts_smp_thr_sigmask(int how,
 ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
 
+/*
+ * See "Documentation of atomics and memory barriers" at the top
+ * of erl_threads.h for info on atomics.
+ */
+
+#ifdef ERTS_SMP
+
+/* Double word size atomics */
+
+#define erts_smp_dw_atomic_init_nob erts_dw_atomic_init_nob
+#define erts_smp_dw_atomic_set_nob erts_dw_atomic_set_nob
+#define erts_smp_dw_atomic_read_nob erts_dw_atomic_read_nob
+#define erts_smp_dw_atomic_cmpxchg_nob erts_dw_atomic_cmpxchg_nob
+
+#define erts_smp_dw_atomic_init_mb erts_dw_atomic_init_mb
+#define erts_smp_dw_atomic_set_mb erts_dw_atomic_set_mb
+#define erts_smp_dw_atomic_read_mb erts_dw_atomic_read_mb
+#define erts_smp_dw_atomic_cmpxchg_mb erts_dw_atomic_cmpxchg_mb
+
+#define erts_smp_dw_atomic_init_acqb erts_dw_atomic_init_acqb
+#define erts_smp_dw_atomic_set_acqb erts_dw_atomic_set_acqb
+#define erts_smp_dw_atomic_read_acqb erts_dw_atomic_read_acqb
+#define erts_smp_dw_atomic_cmpxchg_acqb erts_dw_atomic_cmpxchg_acqb
+
+#define erts_smp_dw_atomic_init_relb erts_dw_atomic_init_relb
+#define erts_smp_dw_atomic_set_relb erts_dw_atomic_set_relb
+#define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb
+#define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb
+
+#define erts_smp_dw_atomic_init_ddrb erts_dw_atomic_init_ddrb
+#define erts_smp_dw_atomic_set_ddrb erts_dw_atomic_set_ddrb
+#define erts_smp_dw_atomic_read_ddrb erts_dw_atomic_read_ddrb
+#define erts_smp_dw_atomic_cmpxchg_ddrb erts_dw_atomic_cmpxchg_ddrb
+
+#define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb
+#define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb
+#define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb
+#define erts_smp_dw_atomic_cmpxchg_rb erts_dw_atomic_cmpxchg_rb
+
+#define erts_smp_dw_atomic_init_wb erts_dw_atomic_init_wb
+#define erts_smp_dw_atomic_set_wb erts_dw_atomic_set_wb
+#define erts_smp_dw_atomic_read_wb erts_dw_atomic_read_wb
+#define erts_smp_dw_atomic_cmpxchg_wb erts_dw_atomic_cmpxchg_wb
+
+/* Word size atomics */
+
+#define erts_smp_atomic_init_nob erts_atomic_init_nob
+#define erts_smp_atomic_set_nob erts_atomic_set_nob
+#define erts_smp_atomic_read_nob erts_atomic_read_nob
+#define erts_smp_atomic_inc_read_nob erts_atomic_inc_read_nob
+#define erts_smp_atomic_dec_read_nob erts_atomic_dec_read_nob
+#define erts_smp_atomic_inc_nob erts_atomic_inc_nob
+#define erts_smp_atomic_dec_nob erts_atomic_dec_nob
+#define erts_smp_atomic_add_read_nob erts_atomic_add_read_nob
+#define erts_smp_atomic_add_nob erts_atomic_add_nob
+#define erts_smp_atomic_read_bor_nob erts_atomic_read_bor_nob
+#define erts_smp_atomic_read_band_nob erts_atomic_read_band_nob
+#define erts_smp_atomic_xchg_nob erts_atomic_xchg_nob
+#define erts_smp_atomic_cmpxchg_nob erts_atomic_cmpxchg_nob
+
+#define erts_smp_atomic_init_mb erts_atomic_init_mb
+#define erts_smp_atomic_set_mb erts_atomic_set_mb
+#define erts_smp_atomic_read_mb erts_atomic_read_mb
+#define erts_smp_atomic_inc_read_mb erts_atomic_inc_read_mb
+#define erts_smp_atomic_dec_read_mb erts_atomic_dec_read_mb
+#define erts_smp_atomic_inc_mb erts_atomic_inc_mb
+#define erts_smp_atomic_dec_mb erts_atomic_dec_mb
+#define erts_smp_atomic_add_read_mb erts_atomic_add_read_mb
+#define erts_smp_atomic_add_mb erts_atomic_add_mb
+#define erts_smp_atomic_read_bor_mb erts_atomic_read_bor_mb
+#define erts_smp_atomic_read_band_mb erts_atomic_read_band_mb
+#define erts_smp_atomic_xchg_mb erts_atomic_xchg_mb
+#define erts_smp_atomic_cmpxchg_mb erts_atomic_cmpxchg_mb
+
+#define erts_smp_atomic_init_acqb erts_atomic_init_acqb
+#define erts_smp_atomic_set_acqb erts_atomic_set_acqb
+#define erts_smp_atomic_read_acqb erts_atomic_read_acqb
+#define erts_smp_atomic_inc_read_acqb erts_atomic_inc_read_acqb
+#define erts_smp_atomic_dec_read_acqb erts_atomic_dec_read_acqb
+#define erts_smp_atomic_inc_acqb erts_atomic_inc_acqb
+#define erts_smp_atomic_dec_acqb erts_atomic_dec_acqb
+#define erts_smp_atomic_add_read_acqb erts_atomic_add_read_acqb
+#define erts_smp_atomic_add_acqb erts_atomic_add_acqb
+#define erts_smp_atomic_read_bor_acqb erts_atomic_read_bor_acqb
+#define erts_smp_atomic_read_band_acqb erts_atomic_read_band_acqb
+#define erts_smp_atomic_xchg_acqb erts_atomic_xchg_acqb
+#define erts_smp_atomic_cmpxchg_acqb erts_atomic_cmpxchg_acqb
+
+#define erts_smp_atomic_init_relb erts_atomic_init_relb
+#define erts_smp_atomic_set_relb erts_atomic_set_relb
+#define erts_smp_atomic_read_relb erts_atomic_read_relb
+#define erts_smp_atomic_inc_read_relb erts_atomic_inc_read_relb
+#define erts_smp_atomic_dec_read_relb erts_atomic_dec_read_relb
+#define erts_smp_atomic_inc_relb erts_atomic_inc_relb
+#define erts_smp_atomic_dec_relb erts_atomic_dec_relb
+#define erts_smp_atomic_add_read_relb erts_atomic_add_read_relb
+#define erts_smp_atomic_add_relb erts_atomic_add_relb
+#define erts_smp_atomic_read_bor_relb erts_atomic_read_bor_relb
+#define erts_smp_atomic_read_band_relb erts_atomic_read_band_relb
+#define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb
+#define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb
+
+#define erts_smp_atomic_init_ddrb erts_atomic_init_ddrb
+#define erts_smp_atomic_set_ddrb erts_atomic_set_ddrb
+#define erts_smp_atomic_read_ddrb erts_atomic_read_ddrb
+#define erts_smp_atomic_inc_read_ddrb erts_atomic_inc_read_ddrb
+#define erts_smp_atomic_dec_read_ddrb erts_atomic_dec_read_ddrb
+#define erts_smp_atomic_inc_ddrb erts_atomic_inc_ddrb
+#define erts_smp_atomic_dec_ddrb erts_atomic_dec_ddrb
+#define erts_smp_atomic_add_read_ddrb erts_atomic_add_read_ddrb
+#define erts_smp_atomic_add_ddrb erts_atomic_add_ddrb
+#define erts_smp_atomic_read_bor_ddrb erts_atomic_read_bor_ddrb
+#define erts_smp_atomic_read_band_ddrb erts_atomic_read_band_ddrb
+#define erts_smp_atomic_xchg_ddrb erts_atomic_xchg_ddrb
+#define erts_smp_atomic_cmpxchg_ddrb erts_atomic_cmpxchg_ddrb
+
+#define erts_smp_atomic_init_rb erts_atomic_init_rb
+#define erts_smp_atomic_set_rb erts_atomic_set_rb
+#define erts_smp_atomic_read_rb erts_atomic_read_rb
+#define erts_smp_atomic_inc_read_rb erts_atomic_inc_read_rb
+#define erts_smp_atomic_dec_read_rb erts_atomic_dec_read_rb
+#define erts_smp_atomic_inc_rb erts_atomic_inc_rb
+#define erts_smp_atomic_dec_rb erts_atomic_dec_rb
+#define erts_smp_atomic_add_read_rb erts_atomic_add_read_rb
+#define erts_smp_atomic_add_rb erts_atomic_add_rb
+#define erts_smp_atomic_read_bor_rb erts_atomic_read_bor_rb
+#define erts_smp_atomic_read_band_rb erts_atomic_read_band_rb
+#define erts_smp_atomic_xchg_rb erts_atomic_xchg_rb
+#define erts_smp_atomic_cmpxchg_rb erts_atomic_cmpxchg_rb
+
+#define erts_smp_atomic_init_wb erts_atomic_init_wb
+#define erts_smp_atomic_set_wb erts_atomic_set_wb
+#define erts_smp_atomic_read_wb erts_atomic_read_wb
+#define erts_smp_atomic_inc_read_wb erts_atomic_inc_read_wb
+#define erts_smp_atomic_dec_read_wb erts_atomic_dec_read_wb
+#define erts_smp_atomic_inc_wb erts_atomic_inc_wb
+#define erts_smp_atomic_dec_wb erts_atomic_dec_wb
+#define erts_smp_atomic_add_read_wb erts_atomic_add_read_wb
+#define erts_smp_atomic_add_wb erts_atomic_add_wb
+#define erts_smp_atomic_read_bor_wb erts_atomic_read_bor_wb
+#define erts_smp_atomic_read_band_wb erts_atomic_read_band_wb
+#define erts_smp_atomic_xchg_wb erts_atomic_xchg_wb
+#define erts_smp_atomic_cmpxchg_wb erts_atomic_cmpxchg_wb
+
+/* 32-bit atomics */
+
+#define erts_smp_atomic32_init_nob erts_atomic32_init_nob
+#define erts_smp_atomic32_set_nob erts_atomic32_set_nob
+#define erts_smp_atomic32_read_nob erts_atomic32_read_nob
+#define erts_smp_atomic32_inc_read_nob erts_atomic32_inc_read_nob
+#define erts_smp_atomic32_dec_read_nob erts_atomic32_dec_read_nob
+#define erts_smp_atomic32_inc_nob erts_atomic32_inc_nob
+#define erts_smp_atomic32_dec_nob erts_atomic32_dec_nob
+#define erts_smp_atomic32_add_read_nob erts_atomic32_add_read_nob
+#define erts_smp_atomic32_add_nob erts_atomic32_add_nob
+#define erts_smp_atomic32_read_bor_nob erts_atomic32_read_bor_nob
+#define erts_smp_atomic32_read_band_nob erts_atomic32_read_band_nob
+#define erts_smp_atomic32_xchg_nob erts_atomic32_xchg_nob
+#define erts_smp_atomic32_cmpxchg_nob erts_atomic32_cmpxchg_nob
+
+#define erts_smp_atomic32_init_mb erts_atomic32_init_mb
+#define erts_smp_atomic32_set_mb erts_atomic32_set_mb
+#define erts_smp_atomic32_read_mb erts_atomic32_read_mb
+#define erts_smp_atomic32_inc_read_mb erts_atomic32_inc_read_mb
+#define erts_smp_atomic32_dec_read_mb erts_atomic32_dec_read_mb
+#define erts_smp_atomic32_inc_mb erts_atomic32_inc_mb
+#define erts_smp_atomic32_dec_mb erts_atomic32_dec_mb
+#define erts_smp_atomic32_add_read_mb erts_atomic32_add_read_mb
+#define erts_smp_atomic32_add_mb erts_atomic32_add_mb
+#define erts_smp_atomic32_read_bor_mb erts_atomic32_read_bor_mb
+#define erts_smp_atomic32_read_band_mb erts_atomic32_read_band_mb
+#define erts_smp_atomic32_xchg_mb erts_atomic32_xchg_mb
+#define erts_smp_atomic32_cmpxchg_mb erts_atomic32_cmpxchg_mb
+
+#define erts_smp_atomic32_init_acqb erts_atomic32_init_acqb
+#define erts_smp_atomic32_set_acqb erts_atomic32_set_acqb
+#define erts_smp_atomic32_read_acqb erts_atomic32_read_acqb
+#define erts_smp_atomic32_inc_read_acqb erts_atomic32_inc_read_acqb
+#define erts_smp_atomic32_dec_read_acqb erts_atomic32_dec_read_acqb
+#define erts_smp_atomic32_inc_acqb erts_atomic32_inc_acqb
+#define erts_smp_atomic32_dec_acqb erts_atomic32_dec_acqb
+#define erts_smp_atomic32_add_read_acqb erts_atomic32_add_read_acqb
+#define erts_smp_atomic32_add_acqb erts_atomic32_add_acqb
+#define erts_smp_atomic32_read_bor_acqb erts_atomic32_read_bor_acqb
+#define erts_smp_atomic32_read_band_acqb erts_atomic32_read_band_acqb
+#define erts_smp_atomic32_xchg_acqb erts_atomic32_xchg_acqb
+#define erts_smp_atomic32_cmpxchg_acqb erts_atomic32_cmpxchg_acqb
+
+#define erts_smp_atomic32_init_relb erts_atomic32_init_relb
+#define erts_smp_atomic32_set_relb erts_atomic32_set_relb
+#define erts_smp_atomic32_read_relb erts_atomic32_read_relb
+#define erts_smp_atomic32_inc_read_relb erts_atomic32_inc_read_relb
+#define erts_smp_atomic32_dec_read_relb erts_atomic32_dec_read_relb
+#define erts_smp_atomic32_inc_relb erts_atomic32_inc_relb
+#define erts_smp_atomic32_dec_relb erts_atomic32_dec_relb
+#define erts_smp_atomic32_add_read_relb erts_atomic32_add_read_relb
+#define erts_smp_atomic32_add_relb erts_atomic32_add_relb
+#define erts_smp_atomic32_read_bor_relb erts_atomic32_read_bor_relb
+#define erts_smp_atomic32_read_band_relb erts_atomic32_read_band_relb
+#define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb
+#define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb
+
+#define erts_smp_atomic32_init_ddrb erts_atomic32_init_ddrb
+#define erts_smp_atomic32_set_ddrb erts_atomic32_set_ddrb
+#define erts_smp_atomic32_read_ddrb erts_atomic32_read_ddrb
+#define erts_smp_atomic32_inc_read_ddrb erts_atomic32_inc_read_ddrb
+#define erts_smp_atomic32_dec_read_ddrb erts_atomic32_dec_read_ddrb
+#define erts_smp_atomic32_inc_ddrb erts_atomic32_inc_ddrb
+#define erts_smp_atomic32_dec_ddrb erts_atomic32_dec_ddrb
+#define erts_smp_atomic32_add_read_ddrb erts_atomic32_add_read_ddrb
+#define erts_smp_atomic32_add_ddrb erts_atomic32_add_ddrb
+#define erts_smp_atomic32_read_bor_ddrb erts_atomic32_read_bor_ddrb
+#define erts_smp_atomic32_read_band_ddrb erts_atomic32_read_band_ddrb
+#define erts_smp_atomic32_xchg_ddrb erts_atomic32_xchg_ddrb
+#define erts_smp_atomic32_cmpxchg_ddrb erts_atomic32_cmpxchg_ddrb
+
+#define erts_smp_atomic32_init_rb erts_atomic32_init_rb
+#define erts_smp_atomic32_set_rb erts_atomic32_set_rb
+#define erts_smp_atomic32_read_rb erts_atomic32_read_rb
+#define erts_smp_atomic32_inc_read_rb erts_atomic32_inc_read_rb
+#define erts_smp_atomic32_dec_read_rb erts_atomic32_dec_read_rb
+#define erts_smp_atomic32_inc_rb erts_atomic32_inc_rb
+#define erts_smp_atomic32_dec_rb erts_atomic32_dec_rb
+#define erts_smp_atomic32_add_read_rb erts_atomic32_add_read_rb
+#define erts_smp_atomic32_add_rb erts_atomic32_add_rb
+#define erts_smp_atomic32_read_bor_rb erts_atomic32_read_bor_rb
+#define erts_smp_atomic32_read_band_rb erts_atomic32_read_band_rb
+#define erts_smp_atomic32_xchg_rb erts_atomic32_xchg_rb
+#define erts_smp_atomic32_cmpxchg_rb erts_atomic32_cmpxchg_rb
+
+#define erts_smp_atomic32_init_wb erts_atomic32_init_wb
+#define erts_smp_atomic32_set_wb erts_atomic32_set_wb
+#define erts_smp_atomic32_read_wb erts_atomic32_read_wb
+#define erts_smp_atomic32_inc_read_wb erts_atomic32_inc_read_wb
+#define erts_smp_atomic32_dec_read_wb erts_atomic32_dec_read_wb
+#define erts_smp_atomic32_inc_wb erts_atomic32_inc_wb
+#define erts_smp_atomic32_dec_wb erts_atomic32_dec_wb
+#define erts_smp_atomic32_add_read_wb erts_atomic32_add_read_wb
+#define erts_smp_atomic32_add_wb erts_atomic32_add_wb
+#define erts_smp_atomic32_read_bor_wb erts_atomic32_read_bor_wb
+#define erts_smp_atomic32_read_band_wb erts_atomic32_read_band_wb
+#define erts_smp_atomic32_xchg_wb erts_atomic32_xchg_wb
+#define erts_smp_atomic32_cmpxchg_wb erts_atomic32_cmpxchg_wb
+
+#else /* !ERTS_SMP */
+
+/* Double word size atomics */
+
+#define erts_smp_dw_atomic_init_nob erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_set_nob erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_nob erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg
+
+#define erts_smp_dw_atomic_init_mb erts_no_dw_atomic_init
+#define erts_smp_dw_atomic_set_mb erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_mb erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg
+
+#define erts_smp_dw_atomic_init_acqb erts_no_dw_atomic_init
+#define erts_smp_dw_atomic_set_acqb erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_acqb erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg
+
+#define erts_smp_dw_atomic_init_relb erts_no_dw_atomic_init
+#define erts_smp_dw_atomic_set_relb erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg
+
+#define erts_smp_dw_atomic_init_ddrb erts_no_dw_atomic_init
+#define erts_smp_dw_atomic_set_ddrb erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_ddrb erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg
+
+#define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init
+#define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg
+
+#define erts_smp_dw_atomic_init_wb erts_no_dw_atomic_init
+#define erts_smp_dw_atomic_set_wb erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_wb erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg
+
+/* Word size atomics */
+
+#define erts_smp_atomic_init_nob erts_no_atomic_set
+#define erts_smp_atomic_set_nob erts_no_atomic_set
+#define erts_smp_atomic_read_nob erts_no_atomic_read
+#define erts_smp_atomic_inc_read_nob erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_nob erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_nob erts_no_atomic_inc
+#define erts_smp_atomic_dec_nob erts_no_atomic_dec
+#define erts_smp_atomic_add_read_nob erts_no_atomic_add_read
+#define erts_smp_atomic_add_nob erts_no_atomic_add
+#define erts_smp_atomic_read_bor_nob erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_nob erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_nob erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
+
+#define erts_smp_atomic_init_mb erts_no_atomic_set
+#define erts_smp_atomic_set_mb erts_no_atomic_set
+#define erts_smp_atomic_read_mb erts_no_atomic_read
+#define erts_smp_atomic_inc_read_mb erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_mb erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_mb erts_no_atomic_inc
+#define erts_smp_atomic_dec_mb erts_no_atomic_dec
+#define erts_smp_atomic_add_read_mb erts_no_atomic_add_read
+#define erts_smp_atomic_add_mb erts_no_atomic_add
+#define erts_smp_atomic_read_bor_mb erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_mb erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_mb erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
+
+#define erts_smp_atomic_init_acqb erts_no_atomic_set
+#define erts_smp_atomic_set_acqb erts_no_atomic_set
+#define erts_smp_atomic_read_acqb erts_no_atomic_read
+#define erts_smp_atomic_inc_read_acqb erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_acqb erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_acqb erts_no_atomic_inc
+#define erts_smp_atomic_dec_acqb erts_no_atomic_dec
+#define erts_smp_atomic_add_read_acqb erts_no_atomic_add_read
+#define erts_smp_atomic_add_acqb erts_no_atomic_add
+#define erts_smp_atomic_read_bor_acqb erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_acqb erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_acqb erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
+
+#define erts_smp_atomic_init_relb erts_no_atomic_set
+#define erts_smp_atomic_set_relb erts_no_atomic_set
+#define erts_smp_atomic_read_relb erts_no_atomic_read
+#define erts_smp_atomic_inc_read_relb erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_relb erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_relb erts_no_atomic_inc
+#define erts_smp_atomic_dec_relb erts_no_atomic_dec
+#define erts_smp_atomic_add_read_relb erts_no_atomic_add_read
+#define erts_smp_atomic_add_relb erts_no_atomic_add
+#define erts_smp_atomic_read_bor_relb erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_relb erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_relb erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
+
+#define erts_smp_atomic_init_ddrb erts_no_atomic_set
+#define erts_smp_atomic_set_ddrb erts_no_atomic_set
+#define erts_smp_atomic_read_ddrb erts_no_atomic_read
+#define erts_smp_atomic_inc_read_ddrb erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_ddrb erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_ddrb erts_no_atomic_inc
+#define erts_smp_atomic_dec_ddrb erts_no_atomic_dec
+#define erts_smp_atomic_add_read_ddrb erts_no_atomic_add_read
+#define erts_smp_atomic_add_ddrb erts_no_atomic_add
+#define erts_smp_atomic_read_bor_ddrb erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_ddrb erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_ddrb erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
+
+#define erts_smp_atomic_init_rb erts_no_atomic_set
+#define erts_smp_atomic_set_rb erts_no_atomic_set
+#define erts_smp_atomic_read_rb erts_no_atomic_read
+#define erts_smp_atomic_inc_read_rb erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_rb erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_rb erts_no_atomic_inc
+#define erts_smp_atomic_dec_rb erts_no_atomic_dec
+#define erts_smp_atomic_add_read_rb erts_no_atomic_add_read
+#define erts_smp_atomic_add_rb erts_no_atomic_add
+#define erts_smp_atomic_read_bor_rb erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_rb erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_rb erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
+
+#define erts_smp_atomic_init_wb erts_no_atomic_set
+#define erts_smp_atomic_set_wb erts_no_atomic_set
+#define erts_smp_atomic_read_wb erts_no_atomic_read
+#define erts_smp_atomic_inc_read_wb erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_wb erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_wb erts_no_atomic_inc
+#define erts_smp_atomic_dec_wb erts_no_atomic_dec
+#define erts_smp_atomic_add_read_wb erts_no_atomic_add_read
+#define erts_smp_atomic_add_wb erts_no_atomic_add
+#define erts_smp_atomic_read_bor_wb erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_wb erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_wb erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
+
+/* 32-bit atomics */
+
+#define erts_smp_atomic32_init_nob erts_no_atomic32_set
+#define erts_smp_atomic32_set_nob erts_no_atomic32_set
+#define erts_smp_atomic32_read_nob erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_nob erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_nob erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_nob erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_nob erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_nob erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_nob erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_nob erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_nob erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_nob erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
+
+#define erts_smp_atomic32_init_mb erts_no_atomic32_set
+#define erts_smp_atomic32_set_mb erts_no_atomic32_set
+#define erts_smp_atomic32_read_mb erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_mb erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_mb erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_mb erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_mb erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_mb erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_mb erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_mb erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_mb erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_mb erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
+
+#define erts_smp_atomic32_init_acqb erts_no_atomic32_set
+#define erts_smp_atomic32_set_acqb erts_no_atomic32_set
+#define erts_smp_atomic32_read_acqb erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_acqb erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_acqb erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_acqb erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_acqb erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_acqb erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_acqb erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_acqb erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_acqb erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_acqb erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
+
+#define erts_smp_atomic32_init_relb erts_no_atomic32_set
+#define erts_smp_atomic32_set_relb erts_no_atomic32_set
+#define erts_smp_atomic32_read_relb erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_relb erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_relb erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_relb erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_relb erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_relb erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_relb erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_relb erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_relb erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
+
+#define erts_smp_atomic32_init_ddrb erts_no_atomic32_set
+#define erts_smp_atomic32_set_ddrb erts_no_atomic32_set
+#define erts_smp_atomic32_read_ddrb erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_ddrb erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_ddrb erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_ddrb erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_ddrb erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_ddrb erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_ddrb erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
+
+#define erts_smp_atomic32_init_rb erts_no_atomic32_set
+#define erts_smp_atomic32_set_rb erts_no_atomic32_set
+#define erts_smp_atomic32_read_rb erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_rb erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_rb erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_rb erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_rb erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_rb erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_rb erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_rb erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_rb erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_rb erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
+
+#define erts_smp_atomic32_init_wb erts_no_atomic32_set
+#define erts_smp_atomic32_set_wb erts_no_atomic32_set
+#define erts_smp_atomic32_read_wb erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_wb erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_wb erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_wb erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_wb erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_wb erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_wb erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_wb erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_wb erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_wb erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
+
+#endif /* !ERTS_SMP */
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
@@ -331,22 +825,6 @@ erts_smp_mtx_destroy(erts_smp_mtx_t *mtx)
 #endif
 }
 
-ERTS_GLB_INLINE void
-erts_smp_mtx_set_forksafe(erts_smp_mtx_t *mtx)
-{
-#ifdef ERTS_SMP
-    erts_mtx_set_forksafe(mtx);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_mtx_unset_forksafe(erts_smp_mtx_t *mtx)
-{
-#ifdef ERTS_SMP
-    erts_mtx_unset_forksafe(mtx);
-#endif
-}
-
 ERTS_GLB_INLINE int
 erts_smp_mtx_trylock(erts_smp_mtx_t *mtx)
 {
@@ -415,6 +893,16 @@ erts_smp_cnd_wait(erts_smp_cnd_t *cnd, erts_smp_mtx_t *mtx)
 #endif
 }
 
+/*
+ * IMPORTANT note about erts_smp_cnd_signal() and erts_smp_cnd_broadcast()
+ *
+ * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast'
+ * even though the associated mutex/mutexes isn't/aren't locked by the
+ * caller. Our implementation do not allow that in order to avoid a
+ * performance penalty. That is, all associated mutexes *need* to be
+ * locked by the caller of erts_smp_cnd_signal()/erts_smp_cnd_broadcast()!
+ */
+
 ERTS_GLB_INLINE void
 erts_smp_cnd_signal(erts_smp_cnd_t *cnd)
 {
@@ -433,6 +921,25 @@ erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd)
 }
 
 ERTS_GLB_INLINE void
+erts_smp_rwmtx_set_reader_group(int no)
+{
+#ifdef ERTS_SMP
+    erts_rwmtx_set_reader_group(no);
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
+			  erts_smp_rwmtx_opt_t *opt,
+			  char *name,
+			  Eterm extra)
+{
+#ifdef ERTS_SMP
+    erts_rwmtx_init_opt_x(rwmtx, opt, name, extra);
+#endif
+}
+
+ERTS_GLB_INLINE void
 erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra)
 {
 #ifdef ERTS_SMP
@@ -441,6 +948,16 @@ erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra)
 }
 
 ERTS_GLB_INLINE void
+erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
+			erts_smp_rwmtx_opt_t *opt,
+			char *name)
+{
+#ifdef ERTS_SMP
+    erts_rwmtx_init_opt(rwmtx, opt, name);
+#endif
+}
+
+ERTS_GLB_INLINE void
 erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name)
 {
 #ifdef ERTS_SMP
@@ -568,148 +1085,6 @@ erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx)
 }
 
 ERTS_GLB_INLINE void
-erts_smp_atomic_init(erts_smp_atomic_t *var, long i)
-{
-#ifdef ERTS_SMP
-    erts_atomic_init(var, i);
-#else
-    *var = i;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic_set(erts_smp_atomic_t *var, long i)
-{
-#ifdef ERTS_SMP
-    erts_atomic_set(var, i);
-#else
-    *var = i;
-#endif
-}
-
-ERTS_GLB_INLINE long
-erts_smp_atomic_read(erts_smp_atomic_t *var)
-{
-#ifdef ERTS_SMP
-    return erts_atomic_read(var);
-#else
-    return *var;
-#endif
-}
-
-ERTS_GLB_INLINE long
-erts_smp_atomic_inctest(erts_smp_atomic_t *incp)
-{
-#ifdef ERTS_SMP
-    return erts_atomic_inctest(incp);
-#else
-    return ++(*incp);
-#endif
-}
-
-ERTS_GLB_INLINE long
-erts_smp_atomic_dectest(erts_smp_atomic_t *decp)
-{
-#ifdef ERTS_SMP
-    return erts_atomic_dectest(decp);
-#else
-    return --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic_inc(erts_smp_atomic_t *incp)
-{
-#ifdef ERTS_SMP
-    erts_atomic_inc(incp);
-#else
-    ++(*incp);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic_dec(erts_smp_atomic_t *decp)
-{
-#ifdef ERTS_SMP
-    erts_atomic_dec(decp);
-#else
-    --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE long
-erts_smp_atomic_addtest(erts_smp_atomic_t *addp, long i)
-{
-#ifdef ERTS_SMP
-    return erts_atomic_addtest(addp, i);
-#else
-    return *addp += i;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic_add(erts_smp_atomic_t *addp, long i)
-{
-#ifdef ERTS_SMP
-    erts_atomic_add(addp, i);
-#else
-    *addp += i;
-#endif
-}
-
-ERTS_GLB_INLINE long
-erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, long new)
-{
-#ifdef ERTS_SMP
-    return erts_atomic_xchg(xchgp, new);
-#else
-    long old;
-    old = *xchgp;
-    *xchgp = new;
-    return old;
-#endif
-}
-
-ERTS_GLB_INLINE long
-erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, long new, long expected)
-{
-#ifdef ERTS_SMP
-    return erts_atomic_cmpxchg(xchgp, new, expected);
-#else
-    long old = *xchgp;
-    if (old == expected)
-        *xchgp = new;
-    return old;
-#endif
-}
-
-ERTS_GLB_INLINE long
-erts_smp_atomic_bor(erts_smp_atomic_t *var, long mask)
-{
-#ifdef ERTS_SMP
-    return erts_atomic_bor(var, mask);
-#else
-    long old;
-    old = *var;
-    *var |= mask;
-    return old;
-#endif
-}
-
-ERTS_GLB_INLINE long
-erts_smp_atomic_band(erts_smp_atomic_t *var, long mask)
-{
-#ifdef ERTS_SMP
-    return erts_atomic_band(var, mask);
-#else
-    long old;
-    old = *var;
-    *var &= mask;
-    return old;
-#endif
-}
-
-ERTS_GLB_INLINE void
 erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra)
 {
 #ifdef ERTS_SMP
@@ -878,14 +1253,6 @@ erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock)
 }
 
 ERTS_GLB_INLINE void
-erts_smp_thr_time_now(erts_smp_thr_timeval_t *time)
-{
-#ifdef ERTS_SMP
-    erts_thr_time_now(time);
-#endif
-}
-
-ERTS_GLB_INLINE void
 erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp)
 {
 #ifdef ERTS_SMP
@@ -919,54 +1286,6 @@ erts_smp_tsd_get(erts_smp_tsd_key_t key)
 #endif
 }
 
-ERTS_GLB_INLINE void
-erts_smp_gate_init(erts_smp_gate_t *gp)
-{
-#ifdef ERTS_SMP
-    erts_gate_init((erts_gate_t *) gp);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_gate_destroy(erts_smp_gate_t *gp)
-{
-#ifdef ERTS_SMP
-    erts_gate_destroy((erts_gate_t *) gp);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_gate_close(erts_smp_gate_t *gp)
-{
-#ifdef ERTS_SMP
-    erts_gate_close((erts_gate_t *) gp);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_gate_let_through(erts_smp_gate_t *gp, unsigned no)
-{
-#ifdef ERTS_SMP
-    erts_gate_let_through((erts_gate_t *) gp, no);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_gate_wait(erts_smp_gate_t *gp)
-{
-#ifdef ERTS_SMP
-    erts_gate_wait((erts_gate_t *) gp);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_gate_swait(erts_smp_gate_t *gp, int spincount)
-{
-#ifdef ERTS_SMP
-    erts_gate_swait((erts_gate_t *) gp, spincount);
-#endif
-}
-
 #ifdef ERTS_THR_HAVE_SIG_FUNCS
 #define ERTS_SMP_THR_HAVE_SIG_FUNCS 1
 
@@ -991,3 +1310,37 @@ erts_smp_thr_sigwait(const sigset_t *set, int *sig)
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 #endif /* ERL_SMP_H */
+
+#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS
+
+/* Deprecated functions to replace */
+
+#undef erts_smp_atomic_init
+#undef erts_smp_atomic_set
+#undef erts_smp_atomic_read
+#undef erts_smp_atomic_inctest
+#undef erts_smp_atomic_dectest
+#undef erts_smp_atomic_inc
+#undef erts_smp_atomic_dec
+#undef erts_smp_atomic_addtest
+#undef erts_smp_atomic_add
+#undef erts_smp_atomic_xchg
+#undef erts_smp_atomic_cmpxchg
+#undef erts_smp_atomic_bor
+#undef erts_smp_atomic_band
+
+#undef erts_smp_atomic32_init
+#undef erts_smp_atomic32_set
+#undef erts_smp_atomic32_read
+#undef erts_smp_atomic32_inctest
+#undef erts_smp_atomic32_dectest
+#undef erts_smp_atomic32_inc
+#undef erts_smp_atomic32_dec
+#undef erts_smp_atomic32_addtest
+#undef erts_smp_atomic32_add
+#undef erts_smp_atomic32_xchg
+#undef erts_smp_atomic32_cmpxchg
+#undef erts_smp_atomic32_bor
+#undef erts_smp_atomic32_band
+
+#endif
diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c
index 2924abbd51..f77e8b798f 100644
--- a/erts/emulator/beam/erl_term.c
+++ b/erts/emulator/beam/erl_term.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2000-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2000-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -58,9 +58,9 @@ do { \
 #endif
 
 #if ET_DEBUG
-unsigned tag_val_def_debug(Eterm x, const char *file, unsigned line)
+unsigned tag_val_def_debug(Wterm x, const char *file, unsigned line)
 #else
-unsigned tag_val_def(Eterm x)
+unsigned tag_val_def(Wterm x)
 #define file __FILE__
 #define line __LINE__
 #endif
@@ -68,7 +68,9 @@ unsigned tag_val_def(Eterm x)
     static char msg[32];
 
     switch (x & _TAG_PRIMARY_MASK) {
-      case TAG_PRIMARY_LIST:					return LIST_DEF;
+    case TAG_PRIMARY_LIST:
+	ET_ASSERT(_list_precond(x),file,line);
+	return LIST_DEF;
       case TAG_PRIMARY_BOXED: {
 	  Eterm hdr = *boxed_val(x);
 	  ET_ASSERT(is_header(hdr),file,line);
@@ -103,7 +105,7 @@ unsigned tag_val_def(Eterm x)
 	  break;
       }
     }
-    sprintf(msg, "tag_val_def: %#lx", x);
+    sprintf(msg, "tag_val_def: %#lx", (unsigned long) x);
     et_abort(msg, file, line);
 #undef file
 #undef line
@@ -121,12 +123,12 @@ FUNTY checked_##FUN(ARGTY x, const char *file, unsigned line) \
     return _unchecked_##FUN(x); \
 }
 
-ET_DEFINE_CHECKED(Eterm,make_boxed,Eterm*,_is_aligned);
+ET_DEFINE_CHECKED(Eterm,make_boxed,Eterm*,_is_taggable_pointer);
 ET_DEFINE_CHECKED(int,is_boxed,Eterm,!is_header);
-ET_DEFINE_CHECKED(Eterm*,boxed_val,Eterm,is_boxed);
-ET_DEFINE_CHECKED(Eterm,make_list,Eterm*,_is_aligned);
+ET_DEFINE_CHECKED(Eterm*,boxed_val,Wterm,_boxed_precond);
+ET_DEFINE_CHECKED(Eterm,make_list,Eterm*,_is_taggable_pointer);
 ET_DEFINE_CHECKED(int,is_not_list,Eterm,!is_header);
-ET_DEFINE_CHECKED(Eterm*,list_val,Eterm,is_list);
+ET_DEFINE_CHECKED(Eterm*,list_val,Wterm,_list_precond);
 ET_DEFINE_CHECKED(Uint,unsigned_val,Eterm,is_small);
 ET_DEFINE_CHECKED(Sint,signed_val,Eterm,is_small);
 ET_DEFINE_CHECKED(Uint,atom_val,Eterm,is_atom);
@@ -134,37 +136,38 @@ ET_DEFINE_CHECKED(Uint,header_arity,Eterm,is_header);
 ET_DEFINE_CHECKED(Uint,arityval,Eterm,is_arity_value);
 ET_DEFINE_CHECKED(Uint,thing_arityval,Eterm,is_thing);
 ET_DEFINE_CHECKED(Uint,thing_subtag,Eterm,is_thing);
-ET_DEFINE_CHECKED(Eterm*,binary_val,Eterm,is_binary);
-ET_DEFINE_CHECKED(Eterm*,fun_val,Eterm,is_fun);
+ET_DEFINE_CHECKED(Eterm*,binary_val,Wterm,is_binary);
+ET_DEFINE_CHECKED(Eterm*,fun_val,Wterm,is_fun);
 ET_DEFINE_CHECKED(int,bignum_header_is_neg,Eterm,_is_bignum_header);
 ET_DEFINE_CHECKED(Eterm,bignum_header_neg,Eterm,_is_bignum_header);
 ET_DEFINE_CHECKED(Uint,bignum_header_arity,Eterm,_is_bignum_header);
-ET_DEFINE_CHECKED(Eterm*,big_val,Eterm,is_big);
-ET_DEFINE_CHECKED(Eterm*,float_val,Eterm,is_float);
-ET_DEFINE_CHECKED(Eterm*,tuple_val,Eterm,is_tuple);
+ET_DEFINE_CHECKED(Eterm*,big_val,Wterm,is_big);
+ET_DEFINE_CHECKED(Eterm*,float_val,Wterm,is_float);
+ET_DEFINE_CHECKED(Eterm*,tuple_val,Wterm,is_tuple);
 ET_DEFINE_CHECKED(Uint,internal_pid_data,Eterm,is_internal_pid);
 ET_DEFINE_CHECKED(struct erl_node_*,internal_pid_node,Eterm,is_internal_pid);
 ET_DEFINE_CHECKED(Uint,internal_port_data,Eterm,is_internal_port);
 ET_DEFINE_CHECKED(struct erl_node_*,internal_port_node,Eterm,is_internal_port);
-ET_DEFINE_CHECKED(Eterm*,internal_ref_val,Eterm,is_internal_ref);
-ET_DEFINE_CHECKED(Uint,internal_ref_data_words,Eterm,is_internal_ref);
-ET_DEFINE_CHECKED(Uint32*,internal_ref_data,Eterm,is_internal_ref);
+ET_DEFINE_CHECKED(Eterm*,internal_ref_val,Wterm,is_internal_ref);
+ET_DEFINE_CHECKED(Uint,internal_ref_data_words,Wterm,is_internal_ref);
+ET_DEFINE_CHECKED(Uint32*,internal_ref_data,Wterm,is_internal_ref);
 ET_DEFINE_CHECKED(struct erl_node_*,internal_ref_node,Eterm,is_internal_ref);
-ET_DEFINE_CHECKED(Eterm*,external_val,Eterm,is_external);
-ET_DEFINE_CHECKED(Uint,external_data_words,Eterm,is_external);
-ET_DEFINE_CHECKED(Uint,external_pid_data_words,Eterm,is_external_pid);
-ET_DEFINE_CHECKED(Uint,external_pid_data,Eterm,is_external_pid);
-ET_DEFINE_CHECKED(struct erl_node_*,external_pid_node,Eterm,is_external_pid);
-ET_DEFINE_CHECKED(Uint,external_port_data_words,Eterm,is_external_port);
-ET_DEFINE_CHECKED(Uint,external_port_data,Eterm,is_external_port);
-ET_DEFINE_CHECKED(struct erl_node_*,external_port_node,Eterm,is_external_port);
-ET_DEFINE_CHECKED(Uint,external_ref_data_words,Eterm,is_external_ref);
-ET_DEFINE_CHECKED(Uint32*,external_ref_data,Eterm,is_external_ref);
+ET_DEFINE_CHECKED(Eterm*,external_val,Wterm,is_external);
+ET_DEFINE_CHECKED(Uint,external_data_words,Wterm,is_external);
+ET_DEFINE_CHECKED(Uint,external_pid_data_words,Wterm,is_external_pid);
+ET_DEFINE_CHECKED(Uint,external_pid_data,Wterm,is_external_pid);
+ET_DEFINE_CHECKED(struct erl_node_*,external_pid_node,Wterm,is_external_pid);
+ET_DEFINE_CHECKED(Uint,external_port_data_words,Wterm,is_external_port);
+ET_DEFINE_CHECKED(Uint,external_port_data,Wterm,is_external_port);
+ET_DEFINE_CHECKED(struct erl_node_*,external_port_node,Wterm,is_external_port);
+ET_DEFINE_CHECKED(Uint,external_ref_data_words,Wterm,is_external_ref);
+ET_DEFINE_CHECKED(Uint32*,external_ref_data,Wterm,is_external_ref);
 ET_DEFINE_CHECKED(struct erl_node_*,external_ref_node,Eterm,is_external_ref);
-ET_DEFINE_CHECKED(Eterm*,export_val,Eterm,is_export);
+ET_DEFINE_CHECKED(Eterm*,export_val,Wterm,is_export);
+ET_DEFINE_CHECKED(Uint,external_thing_data_words,ExternalThing*,is_thing_ptr);
 
-ET_DEFINE_CHECKED(Eterm,make_cp,Uint*,_is_aligned);
-ET_DEFINE_CHECKED(Uint*,cp_val,Eterm,is_CP);
+ET_DEFINE_CHECKED(Eterm,make_cp,UWord *,_is_taggable_pointer);
+ET_DEFINE_CHECKED(UWord *,cp_val,Eterm,is_CP);
 ET_DEFINE_CHECKED(Uint,catch_val,Eterm,is_catch);
 ET_DEFINE_CHECKED(Uint,x_reg_offset,Uint,_is_xreg);
 ET_DEFINE_CHECKED(Uint,y_reg_offset,Uint,_is_yreg);
diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h
index a6596558fa..c270d13365 100644
--- a/erts/emulator/beam/erl_term.h
+++ b/erts/emulator/beam/erl_term.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2000-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2000-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -20,6 +20,34 @@
 #ifndef __ERL_TERM_H
 #define __ERL_TERM_H
 
+#include "sys.h" /* defines HALFWORD_HEAP */
+
+typedef UWord Wterm;  /* Full word terms */
+
+#if HALFWORD_HEAP
+#  define HEAP_ON_C_STACK 0
+#  if HALFWORD_ASSERT
+#    ifdef ET_DEBUG
+#      undef ET_DEBUG
+#    endif
+#    define ET_DEBUG 1
+#  endif
+#  if 1
+#    define CHECK_POINTER_MASK 0xFFFFFFFF00000000UL
+#    define COMPRESS_POINTER(APointer) ((Eterm) (UWord) (APointer))
+#    define EXPAND_POINTER(AnEterm) ((UWord) (AnEterm))
+#  else
+#    define CHECK_POINTER_MASK 0x0UL
+#    define COMPRESS_POINTER(AnUint) (AnUint)
+#    define EXPAND_POINTER(APointer) (APointer)
+#  endif
+#else
+#  define HEAP_ON_C_STACK 1
+#  define CHECK_POINTER_MASK 0x0UL
+#  define COMPRESS_POINTER(AnUint) (AnUint)
+#  define EXPAND_POINTER(APointer) (APointer)
+#endif
+
 struct erl_node_; /* Declared in erl_node_tables.h */
 
 /*
@@ -158,8 +186,15 @@ struct erl_node_; /* Declared in erl_node_tables.h */
 
 
 /* boxed object access methods */
+#if HALFWORD_HEAP
+#define _is_taggable_pointer(x)	 (((UWord)(x) & (CHECK_POINTER_MASK | 0x3)) == 0)
+#define _boxed_precond(x)        (is_boxed(x))
+#else
+#define _is_taggable_pointer(x)	 (((Uint)(x) & 0x3) == 0)
+#define  _boxed_precond(x)       (is_boxed(x))
+#endif
 #define _is_aligned(x)		(((Uint)(x) & 0x3) == 0)
-#define _unchecked_make_boxed(x)	((Uint)(x) + TAG_PRIMARY_BOXED)
+#define _unchecked_make_boxed(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_BOXED)
 _ET_DECLARE_CHECKED(Eterm,make_boxed,Eterm*)
 #define make_boxed(x)		_ET_APPLY(make_boxed,(x))
 #if 1
@@ -170,12 +205,12 @@ _ET_DECLARE_CHECKED(int,is_boxed,Eterm)
 #else
 #define is_boxed(x)		(((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_BOXED)
 #endif
-#define _unchecked_boxed_val(x) ((Eterm*)((x) - TAG_PRIMARY_BOXED))
-_ET_DECLARE_CHECKED(Eterm*,boxed_val,Eterm)
+#define _unchecked_boxed_val(x) ((Eterm*) EXPAND_POINTER(((x) - TAG_PRIMARY_BOXED)))
+_ET_DECLARE_CHECKED(Eterm*,boxed_val,Wterm)
 #define boxed_val(x)		_ET_APPLY(boxed_val,(x))
 
 /* cons cell ("list") access methods */
-#define _unchecked_make_list(x)	((Uint)(x) + TAG_PRIMARY_LIST)
+#define _unchecked_make_list(x)	((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_LIST)
 _ET_DECLARE_CHECKED(Eterm,make_list,Eterm*)
 #define make_list(x)		_ET_APPLY(make_list,(x))
 #if 1
@@ -187,8 +222,13 @@ _ET_DECLARE_CHECKED(int,is_not_list,Eterm)
 #define is_list(x)		(((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_LIST)
 #define is_not_list(x)		(!is_list((x)))
 #endif
-#define _unchecked_list_val(x)	((Eterm*)((x) - TAG_PRIMARY_LIST))
-_ET_DECLARE_CHECKED(Eterm*,list_val,Eterm)
+#if HALFWORD_HEAP
+#define _list_precond(x)        (is_list(x))
+#else
+#define _list_precond(x)       (is_list(x))
+#endif
+#define _unchecked_list_val(x) ((Eterm*) EXPAND_POINTER((x) - TAG_PRIMARY_LIST))
+_ET_DECLARE_CHECKED(Eterm*,list_val,Wterm)
 #define list_val(x)		_ET_APPLY(list_val,(x))
 
 #define CONS(hp, car, cdr) \
@@ -198,28 +238,30 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Eterm)
 #define CDR(x)  ((x)[1])
 
 /* generic tagged pointer (boxed or list) access methods */
-#define _unchecked_ptr_val(x)	((Eterm*)((x) & ~((Uint) 0x3)))
+#define _unchecked_ptr_val(x)	((Eterm*) EXPAND_POINTER((x) & ~((Uint) 0x3)))
 #define ptr_val(x)		_unchecked_ptr_val((x))	/*XXX*/
 #define _unchecked_offset_ptr(x,offs)	((x)+((offs)*sizeof(Eterm)))
 #define offset_ptr(x,offs)	_unchecked_offset_ptr(x,offs)	/*XXX*/
+#define _unchecked_byte_offset_ptr(x,byte_offs)	((x)+(offs))
+#define byte_offset_ptr(x,offs) _unchecked_byte_offset_ptr(x,offs)  /*XXX*/
 
 /* fixnum ("small") access methods */
-#if defined(ARCH_64)
+#if defined(ARCH_64) && !HALFWORD_HEAP
 #define SMALL_BITS	(64-4)
 #define SMALL_DIGITS	(17)
 #else
 #define SMALL_BITS	(28)
 #define SMALL_DIGITS	(8)
 #endif
-#define MAX_SMALL	((1L << (SMALL_BITS-1))-1)
-#define MIN_SMALL	(-(1L << (SMALL_BITS-1)))
+#define MAX_SMALL	((SWORD_CONSTANT(1) << (SMALL_BITS-1))-1)
+#define MIN_SMALL	(-(SWORD_CONSTANT(1) << (SMALL_BITS-1)))
 #define make_small(x)	(((Uint)(x) << _TAG_IMMED1_SIZE) + _TAG_IMMED1_SMALL)
 #define is_small(x)	(((x) & _TAG_IMMED1_MASK) == _TAG_IMMED1_SMALL)
 #define is_not_small(x)	(!is_small((x)))
 #define is_byte(x)	(((x) & ((~(Uint)0 << (_TAG_IMMED1_SIZE+8)) + _TAG_IMMED1_MASK)) == _TAG_IMMED1_SMALL)
 #define is_valid_bit_size(x) (((Sint)(x)) >= 0 && ((x) & 0x7F) == _TAG_IMMED1_SMALL)
 #define is_not_valid_bit_size(x) (!is_valid_bit_size((x)))
-#define MY_IS_SSMALL(x) (((Uint) (((x) >> (SMALL_BITS-1)) + 1)) < 2)
+#define MY_IS_SSMALL(x) (((Uint) ((((x)) >> (SMALL_BITS-1)) + 1)) < 2)
 #define _unchecked_unsigned_val(x)	((x) >> _TAG_IMMED1_SIZE)
 _ET_DECLARE_CHECKED(Uint,unsigned_val,Eterm)
 #define unsigned_val(x)	_ET_APPLY(unsigned_val,(x))
@@ -267,6 +309,7 @@ _ET_DECLARE_CHECKED(Uint,arityval,Eterm)
 
 /* thing access methods */
 #define is_thing(x)	(is_header((x)) && header_is_thing((x)))
+#define is_thing_ptr(t) (is_thing((t)->header))
 #define _unchecked_thing_arityval(x)	_unchecked_header_arity((x))
 _ET_DECLARE_CHECKED(Uint,thing_arityval,Eterm)
 #define thing_arityval(x)	_ET_APPLY(thing_arityval,(x))
@@ -288,7 +331,13 @@ _ET_DECLARE_CHECKED(Uint,thing_subtag,Eterm)
  * we now use a non-zero bit-pattern in debug mode.
  */
 #if ET_DEBUG
-#define THE_NON_VALUE	_make_header(0,_TAG_HEADER_FLOAT)
+# ifdef HIPE
+   /* A very large (or negative) value as work-around for ugly hipe-bifs
+      that return untagged integers (eg hipe_bs_put_utf8) */
+#  define THE_NON_VALUE	_make_header((Uint)~0,_TAG_HEADER_FLOAT)
+# else
+#  define THE_NON_VALUE	_make_header(0,_TAG_HEADER_FLOAT)
+# endif
 #else
 #define THE_NON_VALUE	(0)
 #endif
@@ -301,7 +350,7 @@ _ET_DECLARE_CHECKED(Uint,thing_subtag,Eterm)
 #define is_binary(x)	(is_boxed((x)) && is_binary_header(*boxed_val((x))))
 #define is_not_binary(x) (!is_binary((x)))
 #define _unchecked_binary_val(x) _unchecked_boxed_val((x))
-_ET_DECLARE_CHECKED(Eterm*,binary_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,binary_val,Wterm)
 #define binary_val(x)	_ET_APPLY(binary_val,(x))
 
 /* process binaries stuff (special case of binaries) */
@@ -318,7 +367,7 @@ _ET_DECLARE_CHECKED(Eterm*,binary_val,Eterm)
 #define is_fun(x)		(is_boxed((x)) && is_fun_header(*boxed_val((x))))
 #define is_not_fun(x)		(!is_fun((x)))
 #define _unchecked_fun_val(x)   _unchecked_boxed_val((x))
-_ET_DECLARE_CHECKED(Eterm*,fun_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,fun_val,Wterm)
 #define fun_val(x)		_ET_APPLY(fun_val,(x))
 
 /* export access methods */
@@ -326,10 +375,14 @@ _ET_DECLARE_CHECKED(Eterm*,fun_val,Eterm)
 #define is_export(x)     (is_boxed((x)) && is_export_header(*boxed_val((x))))
 #define is_not_export(x) (!is_export((x)))
 #define _unchecked_export_val(x)   _unchecked_boxed_val(x)
-_ET_DECLARE_CHECKED(Eterm*,export_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,export_val,Wterm)
 #define export_val(x)	_ET_APPLY(export_val,(x))
 #define is_export_header(x)	((x) == HEADER_EXPORT)
+#if HALFWORD_HEAP
+#define HEADER_EXPORT   _make_header(2,_TAG_HEADER_EXPORT)
+#else
 #define HEADER_EXPORT   _make_header(1,_TAG_HEADER_EXPORT)
+#endif
 
 /* bignum access methods */
 #define make_pos_bignum_header(sz)	_make_header((sz),_TAG_HEADER_POS_BIG)
@@ -349,11 +402,11 @@ _ET_DECLARE_CHECKED(Uint,bignum_header_arity,Eterm)
 #define is_big(x)	(is_boxed((x)) && _is_bignum_header(*boxed_val((x))))
 #define is_not_big(x)	(!is_big((x)))
 #define _unchecked_big_val(x)	_unchecked_boxed_val((x))
-_ET_DECLARE_CHECKED(Eterm*,big_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,big_val,Wterm)
 #define big_val(x)		_ET_APPLY(big_val,(x))
 
 /* flonum ("float") access methods */
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 #define HEADER_FLONUM   _make_header(1,_TAG_HEADER_FLOAT)
 #else
 #define HEADER_FLONUM	_make_header(2,_TAG_HEADER_FLOAT)
@@ -362,7 +415,7 @@ _ET_DECLARE_CHECKED(Eterm*,big_val,Eterm)
 #define is_float(x)	(is_boxed((x)) && *boxed_val((x)) == HEADER_FLONUM)
 #define is_not_float(x)	(!is_float(x))
 #define _unchecked_float_val(x)	_unchecked_boxed_val((x))
-_ET_DECLARE_CHECKED(Eterm*,float_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,float_val,Wterm)
 #define float_val(x)	_ET_APPLY(float_val,(x))
 
 /* Float definition for byte and word access */
@@ -374,21 +427,22 @@ typedef union float_def
     byte   fb[sizeof(ieee754_8)];
     Uint16 fs[sizeof(ieee754_8) / sizeof(Uint16)];
     Uint32 fw[sizeof(ieee754_8) / sizeof(Uint32)];
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
     Uint   fdw;
 #endif
 } FloatDef;
 
-#ifdef ARCH_64
-#define GET_DOUBLE(x, f) (f).fdw = *(float_val(x)+1)
+#if defined(ARCH_64) && !HALFWORD_HEAP
+
+#define FLOAT_VAL_GET_DOUBLE(fval, f) (f).fdw = *((fval)+1)
 
 #define PUT_DOUBLE(f, x)  *(x) = HEADER_FLONUM, \
                           *((x)+1) = (f).fdw
 #define GET_DOUBLE_DATA(p, f) (f).fdw = *((Uint *) (p))
 #define PUT_DOUBLE_DATA(f,p) *((Uint *) (p)) = (f).fdw
 #else
-#define GET_DOUBLE(x, f) (f).fw[0] = *(float_val(x)+1), \
-                         (f).fw[1] = *(float_val(x)+2)
+#define FLOAT_VAL_GET_DOUBLE(fval, f) (f).fw[0] = *((fval)+1), \
+				      (f).fw[1] = *((fval)+2)
 
 #define PUT_DOUBLE(f, x)  *(x) = HEADER_FLONUM, \
                           *((x)+1) = (f).fw[0], \
@@ -398,6 +452,9 @@ typedef union float_def
 #define PUT_DOUBLE_DATA(f,p) *((Uint *) (p)) = (f).fw[0],\
                              *(((Uint *) (p))+1) = (f).fw[1]
 #endif
+
+#define GET_DOUBLE(x, f) FLOAT_VAL_GET_DOUBLE(float_val(x), f)
+
 #define DOUBLE_DATA_WORDS (sizeof(ieee754_8)/sizeof(Eterm))
 #define FLOAT_SIZE_OBJECT (DOUBLE_DATA_WORDS+1)
 
@@ -409,7 +466,7 @@ typedef union float_def
    (is_boxed((x)) && *boxed_val((x)) == make_arityval((a)))
 #define is_not_tuple_arity(x, a) (!is_tuple_arity((x),(a)))
 #define _unchecked_tuple_val(x)	_unchecked_boxed_val(x)
-_ET_DECLARE_CHECKED(Eterm*,tuple_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,tuple_val,Wterm)
 #define tuple_val(x)	_ET_APPLY(tuple_val,(x))
 
 #define TUPLE0(t) \
@@ -679,7 +736,7 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_port_node,Eterm)
 #define ERTS_MAX_REF_NUMBERS	3
 #define ERTS_REF_NUMBERS	ERTS_MAX_REF_NUMBERS
 
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 #  define ERTS_REF_WORDS	(ERTS_REF_NUMBERS/2 + 1)
 #  define ERTS_REF_32BIT_WORDS  (ERTS_REF_NUMBERS+1)
 #else
@@ -701,7 +758,7 @@ typedef struct {
 #define make_ref_thing_header(DW) \
   _make_header((DW)+REF_THING_HEAD_SIZE-1,_TAG_HEADER_REF)
 
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 
 /*
  * Ref layout on a 64-bit little endian machine:
@@ -748,21 +805,24 @@ do {									\
   ((RefThing*) internal_ref_val(x))
 
 #define is_internal_ref(x) \
-  (_unchecked_is_boxed((x)) && is_ref_thing_header(*boxed_val((x))))
+    (_unchecked_is_boxed((x)) && is_ref_thing_header(*boxed_val((x))))
+
 #define is_not_internal_ref(x) \
   (!is_internal_ref((x)))
 
 #define _unchecked_internal_ref_val(x) _unchecked_boxed_val((x))
-_ET_DECLARE_CHECKED(Eterm*,internal_ref_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,internal_ref_val,Wterm)
 #define internal_ref_val(x) _ET_APPLY(internal_ref_val,(x))
 
+#define internal_thing_ref_data_words(t) (thing_arityval(*(Eterm*)(t)))
 #define _unchecked_internal_ref_data_words(x) \
  (_unchecked_thing_arityval(*_unchecked_internal_ref_val(x)))
-_ET_DECLARE_CHECKED(Uint,internal_ref_data_words,Eterm)
+_ET_DECLARE_CHECKED(Uint,internal_ref_data_words,Wterm)
 #define internal_ref_data_words(x) _ET_APPLY(internal_ref_data_words,(x))
 
-#define _unchecked_internal_ref_data(x) (_unchecked_ref_thing_ptr(x)->data.ui32)
-_ET_DECLARE_CHECKED(Uint32*,internal_ref_data,Eterm)
+#define internal_thing_ref_data(thing) ((thing)->data.ui32)
+#define _unchecked_internal_ref_data(x) (internal_thing_ref_data(_unchecked_ref_thing_ptr(x)))
+_ET_DECLARE_CHECKED(Uint32*,internal_ref_data,Wterm)
 #define internal_ref_data(x) _ET_APPLY(internal_ref_data,(x))
 
 #define _unchecked_internal_ref_node(x) erts_this_node
@@ -779,10 +839,10 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_ref_node,Eterm)
  *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  *  |A A A A A A A A A A A A A A A A A A A A A A A A A A|t t t t|0 0| Thing
  *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- *  |N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N| Next
- *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  *  |E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E| ErlNode
  *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ *  |N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N| Next
+ *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  *  |X X X X X X X X X X X X X X X X X X X X X X X X X X X X X X X X| Data 0
  *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  *  .                                                               .   .
@@ -793,7 +853,7 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_ref_node,Eterm)
  *  t : External pid thing tag  (1100)
  *  t : External port thing tag (1101)
  *  t : External ref thing tag  (1110)
- *  N : Next (external thing) pointer
+ *  N : Next (off_heap) pointer
  *  E : ErlNode pointer
  *  X : Type specific data
  *
@@ -807,11 +867,19 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_ref_node,Eterm)
  *
  */
 
+/* XXX:PaN - this structure is not perfect for halfword heap, it takes
+   a lot of memory due to padding, and the array will not begin at the end of the
+   structure, as otherwise expected. Be sure to access data.ui32 array and not try
+   to do pointer manipulation on an Eterm * to reach the actual data...
+
+   XXX:Sverk - Problem made worse by "one off-heap list" when 'next' pointer
+     must align with 'next' in ProcBin, erl_fun_thing and erl_off_heap_header.
+*/
 typedef struct external_thing_ {
     /*                                 ----+                        */
     Eterm                   header;     /* |                        */
-    struct external_thing_ *next;       /*  > External thing head   */
-    struct erl_node_       *node;       /* |                        */
+    struct erl_node_*       node;       /*  > External thing head   */
+    struct erl_off_heap_header* next;   /* |                        */
     /*                                 ----+                        */
     union {
 	Uint32              ui32[1];
@@ -839,14 +907,14 @@ typedef struct external_thing_ {
 #define is_external_header(x) \
   (((x) & (_TAG_HEADER_MASK-_BINARY_XXX_MASK)) == _TAG_HEADER_EXTERNAL_PID)
 
-#define is_external(x) \
-  (is_boxed((x)) && is_external_header(*boxed_val((x))))
+#define is_external(x) (is_boxed((x)) && is_external_header(*boxed_val((x))))
+
 #define is_external_pid(x) \
   (is_boxed((x)) && is_external_pid_header(*boxed_val((x))))
 #define is_external_port(x) \
-  (is_boxed((x)) && is_external_port_header(*boxed_val((x))))
-#define is_external_ref(x) \
-  (_unchecked_is_boxed((x)) && is_external_ref_header(*boxed_val((x))))
+    (is_boxed((x)) && is_external_port_header(*boxed_val((x))))
+
+#define is_external_ref(x) (_unchecked_is_boxed((x)) && is_external_ref_header(*boxed_val((x))))
 
 #define _unchecked_is_external(x) \
   (_unchecked_is_boxed((x)) && is_external_header(*_unchecked_boxed_val((x))))
@@ -864,17 +932,21 @@ typedef struct external_thing_ {
 #define make_external_ref		make_external
 
 #define _unchecked_external_val(x) _unchecked_boxed_val((x))
-_ET_DECLARE_CHECKED(Eterm*,external_val,Eterm)
+_ET_DECLARE_CHECKED(Eterm*,external_val,Wterm)
 #define external_val(x) _ET_APPLY(external_val,(x))
 
 #define external_thing_ptr(x) ((ExternalThing *) external_val((x)))
 #define _unchecked_external_thing_ptr(x) \
   ((ExternalThing *) _unchecked_external_val((x)))
 
+#define _unchecked_external_thing_data_words(thing) \
+    (_unchecked_thing_arityval((thing)->header) + (1 - EXTERNAL_THING_HEAD_SIZE))
+_ET_DECLARE_CHECKED(Uint,external_thing_data_words,ExternalThing*)
+#define external_thing_data_words(thing) _ET_APPLY(external_thing_data_words,(thing))
+
 #define _unchecked_external_data_words(x) \
-  (_unchecked_thing_arityval(_unchecked_external_thing_ptr((x))->header) \
-   + (1 - EXTERNAL_THING_HEAD_SIZE))
-_ET_DECLARE_CHECKED(Uint,external_data_words,Eterm)
+    _unchecked_external_thing_data_words(_unchecked_external_thing_ptr((x)))
+_ET_DECLARE_CHECKED(Uint,external_data_words,Wterm)
 #define external_data_words(x) _ET_APPLY(external_data_words,(x))
 
 #define _unchecked_external_data(x) (_unchecked_external_thing_ptr((x))->data.ui)
@@ -885,15 +957,15 @@ _ET_DECLARE_CHECKED(Uint,external_data_words,Eterm)
 
 #define _unchecked_external_pid_data_words(x) \
   _unchecked_external_data_words((x))
-_ET_DECLARE_CHECKED(Uint,external_pid_data_words,Eterm)
+_ET_DECLARE_CHECKED(Uint,external_pid_data_words,Wterm)
 #define external_pid_data_words(x) _ET_APPLY(external_pid_data_words,(x))
 
 #define _unchecked_external_pid_data(x) _unchecked_external_data((x))[0]
-_ET_DECLARE_CHECKED(Uint,external_pid_data,Eterm)
+_ET_DECLARE_CHECKED(Uint,external_pid_data,Wterm)
 #define external_pid_data(x) _ET_APPLY(external_pid_data,(x))
 
 #define _unchecked_external_pid_node(x) _unchecked_external_node((x))
-_ET_DECLARE_CHECKED(struct erl_node_*,external_pid_node,Eterm)
+_ET_DECLARE_CHECKED(struct erl_node_*,external_pid_node,Wterm)
 #define external_pid_node(x) _ET_APPLY(external_pid_node,(x))
 
 #define external_pid_number(x) _GET_PID_NUM(external_pid_data((x)))
@@ -901,27 +973,29 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_pid_node,Eterm)
 
 #define _unchecked_external_port_data_words(x) \
   _unchecked_external_data_words((x))
-_ET_DECLARE_CHECKED(Uint,external_port_data_words,Eterm)
+_ET_DECLARE_CHECKED(Uint,external_port_data_words,Wterm)
 #define external_port_data_words(x) _ET_APPLY(external_port_data_words,(x))
 
 #define _unchecked_external_port_data(x) _unchecked_external_data((x))[0]
-_ET_DECLARE_CHECKED(Uint,external_port_data,Eterm)
+_ET_DECLARE_CHECKED(Uint,external_port_data,Wterm)
 #define external_port_data(x) _ET_APPLY(external_port_data,(x))
 
 #define _unchecked_external_port_node(x) _unchecked_external_node((x))
-_ET_DECLARE_CHECKED(struct erl_node_*,external_port_node,Eterm)
+_ET_DECLARE_CHECKED(struct erl_node_*,external_port_node,Wterm)
 #define external_port_node(x) _ET_APPLY(external_port_node,(x))
 
 #define external_port_number(x) _GET_PORT_NUM(external_port_data((x)))
 
 #define _unchecked_external_ref_data_words(x) \
   _unchecked_external_data_words((x))
-_ET_DECLARE_CHECKED(Uint,external_ref_data_words,Eterm)
+_ET_DECLARE_CHECKED(Uint,external_ref_data_words,Wterm)
 #define external_ref_data_words(x) _ET_APPLY(external_ref_data_words,(x))
+#define external_thing_ref_data_words(thing) external_thing_data_words(thing)
 
 #define _unchecked_external_ref_data(x) (_unchecked_external_thing_ptr((x))->data.ui32)
-_ET_DECLARE_CHECKED(Uint32*,external_ref_data,Eterm)
+_ET_DECLARE_CHECKED(Uint32*,external_ref_data,Wterm)
 #define external_ref_data(x) _ET_APPLY(external_ref_data,(x))
+#define external_thing_ref_data(thing) ((thing)->data.ui32)
 
 #define _unchecked_external_ref_node(x) _unchecked_external_node((x))
 _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm)
@@ -944,15 +1018,15 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm)
 #error "fix yer arch, like"
 #endif
 
-#define _unchecked_make_cp(x)	((Eterm)(x))
-_ET_DECLARE_CHECKED(Eterm,make_cp,Uint*)
+#define _unchecked_make_cp(x)	((Eterm) COMPRESS_POINTER(x))
+_ET_DECLARE_CHECKED(Eterm,make_cp,BeamInstr*)
 #define make_cp(x)	_ET_APPLY(make_cp,(x))
 
 #define is_not_CP(x)	((x) & _CPMASK)
 #define is_CP(x)	(!is_not_CP(x))
 
-#define _unchecked_cp_val(x)	((Uint*)(x))
-_ET_DECLARE_CHECKED(Uint*,cp_val,Eterm)
+#define _unchecked_cp_val(x)	((BeamInstr*) EXPAND_POINTER(x))
+_ET_DECLARE_CHECKED(BeamInstr*,cp_val,Eterm)
 #define cp_val(x)	_ET_APPLY(cp_val,(x))
 
 #define make_catch(x)	(((x) << _TAG_IMMED2_SIZE) | _TAG_IMMED2_CATCH)
@@ -1033,10 +1107,10 @@ _ET_DECLARE_CHECKED(Uint,y_reg_index,Uint)
 #define SMALL_DEF		0xf
 
 #if ET_DEBUG
-extern unsigned tag_val_def_debug(Eterm, const char*, unsigned);
+extern unsigned tag_val_def_debug(Wterm, const char*, unsigned);
 #define tag_val_def(x)	tag_val_def_debug((x),__FILE__,__LINE__)
 #else
-extern unsigned tag_val_def(Eterm);
+extern unsigned tag_val_def(Wterm);
 #endif
 #define not_eq_tags(X,Y)	(tag_val_def((X)) ^ tag_val_def((Y)))
 
@@ -1052,5 +1126,81 @@ extern unsigned tag_val_def(Eterm);
 #define FLOAT_BIG 	_NUMBER_CODE(FLOAT_DEF,BIG_DEF)
 #define FLOAT_FLOAT	_NUMBER_CODE(FLOAT_DEF,FLOAT_DEF)
 
+#if HALFWORD_HEAP
+#define ptr2rel(PTR,BASE) ((Eterm*)((char*)(PTR) - (char*)(BASE)))
+#define rterm2wterm(REL,BASE) ((Wterm)(REL) + (Wterm)(BASE))
+
+#else /* HALFWORD_HEAP */
+
+#define ptr2rel(PTR,BASE) (PTR)
+#define rterm2wterm(REL,BASE) (REL)
+
+#endif /* !HALFWORD_HEAP */
+
+#define make_list_rel(PTR, BASE) make_list(ptr2rel(PTR,BASE))
+#define make_boxed_rel(PTR, BASE) make_boxed(ptr2rel(PTR,BASE))
+#define make_fun_rel make_boxed_rel
+#define make_binary_rel make_boxed_rel
+#define make_tuple_rel make_boxed_rel
+#define make_external_rel make_boxed_rel
+#define make_internal_ref_rel make_boxed_rel
+
+#define binary_val_rel(RTERM, BASE) binary_val(rterm2wterm(RTERM, BASE))
+#define list_val_rel(RTERM, BASE) list_val(rterm2wterm(RTERM, BASE))
+#define boxed_val_rel(RTERM, BASE) boxed_val(rterm2wterm(RTERM, BASE))
+#define tuple_val_rel(RTERM, BASE) tuple_val(rterm2wterm(RTERM, BASE))
+#define export_val_rel(RTERM, BASE) export_val(rterm2wterm(RTERM, BASE))
+#define fun_val_rel(RTERM, BASE) fun_val(rterm2wterm(RTERM, BASE))
+#define big_val_rel(RTERM,BASE)	big_val(rterm2wterm(RTERM,BASE))
+#define float_val_rel(RTERM,BASE) float_val(rterm2wterm(RTERM,BASE))
+#define internal_ref_val_rel(RTERM,BASE) internal_ref_val(rterm2wterm(RTERM,BASE))
+
+#define external_thing_ptr_rel(RTERM, BASE) external_thing_ptr(rterm2wterm(RTERM, BASE))
+#define external_data_words_rel(RTERM,BASE) external_data_words(rterm2wterm(RTERM,BASE))
+
+#define external_port_node_rel(RTERM,BASE) external_port_node(rterm2wterm(RTERM,BASE))
+#define external_port_data_rel(RTERM,BASE) external_port_data(rterm2wterm(RTERM,BASE))
+
+#define is_external_pid_rel(RTERM,BASE) is_external_pid(rterm2wterm(RTERM,BASE))
+#define external_pid_node_rel(RTERM,BASE) external_pid_node(rterm2wterm(RTERM,BASE))
+#define external_pid_data_rel(RTERM,BASE) external_pid_data(rterm2wterm(RTERM,BASE))
+
+#define is_binary_rel(RTERM,BASE) is_binary(rterm2wterm(RTERM,BASE))
+#define is_float_rel(RTERM,BASE) is_float(rterm2wterm(RTERM,BASE))
+#define is_fun_rel(RTERM,BASE) is_fun(rterm2wterm(RTERM,BASE))
+#define is_big_rel(RTERM,BASE) is_big(rterm2wterm(RTERM,BASE))
+#define is_export_rel(RTERM,BASE) is_export(rterm2wterm(RTERM,BASE))
+#define is_tuple_rel(RTERM,BASE) is_tuple(rterm2wterm(RTERM,BASE))
+
+#define GET_DOUBLE_REL(RTERM, f, BASE) GET_DOUBLE(rterm2wterm(RTERM,BASE), f)
+
+#define ref_thing_ptr_rel(RTERM,BASE) ref_thing_ptr(rterm2wterm(RTERM,BASE))
+#define is_internal_ref_rel(RTERM,BASE) is_internal_ref(rterm2wterm(RTERM,BASE))
+#define is_external_rel(RTERM,BASE) is_external(rterm2wterm(RTERM,BASE))
+#define is_external_port_rel(RTERM,BASE) is_external_port(rterm2wterm(RTERM,BASE))
+#define is_external_ref_rel(RTERM,BASE) is_external_ref(rterm2wterm(RTERM,BASE))
+
+#define external_node_rel(RTERM,BASE) external_node(rterm2wterm(RTERM,BASE))
+
+
+#if HALFWORD_HEAP
+ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base)
+{
+    /* If bases differ, assume a and b are on different "heaps",
+       ie can only be same if immed */
+    ASSERT(a_base == b_base || is_immed(a) || is_immed(b)
+	   || rterm2wterm(a,a_base) != rterm2wterm(b,b_base));
+
+    return a == b && (a_base == b_base || is_immed(a));
+}
+#endif
+
+#else /* !HALFWORD_HEAP */
+#define is_same(A,A_BASE,B,B_BASE) ((A)==(B))
+#endif
+
 #endif	/* __ERL_TERM_H */
 
diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
new file mode 100644
index 0000000000..88524bdd4c
--- /dev/null
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -0,0 +1,1377 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Thread progress information. Used by lock free algorithms
+ *              to determine when all involved threads are guaranteed to
+ *              have passed a specific point of execution.
+ *
+ *              Usage instructions below.
+ *
+ * Author: 	Rickard Green
+ */
+
+/*
+ * ------ Usage instructions -----------------------------------------------
+ *
+ * This module keeps track of the progress of a set of managed threads. Only
+ * threads that behave well can be allowed to be managed. A managed thread
+ * should update its thread progress frequently. Currently only scheduler
+ * threads, the system-message-dispatcher threads, and the aux-thread are
+ * managed threads. We typically do not want any async threads as managed
+ * threads since they cannot guarantee a frequent update of thread progress,
+ * since they execute user implemented driver code that is assumed to be
+ * time consuming.
+ *
+ * erts_thr_progress_current() returns the global current thread progress
+ * value of managed threads. I.e., the latest progress value that all
+ * managed threads have reached. Thread progress values are opaque.
+ *
+ * erts_thr_progress_has_reached(VAL) returns a value != 0 if current
+ * global thread progress has reached or passed VAL.
+ *
+ * erts_thr_progress_later() returns a thread progress value in the future
+ * which no managed thread have yet reached.
+ *
+ * All threads issue a full memory barrier when reaching a new thread
+ * progress value. They only reach new thread progress values in specific
+ * controlled states when calling erts_thr_progress_update(). Schedulers
+ * call erts_thr_progress_update() in between execution of processes,
+ * when going to sleep and when waking up.
+ *
+ * Sleeping managed threads are considered to have reached next thread
+ * progress value immediately. They are not woken and do therefore not
+ * issue any memory barriers when reaching a new thread progress value.
+ * A sleeping thread do however immediately issue a memory barrier upon
+ * wakeup.
+ *
+ * Both managed and registered unmanaged threads may request wakeup when
+ * the global thread progress reach a certain value using
+ * erts_thr_progress_wakeup().
+ *
+ * Note that thread progress values are opaque, and that you are only
+ * allowed to use thread progress values retrieved from this API!
+ *
+ * -------------------------------------------------------------------------
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include <stddef.h> /* offsetof() */
+#include "erl_thr_progress.h"
+#include "global.h"
+
+#ifdef ERTS_SMP
+
+#define ERTS_THR_PRGR_DBG_CHK_WAKEUP_REQUEST_VALUE 0
+
+#ifdef DEBUG
+#undef ERTS_THR_PRGR_DBG_CHK_WAKEUP_REQUEST_VALUE
+#define ERTS_THR_PRGR_DBG_CHK_WAKEUP_REQUEST_VALUE 1
+#endif
+
+#define ERTS_THR_PRGR_PRINT_LEADER 0
+#define ERTS_THR_PRGR_PRINT_VAL 0
+#define ERTS_THR_PRGR_PRINT_BLOCKERS 0
+
+#define ERTS_THR_PRGR_FTL_ERR_BLCK_POLL_INTERVAL 100
+
+#define ERTS_THR_PRGR_LFLG_BLOCK	(((erts_aint32_t) 1) << 31)
+#define ERTS_THR_PRGR_LFLG_NO_LEADER	(((erts_aint32_t) 1) << 30)
+#define ERTS_THR_PRGR_LFLG_ACTIVE_MASK	(~(ERTS_THR_PRGR_LFLG_NO_LEADER \
+					   | ERTS_THR_PRGR_LFLG_BLOCK))
+
+#define ERTS_THR_PRGR_LFLGS_ACTIVE(LFLGS) \
+    ((LFLGS) & ERTS_THR_PRGR_LFLG_ACTIVE_MASK)
+
+#define ERTS_THR_PRGR_LFLGS_ALL_WAITING(LFLGS) \
+    (((LFLGS) & (ERTS_THR_PRGR_LFLG_NO_LEADER \
+		 |ERTS_THR_PRGR_LFLG_ACTIVE_MASK)) \
+     == ERTS_THR_PRGR_LFLG_NO_LEADER)
+
+/*
+ * We use a 64-bit value for thread progress. By this wrapping of
+ * the thread progress will more or less never occur.
+ *
+ * On 32-bit systems we therefore need a double word atomic.
+ */
+#undef read_acqb
+#define read_acqb erts_thr_prgr_read_acqb__
+#undef read_nob
+#define read_nob erts_thr_prgr_read_nob__
+
+#ifdef ARCH_64
+
+static ERTS_INLINE void
+set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
+{
+    erts_atomic_set_mb(atmc, val);
+}
+
+static ERTS_INLINE void
+set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
+{
+    erts_atomic_set_nob(atmc, val);
+}
+
+static ERTS_INLINE void
+init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
+{
+    erts_atomic_init_nob(atmc, val);
+}
+
+#else
+
+#undef dw_aint_to_val
+#define dw_aint_to_val erts_thr_prgr_dw_aint_to_val__
+
+static void
+val_to_dw_aint(erts_dw_aint_t *dw_aint, ErtsThrPrgrVal val)
+{
+#ifdef ETHR_SU_DW_NAINT_T__
+    dw_aint->dw_sint = (ETHR_SU_DW_NAINT_T__) val;
+#else
+    dw_aint->sint[ERTS_DW_AINT_LOW_WORD]
+	= (erts_aint_t) (val & 0xffffffff);
+    dw_aint->sint[ERTS_DW_AINT_HIGH_WORD]
+	= (erts_aint_t) ((val >> 32) & 0xffffffff);
+#endif
+}
+
+static ERTS_INLINE void
+set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
+{
+    erts_dw_aint_t dw_aint;
+    val_to_dw_aint(&dw_aint, val);
+    erts_dw_atomic_set_mb(atmc, &dw_aint);
+}
+
+static ERTS_INLINE void
+set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
+{
+    erts_dw_aint_t dw_aint;
+    val_to_dw_aint(&dw_aint, val);
+    erts_dw_atomic_set_nob(atmc, &dw_aint);
+}
+
+static ERTS_INLINE void
+init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
+{
+    erts_dw_aint_t dw_aint;
+    val_to_dw_aint(&dw_aint, val);
+    erts_dw_atomic_init_nob(atmc, &dw_aint);
+}
+
+#endif
+
+/* #define ERTS_THR_PROGRESS_STATE_DEBUG */
+
+#ifdef ERTS_THR_PROGRESS_STATE_DEBUG
+
+#ifdef __GNUC__
+#warning "Thread progress state debug is on"
+#endif
+
+#define ERTS_THR_PROGRESS_STATE_DEBUG_LEADER	(((erts_aint32_t) 1) << 0)
+#define ERTS_THR_PROGRESS_STATE_DEBUG_ACTIVE	(((erts_aint32_t) 1) << 1)
+
+#define ERTS_THR_PROGRESS_STATE_DEBUG_INIT(ID)						\
+    erts_atomic32_init_nob(&intrnl->thr[(ID)].data.state_debug,				\
+			   ERTS_THR_PROGRESS_STATE_DEBUG_ACTIVE)
+
+#define ERTS_THR_PROGRESS_STATE_DEBUG_SET_ACTIVE(ID, ON)				\
+do {											\
+    erts_aint32_t state_debug__;							\
+    state_debug__ = erts_atomic32_read_nob(&intrnl->thr[(ID)].data.state_debug);	\
+    if ((ON))										\
+	state_debug__ |= ERTS_THR_PROGRESS_STATE_DEBUG_ACTIVE;				\
+    else										\
+	state_debug__ &= ~ERTS_THR_PROGRESS_STATE_DEBUG_ACTIVE;				\
+    erts_atomic32_set_nob(&intrnl->thr[(ID)].data.state_debug, state_debug__);		\
+} while (0)
+
+#define ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(ID, ON)				\
+do {											\
+    erts_aint32_t state_debug__;							\
+    state_debug__ = erts_atomic32_read_nob(&intrnl->thr[(ID)].data.state_debug);	\
+    if ((ON))										\
+	state_debug__ |= ERTS_THR_PROGRESS_STATE_DEBUG_LEADER;				\
+    else										\
+	state_debug__ &= ~ERTS_THR_PROGRESS_STATE_DEBUG_LEADER;				\
+    erts_atomic32_set_nob(&intrnl->thr[(ID)].data.state_debug, state_debug__);		\
+} while (0)
+
+#else
+
+#define ERTS_THR_PROGRESS_STATE_DEBUG_INIT(ID)
+#define ERTS_THR_PROGRESS_STATE_DEBUG_SET_ACTIVE(ID, ON)
+#define ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(ID, ON)
+
+#endif /* ERTS_THR_PROGRESS_STATE_DEBUG */
+
+#define ERTS_THR_PRGR_BLCKR_INVALID (~((erts_aint32_t) 0))
+#define ERTS_THR_PRGR_BLCKR_UNMANAGED (((erts_aint32_t) 1) << 31)
+
+#define ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING (((erts_aint32_t) 1) << 31)
+
+#define ERTS_THR_PRGR_BM_BITS 32
+#define ERTS_THR_PRGR_BM_SHIFT 5
+#define ERTS_THR_PRGR_BM_MASK 0x1f
+
+#define ERTS_THR_PRGR_WAKEUP_DATA_MASK (ERTS_THR_PRGR_WAKEUP_DATA_SIZE - 1)
+
+#define ERTS_THR_PRGR_WAKEUP_IX(V) \
+    ((int) ((V) & ERTS_THR_PRGR_WAKEUP_DATA_MASK))
+
+typedef struct {
+    erts_atomic32_t len;
+    int id[1];
+} ErtsThrPrgrManagedWakeupData;
+
+typedef struct {
+    erts_atomic32_t len;
+    int high_sz;
+    int low_sz;
+    erts_atomic32_t *high;
+    erts_atomic32_t *low;
+} ErtsThrPrgrUnmanagedWakeupData;
+
+typedef struct {
+    erts_atomic32_t lflgs;
+    erts_atomic32_t block_count;
+    erts_atomic_t blocker_event;
+    erts_atomic32_t pref_wakeup_used;
+    erts_atomic32_t managed_count;
+    erts_atomic32_t managed_id;
+    erts_atomic32_t unmanaged_id;    
+} ErtsThrPrgrMiscData;
+
+typedef struct {
+    ERTS_THR_PRGR_ATOMIC current;
+#ifdef ERTS_THR_PROGRESS_STATE_DEBUG
+    erts_atomic32_t state_debug;
+#endif
+} ErtsThrPrgrElement;
+
+typedef union {
+    ErtsThrPrgrElement data;
+    char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsThrPrgrElement))];
+} ErtsThrPrgrArray;
+
+typedef struct {
+    union {
+	ErtsThrPrgrMiscData data;
+	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(
+		sizeof(ErtsThrPrgrMiscData))];
+    } misc;
+    ErtsThrPrgrArray *thr;
+    struct {
+	int no;
+	ErtsThrPrgrCallbacks *callbacks;
+	ErtsThrPrgrManagedWakeupData *data[ERTS_THR_PRGR_WAKEUP_DATA_SIZE];
+    } managed;
+    struct {
+	int no;
+	ErtsThrPrgrCallbacks *callbacks;
+	ErtsThrPrgrUnmanagedWakeupData *data[ERTS_THR_PRGR_WAKEUP_DATA_SIZE];
+    } unmanaged;
+} ErtsThrPrgrInternalData;
+
+static ErtsThrPrgrInternalData *intrnl;
+
+ErtsThrPrgr erts_thr_prgr__;
+
+erts_tsd_key_t erts_thr_prgr_data_key__;
+
+static void handle_wakeup_requests(ErtsThrPrgrVal current);
+static int got_sched_wakeups(void);
+static erts_aint32_t block_thread(ErtsThrPrgrData *tpd);
+
+static ERTS_INLINE void
+wakeup_managed(int id)
+{
+    ErtsThrPrgrCallbacks *cbp = &intrnl->managed.callbacks[id];
+    ASSERT(0 <= id && id < intrnl->managed.no);
+    cbp->wakeup(cbp->arg);
+}
+
+
+static ERTS_INLINE void
+wakeup_unmanaged(int id)
+{
+    ErtsThrPrgrCallbacks *cbp = &intrnl->unmanaged.callbacks[id];
+    ASSERT(0 <= id && id < intrnl->unmanaged.no);
+    cbp->wakeup(cbp->arg);
+}
+
+static ERTS_INLINE ErtsThrPrgrData *
+perhaps_thr_prgr_data(ErtsSchedulerData *esdp)
+{
+    if (esdp)
+	return &esdp->thr_progress_data;
+    else
+	return erts_tsd_get(erts_thr_prgr_data_key__);
+}
+
+static ERTS_INLINE ErtsThrPrgrData *
+thr_prgr_data(ErtsSchedulerData *esdp)
+{
+    ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(esdp);
+    ASSERT(tpd);
+    return tpd;
+}
+
+static void
+init_tmp_thr_prgr_data(ErtsThrPrgrData *tpd)
+{
+    tpd->id = -1;
+    tpd->is_managed = 0;
+    tpd->is_blocking = 0;
+    tpd->is_temporary = 1;
+
+    erts_tsd_set(erts_thr_prgr_data_key__, (void *) tpd);
+}
+
+static ERTS_INLINE ErtsThrPrgrData *
+tmp_thr_prgr_data(ErtsSchedulerData *esdp)
+{
+    ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(esdp);
+
+    if (!tpd) {
+	/*
+	 * We only allocate the part up to the wakeup_request field
+	 * which is the first field only used by registered threads
+	 */
+	tpd = erts_alloc(ERTS_ALC_T_T_THR_PRGR_DATA,
+			 offsetof(ErtsThrPrgrData, wakeup_request));
+	init_tmp_thr_prgr_data(tpd);
+    }
+
+    return tpd;
+}
+
+static ERTS_INLINE void
+return_tmp_thr_prgr_data(ErtsThrPrgrData *tpd)
+{
+    if (tpd->is_temporary) {
+	erts_tsd_set(erts_thr_prgr_data_key__, NULL);
+	erts_free(ERTS_ALC_T_T_THR_PRGR_DATA, tpd);
+    }
+}
+
+static ERTS_INLINE int
+block_count_dec(void)
+{
+    erts_aint32_t block_count;
+    block_count = erts_atomic32_dec_read_mb(&intrnl->misc.data.block_count);
+    if (block_count == 0) {
+	erts_tse_t *event;
+	event = ((erts_tse_t*)
+		 erts_atomic_read_nob(&intrnl->misc.data.blocker_event));
+	if (event)
+	    erts_tse_set(event);
+	return 1;
+    }
+
+    return (block_count & ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING) == 0;
+}
+
+static ERTS_INLINE int
+block_count_inc(void)
+{
+    erts_aint32_t block_count;
+    block_count = erts_atomic32_inc_read_mb(&intrnl->misc.data.block_count);
+    return (block_count & ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING) == 0;
+}
+
+
+void
+erts_thr_progress_pre_init(void)
+{
+    intrnl = NULL;
+    erts_tsd_key_create(&erts_thr_prgr_data_key__);
+    init_nob(&erts_thr_prgr__.current, 0);
+}
+
+void
+erts_thr_progress_init(int no_schedulers, int managed, int unmanaged)
+{
+    int i, j, um_low, um_high;
+    char *ptr;
+    size_t cb_sz, intrnl_sz, thr_arr_sz, m_wakeup_size, um_wakeup_size,
+	tot_size;
+
+    intrnl_sz = sizeof(ErtsThrPrgrInternalData);
+    intrnl_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(intrnl_sz);
+
+    cb_sz = sizeof(ErtsThrPrgrCallbacks)*(managed+unmanaged);
+    cb_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(cb_sz);
+
+    thr_arr_sz = sizeof(ErtsThrPrgrArray)*managed;
+    ASSERT(thr_arr_sz == ERTS_ALC_CACHE_LINE_ALIGN_SIZE(thr_arr_sz));
+
+    m_wakeup_size = sizeof(ErtsThrPrgrManagedWakeupData);
+    m_wakeup_size += (managed - 1)*sizeof(int);
+    m_wakeup_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(m_wakeup_size);
+
+    um_low = (unmanaged - 1)/ERTS_THR_PRGR_BM_BITS + 1;
+    um_high = (um_low - 1)/ERTS_THR_PRGR_BM_BITS + 1;
+
+    um_wakeup_size = sizeof(ErtsThrPrgrUnmanagedWakeupData);
+    um_wakeup_size += (um_high + um_low)*sizeof(erts_atomic32_t);
+    um_wakeup_size = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(um_wakeup_size);
+
+    tot_size = intrnl_sz;
+    tot_size += cb_sz;
+    tot_size += thr_arr_sz;
+    tot_size += m_wakeup_size*ERTS_THR_PRGR_WAKEUP_DATA_SIZE;
+    tot_size += um_wakeup_size*ERTS_THR_PRGR_WAKEUP_DATA_SIZE;
+
+    ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_THR_PRGR_IDATA,
+					     tot_size);
+
+    intrnl = (ErtsThrPrgrInternalData *) ptr;
+    ptr += intrnl_sz;
+
+    erts_atomic32_init_nob(&intrnl->misc.data.lflgs,
+			   ERTS_THR_PRGR_LFLG_NO_LEADER);
+    erts_atomic32_init_nob(&intrnl->misc.data.block_count,
+			   (ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING
+			    | (erts_aint32_t) managed));
+    erts_atomic_init_nob(&intrnl->misc.data.blocker_event, ERTS_AINT_NULL);
+    erts_atomic32_init_nob(&intrnl->misc.data.pref_wakeup_used, 0);
+    erts_atomic32_init_nob(&intrnl->misc.data.managed_count, 0);
+    erts_atomic32_init_nob(&intrnl->misc.data.managed_id, no_schedulers);
+    erts_atomic32_init_nob(&intrnl->misc.data.unmanaged_id, -1);
+
+    intrnl->thr = (ErtsThrPrgrArray *) ptr;
+    ptr += thr_arr_sz;
+    for (i = 0; i < managed; i++)
+	init_nob(&intrnl->thr[i].data.current, 0);
+
+    intrnl->managed.callbacks = (ErtsThrPrgrCallbacks *) ptr;
+    intrnl->unmanaged.callbacks = &intrnl->managed.callbacks[managed];
+    ptr += cb_sz;
+
+    intrnl->managed.no = managed;
+    for (i = 0; i < managed; i++) {
+	intrnl->managed.callbacks[i].arg = NULL;
+	intrnl->managed.callbacks[i].wakeup = NULL;
+    }
+
+    intrnl->unmanaged.no = unmanaged;
+    for (i = 0; i < unmanaged; i++) {
+	intrnl->unmanaged.callbacks[i].arg = NULL;
+	intrnl->unmanaged.callbacks[i].wakeup = NULL;
+    }
+
+    for (i = 0; i < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; i++) {
+	intrnl->managed.data[i] = (ErtsThrPrgrManagedWakeupData *) ptr;
+	erts_atomic32_init_nob(&intrnl->managed.data[i]->len, 0);
+	ptr += m_wakeup_size;
+    }
+
+    for (i = 0; i < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; i++) {
+	erts_atomic32_t *bm;
+	intrnl->unmanaged.data[i] = (ErtsThrPrgrUnmanagedWakeupData *) ptr;
+	erts_atomic32_init_nob(&intrnl->unmanaged.data[i]->len, 0);
+	bm = (erts_atomic32_t *) (ptr + sizeof(ErtsThrPrgrUnmanagedWakeupData));
+	intrnl->unmanaged.data[i]->high = bm;
+	intrnl->unmanaged.data[i]->high_sz = um_high;
+	for (j = 0; j < um_high; j++)
+	    erts_atomic32_init_nob(&intrnl->unmanaged.data[i]->high[j], 0);
+	intrnl->unmanaged.data[i]->low
+	    = &intrnl->unmanaged.data[i]->high[um_high];	
+	intrnl->unmanaged.data[i]->low_sz = um_low;
+	for (j = 0; j < um_low; j++)
+	    erts_atomic32_init_nob(&intrnl->unmanaged.data[i]->low[j], 0);
+	ptr += um_wakeup_size;
+    }
+    ERTS_THR_MEMORY_BARRIER;
+}
+
+static void
+init_wakeup_request_array(ErtsThrPrgrVal *w)
+{
+    int i;
+    ErtsThrPrgrVal current;
+
+    current = read_acqb(&erts_thr_prgr__.current);
+    for (i = 0; i < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; i++) {
+	w[i] = current - ((ErtsThrPrgrVal) (ERTS_THR_PRGR_WAKEUP_DATA_SIZE + i));
+	if (w[i] > current)
+	    w[i]--;
+    }
+}
+
+void
+erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *callbacks)
+{
+    ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL);
+    int is_blocking = 0;
+
+    if (tpd) {
+	if (!tpd->is_temporary)
+	    erl_exit(ERTS_ABORT_EXIT,
+		     "%s:%d:%s(): Double register of thread\n",
+		     __FILE__, __LINE__, __func__);
+	is_blocking = tpd->is_blocking;
+	return_tmp_thr_prgr_data(tpd);
+    }
+
+    /*
+     * We only allocate the part up to the leader field
+     * which is the first field only used by managed threads
+     */
+    tpd = erts_alloc(ERTS_ALC_T_THR_PRGR_DATA,
+		     offsetof(ErtsThrPrgrData, leader));
+    tpd->id = (int) erts_atomic32_inc_read_nob(&intrnl->misc.data.unmanaged_id);
+    tpd->is_managed = 0;
+    tpd->is_blocking = is_blocking;
+    tpd->is_temporary = 0;
+    ASSERT(tpd->id >= 0);
+    if (tpd->id >= intrnl->unmanaged.no)
+	erl_exit(ERTS_ABORT_EXIT,
+		 "%s:%d:%s(): Too many unmanaged registered threads\n",
+		 __FILE__, __LINE__, __func__);
+
+    init_wakeup_request_array(&tpd->wakeup_request[0]);
+    erts_tsd_set(erts_thr_prgr_data_key__, (void *) tpd);
+
+    ASSERT(callbacks->wakeup);
+
+    intrnl->unmanaged.callbacks[tpd->id] = *callbacks;
+}
+
+
+void
+erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp,
+					  ErtsThrPrgrCallbacks *callbacks,
+					  int pref_wakeup)
+{
+    ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL);
+    int is_blocking = 0, managed;
+
+    if (tpd) {
+	if (!tpd->is_temporary)
+	    erl_exit(ERTS_ABORT_EXIT,
+		     "%s:%d:%s(): Double register of thread\n",
+		     __FILE__, __LINE__, __func__);
+	is_blocking = tpd->is_blocking;
+	return_tmp_thr_prgr_data(tpd);
+    }
+
+    if (esdp)
+	tpd = &esdp->thr_progress_data;
+    else
+	tpd = erts_alloc(ERTS_ALC_T_THR_PRGR_DATA, sizeof(ErtsThrPrgrData));
+
+    if (pref_wakeup
+	&& !erts_atomic32_xchg_nob(&intrnl->misc.data.pref_wakeup_used, 1))
+	tpd->id = 0;
+    else if (esdp)
+	tpd->id = (int) esdp->no;
+    else
+	tpd->id = erts_atomic32_inc_read_nob(&intrnl->misc.data.managed_id);
+    ASSERT(tpd->id >= 0);
+    if (tpd->id >= intrnl->managed.no)
+	erl_exit(ERTS_ABORT_EXIT,
+		 "%s:%d:%s(): Too many managed registered threads\n",
+		 __FILE__, __LINE__, __func__);
+
+    tpd->is_managed = 1;
+    tpd->is_blocking = is_blocking;
+    tpd->is_temporary = 0;
+
+    init_wakeup_request_array(&tpd->wakeup_request[0]);
+
+    ERTS_THR_PROGRESS_STATE_DEBUG_INIT(tpd->id);
+
+    tpd->leader = 0;
+    tpd->active = 1;
+    tpd->previous.local = 0;
+    tpd->previous.current = ERTS_THR_PRGR_VAL_WAITING;
+    erts_tsd_set(erts_thr_prgr_data_key__, (void *) tpd);
+
+    erts_atomic32_inc_nob(&intrnl->misc.data.lflgs);
+
+    ASSERT(callbacks->wakeup);
+    ASSERT(callbacks->prepare_wait);
+    ASSERT(callbacks->wait);
+    ASSERT(callbacks->finalize_wait);
+
+    intrnl->managed.callbacks[tpd->id] = *callbacks;
+
+    callbacks->prepare_wait(callbacks->arg);
+    managed = erts_atomic32_inc_read_relb(&intrnl->misc.data.managed_count);
+    if (managed != intrnl->managed.no) {
+	/* Wait until all managed threads have registered... */
+	do {
+	    callbacks->wait(callbacks->arg);
+	    callbacks->prepare_wait(callbacks->arg);
+	    managed = erts_atomic32_read_acqb(&intrnl->misc.data.managed_count);
+	} while (managed != intrnl->managed.no);
+    }
+    else {
+	int id;
+	/* All managed threads have registered; lets go... */
+	for (id = 0; id < managed; id++)
+	    if (id != tpd->id)
+		wakeup_managed(id);
+    }
+    callbacks->finalize_wait(callbacks->arg);
+}
+
+static ERTS_INLINE int
+leader_update(ErtsThrPrgrData *tpd)
+{
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    erts_lc_check_exact(NULL, 0);
+#endif
+    if (!tpd->leader) {
+	/* Probably need to block... */
+	block_thread(tpd);
+    }
+    else {
+	erts_aint32_t lflgs;
+	ErtsThrPrgrVal next;
+	int ix, sz, make_progress;
+
+	if (tpd->previous.current == ERTS_THR_PRGR_VAL_WAITING) {
+	    /* Took over as leader from another thread */
+	    tpd->previous.current = read_acqb(&erts_thr_prgr__.current);
+	    tpd->previous.next = tpd->previous.current;
+	    tpd->previous.next++;
+	    if (tpd->previous.next == ERTS_THR_PRGR_VAL_WAITING)
+		tpd->previous.next = 0;
+	}
+
+	if (tpd->previous.local == tpd->previous.current) {
+	    ErtsThrPrgrVal val = tpd->previous.current + 1;
+	    if (val == ERTS_THR_PRGR_VAL_WAITING)
+		val = 0;
+	    tpd->previous.local = val;
+	    set_mb(&intrnl->thr[tpd->id].data.current, val);
+	}
+
+	next = tpd->previous.next;
+
+	make_progress = 1;
+	sz = intrnl->managed.no;
+	for (ix = 0; ix < sz; ix++) {
+	    ErtsThrPrgrVal tmp;
+	    tmp = read_nob(&intrnl->thr[ix].data.current);
+	    if (tmp != next && tmp != ERTS_THR_PRGR_VAL_WAITING) {
+		make_progress = 0;
+		ASSERT(erts_thr_progress_has_passed__(next, tmp));
+		break;
+	    }
+	}
+
+	if (make_progress) {
+	    ErtsThrPrgrVal current = next;
+
+	    next++;
+	    if (next == ERTS_THR_PRGR_VAL_WAITING)
+		next = 0;
+
+	    set_nob(&intrnl->thr[tpd->id].data.current, next);
+	    set_mb(&erts_thr_prgr__.current, current);
+	    tpd->previous.local = next;
+	    tpd->previous.next = next;
+	    tpd->previous.current = current;
+
+#if ERTS_THR_PRGR_PRINT_VAL
+	    if (current % 1000 == 0)
+		erts_fprintf(stderr, "%b64u\n", current);
+#endif
+	    handle_wakeup_requests(current);
+	}
+
+	if (tpd->active) {
+	    lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs);
+	    if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK)
+		(void) block_thread(tpd);
+	}
+	else {
+	    tpd->leader = 0;
+	    tpd->previous.current = ERTS_THR_PRGR_VAL_WAITING;
+#if ERTS_THR_PRGR_PRINT_LEADER
+	    erts_fprintf(stderr, "L <- %d\n", tpd->id);
+#endif
+
+	    ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(tpd->id, 0);
+
+	    lflgs = erts_atomic32_read_bor_relb(&intrnl->misc.data.lflgs,
+						ERTS_THR_PRGR_LFLG_NO_LEADER);
+	    if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK)
+		lflgs = block_thread(tpd);
+	    if (ERTS_THR_PRGR_LFLGS_ACTIVE(lflgs) == 0 && got_sched_wakeups())
+		wakeup_managed(0);
+	}
+    }
+
+    return tpd->leader;
+}
+
+static int
+update(ErtsThrPrgrData *tpd)
+{
+    int res;
+    ErtsThrPrgrVal val;
+
+    if (tpd->leader)
+	res = 1;
+    else {
+	erts_aint32_t lflgs;
+	res = 0;
+	val = read_acqb(&erts_thr_prgr__.current);
+	if (tpd->previous.local == val) {
+	    val++;
+	    if (val == ERTS_THR_PRGR_VAL_WAITING)
+		val = 0;
+	    tpd->previous.local = val;
+	    set_mb(&intrnl->thr[tpd->id].data.current, val);
+	}
+
+	lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs);
+	if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK)
+	    res = 1; /* Need to block in leader_update() */
+
+	if ((lflgs & ERTS_THR_PRGR_LFLG_NO_LEADER)
+	    && (tpd->active || ERTS_THR_PRGR_LFLGS_ACTIVE(lflgs) == 0)) {
+	    /* Try to take over leadership... */
+	    erts_aint32_t olflgs;
+	    olflgs = erts_atomic32_read_band_acqb(
+		&intrnl->misc.data.lflgs,
+		~ERTS_THR_PRGR_LFLG_NO_LEADER);
+	    if (olflgs & ERTS_THR_PRGR_LFLG_NO_LEADER) {
+		tpd->leader = 1;
+#if ERTS_THR_PRGR_PRINT_LEADER
+		erts_fprintf(stderr, "L -> %d\n", tpd->id);
+#endif
+		ERTS_THR_PROGRESS_STATE_DEBUG_SET_LEADER(tpd->id, 1);
+	    }
+	}
+	res |= tpd->leader;
+    }
+    return res;
+}
+
+int
+erts_thr_progress_update(ErtsSchedulerData *esdp)
+{
+    return update(thr_prgr_data(esdp));
+}
+
+
+int
+erts_thr_progress_leader_update(ErtsSchedulerData *esdp)
+{
+    return leader_update(thr_prgr_data(esdp));
+}
+
+void
+erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp)
+{
+    erts_aint32_t lflgs;
+    ErtsThrPrgrData *tpd = thr_prgr_data(esdp);
+
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    erts_lc_check_exact(NULL, 0);
+#endif
+
+    block_count_dec();
+
+    tpd->previous.local = ERTS_THR_PRGR_VAL_WAITING;
+    set_mb(&intrnl->thr[tpd->id].data.current, ERTS_THR_PRGR_VAL_WAITING);
+
+    lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs);
+    if (ERTS_THR_PRGR_LFLGS_ALL_WAITING(lflgs) && got_sched_wakeups())
+	wakeup_managed(0); /* Someone need to make progress */
+}
+
+void
+erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp)
+{
+    ErtsThrPrgrData *tpd = thr_prgr_data(esdp);
+    ErtsThrPrgrVal current, val;
+
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    erts_lc_check_exact(NULL, 0);
+#endif
+
+    /*
+     * We aren't allowed to continue until our thread
+     * progress is past global current.
+     */
+    val = current = read_acqb(&erts_thr_prgr__.current);
+    while (1) {
+	val++;
+	if (val == ERTS_THR_PRGR_VAL_WAITING)
+	    val = 0;
+	tpd->previous.local = val;
+	set_mb(&intrnl->thr[tpd->id].data.current, val);
+	val = read_acqb(&erts_thr_prgr__.current);
+	if (current == val)
+	    break;
+	current = val;
+    }
+    if (block_count_inc())
+	block_thread(tpd);
+    if (update(tpd))
+	leader_update(tpd);
+}
+
+void
+erts_thr_progress_active(ErtsSchedulerData *esdp, int on)
+{
+    ErtsThrPrgrData *tpd = thr_prgr_data(esdp);
+
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    erts_lc_check_exact(NULL, 0);
+#endif
+
+    ERTS_THR_PROGRESS_STATE_DEBUG_SET_ACTIVE(tpd->id, on);
+
+    if (on) {
+	ASSERT(!tpd->active);
+	tpd->active = 1;
+	erts_atomic32_inc_nob(&intrnl->misc.data.lflgs);
+    }
+    else {
+	ASSERT(tpd->active);
+	tpd->active = 0;
+	erts_atomic32_dec_nob(&intrnl->misc.data.lflgs);
+	if (update(tpd))
+	    leader_update(tpd);
+    }
+
+#ifdef DEBUG
+    {
+	erts_aint32_t n = erts_atomic32_read_nob(&intrnl->misc.data.lflgs);
+	n &= ERTS_THR_PRGR_LFLG_ACTIVE_MASK;
+	ASSERT(tpd->active <= n && n <= intrnl->managed.no);
+    }
+#endif
+
+}
+
+static ERTS_INLINE int
+has_reached_wakeup(ErtsThrPrgrVal wakeup)
+{
+    /*
+     * Exactly the same as erts_thr_progress_has_reached(), but
+     * also verify valid wakeup requests in debug mode.
+     */
+    ErtsThrPrgrVal current;
+
+    current = read_acqb(&erts_thr_prgr__.current);
+
+#if ERTS_THR_PRGR_DBG_CHK_WAKEUP_REQUEST_VALUE
+    {
+	ErtsThrPrgrVal limit;
+	/*
+	 * erts_thr_progress_later() returns values which are
+	 * equal to 'current + 2', or 'current + 3'. That is, users
+	 * should never get a hold of values larger than that.
+	 *
+	 * That is, valid values are values less than 'current + 4'.
+	 *
+	 * Values larger than this won't work with the wakeup
+	 * algorithm.
+	 */
+
+	limit = current + 4;
+	if (limit == ERTS_THR_PRGR_VAL_WAITING)
+	    limit = 0;
+	else if (limit < current) /* Wrapped */
+	    limit += 1;
+
+	if (!erts_thr_progress_has_passed__(limit, wakeup))
+	    erl_exit(ERTS_ABORT_EXIT,
+		     "Invalid wakeup request value found:"
+		     " current=%b64u, wakeup=%b64u, limit=%b64u",
+		     current, wakeup, limit);
+    }
+#endif
+
+    if (current == wakeup)
+	return 1;
+    return erts_thr_progress_has_passed__(current, wakeup);
+}
+
+static void
+request_wakeup_managed(ErtsThrPrgrData *tpd, ErtsThrPrgrVal value)
+{
+    ErtsThrPrgrManagedWakeupData *mwd;
+    int ix, wix;
+
+    /*
+     * Only managed threads that aren't in waiting state
+     * are allowed to call this function.
+     */
+
+    ASSERT(tpd->is_managed);
+    ASSERT(tpd->previous.local != ERTS_THR_PRGR_VAL_WAITING);
+
+    if (has_reached_wakeup(value)) {
+	wakeup_managed(tpd->id);
+	return;
+    }
+
+    wix = ERTS_THR_PRGR_WAKEUP_IX(value);
+    if (tpd->wakeup_request[wix] == value)
+	return; /* Already got a request registered */
+
+    ASSERT(erts_thr_progress_has_passed__(value,
+					  tpd->wakeup_request[wix]));
+
+
+    if (tpd->previous.local == value) {
+	/*
+	 * We have already confirmed this value. We need to request
+	 * wakeup for a value later than our latest confirmed value in
+	 * order to prevent progress from reaching the requested value
+	 * while we are writing the request.
+	 *
+	 * It is ok to move the wakeup request forward since the only
+	 * guarantee we make (and can make) is that the thread will be
+	 * woken some time *after* the requested value has been reached.
+	 */
+	value++;
+	if (value == ERTS_THR_PRGR_VAL_WAITING)
+	    value = 0;
+
+	wix = ERTS_THR_PRGR_WAKEUP_IX(value);
+	if (tpd->wakeup_request[wix] == value)
+	    return; /* Already got a request registered */
+
+	ASSERT(erts_thr_progress_has_passed__(value,
+					      tpd->wakeup_request[wix]));
+    }
+
+    tpd->wakeup_request[wix] = value;
+
+    mwd = intrnl->managed.data[wix];
+
+    ix = erts_atomic32_inc_read_nob(&mwd->len) - 1;
+#if ERTS_THR_PRGR_DBG_CHK_WAKEUP_REQUEST_VALUE
+    if (ix >= intrnl->managed.no)
+	erl_exit(ERTS_ABORT_EXIT, "Internal error: Too many wakeup requests\n");
+#endif
+    mwd->id[ix] = tpd->id;
+
+    ASSERT(!erts_thr_progress_has_reached(value));
+
+    /*
+     * This thread is guarranteed to issue a full memory barrier:
+     * - after the request has been written, but
+     * - before the global thread progress reach the (possibly
+     *   increased) requested wakeup value.
+     */
+}
+
+static void
+request_wakeup_unmanaged(ErtsThrPrgrData *tpd, ErtsThrPrgrVal value)
+{
+    int wix, ix, id, bit;
+    ErtsThrPrgrUnmanagedWakeupData *umwd;
+
+    ASSERT(!tpd->is_managed);
+
+    /*
+     * Thread progress *can* reach and pass our requested value while
+     * we are writing the request.
+     */
+
+    if (has_reached_wakeup(value)) {
+	wakeup_unmanaged(tpd->id);
+	return;
+    }
+
+    wix = ERTS_THR_PRGR_WAKEUP_IX(value);
+
+    if (tpd->wakeup_request[wix] == value)
+	return; /* Already got a request registered */
+
+    ASSERT(erts_thr_progress_has_passed__(value,
+					  tpd->wakeup_request[wix]));
+
+    umwd = intrnl->unmanaged.data[wix];
+
+    id = tpd->id;
+
+    bit = id & ERTS_THR_PRGR_BM_MASK;
+    ix = id >> ERTS_THR_PRGR_BM_SHIFT;
+    ASSERT(0 <= ix && ix < umwd->low_sz);
+    erts_atomic32_read_bor_nob(&umwd->low[ix], 1 << bit);
+
+    bit = ix & ERTS_THR_PRGR_BM_MASK;
+    ix >>= ERTS_THR_PRGR_BM_SHIFT;
+    ASSERT(0 <= ix && ix < umwd->high_sz);
+    erts_atomic32_read_bor_nob(&umwd->high[ix], 1 << bit);
+
+    erts_atomic32_inc_mb(&umwd->len);
+
+    if (erts_thr_progress_has_reached(value))
+	wakeup_unmanaged(tpd->id);
+    else
+	tpd->wakeup_request[wix] = value;
+}
+
+void
+erts_thr_progress_wakeup(ErtsSchedulerData *esdp,
+			 ErtsThrPrgrVal value)
+{
+    ErtsThrPrgrData *tpd = thr_prgr_data(esdp);
+    ASSERT(!tpd->is_temporary);
+    if (tpd->is_managed)
+	request_wakeup_managed(tpd, value);
+    else
+	request_wakeup_unmanaged(tpd, value);
+}
+
+static void
+wakeup_unmanaged_threads(ErtsThrPrgrUnmanagedWakeupData *umwd)
+{
+    int hix;
+    for (hix = 0; hix < umwd->high_sz; hix++) {
+	erts_aint32_t hmask = erts_atomic32_read_nob(&umwd->high[hix]);
+	if (hmask) {
+	    int hbase = hix << ERTS_THR_PRGR_BM_SHIFT;
+	    int hbit;
+	    for (hbit = 0; hbit < ERTS_THR_PRGR_BM_BITS; hbit++) {
+		if (hmask & (1 << hbit)) {
+		    erts_aint_t lmask;
+		    int lix = hbase + hbit;
+		    ASSERT(0 <= lix && lix < umwd->low_sz);
+		    lmask = erts_atomic32_read_nob(&umwd->low[lix]);
+		    if (lmask) {
+			int lbase = lix << ERTS_THR_PRGR_BM_SHIFT;
+			int lbit;
+			for (lbit = 0; lbit < ERTS_THR_PRGR_BM_BITS; lbit++) {
+			    if (lmask & (1 << lbit)) {
+				int id = lbase + lbit;
+				wakeup_unmanaged(id);
+			    }
+			}
+			erts_atomic32_set_nob(&umwd->low[lix], 0);
+		    }
+		}
+	    }
+	    erts_atomic32_set_nob(&umwd->high[hix], 0);
+	}
+    }
+}
+
+
+static void
+handle_wakeup_requests(ErtsThrPrgrVal current)
+{
+    ErtsThrPrgrManagedWakeupData *mwd;
+    ErtsThrPrgrUnmanagedWakeupData *umwd;
+    int wix, len, i;
+
+    wix = ERTS_THR_PRGR_WAKEUP_IX(current);
+
+    mwd = intrnl->managed.data[wix];
+    len = erts_atomic32_read_nob(&mwd->len);
+    ASSERT(len >= 0);
+    if (len) {
+	for (i = 0; i < len; i++)
+	    wakeup_managed(mwd->id[i]);
+	erts_atomic32_set_nob(&mwd->len, 0);
+    }
+
+    umwd = intrnl->unmanaged.data[wix];
+    len = erts_atomic32_read_nob(&umwd->len);
+    ASSERT(len >= 0);
+    if (len) {
+	wakeup_unmanaged_threads(umwd);
+	erts_atomic32_set_nob(&umwd->len, 0);
+    }
+
+}
+
+static int
+got_sched_wakeups(void)
+{
+    int wix;
+
+    ERTS_THR_MEMORY_BARRIER;
+
+    for (wix = 0; wix < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; wix++) {
+ 	ErtsThrPrgrManagedWakeupData **mwd = intrnl->managed.data;
+	if (erts_atomic32_read_nob(&mwd[wix]->len))
+	    return 1;
+    }
+    for (wix = 0; wix < ERTS_THR_PRGR_WAKEUP_DATA_SIZE; wix++) {
+ 	ErtsThrPrgrUnmanagedWakeupData **umwd = intrnl->unmanaged.data;
+	if (erts_atomic32_read_nob(&umwd[wix]->len))
+	    return 1;
+    }
+    return 0;
+}
+
+static erts_aint32_t
+block_thread(ErtsThrPrgrData *tpd)
+{
+    erts_aint32_t lflgs;
+    ErtsThrPrgrCallbacks *cbp = &intrnl->managed.callbacks[tpd->id];
+
+    do {
+	block_count_dec();
+
+	while (1) {
+	    cbp->prepare_wait(cbp->arg);
+	    lflgs = erts_atomic32_read_nob(&intrnl->misc.data.lflgs);
+	    if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK)
+		cbp->wait(cbp->arg);
+	    else
+		break;
+	}
+
+    } while (block_count_inc());
+
+    cbp->finalize_wait(cbp->arg);
+
+    return lflgs;
+}
+
+static erts_aint32_t
+thr_progress_block(ErtsThrPrgrData *tpd, int wait)
+{
+    erts_tse_t *event = NULL; /* Remove erroneous warning... sigh... */
+    erts_aint32_t lflgs, bc;
+
+    if (tpd->is_blocking++)
+	return (erts_aint32_t) 0;
+
+    while (1) {
+	lflgs = erts_atomic32_read_bor_nob(&intrnl->misc.data.lflgs,
+					   ERTS_THR_PRGR_LFLG_BLOCK);
+	if (lflgs & ERTS_THR_PRGR_LFLG_BLOCK)
+	    block_thread(tpd);
+	else
+	    break;
+    }
+
+#if ERTS_THR_PRGR_PRINT_BLOCKERS
+    erts_fprintf(stderr, "block(%d)\n", tpd->id);
+#endif
+
+    ASSERT(ERTS_AINT_NULL
+	   == erts_atomic_read_nob(&intrnl->misc.data.blocker_event));
+
+    if (wait) {
+	event = erts_tse_fetch();
+	erts_tse_reset(event);
+	erts_atomic_set_nob(&intrnl->misc.data.blocker_event,
+			    (erts_aint_t) event);
+    }
+    if (tpd->is_managed)
+	erts_atomic32_dec_nob(&intrnl->misc.data.block_count);
+    bc = erts_atomic32_read_band_mb(&intrnl->misc.data.block_count,
+				    ~ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING);
+    bc &= ~ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING;
+    if (wait) {
+	while (bc != 0) {
+	    erts_tse_wait(event);
+	    erts_tse_reset(event);
+	    bc = erts_atomic32_read_acqb(&intrnl->misc.data.block_count);
+	}
+    }
+    return bc;
+
+}
+
+void
+erts_thr_progress_block(void)
+{
+    thr_progress_block(tmp_thr_prgr_data(NULL), 1);
+}
+
+void
+erts_thr_progress_fatal_error_block(SWord timeout,
+				    ErtsThrPrgrData *tmp_tpd_bufp)
+{
+    ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL);
+    erts_aint32_t bc;
+    SWord time_left = timeout;
+    SysTimeval to;
+
+    /*
+     * Counting poll intervals may give us a too long timeout
+     * if cpu is busy. If we got tolerant time of day we use it
+     * to prevent this.
+     */
+    if (!erts_disable_tolerant_timeofday) {
+	erts_get_timeval(&to);
+	to.tv_sec += timeout / 1000;
+	to.tv_sec += timeout % 1000;
+    }
+
+    if (!tpd) {
+	/*
+	 * We stack allocate since failure to allocate memory may
+	 * have caused the problem in the first place. This is ok
+	 * since we never complete an unblock after a fatal error
+	 * block.
+	 */
+	tpd = tmp_tpd_bufp;
+	init_tmp_thr_prgr_data(tpd);
+    }
+
+    bc = thr_progress_block(tpd, 0);
+    if (bc == 0)
+	return; /* Succefully blocked all managed threads */
+
+    while (1) {
+	if (erts_milli_sleep(ERTS_THR_PRGR_FTL_ERR_BLCK_POLL_INTERVAL) == 0)
+	    time_left -= ERTS_THR_PRGR_FTL_ERR_BLCK_POLL_INTERVAL;
+	bc = erts_atomic32_read_acqb(&intrnl->misc.data.block_count);
+	if (bc == 0)
+	    break; /* Succefully blocked all managed threads */
+	if (time_left <= 0)
+	    break; /* Timeout */
+	if (!erts_disable_tolerant_timeofday) {
+	    SysTimeval now;
+	    erts_get_timeval(&now);
+	    if (now.tv_sec > to.tv_sec)
+		break; /* Timeout */
+	    if (now.tv_sec == to.tv_sec && now.tv_usec >= to.tv_usec)
+		break; /* Timeout */
+	}
+    }
+}
+
+void
+erts_thr_progress_unblock(void)
+{
+    erts_tse_t *event;
+    int id, break_id, sz, wakeup;
+    ErtsThrPrgrData *tpd = thr_prgr_data(NULL);
+
+    ASSERT(tpd->is_blocking);
+    if (--tpd->is_blocking)
+	return;
+
+    sz = intrnl->managed.no;
+
+    wakeup = 1;
+    if (!tpd->is_managed)
+	id = break_id = tpd->id < 0 ? 0 : tpd->id % sz;
+    else {
+	break_id = tpd->id;
+	id = break_id + 1;
+	if (id >= sz)
+	    id = 0;
+	if (id == break_id)
+	    wakeup = 0;
+	erts_atomic32_inc_nob(&intrnl->misc.data.block_count);
+    }
+
+    event = ((erts_tse_t *)
+	     erts_atomic_read_nob(&intrnl->misc.data.blocker_event));
+    ASSERT(event);
+    erts_atomic_set_nob(&intrnl->misc.data.blocker_event, ERTS_AINT_NULL);
+
+    erts_atomic32_read_bor_relb(&intrnl->misc.data.block_count,
+				ERTS_THR_PRGR_BC_FLG_NOT_BLOCKING);
+#if ERTS_THR_PRGR_PRINT_BLOCKERS
+    erts_fprintf(stderr, "unblock(%d)\n", tpd->id);
+#endif
+    erts_atomic32_read_band_mb(&intrnl->misc.data.lflgs,
+			       ~ERTS_THR_PRGR_LFLG_BLOCK);
+
+    if (wakeup) {
+	do {
+	    ErtsThrPrgrVal tmp;
+	    tmp = read_nob(&intrnl->thr[id].data.current);
+	    if (tmp != ERTS_THR_PRGR_VAL_WAITING)
+		wakeup_managed(id);
+	    if (++id >= sz)
+		id = 0;
+	} while (id != break_id);
+    }
+
+    return_tmp_thr_prgr_data(tpd);
+    erts_tse_return(event);
+}
+
+int
+erts_thr_progress_is_blocking(void)
+{
+    ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL);
+    return tpd && tpd->is_blocking;
+}
+
+void erts_thr_progress_dbg_print_state(void)
+{
+    int id;
+    int sz = intrnl->managed.no;
+
+    erts_fprintf(stderr, "--- thread progress ---\n");
+    erts_fprintf(stderr,"current=%b64u\n", erts_thr_progress_current());
+    for (id = 0; id < sz; id++) {
+	ErtsThrPrgrVal current = read_nob(&intrnl->thr[id].data.current);
+#ifdef ERTS_THR_PROGRESS_STATE_DEBUG
+	erts_aint32_t state_debug;
+	char *active, *leader;
+
+	state_debug = erts_atomic32_read_nob(&intrnl->thr[id].data.state_debug);
+	active = (state_debug & ERTS_THR_PROGRESS_STATE_DEBUG_ACTIVE
+		  ? "true"
+		  : "false");
+	leader = (state_debug & ERTS_THR_PROGRESS_STATE_DEBUG_LEADER
+		  ? "true"
+		  : "false");
+#endif
+	if (current == ERTS_THR_PRGR_VAL_WAITING)
+	    erts_fprintf(stderr,
+			 "  id=%d, current=WAITING"
+#ifdef ERTS_THR_PROGRESS_STATE_DEBUG
+			 ", active=%s, leader=%s"
+#endif
+			 "\n", id
+#ifdef ERTS_THR_PROGRESS_STATE_DEBUG
+			 , active, leader
+#endif
+		);
+	else
+	    erts_fprintf(stderr,
+			 "  id=%d, current=%b64u"
+#ifdef ERTS_THR_PROGRESS_STATE_DEBUG
+			 ", active=%s, leader=%s"
+#endif
+			 "\n", id, current
+#ifdef ERTS_THR_PROGRESS_STATE_DEBUG
+			 , active, leader
+#endif
+		);
+    }
+    erts_fprintf(stderr, "-----------------------\n");
+    
+
+}
+
+#endif
diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h
new file mode 100644
index 0000000000..89486b065b
--- /dev/null
+++ b/erts/emulator/beam/erl_thr_progress.h
@@ -0,0 +1,317 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Thread progress information. Used by lock free algorithms
+ *              to determine when all involved threads are guaranteed to
+ *              have passed a specific point of execution.
+ *
+ *              Usage instructions can be found in ert_thr_progress.c
+ *
+ * Author: 	Rickard Green
+ */
+
+#if !defined(ERL_THR_PROGRESS_H__TSD_TYPE__)
+#define ERL_THR_PROGRESS_H__TSD_TYPE__
+
+#include "sys.h"
+
+#ifndef ERTS_SMP
+
+#define erts_smp_thr_progress_block() ((void) 0)
+#define erts_smp_thr_progress_unblock() ((void) 0)
+#define erts_smp_thr_progress_is_blocking() 1
+
+#else /* ERTS_SMP */
+
+#define erts_smp_thr_progress_block erts_thr_progress_block
+#define erts_smp_thr_progress_unblock erts_thr_progress_unblock
+#define erts_smp_thr_progress_is_blocking erts_thr_progress_is_blocking
+
+void erts_thr_progress_block(void);
+void erts_thr_progress_unblock(void);
+int erts_thr_progress_is_blocking(void);
+
+typedef Uint64 ErtsThrPrgrVal;
+
+#define ERTS_THR_PRGR_WAKEUP_DATA_SIZE 4 /* Need to be an even power of 2. */
+
+typedef struct {
+    int id;
+    int is_managed;
+    int is_blocking;
+    int is_temporary;
+
+    /* --- Part below only for registered threads --- */
+
+    ErtsThrPrgrVal wakeup_request[ERTS_THR_PRGR_WAKEUP_DATA_SIZE];
+
+    /* --- Part below only for managed threads --- */
+
+    int leader; /* Needs to be first in the managed threads part */
+    int active;
+    struct {
+	ErtsThrPrgrVal local;
+	ErtsThrPrgrVal next;
+	ErtsThrPrgrVal current;
+    } previous;
+} ErtsThrPrgrData;
+
+void erts_thr_progress_fatal_error_block(SWord timeout,
+					 ErtsThrPrgrData *tmp_tpd_bufp);
+
+#endif /* ERTS_SMP */
+
+#endif
+
+#if !defined(ERL_THR_PROGRESS_H__) && !defined(ERL_THR_PROGRESS_TSD_TYPE_ONLY)
+#define ERL_THR_PROGRESS_H__
+
+#include "erl_threads.h"
+#include "erl_process.h"
+
+#ifdef ERTS_SMP
+
+#define ERTS_THR_PRGR_VAL_WAITING (~((ErtsThrPrgrVal) 0))
+#define ERTS_THR_PRGR_INVALID (~((ErtsThrPrgrVal) 0))
+
+extern erts_tsd_key_t erts_thr_prgr_data_key__;
+
+#ifdef ARCH_64
+#  define ERTS_THR_PRGR_ATOMIC erts_atomic_t
+#else /* ARCH_32 */
+#  define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t
+#endif
+
+typedef struct {
+    void *arg;
+    void (*wakeup)(void *);
+    void (*prepare_wait)(void *);
+    void (*wait)(void *);
+    void (*finalize_wait)(void *);
+} ErtsThrPrgrCallbacks;
+
+typedef struct {
+    ERTS_THR_PRGR_ATOMIC current;
+} ErtsThrPrgr;
+
+extern ErtsThrPrgr erts_thr_prgr__;
+
+void erts_thr_progress_pre_init(void);
+void erts_thr_progress_init(int no_schedulers, int managed, int unmanaged);
+void erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp,
+					       ErtsThrPrgrCallbacks *,
+					       int);
+void erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *);
+void erts_thr_progress_active(ErtsSchedulerData *esdp, int on);
+void erts_thr_progress_wakeup(ErtsSchedulerData *esdp,
+			      ErtsThrPrgrVal value);
+int erts_thr_progress_update(ErtsSchedulerData *esdp);
+int erts_thr_progress_leader_update(ErtsSchedulerData *esdp);
+void erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp);
+void erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp);
+
+void erts_thr_progress_dbg_print_state(void);
+
+#ifdef ARCH_32
+#define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint);
+#endif
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc);
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc);
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc);
+
+ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void);
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current_to_later__(ErtsThrPrgrVal val);
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later(ErtsSchedulerData *);
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void);
+ERTS_GLB_INLINE int erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
+ERTS_GLB_INLINE int erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val);
+ERTS_GLB_INLINE int erts_thr_progress_equal(ErtsThrPrgrVal val1,
+					    ErtsThrPrgrVal val2);
+ERTS_GLB_INLINE int erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
+ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+#ifdef ARCH_64
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
+{
+    return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc);
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc)
+{
+    return (ErtsThrPrgrVal) erts_atomic_read_acqb(atmc);
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc)
+{
+    return (ErtsThrPrgrVal) erts_atomic_read_mb(atmc);
+}
+
+#else /* ARCH_32 */
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint)
+{
+#ifdef ETHR_SU_DW_NAINT_T__
+    return (ErtsThrPrgrVal) dw_aint->dw_sint;
+#else
+    ErtsThrPrgrVal res;
+    res = (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_HIGH_WORD]);
+    res <<= 32;
+    res |= (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_LOW_WORD]);
+    return res;
+#endif
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
+{
+    erts_dw_aint_t dw_aint;
+    erts_dw_atomic_read_nob(atmc, &dw_aint);
+    return erts_thr_prgr_dw_aint_to_val__(&dw_aint);
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc)
+{
+    erts_dw_aint_t dw_aint;
+    erts_dw_atomic_read_acqb(atmc, &dw_aint);
+    return erts_thr_prgr_dw_aint_to_val__(&dw_aint);
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc)
+{
+    erts_dw_aint_t dw_aint;
+    erts_dw_atomic_read_mb(atmc, &dw_aint);
+    return erts_thr_prgr_dw_aint_to_val__(&dw_aint);
+}
+
+#endif
+
+ERTS_GLB_INLINE int
+erts_thr_progress_is_managed_thread(void)
+{
+    ErtsThrPrgrData *tpd = erts_tsd_get(erts_thr_prgr_data_key__);
+    return tpd && tpd->is_managed;
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_progress_current_to_later__(ErtsThrPrgrVal val)
+{
+    if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)2)))
+	return ((ErtsThrPrgrVal) 0);
+    else if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)1)))
+	return ((ErtsThrPrgrVal) 1);
+    else
+	return val + ((ErtsThrPrgrVal) 2);
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_progress_later(ErtsSchedulerData *esdp)
+{
+    ErtsThrPrgrData *tpd;
+    ErtsThrPrgrVal val;
+    if (esdp) {
+	tpd = &esdp->thr_progress_data;
+    managed_thread:
+	val = tpd->previous.local;
+	ERTS_THR_MEMORY_BARRIER;
+    }
+    else {
+	tpd = erts_tsd_get(erts_thr_prgr_data_key__);
+	if (tpd && tpd->is_managed)
+	    goto managed_thread;
+	val = erts_thr_prgr_read_mb__(&erts_thr_prgr__.current);
+    }
+    ASSERT(val != ERTS_THR_PRGR_VAL_WAITING);
+    return erts_thr_progress_current_to_later__(val);
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_progress_current(void)
+{
+    if (erts_thr_progress_is_managed_thread())
+	return erts_thr_prgr_read_nob__(&erts_thr_prgr__.current);
+    else
+	return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
+}
+
+ERTS_GLB_INLINE int
+erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val0)
+{
+    if ((((((ErtsThrPrgrVal) 1) << 63) & val1)
+	 ^ ((((ErtsThrPrgrVal) 1) << 63) & val0)) != 0) {
+	/* May have wrapped... */
+	if (val1 < (((ErtsThrPrgrVal) 1) << 62)
+	    && val0 > (((ErtsThrPrgrVal) 3) << 62)) {
+	    /*
+	     * 'val1' has wrapped but 'val0' has not yet wrapped. While in
+	     * these ranges 'current' is considered later than 'val0'.
+	     */
+	    return 1;
+	}
+    }
+    return val1 > val0;
+}
+
+ERTS_GLB_INLINE int
+erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val)
+{
+    if (this == val)
+	return 1;
+    return erts_thr_progress_has_passed__(this, val);
+}
+
+ERTS_GLB_INLINE int
+erts_thr_progress_equal(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2)
+{
+    return val1 == val2 && val1 != ERTS_THR_PRGR_INVALID;
+}
+
+ERTS_GLB_INLINE int
+erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2)
+{
+    if (val1 == val2)
+	return 0;
+    if (erts_thr_progress_has_passed__(val1, val2))
+	return 1;
+    else
+	return -1;
+}	
+
+ERTS_GLB_INLINE int
+erts_thr_progress_has_reached(ErtsThrPrgrVal val)
+{
+    ErtsThrPrgrVal current = erts_thr_progress_current();
+    return erts_thr_progress_has_reached_this(current, val);
+}
+
+#endif
+
+#endif /* ERTS_SMP */
+
+#endif
diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c
new file mode 100644
index 0000000000..f07964a265
--- /dev/null
+++ b/erts/emulator/beam/erl_thr_queue.c
@@ -0,0 +1,762 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Lock-free queue for communication between threads.
+ *
+ *              Currently only a many-to-one version has been,
+ *              implemented, i.e., many threads can enqueue but
+ *              only one thread can dequeue at a time. It doesn't
+ *              have to be the same thread dequeuing every time, but
+ *              synchronization so that only one thread dequeues
+ *              at a time has to be provided by other means.
+ *
+ *              When/If the need for a many-to-many queue arises,
+ *              this implementation can relatively easy be extended
+ *              to support that too.
+ *
+ *              Usage instructions below.
+ *
+ * Author: 	Rickard Green
+ */
+
+/*
+ * ------ Usage instructions -----------------------------------------------
+ *
+ * Dequeuing generates garbage that needs to be cleaned up.
+ * erts_thr_q_dequeue() automatically cleans, but garbage may have to be
+ * cleaned up also when the queue is empty. This is done by calling
+ * erts_thr_q_clean(). In the SMP case thread progress may have to be made
+ * before cleaning can continue. If so, erts_thr_q_need_thr_progress() in
+ * combination with erts_thr_progress_wakeup() can be used in order to
+ * request a wakeup at appropriate time.
+ *
+ * Enqueuing implies memory allocation and dequeuing implies memory
+ * deallocation. Memory allocation can be moved to another more suitable
+ * thread using  erts_thr_q_prepare_enqueue() together with
+ * erts_thr_q_enqueue_prepared() instead of using erts_thr_q_enqueue().
+ * Memory deallocation can can be moved to another more suitable thread by
+ * disabling auto_finalize_dequeue when initializing the queue and then use
+ * erts_thr_q_get_finalize_dequeue_data() together
+ * erts_thr_q_finalize_dequeue() after dequeuing or cleaning.
+ *
+ * Ending the life of the queue using either erts_thr_q_destroy()
+ * or erts_thr_q_finalize() impies cleaning the queue. Both functions
+ * return the cleaning result and may have to be called multiple times
+ * until the queue is clean. Once one of these functions have been called
+ * enqueuing is not allowed. This has to be synchronized by the user.
+ * If auto_finalize_dequeue has been disabled, the finalize dequeue
+ * functionality has to be called after ending the life of the queue just
+ * as when dequeuing or cleaning on a queue that is alive.
+ *
+ * -------------------------------------------------------------------------
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "erl_thr_queue.h"
+
+#if defined(DEBUG)
+#define ERTS_THR_Q_DBG_CHK_DATA 1
+#else
+#define ERTS_THR_Q_DBG_CHK_DATA 0
+#endif
+
+#define ERTS_THR_Q_MAX_CLEAN_REACHED_HEAD_COUNT 100
+#define ERTS_THR_Q_MAX_SCHED_CLEAN_OPS 50
+#define ERTS_THR_Q_MAX_DEQUEUE_CLEAN_OPS 3
+
+#define ERTS_THR_Q_MAX_FINI_DEQ_OPS 50
+
+#ifdef ERTS_SMP
+ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(sl_element,
+				 ErtsThrQElement_t,
+				 1000,
+				 ERTS_ALC_T_THR_Q_EL_SL)
+#else
+
+static void
+init_sl_element_alloc(void)
+{
+}
+
+static ErtsThrQElement_t *
+sl_element_alloc(void)
+{
+    return erts_alloc(ERTS_ALC_T_THR_Q_EL_SL,
+		      sizeof(ErtsThrQElement_t));
+}
+
+static void
+sl_element_free(ErtsThrQElement_t *p)
+{
+    erts_free(ERTS_ALC_T_THR_Q_EL_SL, p);
+}
+
+#endif
+
+typedef union {
+    ErtsThrQ_t q;
+    char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsThrQ_t))];
+} ErtsAlignedThrQ_t;
+
+void
+erts_thr_q_init(void)
+{
+    init_sl_element_alloc();
+}
+
+static void noop_callback(void *arg) { }
+
+void
+erts_thr_q_initialize(ErtsThrQ_t *q, ErtsThrQInit_t *qi)
+{
+#ifndef USE_THREADS
+    q->init = *qi;
+    if (!q->init.notify)
+	q->init.notify = noop_callback;
+    q->first = NULL;
+    q->last = NULL;
+    q->q.blk = NULL;
+#else
+    erts_atomic_init_nob(&q->tail.data.marker.next.atmc, ERTS_AINT_NULL);
+    q->tail.data.marker.data.ptr = NULL;
+    erts_atomic_init_nob(&q->tail.data.last,
+			 (erts_aint_t) &q->tail.data.marker);
+    erts_atomic_init_nob(&q->tail.data.um_refc[0], 0);
+    erts_atomic_init_nob(&q->tail.data.um_refc[1], 0);
+    erts_atomic32_init_nob(&q->tail.data.um_refc_ix, 0);
+    q->tail.data.live = qi->live.objects;
+    q->tail.data.arg = qi->arg;
+    q->tail.data.notify = qi->notify;
+    if (!q->tail.data.notify)
+	q->tail.data.notify = noop_callback;
+
+    q->head.head.ptr = &q->tail.data.marker;
+    q->head.live = qi->live.objects;
+    q->head.first = &q->tail.data.marker;
+    q->head.unref_end = &q->tail.data.marker;
+    q->head.clean_reached_head_count = 0;
+    q->head.deq_fini.automatic = qi->auto_finalize_dequeue;
+    q->head.deq_fini.start = NULL;
+    q->head.deq_fini.end = NULL;
+#ifdef ERTS_SMP
+    q->head.next.thr_progress = erts_thr_progress_current();
+    q->head.next.thr_progress_reached = 1;
+#endif
+    q->head.next.um_refc_ix = 1;
+    q->head.next.unref_end = &q->tail.data.marker;
+    q->head.used_marker = 1;
+    q->head.arg = qi->arg;
+    q->head.notify = q->tail.data.notify;
+    q->q.finalizing = 0;
+    q->q.live = qi->live.queue;
+    q->q.blk = NULL;
+#endif
+}
+
+ErtsThrQCleanState_t
+erts_thr_q_finalize(ErtsThrQ_t *q)
+{
+#ifdef USE_THREADS
+    q->q.finalizing = 1;
+#endif
+    while (erts_thr_q_dequeue(q));
+    return erts_thr_q_clean(q);
+}
+
+ErtsThrQ_t *
+erts_thr_q_create(ErtsThrQInit_t *qi)
+{
+    ErtsAlcType_t atype;
+    ErtsThrQ_t *q, *qblk;
+    UWord qw;
+
+    switch (qi->live.queue) {
+    case ERTS_THR_Q_LIVE_SHORT:
+	atype = ERTS_ALC_T_THR_Q_SL;
+	break;
+    case ERTS_THR_Q_LIVE_LONG:
+	atype = ERTS_ALC_T_THR_Q_LL;
+	break;
+    default:
+	atype = ERTS_ALC_T_THR_Q;
+	break;
+    }
+
+    qw = (UWord) erts_alloc(atype,
+			    sizeof(ErtsThrQ_t) + (ERTS_CACHE_LINE_SIZE-1));
+    qblk = (ErtsThrQ_t *) qw;
+    if (qw & ERTS_CACHE_LINE_MASK)
+	qw = (qw & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE;
+    ASSERT((qw & ERTS_CACHE_LINE_MASK) == 0);
+    q = (ErtsThrQ_t *) qw;
+    erts_thr_q_initialize(q, qi);
+    q->q.blk = qblk;
+    return q;
+}
+
+ErtsThrQCleanState_t
+erts_thr_q_destroy(ErtsThrQ_t *q)
+{
+    if (!q->q.blk)
+	erl_exit(ERTS_ABORT_EXIT,
+		 "Trying to destroy not created thread queue\n");
+    return erts_thr_q_finalize(q);
+}
+
+#ifdef USE_THREADS
+
+static void
+destroy(ErtsThrQ_t *q)
+{
+    ErtsAlcType_t atype;
+    switch (q->q.live) {
+    case ERTS_THR_Q_LIVE_SHORT:
+	atype = ERTS_ALC_T_THR_Q_SL;
+	break;
+    case ERTS_THR_Q_LIVE_LONG:
+	atype = ERTS_ALC_T_THR_Q_LL;
+	break;
+    default:
+	atype = ERTS_ALC_T_THR_Q;
+	break;
+    }
+    erts_free(atype, q->q.blk);
+}
+
+#endif
+
+static ERTS_INLINE ErtsThrQElement_t *
+element_live_alloc(ErtsThrQLive_t live)
+{
+    switch (live) {
+    case ERTS_THR_Q_LIVE_SHORT:
+	return sl_element_alloc();
+    default:
+	return (ErtsThrQElement_t *) erts_alloc(ERTS_ALC_T_THR_Q_EL,
+						sizeof(ErtsThrQElement_t));
+    }
+}
+
+static ERTS_INLINE ErtsThrQElement_t *
+element_alloc(ErtsThrQ_t *q)
+{
+    ErtsThrQLive_t live;
+#ifdef USE_THREADS
+    live = q->tail.data.live;
+#else
+    live = q->init.live.objects;
+#endif
+    return element_live_alloc(live);
+}
+
+static ERTS_INLINE void
+element_live_free(ErtsThrQLive_t live, ErtsThrQElement_t *el)
+{
+    switch (live) {
+    case ERTS_THR_Q_LIVE_SHORT:
+	sl_element_free(el);
+	break;
+    default:
+	erts_free(ERTS_ALC_T_THR_Q_EL, el);
+    }
+}
+
+static ERTS_INLINE void
+element_free(ErtsThrQ_t *q, ErtsThrQElement_t *el)
+{
+    ErtsThrQLive_t live;
+#ifdef USE_THREADS
+    live = q->head.live;
+#else
+    live = q->init.live.objects;
+#endif
+    element_live_free(live, el);
+}
+
+#ifdef USE_THREADS
+
+static ERTS_INLINE ErtsThrQElement_t *
+enqueue_managed(ErtsThrQ_t *q, ErtsThrQElement_t *this, int want_last)
+{
+    erts_aint_t ilast, itmp;
+
+    erts_atomic_init_nob(&this->next.atmc, ERTS_AINT_NULL);
+    /* Enqueue at end of list... */
+
+    ilast = erts_atomic_read_nob(&q->tail.data.last);
+    while (1) {
+	ErtsThrQElement_t *last = (ErtsThrQElement_t *) ilast;
+	itmp = erts_atomic_cmpxchg_mb(&last->next.atmc,
+				      (erts_aint_t) this,
+				      ERTS_AINT_NULL);
+	if (itmp == ERTS_AINT_NULL)
+	    break;
+	ilast = itmp;
+    }
+
+    /* Move last pointer forward... */
+    while (1) {
+	if (want_last) {
+	    if (erts_atomic_read_rb(&this->next.atmc) != ERTS_AINT_NULL) {
+		/* Someone else will move it forward */
+		ilast = erts_atomic_read_rb(&q->tail.data.last);
+		return (ErtsThrQElement_t *) ilast;
+	    }
+	}
+	else {
+	    if (erts_atomic_read_nob(&this->next.atmc) != ERTS_AINT_NULL) {
+		/* Someone else will move it forward */
+		return NULL;
+	    }
+	}
+	itmp = erts_atomic_cmpxchg_mb(&q->tail.data.last,
+				      (erts_aint_t) this,
+				      ilast);
+	if (ilast == itmp)
+	    return want_last ? this : NULL;
+	ilast = itmp;
+    }
+}
+
+static ErtsThrQCleanState_t
+clean(ErtsThrQ_t *q, int max_ops, int do_notify)
+{
+    erts_aint_t ilast;
+    int um_refc_ix;
+    int ops;
+
+    for (ops = 0; ops < max_ops; ops++) {
+	ErtsThrQElement_t *tmp;
+    restart:
+	ASSERT(q->head.first);
+	if (q->head.first == q->head.head.ptr) {
+	    q->head.clean_reached_head_count++;
+	    if (q->head.clean_reached_head_count
+		>= ERTS_THR_Q_MAX_CLEAN_REACHED_HEAD_COUNT) {
+		q->head.clean_reached_head_count = 0;
+		break;
+	    }
+	    goto inspect_head;
+	}
+	if (q->head.first == q->head.unref_end)	    
+	    break;
+	if (q->head.first == &q->tail.data.marker) {
+	    q->head.used_marker = 0;
+	    q->head.first = q->head.first->next.ptr;
+	    goto restart;
+	}
+	tmp = q->head.first;
+	q->head.first = q->head.first->next.ptr;
+	if (q->head.deq_fini.automatic)
+	    element_free(q, tmp);
+	else {
+	    tmp->data.ptr = (void *) (UWord) q->head.live;
+	    if (!q->head.deq_fini.start)
+		q->head.deq_fini.start = tmp;
+	    else if (q->head.deq_fini.end->next.ptr == &q->tail.data.marker)
+		q->head.deq_fini.end->next.ptr = tmp;
+	    q->head.deq_fini.end = tmp;
+	}
+    }
+
+    ilast = erts_atomic_read_nob(&q->tail.data.last);
+    if (q->head.first == ((ErtsThrQElement_t *) ilast)
+	&& ((ErtsThrQElement_t *) ilast) == &q->tail.data.marker
+	&& q->head.first == &q->tail.data.marker) {
+	/* Empty and clean queue */
+	if (q->q.finalizing)
+	    destroy(q);
+	return ERTS_THR_Q_CLEAN;
+    }
+
+#ifdef ERTS_SMP
+    if (q->head.next.thr_progress_reached
+	|| erts_thr_progress_has_reached(q->head.next.thr_progress)) {
+	q->head.next.thr_progress_reached = 1;
+#endif
+	um_refc_ix = q->head.next.um_refc_ix;
+	if (erts_atomic_read_acqb(&q->tail.data.um_refc[um_refc_ix]) == 0) {
+	    /* Move unreferenced end pointer forward... */
+	    q->head.clean_reached_head_count = 0;
+	    q->head.unref_end = q->head.next.unref_end;
+
+	    if (!q->head.used_marker
+		&& q->head.unref_end == (ErtsThrQElement_t *) ilast) {
+		q->head.used_marker = 1;
+		ilast = (erts_aint_t) enqueue_managed(q,
+						      &q->tail.data.marker,
+						      1);
+		if (q->head.head.ptr == q->head.unref_end) {
+		    ErtsThrQElement_t *next;
+		    next = ((ErtsThrQElement_t *)
+			    erts_atomic_read_acqb(&q->head.head.ptr->next.atmc));
+		    if (next == &q->tail.data.marker) {
+			q->head.head.ptr->next.ptr = &q->tail.data.marker;
+			q->head.head.ptr = &q->tail.data.marker;
+		    }
+		}
+	    }
+
+	    if (q->head.unref_end == (ErtsThrQElement_t *) ilast)
+		ERTS_SMP_MEMORY_BARRIER;
+	    else {
+		q->head.next.unref_end = (ErtsThrQElement_t *) ilast;
+#ifdef ERTS_SMP
+		q->head.next.thr_progress = erts_thr_progress_later(NULL);
+#endif
+		erts_atomic32_set_relb(&q->tail.data.um_refc_ix,
+				       um_refc_ix);
+		q->head.next.um_refc_ix = um_refc_ix == 0 ? 1 : 0;
+#ifdef ERTS_SMP
+		q->head.next.thr_progress_reached = 0;
+#endif
+	    }
+	}
+#ifdef ERTS_SMP
+    }
+#endif
+
+    if (q->head.first == q->head.head.ptr) {
+    inspect_head:
+	if (!q->head.used_marker) {
+	    erts_aint_t inext;
+	    inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc);
+	    if (inext == ERTS_AINT_NULL) {
+		q->head.used_marker = 1;
+		(void) enqueue_managed(q, &q->tail.data.marker, 0);
+		inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc);
+		if (inext == (erts_aint_t) &q->tail.data.marker) {
+		    q->head.head.ptr->next.ptr = &q->tail.data.marker;
+		    q->head.head.ptr = &q->tail.data.marker;
+		    goto check_thr_progress;
+		}
+	    }
+	}
+
+	if (q->q.finalizing) {
+	    ilast = erts_atomic_read_nob(&q->tail.data.last);
+	    if (q->head.first == ((ErtsThrQElement_t *) ilast)
+		&& ((ErtsThrQElement_t *) ilast) == &q->tail.data.marker
+		&& q->head.first == &q->tail.data.marker) {
+		destroy(q);
+	    }
+	    else {
+		goto dirty;
+	    }
+	}
+	return ERTS_THR_Q_CLEAN;
+    }
+
+    if (q->head.first != q->head.unref_end)
+	goto dirty;
+
+check_thr_progress:
+
+#ifdef ERTS_SMP
+    if (q->head.next.thr_progress_reached)
+#endif
+    {
+	int um_refc_ix = q->head.next.um_refc_ix;
+	if (erts_atomic_read_acqb(&q->tail.data.um_refc[um_refc_ix]) == 0) {
+	dirty:
+	    if (do_notify)
+		q->head.notify(q->head.arg);
+	    return ERTS_THR_Q_DIRTY;
+	}
+    }
+
+    return ERTS_THR_Q_NEED_THR_PRGR;
+}
+
+#endif
+
+ErtsThrQCleanState_t
+erts_thr_q_clean(ErtsThrQ_t *q)
+{
+#ifdef USE_THREADS
+    return clean(q, ERTS_THR_Q_MAX_SCHED_CLEAN_OPS, 0);
+#else
+    return ERTS_THR_Q_CLEAN;
+#endif
+}
+
+ErtsThrQCleanState_t
+erts_thr_q_inspect(ErtsThrQ_t *q, int ensure_empty)
+{
+#ifndef USE_THREADS
+    return ERTS_THR_Q_CLEAN;
+#else
+    if (ensure_empty) {
+	erts_aint_t inext;
+	inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc);
+	if (inext != ERTS_AINT_NULL) {
+	    if (&q->tail.data.marker != (ErtsThrQElement_t *) inext)
+		return ERTS_THR_Q_DIRTY;
+	    else {
+		q->head.head.ptr->next.ptr = (ErtsThrQElement_t *) inext;
+		q->head.head.ptr = (ErtsThrQElement_t *) inext;
+		inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc);
+		if (inext != ERTS_AINT_NULL)
+		    return ERTS_THR_Q_DIRTY;
+	    }
+	}
+    }
+
+    if (q->head.first == q->head.head.ptr) {
+	if (!q->head.used_marker) {
+	    erts_aint_t inext;
+	    inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc);
+	    if (inext == ERTS_AINT_NULL)
+		return ERTS_THR_Q_DIRTY;
+	}
+	return ERTS_THR_Q_CLEAN;
+    }
+
+    if (q->head.first != q->head.unref_end)
+	return ERTS_THR_Q_DIRTY;
+
+#ifdef ERTS_SMP
+    if (q->head.next.thr_progress_reached)
+#endif
+    {
+	int um_refc_ix = q->head.next.um_refc_ix;
+	if (erts_atomic_read_acqb(&q->tail.data.um_refc[um_refc_ix]) == 0)
+	    return ERTS_THR_Q_DIRTY;
+    }
+    return ERTS_THR_Q_NEED_THR_PRGR;
+#endif
+}
+
+static void
+enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this)
+{
+#ifndef USE_THREADS
+    ASSERT(data);
+
+    this->next.ptr = NULL;
+    this->data.ptr = data;
+
+    if (q->last)
+	q->last->next.ptr = this;
+    else {
+	q->first = q->last = this;
+	q->init.notify(q->init.arg);
+    }
+#else
+    int notify;
+    int um_refc_ix = 0;
+#ifdef ERTS_SMP
+    int unmanaged_thread;
+#endif
+
+#if ERTS_THR_Q_DBG_CHK_DATA
+    if (!data)
+	erl_exit(ERTS_ABORT_EXIT, "Missing data in enqueue\n");
+#endif
+
+    ASSERT(!q->q.finalizing);
+
+    this->data.ptr = data;
+
+#ifdef ERTS_SMP
+    unmanaged_thread = !erts_thr_progress_is_managed_thread();
+    if (unmanaged_thread)
+#endif
+    {
+	um_refc_ix = erts_atomic32_read_acqb(&q->tail.data.um_refc_ix);
+	while (1) {
+	    int tmp_um_refc_ix;
+	    erts_atomic_inc_acqb(&q->tail.data.um_refc[um_refc_ix]);
+	    tmp_um_refc_ix = erts_atomic32_read_acqb(&q->tail.data.um_refc_ix);
+	    if (tmp_um_refc_ix == um_refc_ix)
+		break;
+	    erts_atomic_dec_relb(&q->tail.data.um_refc[um_refc_ix]);
+	    um_refc_ix = tmp_um_refc_ix;
+	}
+    }
+
+    notify = this == enqueue_managed(q, this, 1);
+	
+
+#ifdef ERTS_SMP
+    if (unmanaged_thread)
+#endif
+    {
+	if (notify)
+	    erts_atomic_dec_relb(&q->tail.data.um_refc[um_refc_ix]);
+	else if (erts_atomic_dec_read_relb(&q->tail.data.um_refc[um_refc_ix]) == 0)
+	    notify = 1;
+    }
+    if (notify)
+	q->tail.data.notify(q->tail.data.arg);
+#endif
+}
+
+void
+erts_thr_q_enqueue(ErtsThrQ_t *q, void *data)
+{
+    enqueue(q, data, element_alloc(q));
+}
+
+ErtsThrQPrepEnQ_t *
+erts_thr_q_prepare_enqueue(ErtsThrQ_t *q)
+{
+    return (ErtsThrQPrepEnQ_t *) element_alloc(q);
+}
+
+int
+erts_thr_q_get_finalize_dequeue_data(ErtsThrQ_t *q, ErtsThrQFinDeQ_t *fdp)
+{
+#ifndef USE_THREADS
+    return 0;
+#else
+#ifdef DEBUG
+    if (!q->head.deq_fini.start) {
+	ASSERT(!q->head.deq_fini.end);
+    }
+    else {
+	ErtsThrQElement_t *e = q->head.deq_fini.start;
+	ErtsThrQElement_t *end = q->head.deq_fini.end;
+	while (e != end) {
+	    ASSERT(q->head.head.ptr != e);
+	    ASSERT(q->head.first != e);
+	    ASSERT(q->head.unref_end != e);
+	    e = e->next.ptr;
+	}
+    }	
+#endif
+    fdp->start = q->head.deq_fini.start;
+    fdp->end = q->head.deq_fini.end;
+    if (fdp->end)
+	fdp->end->next.ptr = NULL;
+    q->head.deq_fini.start = NULL;
+    q->head.deq_fini.end = NULL;
+    return fdp->start != NULL;
+#endif
+}
+
+void
+erts_thr_q_append_finalize_dequeue_data(ErtsThrQFinDeQ_t *fdp0,
+					ErtsThrQFinDeQ_t *fdp1)
+{
+#ifdef USE_THREADS
+    if (fdp1->start) {
+	if (fdp0->end)
+	    fdp0->end->next.ptr = fdp1->start;
+	else
+	    fdp0->start = fdp1->start;
+	fdp0->end = fdp1->end;
+    }
+#endif
+}
+
+
+int erts_thr_q_finalize_dequeue(ErtsThrQFinDeQ_t *state)
+{
+#ifdef USE_THREADS
+    ErtsThrQElement_t *start = state->start;
+    if (start) {
+	ErtsThrQLive_t live;
+	int i;
+	for (i = 0; i < ERTS_THR_Q_MAX_FINI_DEQ_OPS; i++) {
+	    ErtsThrQElement_t *tmp;
+	    if (!start)
+		break;
+	    tmp = start;
+	    start = start->next.ptr;
+	    live = (ErtsThrQLive_t) (UWord) tmp->data.ptr;
+	    element_live_free(live, tmp);
+	}
+	state->start = start;
+	if (start)
+	    return 1; /* More to do */
+	state->end = NULL;
+    }
+#endif
+    return 0;
+}
+
+void
+erts_thr_q_finalize_dequeue_state_init(ErtsThrQFinDeQ_t *state)
+{
+#ifdef USE_THREADS
+    state->start = NULL;
+    state->end = NULL;
+#endif
+}
+
+
+void
+erts_thr_q_enqueue_prepared(ErtsThrQ_t *q, void *data, ErtsThrQPrepEnQ_t *prep)
+{
+    ASSERT(prep);
+    enqueue(q, data, (ErtsThrQElement_t *) prep);
+}
+
+void *
+erts_thr_q_dequeue(ErtsThrQ_t *q)
+{
+#ifndef USE_THREADS
+    void *res;
+    ErtsThrQElement_t *tmp;
+
+    if (!q->first)
+	return NULL;
+    tmp = q->first;
+    res = tmp->data.ptr;
+    q->first = tmp->next.ptr;
+    if (!q->first)
+	q->last = NULL;
+
+    element_free(q, tmp);
+
+    return res;
+#else
+    erts_aint_t inext;
+    void *res;
+
+    inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc);
+    if (inext == ERTS_AINT_NULL)
+	return NULL;
+    q->head.head.ptr->next.ptr = (ErtsThrQElement_t *) inext;
+    q->head.head.ptr = (ErtsThrQElement_t *) inext;
+    if (q->head.head.ptr == &q->tail.data.marker) {
+	inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc);
+	if (inext == ERTS_AINT_NULL)
+	    return NULL;
+	q->head.head.ptr->next.ptr = (ErtsThrQElement_t *) inext;
+	q->head.head.ptr = (ErtsThrQElement_t *) inext;
+    }
+    res = q->head.head.ptr->data.ptr;
+#if ERTS_THR_Q_DBG_CHK_DATA
+    q->head.head.ptr->data.ptr = NULL;
+    if (!res)
+	erl_exit(ERTS_ABORT_EXIT, "Missing data in dequeue\n");
+#endif
+    clean(q,
+	  (q->head.deq_fini.automatic
+	   ? ERTS_THR_Q_MAX_DEQUEUE_CLEAN_OPS
+	   : ERTS_THR_Q_MAX_SCHED_CLEAN_OPS), 1);
+    return res;
+#endif
+}
diff --git a/erts/emulator/beam/erl_thr_queue.h b/erts/emulator/beam/erl_thr_queue.h
new file mode 100644
index 0000000000..edcf2c3823
--- /dev/null
+++ b/erts/emulator/beam/erl_thr_queue.h
@@ -0,0 +1,209 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Lock-free queue for communication between threads.
+ *
+ *              Currently only a many-to-one version has been,
+ *              implemented, i.e., many threads can enqueue but
+ *              only one thread can dequeue at a time. It doesn't
+ *              have to be the same thread dequeuing every time, but
+ *              synchronization so that only one thread dequeues
+ *              at a time has to be provided by other means.
+ *
+ *              When/If the need for a many-to-many queue arises,
+ *              this implementation can relatively easy be extended
+ *              to support that too.
+ *
+ *              Usage instructions can be found in erts_thr_queue.c
+ *
+ * Author: 	Rickard Green
+ */
+
+#ifndef ERL_THR_QUEUE_H__
+#define ERL_THR_QUEUE_H__
+
+#include "sys.h"
+#include "erl_threads.h"
+#include "erl_alloc.h"
+#include "erl_thr_progress.h"
+
+typedef enum {
+    ERTS_THR_Q_LIVE_UNDEF,
+    ERTS_THR_Q_LIVE_SHORT,
+    ERTS_THR_Q_LIVE_LONG
+} ErtsThrQLive_t;
+
+#define ERTS_THR_Q_INIT_DEFAULT						\
+{									\
+    {									\
+	ERTS_THR_Q_LIVE_UNDEF,						\
+	ERTS_THR_Q_LIVE_SHORT						\
+    },									\
+    NULL,								\
+    NULL,								\
+    1									\
+}
+
+typedef struct ErtsThrQ_t_ ErtsThrQ_t;
+
+typedef struct {
+    struct {
+	ErtsThrQLive_t queue;
+	ErtsThrQLive_t objects;
+    } live;
+    void *arg;
+    void (*notify)(void *);
+    int auto_finalize_dequeue;
+} ErtsThrQInit_t;
+
+typedef struct ErtsThrQElement_t_ ErtsThrQElement_t;
+typedef struct ErtsThrQElement_t ErtsThrQPrepEnQ_t;
+
+typedef union {
+    erts_atomic_t atmc;
+    ErtsThrQElement_t *ptr;
+} ErtsThrQPtr_t;
+
+struct ErtsThrQElement_t_ {
+    ErtsThrQPtr_t next;
+    union {
+	erts_atomic_t atmc;
+	void *ptr;
+    } data;
+};
+
+typedef struct {
+    ErtsThrQElement_t *start;
+    ErtsThrQElement_t *end;
+} ErtsThrQFinDeQ_t;
+
+typedef enum {
+    ERTS_THR_Q_CLEAN,
+    ERTS_THR_Q_NEED_THR_PRGR,
+    ERTS_THR_Q_DIRTY,
+} ErtsThrQCleanState_t;
+
+#ifdef USE_THREADS
+
+typedef struct {
+    ErtsThrQElement_t marker;
+    erts_atomic_t last;
+    erts_atomic_t um_refc[2];
+    erts_atomic32_t um_refc_ix;
+    ErtsThrQLive_t live;
+#ifdef ERTS_SMP
+    erts_atomic32_t thr_prgr_clean_scheduled;
+#endif
+    void *arg;
+    void (*notify)(void *);
+} ErtsThrQTail_t;
+
+struct ErtsThrQ_t_ {
+    /*
+     * This structure needs to be cache line aligned for best
+     * performance.
+     */
+    union {
+	/* Modified by threads enqueuing */
+	ErtsThrQTail_t data;
+	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsThrQTail_t))];
+    } tail;
+    /*
+     * Everything below this point is *only* accessed by the
+     * thread dequeuing.
+     */
+    struct {
+	ErtsThrQPtr_t head;
+	ErtsThrQLive_t live;
+	ErtsThrQElement_t *first;
+	ErtsThrQElement_t *unref_end;
+	int clean_reached_head_count;
+	struct {
+	    int automatic;
+	    ErtsThrQElement_t *start;
+	    ErtsThrQElement_t *end;
+	} deq_fini;
+	struct {
+#ifdef ERTS_SMP
+	    ErtsThrPrgrVal thr_progress;
+	    int thr_progress_reached;
+#endif
+	    int um_refc_ix;
+	    ErtsThrQElement_t *unref_end;
+	} next;
+	int used_marker;
+	void *arg;
+	void (*notify)(void *);
+    } head;
+    struct {
+	int finalizing;
+	ErtsThrQLive_t live;
+	void *blk;
+    } q;
+};
+
+#else /* !USE_THREADS */
+
+struct ErtsThrQ_t_ {
+    ErtsThrQInit_t init;
+    ErtsThrQElement_t *first;
+    ErtsThrQElement_t *last;
+    struct {
+	void *blk;
+    } q;
+};
+
+#endif
+
+void erts_thr_q_init(void);
+void erts_thr_q_initialize(ErtsThrQ_t *, ErtsThrQInit_t *);
+ErtsThrQCleanState_t erts_thr_q_finalize(ErtsThrQ_t *);
+ErtsThrQ_t *erts_thr_q_create(ErtsThrQInit_t *);
+ErtsThrQCleanState_t erts_thr_q_destroy(ErtsThrQ_t *);
+ErtsThrQCleanState_t erts_thr_q_clean(ErtsThrQ_t *);
+ErtsThrQCleanState_t erts_thr_q_inspect(ErtsThrQ_t *, int);
+ErtsThrQPrepEnQ_t *erts_thr_q_prepare_enqueue(ErtsThrQ_t *);
+void erts_thr_q_enqueue_prepared(ErtsThrQ_t *, void *, ErtsThrQPrepEnQ_t *);
+void erts_thr_q_enqueue(ErtsThrQ_t *, void *);
+void * erts_thr_q_dequeue(ErtsThrQ_t *);
+int erts_thr_q_get_finalize_dequeue_data(ErtsThrQ_t *,
+					 ErtsThrQFinDeQ_t *);
+void erts_thr_q_append_finalize_dequeue_data(ErtsThrQFinDeQ_t *,
+					     ErtsThrQFinDeQ_t *);
+int erts_thr_q_finalize_dequeue(ErtsThrQFinDeQ_t *);
+void erts_thr_q_finalize_dequeue_state_init(ErtsThrQFinDeQ_t *);
+
+#ifdef ERTS_SMP
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_q_need_thr_progress(ErtsThrQ_t *q);
+#endif
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+#ifdef ERTS_SMP
+ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_q_need_thr_progress(ErtsThrQ_t *q)
+{
+    return q->head.next.thr_progress;
+}
+#endif
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+#endif /* ERL_THR_QUEUE_H__ */
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index d635916dd8..ee47c98009 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -25,7 +25,243 @@
 #ifndef ERL_THREAD_H__
 #define ERL_THREAD_H__
 
+/*
+ * --- Documentation of atomics and memory barriers --------------------------
+ *
+ * The following explicit memory barriers exist:
+ *   
+ * - ERTS_THR_MEMORY_BARRIER
+ *      Full memory barrier. Orders both loads, and stores. No
+ *      load or store is allowed to be reordered over the
+ *      barrier.
+ * - ERTS_THR_WRITE_MEMORY_BARRIER
+ *      Write barrier. Orders *only* stores. These are not
+ *      allowed to be reordered over the barrier.
+ * - ERTS_THR_READ_MEMORY_BARRIER
+ *      Read barrier. Orders *only* loads. These are not
+ *      allowed to be reordered over the barrier.
+ * - ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
+ *      Data dependency read barrier. Orders *only* loads
+ *      according to data dependency across the barrier.
+ *
+ * If thread support has been disabled, these barriers will become no-ops.
+ *
+ * If the prefix ERTS_THR_ is replaced with ERTS_SMP_, the barriers will
+ * be enabled only in the SMP enabled runtime system.
+ *
+ * --- Atomic operations ---
+ *
+ * Atomics operations exist for 32-bit, word size, and double word size
+ * integers. Function prototypes are listed below.
+ *
+ * Each function implementing an atomic operation exist with the following
+ * implied memory barrier semantics. Not all combinations are useful, but
+ * all of them exist for simplicity. <B> is suffix in function name:
+ *
+ * - <B>  - Description
+ *
+ * - mb   - Full memory barrier. Orders both loads, and
+ *          stores before, and after the atomic operation.
+ *          No load or store is allowed to be reordered
+ *          over the atomic operation.
+ * - relb - Release barrier. Orders both loads, and
+ *          stores appearing *before* the atomic
+ *          operation. These are not allowed to be
+ *          reordered over the atomic operation.
+ * - acqb - Acquire barrier. Orders both loads, and stores
+ *          appearing *after* the atomic operation. These
+ *          are not allowed to be reordered over the
+ *          atomic operation.
+ * - wb   - Write barrier. Orders *only* stores. These are
+ *          not allowed to be reordered over the barrier.
+ *          Store in atomic operation is ordered *after*
+ *          the barrier.
+ * - rb   - Read barrier. Orders *only* loads. These are
+ *          not allowed to be reordered over the barrier.
+ *          Load in atomic operation is ordered *before*
+ *          the barrier. 
+ * - ddrb - Data dependency read barrier. Orders *only*
+ *          loads according to data dependency across the
+ *          barrier. Load in atomic operation is ordered
+ *          before the barrier.
+ *
+ * If thread support has been disabled, these functions are mapped to
+ * functions that performs the same operation, but aren't atomic
+ * and don't imply any memory barriers.
+ *
+ * If the atomic operations are prefixed with erts_smp_ instead of only
+ * erts_ the atomic operations will only be atomic in the SMP enabled
+ * runtime system, and will be mapped to non-atomic operations without
+ * memory barriers in the runtime system without SMP support. Atomic
+ * operations with erts_smp_ prefix should use the atomic types
+ * erts_smp_atomic32_t, erts_smp_atomic_t, and erts_smp_dw_atomic_t
+ * instead of erts_atomic32_t, erts_atomic_t, and erts_dw_atomic_t. The
+ * integer data types erts_aint32_t, erts_aint_t, and erts_dw_atomic_t
+ * are the same.
+ *
+ * --- 32-bit atomic operations ---
+ *
+ * The following 32-bit atomic operations exist. <B> should be
+ * replaced with a supported memory barrier (see above). Note
+ * that sizeof(erts_atomic32_t) might be larger than 4!
+ *
+ *
+ * Initialize (not necessarily the same as the set operation):
+ *   void erts_atomic32_init_<B>(erts_atomic32_t *atmc,
+ *                               erts_aint32_t val);
+ *
+ * Set value:
+ *   void erts_atomic32_set_<B>(erts_atomic32_t *atmc,
+ *                              erts_aint32_t val);
+ *
+ * Read; returns current value:
+ *   erts_aint32_t erts_atomic32_read_<B>(erts_atomic32_t *atmc);
+ *
+ * Increment; returns resulting value:
+ *   erts_aint32_t erts_atomic32_inc_read_<B>(erts_atomic32_t *atmc);
+ *
+ * Decrement; returns resulting value:
+ *   erts_aint32_t erts_atomic32_dec_read_<B>(erts_atomic32_t *atmc);
+ *
+ * Increment:
+ *   void erts_atomic32_inc_<B>(erts_atomic32_t *atmc);
+ *
+ * Decrement:
+ *   void erts_atomic32_dec_<B>(erts_atomic32_t *atmc);
+ *
+ * Add value; returns resulting value:
+ *   erts_aint32_t erts_atomic32_add_read_<B>(erts_atomic32_t *atmc,
+ *                                            erts_aint32_t val);
+ *
+ * Add value:
+ *   void erts_atomic32_add_<B>(erts_atomic32_t *atmc,
+ *                              erts_aint32_t val);
+ *
+ * Bitwise-or; returns previous value:
+ *   erts_aint32_t erts_atomic32_read_bor_<B>(erts_atomic32_t *atmc,
+ *                                            erts_aint32_t val);
+ *
+ * Bitwise-and; returns previous value:
+ *   erts_aint32_t erts_atomic32_read_band_<B>(erts_atomic32_t *atmc,
+ *                                             erts_aint32_t val);
+ *
+ * Exchange; returns previous value:
+ *   erts_aint32_t erts_atomic32_xchg_<B>(erts_atomic32_t *atmc,
+ *                                        erts_aint32_t val);
+ *
+ * Compare and exchange; returns previous or current value. If
+ * returned value equals 'exp' the value was changed to 'new';
+ * otherwise not:
+ *   erts_aint32_t erts_atomic32_cmpxchg_<B>(erts_atomic32_t *a,
+ *                                           erts_aint32_t new,
+ *                                           erts_aint32_t exp);
+ *
+ * --- Word size atomic operations ---
+ *
+ * The following word size (same size as sizeof(void *)) atomic
+ * operations exist. <B> should be replaced with a supported
+ * memory barrier (see above). Note that sizeof(erts_atomic_t)
+ * might be larger than sizeof(void *)!
+ *
+ * Initialize (not necessarily the same as the set operation):
+ *   void erts_atomic_init_<B>(erts_atomic_t *atmc,
+ *                             erts_aint_t val);
+ *
+ * Set value;
+ *   void erts_atomic_set_<B>(erts_atomic_t *atmc,
+ *                            erts_aint_t val);
+ *
+ * Read; returns current value:
+ *   erts_aint_t erts_atomic_read_<B>(erts_atomic_t *atmc);
+ *
+ * Increment; returns resulting value:
+ *   erts_aint_t erts_atomic_inc_read_<B>(erts_atomic_t *atmc);
+ *
+ * Decrement; returns resulting value:
+ *   erts_aint_t erts_atomic_dec_read_<B>(erts_atomic_t *atmc);
+ *
+ * Increment:
+ *   void erts_atomic_inc_<B>(erts_atomic_t *atmc);
+ *
+ * Decrement:
+ *   void erts_atomic_dec_<B>(erts_atomic_t *atmc);
+ *
+ * Add value; returns resulting value:
+ *   erts_aint_t erts_atomic_add_read_<B>(erts_atomic_t *atmc,
+ *                                        erts_aint_t val);
+ *
+ * Add value:
+ *   void erts_atomic_add_<B>(erts_atomic_t *atmc,
+ *                            erts_aint_t val);
+ *
+ * Bitwise-or; returns previous value:
+ *   erts_aint_t erts_atomic_read_bor_<B>(erts_atomic_t *atmc,
+ *                                        erts_aint_t val);
+ *
+ * Bitwise-and; returns previous value:
+ *   erts_aint_t erts_atomic_read_band_<B>(erts_atomic_t *atmc,
+ *                                         erts_aint_t val);
+ *
+ * Exchange; returns previous value:
+ *   erts_aint_t erts_atomic_xchg_<B>(erts_atomic_t *atmc,
+ *                                    erts_aint_t val);
+ *
+ * Compare and exchange; returns previous or current value. If
+ * returned value equals 'exp' the value was changed to 'new';
+ * otherwise not:
+ *   erts_aint_t erts_atomic_cmpxchg_<B>(erts_atomic_t *a,
+ *                                       erts_aint_t new,
+ *                                       erts_aint_t exp);
+ *
+ * --- Double word size atomic operations ---
+ *
+ * The following double word atomic operations exist. <B> should be
+ * replaced with a supported memory barrier (see above).
+ *
+ * Note that sizeof(erts_dw_atomic_t) usually is larger than
+ * 2*sizeof(void *)!
+ * 
+ * The erts_dw_aint_t data type should be accessed as if it was defined
+ * like this:
+ *
+ *     typedef struct {
+ *         erts_aint_t sint[2];
+ *     } erts_dw_aint_t;
+ *
+ *     Most significant word is 'sint[ERTS_DW_AINT_HIGH_WORD]' and least
+ *     significant word is 'sint[ERTS_DW_AINT_LOW_WORD]'.
+ *
+ *
+ * Initialize (not necessarily the same as the set operation):
+ *   void erts_dw_atomic_init_<B>(erts_dw_atomic_t *var,
+ *                           erts_dw_aint_t *val);
+ *
+ * Set; value is written into 'val':
+ *   void erts_dw_atomic_set_<B>(erts_dw_atomic_t *var,
+ *                               erts_dw_aint_t *val);
+ *
+ * Read; value is written into 'val':
+ *   void erts_dw_atomic_read_<B>(erts_dw_atomic_t *var,
+ *                                erts_dw_aint_t *val);
+ *
+ * Compare and exchange; returns a value != 0 if exchange was
+ * made; otherwise 0. 'new_val' contains new value to set. If 'exp_act'
+ * contains the same value as in memory when the function is called,
+ * 'new' is written to memory; otherwise, not. If exchange was not
+ * made, 'exp_act' contains the actual value in memory:
+ *   int erts_dw_atomic_cmpxchg_<B>(erts_dw_atomic_t *var,
+ *                                  erts_dw_aint_t *new,
+ *                                  erts_dw_aint_t *exp_act);
+ */
+
+#define ERTS_SPIN_BODY ETHR_SPIN_BODY
+
 #include "sys.h"
+
+typedef struct { SWord sint[2]; } erts_no_dw_atomic_t;
+typedef SWord erts_no_atomic_t;
+typedef Sint32 erts_no_atomic32_t;
+
 #ifdef USE_THREADS
 
 #define ETHR_TRY_INLINE_FUNCS
@@ -34,6 +270,21 @@
 #include "erl_lock_count.h"
 #include "erl_term.h"
 
+#if defined(__GLIBC__) && (__GLIBC__ << 16) + __GLIBC_MINOR__ < (2 << 16) + 4
+/*
+ * pthread_mutex_destroy() may return EBUSY when it shouldn't :( We have
+ * only seen this bug in glibc versions before 2.4. Note that condition
+ * variables, rwmutexes, spinlocks, and rwspinlocks also may be effected by
+ * this bug since these implementations may use mutexes internally.
+ */
+#  define ERTS_THR_HAVE_BUSY_DESTROY_BUG
+#endif
+
+#define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER
+#define ERTS_THR_WRITE_MEMORY_BARRIER ETHR_WRITE_MEMORY_BARRIER
+#define ERTS_THR_READ_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER
+#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER ETHR_READ_DEPEND_MEMORY_BARRIER
+
 #ifdef ERTS_ENABLE_LOCK_COUNT
 #define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__)
 #define erts_spin_lock(L) erts_spin_lock_x(L, __FILE__, __LINE__)
@@ -46,6 +297,7 @@
 #define ERTS_THR_OPTS_DEFAULT_INITER ETHR_THR_OPTS_DEFAULT_INITER
 typedef ethr_thr_opts erts_thr_opts_t;
 typedef ethr_init_data erts_thr_init_data_t;
+typedef ethr_late_init_data erts_thr_late_init_data_t;
 typedef ethr_tid erts_tid_t;
 
 /* mutex */
@@ -71,9 +323,30 @@ typedef struct {
     erts_lcnt_lock_t lcnt;
 #endif
 } erts_rwmtx_t;
+
+#define ERTS_MTX_OPT_DEFAULT_INITER ETHR_MUTEX_OPT_DEFAULT_INITER
+#define ERTS_CND_OPT_DEFAULT_INITER ETHR_COND_OPT_DEFAULT_INITER
+#define ERTS_RWMTX_OPT_DEFAULT_INITER ETHR_RWMUTEX_OPT_DEFAULT_INITER
+#define ERTS_RWMTX_TYPE_NORMAL ETHR_RWMUTEX_TYPE_NORMAL
+#define ERTS_RWMTX_TYPE_FREQUENT_READ ETHR_RWMUTEX_TYPE_FREQUENT_READ
+#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \
+  ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ
+#define ERTS_RWMTX_LONG_LIVED ETHR_RWMUTEX_LONG_LIVED
+#define ERTS_RWMTX_SHORT_LIVED ETHR_RWMUTEX_SHORT_LIVED
+#define ERTS_RWMTX_UNKNOWN_LIVED ETHR_RWMUTEX_UNKNOWN_LIVED
+typedef ethr_rwmutex_opt erts_rwmtx_opt_t;
+
 typedef ethr_tsd_key erts_tsd_key_t;
-typedef ethr_gate erts_gate_t;
-typedef ethr_atomic_t erts_atomic_t;
+typedef ethr_ts_event erts_tse_t;
+#define erts_dw_aint_t ethr_dw_sint_t
+#define erts_dw_atomic_t ethr_dw_atomic_t
+#define erts_aint_t ethr_sint_t
+#define erts_atomic_t ethr_atomic_t
+#define erts_aint32_t ethr_sint32_t 
+#define erts_atomic32_t ethr_atomic32_t
+
+#define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD
+#define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD
 
 /* spinlock */
 typedef struct {
@@ -97,41 +370,53 @@ typedef struct {
 #endif
 } erts_rwlock_t;
 
-typedef ethr_timeval erts_thr_timeval_t;
 __decl_noreturn void  __noreturn erts_thr_fatal_error(int, char *); 
                                  /* implemented in erl_init.c */
 
-#ifdef ERTS_ENABLE_LOCK_CHECK
-#define ERTS_REC_MTX_INITER \
-  {ETHR_REC_MUTEX_INITER, \
-   ERTS_LC_LOCK_INIT(-1,THE_NON_VALUE,ERTS_LC_FLG_LT_MUTEX)}
-#define ERTS_MTX_INITER \
-  {ETHR_MUTEX_INITER, \
-   ERTS_LC_LOCK_INIT(-1, THE_NON_VALUE, ERTS_LC_FLG_LT_MUTEX)}
-#else
-#define ERTS_REC_MTX_INITER		{ETHR_REC_MUTEX_INITER}
-#define ERTS_MTX_INITER			{ETHR_MUTEX_INITER}
-#endif
-#define ERTS_CND_INITER			ETHR_COND_INITER
 #define ERTS_THR_INIT_DATA_DEF_INITER	ETHR_INIT_DATA_DEFAULT_INITER
+#define ERTS_THR_LATE_INIT_DATA_DEF_INITER \
+                                        ETHR_LATE_INIT_DATA_DEFAULT_INITER
 
 #ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
 #  define ERTS_HAVE_REC_MTX_INIT	ETHR_HAVE_ETHR_REC_MUTEX_INIT
 #endif
 
-
 #else /* #ifdef USE_THREADS */
 
+#define ERTS_THR_MEMORY_BARRIER
+#define ERTS_THR_WRITE_MEMORY_BARRIER
+#define ERTS_THR_READ_MEMORY_BARRIER
+#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
+
 #define ERTS_THR_OPTS_DEFAULT_INITER 0
 typedef int erts_thr_opts_t;
 typedef int erts_thr_init_data_t;
+typedef int erts_thr_late_init_data_t;
 typedef int erts_tid_t;
 typedef int erts_mtx_t;
 typedef int erts_cnd_t;
+#define ERTS_RWMTX_OPT_DEFAULT_INITER {0}
+#define ERTS_RWMTX_TYPE_NORMAL 0
+#define ERTS_RWMTX_TYPE_FREQUENT_READ 0
+#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0
+#define ERTS_RWMTX_LONG_LIVED 0
+#define ERTS_RWMTX_SHORT_LIVED 0
+#define ERTS_RWMTX_UNKNOWN_LIVED 0
+typedef struct {
+    char type;
+    char lived;
+    int main_spincount;
+    int aux_spincount;
+} erts_rwmtx_opt_t;
 typedef int erts_rwmtx_t;
 typedef int erts_tsd_key_t;
-typedef int erts_gate_t;
-typedef long erts_atomic_t;
+typedef int erts_tse_t;
+#define erts_dw_aint_t erts_no_dw_atomic_t
+#define erts_dw_atomic_t erts_no_dw_atomic_t
+#define erts_aint_t SWord
+#define erts_atomic_t erts_no_atomic_t
+#define erts_aint32_t Sint32
+#define erts_atomic32_t erts_no_atomic32_t
 #if __GNUC__ > 2
 typedef struct { } erts_spinlock_t;
 typedef struct { } erts_rwlock_t;
@@ -139,12 +424,7 @@ typedef struct { } erts_rwlock_t;
 typedef struct { int gcc_is_buggy; } erts_spinlock_t;
 typedef struct { int gcc_is_buggy; } erts_rwlock_t;
 #endif
-typedef struct {
-    long tv_sec;
-    long tv_nsec;
-} erts_thr_timeval_t;
 
-#define ERTS_REC_MTX_INITER		0
 #define ERTS_MTX_INITER			0
 #define ERTS_CND_INITER			0
 #define ERTS_THR_INIT_DATA_DEF_INITER	0
@@ -153,7 +433,15 @@ typedef struct {
 
 #endif /* #ifdef USE_THREADS */
 
+#define ERTS_AINT_NULL ((erts_aint_t) NULL)
+
+#define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1)))
+#define ERTS_AINT_T_MIN ((((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1)))
+#define ERTS_AINT32_T_MAX (~(((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1)))
+#define ERTS_AINT32_T_MIN ((((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1)))
+
 ERTS_GLB_INLINE void erts_thr_init(erts_thr_init_data_t *id);
+ERTS_GLB_INLINE void erts_thr_late_init(erts_thr_late_init_data_t *id);
 ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *),
 				     void *arg, erts_thr_opts_t *opts);
 ERTS_GLB_INLINE void erts_thr_join(erts_tid_t tid, void **thr_res);
@@ -162,9 +450,6 @@ ERTS_GLB_INLINE void erts_thr_exit(void *res);
 ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void));
 ERTS_GLB_INLINE erts_tid_t erts_thr_self(void);
 ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y);
-#ifdef ERTS_HAVE_REC_MTX_INIT
-ERTS_GLB_INLINE void erts_rec_mtx_init(erts_mtx_t *mtx);
-#endif
 ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra);
 ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt);
 ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx,
@@ -173,8 +458,6 @@ ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx,
 ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name);
 ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name);
 ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx);
-ERTS_GLB_INLINE void erts_mtx_set_forksafe(erts_mtx_t *mtx);
-ERTS_GLB_INLINE void erts_mtx_unset_forksafe(erts_mtx_t *mtx);
 ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
 ERTS_GLB_INLINE void erts_mtx_lock_x(erts_mtx_t *mtx, char *file, unsigned int line);
@@ -188,9 +471,17 @@ ERTS_GLB_INLINE void erts_cnd_destroy(erts_cnd_t *cnd);
 ERTS_GLB_INLINE void erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx);
 ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd);
 ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd);
+ERTS_GLB_INLINE void erts_rwmtx_set_reader_group(int no);
+ERTS_GLB_INLINE void erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx,
+					   erts_rwmtx_opt_t *opt,
+					   char *name,
+					   Eterm extra);
 ERTS_GLB_INLINE void erts_rwmtx_init_x(erts_rwmtx_t *rwmtx,
 				       char *name,
 				       Eterm extra);
+ERTS_GLB_INLINE void erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx,
+					 erts_rwmtx_opt_t *opt,
+					 char *name);
 ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx,
 					 char *name);
 ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx);
@@ -207,23 +498,55 @@ ERTS_GLB_INLINE int erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx);
 ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx);
-ERTS_GLB_INLINE void erts_atomic_init(erts_atomic_t *var, long i);
-ERTS_GLB_INLINE void erts_atomic_set(erts_atomic_t *var, long i);
-ERTS_GLB_INLINE long erts_atomic_read(erts_atomic_t *var);
-ERTS_GLB_INLINE long erts_atomic_inctest(erts_atomic_t *incp);
-ERTS_GLB_INLINE long erts_atomic_dectest(erts_atomic_t *decp);
-ERTS_GLB_INLINE void erts_atomic_inc(erts_atomic_t *incp);
-ERTS_GLB_INLINE void erts_atomic_dec(erts_atomic_t *decp);
-ERTS_GLB_INLINE long erts_atomic_addtest(erts_atomic_t *addp,
-					 long i);
-ERTS_GLB_INLINE void erts_atomic_add(erts_atomic_t *addp, long i);
-ERTS_GLB_INLINE long erts_atomic_xchg(erts_atomic_t *xchgp,
-				      long new);
-ERTS_GLB_INLINE long erts_atomic_cmpxchg(erts_atomic_t *xchgp,
-					 long new,
-					 long expected);
-ERTS_GLB_INLINE long erts_atomic_bor(erts_atomic_t *var, long mask);
-ERTS_GLB_INLINE long erts_atomic_band(erts_atomic_t *var, long mask);
+
+ERTS_GLB_INLINE void erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val);
+ERTS_GLB_INLINE void erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val);
+ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var,
+					      erts_no_dw_atomic_t *val,
+					      erts_no_dw_atomic_t *old_val);
+ERTS_GLB_INLINE void erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read(erts_no_atomic_t *var);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_inc_read(erts_no_atomic_t *incp);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_dec_read(erts_no_atomic_t *decp);
+ERTS_GLB_INLINE void erts_no_atomic_inc(erts_no_atomic_t *incp);
+ERTS_GLB_INLINE void erts_no_atomic_dec(erts_no_atomic_t *decp);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_add_read(erts_no_atomic_t *addp,
+						    erts_aint_t i);
+ERTS_GLB_INLINE void erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bor(erts_no_atomic_t *var,
+						    erts_aint_t mask);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_band(erts_no_atomic_t *var,
+						     erts_aint_t mask);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_xchg(erts_no_atomic_t *xchgp,
+						erts_aint_t new);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp,
+						   erts_aint_t new,
+						   erts_aint_t expected);
+ERTS_GLB_INLINE void erts_no_atomic32_set(erts_no_atomic32_t *var,
+					  erts_aint32_t i);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read(erts_no_atomic32_t *var);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_inc_read(erts_no_atomic32_t *incp);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_dec_read(erts_no_atomic32_t *decp);
+ERTS_GLB_INLINE void erts_no_atomic32_inc(erts_no_atomic32_t *incp);
+ERTS_GLB_INLINE void erts_no_atomic32_dec(erts_no_atomic32_t *decp);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_add_read(erts_no_atomic32_t *addp,
+							erts_aint32_t i);
+ERTS_GLB_INLINE void erts_no_atomic32_add(erts_no_atomic32_t *addp,
+					  erts_aint32_t i);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bor(erts_no_atomic32_t *var,
+							erts_aint32_t mask);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_band(erts_no_atomic32_t *var,
+							 erts_aint32_t mask);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp,
+						    erts_aint32_t new);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp,
+						       erts_aint32_t new,
+						       erts_aint32_t expected);
+
+ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock,
+					      char *name,
+					      Eterm extra,
+					      Uint16 opt);
 ERTS_GLB_INLINE void erts_spinlock_init_x(erts_spinlock_t *lock,
 					  char *name,
 					  Eterm extra);
@@ -254,17 +577,20 @@ ERTS_GLB_INLINE void erts_write_lock(erts_rwlock_t *lock);
 ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock);
 ERTS_GLB_INLINE int erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock);
 ERTS_GLB_INLINE int erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock);
-ERTS_GLB_INLINE void erts_thr_time_now(erts_thr_timeval_t *time);
 ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp);
 ERTS_GLB_INLINE void erts_tsd_key_delete(erts_tsd_key_t key);
 ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value);
 ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key);
-ERTS_GLB_INLINE void erts_gate_init(erts_gate_t *gp);
-ERTS_GLB_INLINE void erts_gate_destroy(erts_gate_t *gp);
-ERTS_GLB_INLINE void erts_gate_close(erts_gate_t *gp);
-ERTS_GLB_INLINE void erts_gate_let_through(erts_gate_t *gp, unsigned no);
-ERTS_GLB_INLINE void erts_gate_wait(erts_gate_t *gp);
-ERTS_GLB_INLINE void erts_gate_swait(erts_gate_t *gp, int spincount);
+ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void);
+ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep);
+ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep);
+ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep);
+ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep);
+ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount);
+ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep);
+ERTS_GLB_INLINE void erts_thr_set_main_status(int, int);
+ERTS_GLB_INLINE int erts_thr_get_main_status(void);
+ERTS_GLB_INLINE void erts_thr_yield(void);
 
 #ifdef ETHR_HAVE_ETHR_SIG_FUNCS
 #define ERTS_THR_HAVE_SIG_FUNCS 1
@@ -273,6 +599,491 @@ ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set,
 ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
 #endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
 
+#ifdef USE_THREADS
+
+/*
+ * See "Documentation of atomics and memory barriers" at the top
+ * of this file for info on atomics.
+ */
+
+/* Double word size atomics */
+
+#define erts_dw_atomic_init_nob ethr_dw_atomic_init
+#define erts_dw_atomic_set_nob ethr_dw_atomic_set
+#define erts_dw_atomic_read_nob ethr_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_nob ethr_dw_atomic_cmpxchg
+
+#define erts_dw_atomic_init_mb ethr_dw_atomic_init_mb
+#define erts_dw_atomic_set_mb ethr_dw_atomic_set_mb
+#define erts_dw_atomic_read_mb ethr_dw_atomic_read_mb
+#define erts_dw_atomic_cmpxchg_mb ethr_dw_atomic_cmpxchg_mb
+
+#define erts_dw_atomic_init_acqb ethr_dw_atomic_init_acqb
+#define erts_dw_atomic_set_acqb ethr_dw_atomic_set_acqb
+#define erts_dw_atomic_read_acqb ethr_dw_atomic_read_acqb
+#define erts_dw_atomic_cmpxchg_acqb ethr_dw_atomic_cmpxchg_acqb
+
+#define erts_dw_atomic_init_relb ethr_dw_atomic_init_relb
+#define erts_dw_atomic_set_relb ethr_dw_atomic_set_relb
+#define erts_dw_atomic_read_relb ethr_dw_atomic_read_relb
+#define erts_dw_atomic_cmpxchg_relb ethr_dw_atomic_cmpxchg_relb
+
+#define erts_dw_atomic_init_ddrb ethr_dw_atomic_init_ddrb
+#define erts_dw_atomic_set_ddrb ethr_dw_atomic_set_ddrb
+#define erts_dw_atomic_read_ddrb ethr_dw_atomic_read_ddrb
+#define erts_dw_atomic_cmpxchg_ddrb ethr_dw_atomic_cmpxchg_ddrb
+
+#define erts_dw_atomic_init_rb ethr_dw_atomic_init_rb
+#define erts_dw_atomic_set_rb ethr_dw_atomic_set_rb
+#define erts_dw_atomic_read_rb ethr_dw_atomic_read_rb
+#define erts_dw_atomic_cmpxchg_rb ethr_dw_atomic_cmpxchg_rb
+
+#define erts_dw_atomic_init_wb ethr_dw_atomic_init_wb
+#define erts_dw_atomic_set_wb ethr_dw_atomic_set_wb
+#define erts_dw_atomic_read_wb ethr_dw_atomic_read_wb
+#define erts_dw_atomic_cmpxchg_wb ethr_dw_atomic_cmpxchg_wb
+
+/* Word size atomics */
+
+#define erts_atomic_init_nob ethr_atomic_init
+#define erts_atomic_set_nob ethr_atomic_set
+#define erts_atomic_read_nob ethr_atomic_read
+#define erts_atomic_inc_read_nob ethr_atomic_inc_read
+#define erts_atomic_dec_read_nob ethr_atomic_dec_read
+#define erts_atomic_inc_nob ethr_atomic_inc
+#define erts_atomic_dec_nob ethr_atomic_dec
+#define erts_atomic_add_read_nob ethr_atomic_add_read
+#define erts_atomic_add_nob ethr_atomic_add
+#define erts_atomic_read_bor_nob ethr_atomic_read_bor
+#define erts_atomic_read_band_nob ethr_atomic_read_band
+#define erts_atomic_xchg_nob ethr_atomic_xchg
+#define erts_atomic_cmpxchg_nob ethr_atomic_cmpxchg
+
+#define erts_atomic_init_mb ethr_atomic_init_mb
+#define erts_atomic_set_mb ethr_atomic_set_mb
+#define erts_atomic_read_mb ethr_atomic_read_mb
+#define erts_atomic_inc_read_mb ethr_atomic_inc_read_mb
+#define erts_atomic_dec_read_mb ethr_atomic_dec_read_mb
+#define erts_atomic_inc_mb ethr_atomic_inc_mb
+#define erts_atomic_dec_mb ethr_atomic_dec_mb
+#define erts_atomic_add_read_mb ethr_atomic_add_read_mb
+#define erts_atomic_add_mb ethr_atomic_add_mb
+#define erts_atomic_read_bor_mb ethr_atomic_read_bor_mb
+#define erts_atomic_read_band_mb ethr_atomic_read_band_mb
+#define erts_atomic_xchg_mb ethr_atomic_xchg_mb
+#define erts_atomic_cmpxchg_mb ethr_atomic_cmpxchg_mb
+
+#define erts_atomic_init_acqb ethr_atomic_init_acqb
+#define erts_atomic_set_acqb ethr_atomic_set_acqb
+#define erts_atomic_read_acqb ethr_atomic_read_acqb
+#define erts_atomic_inc_read_acqb ethr_atomic_inc_read_acqb
+#define erts_atomic_dec_read_acqb ethr_atomic_dec_read_acqb
+#define erts_atomic_inc_acqb ethr_atomic_inc_acqb
+#define erts_atomic_dec_acqb ethr_atomic_dec_acqb
+#define erts_atomic_add_read_acqb ethr_atomic_add_read_acqb
+#define erts_atomic_add_acqb ethr_atomic_add_acqb
+#define erts_atomic_read_bor_acqb ethr_atomic_read_bor_acqb
+#define erts_atomic_read_band_acqb ethr_atomic_read_band_acqb
+#define erts_atomic_xchg_acqb ethr_atomic_xchg_acqb
+#define erts_atomic_cmpxchg_acqb ethr_atomic_cmpxchg_acqb
+
+#define erts_atomic_init_relb ethr_atomic_init_relb
+#define erts_atomic_set_relb ethr_atomic_set_relb
+#define erts_atomic_read_relb ethr_atomic_read_relb
+#define erts_atomic_inc_read_relb ethr_atomic_inc_read_relb
+#define erts_atomic_dec_read_relb ethr_atomic_dec_read_relb
+#define erts_atomic_inc_relb ethr_atomic_inc_relb
+#define erts_atomic_dec_relb ethr_atomic_dec_relb
+#define erts_atomic_add_read_relb ethr_atomic_add_read_relb
+#define erts_atomic_add_relb ethr_atomic_add_relb
+#define erts_atomic_read_bor_relb ethr_atomic_read_bor_relb
+#define erts_atomic_read_band_relb ethr_atomic_read_band_relb
+#define erts_atomic_xchg_relb ethr_atomic_xchg_relb
+#define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb
+
+#define erts_atomic_init_ddrb ethr_atomic_init_ddrb
+#define erts_atomic_set_ddrb ethr_atomic_set_ddrb
+#define erts_atomic_read_ddrb ethr_atomic_read_ddrb
+#define erts_atomic_inc_read_ddrb ethr_atomic_inc_read_ddrb
+#define erts_atomic_dec_read_ddrb ethr_atomic_dec_read_ddrb
+#define erts_atomic_inc_ddrb ethr_atomic_inc_ddrb
+#define erts_atomic_dec_ddrb ethr_atomic_dec_ddrb
+#define erts_atomic_add_read_ddrb ethr_atomic_add_read_ddrb
+#define erts_atomic_add_ddrb ethr_atomic_add_ddrb
+#define erts_atomic_read_bor_ddrb ethr_atomic_read_bor_ddrb
+#define erts_atomic_read_band_ddrb ethr_atomic_read_band_ddrb
+#define erts_atomic_xchg_ddrb ethr_atomic_xchg_ddrb
+#define erts_atomic_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb
+
+#define erts_atomic_init_rb ethr_atomic_init_rb
+#define erts_atomic_set_rb ethr_atomic_set_rb
+#define erts_atomic_read_rb ethr_atomic_read_rb
+#define erts_atomic_inc_read_rb ethr_atomic_inc_read_rb
+#define erts_atomic_dec_read_rb ethr_atomic_dec_read_rb
+#define erts_atomic_inc_rb ethr_atomic_inc_rb
+#define erts_atomic_dec_rb ethr_atomic_dec_rb
+#define erts_atomic_add_read_rb ethr_atomic_add_read_rb
+#define erts_atomic_add_rb ethr_atomic_add_rb
+#define erts_atomic_read_bor_rb ethr_atomic_read_bor_rb
+#define erts_atomic_read_band_rb ethr_atomic_read_band_rb
+#define erts_atomic_xchg_rb ethr_atomic_xchg_rb
+#define erts_atomic_cmpxchg_rb ethr_atomic_cmpxchg_rb
+
+#define erts_atomic_init_wb ethr_atomic_init_wb
+#define erts_atomic_set_wb ethr_atomic_set_wb
+#define erts_atomic_read_wb ethr_atomic_read_wb
+#define erts_atomic_inc_read_wb ethr_atomic_inc_read_wb
+#define erts_atomic_dec_read_wb ethr_atomic_dec_read_wb
+#define erts_atomic_inc_wb ethr_atomic_inc_wb
+#define erts_atomic_dec_wb ethr_atomic_dec_wb
+#define erts_atomic_add_read_wb ethr_atomic_add_read_wb
+#define erts_atomic_add_wb ethr_atomic_add_wb
+#define erts_atomic_read_bor_wb ethr_atomic_read_bor_wb
+#define erts_atomic_read_band_wb ethr_atomic_read_band_wb
+#define erts_atomic_xchg_wb ethr_atomic_xchg_wb
+#define erts_atomic_cmpxchg_wb ethr_atomic_cmpxchg_wb
+
+/* 32-bit atomics */
+
+#define erts_atomic32_init_nob ethr_atomic32_init
+#define erts_atomic32_set_nob ethr_atomic32_set
+#define erts_atomic32_read_nob ethr_atomic32_read
+#define erts_atomic32_inc_read_nob ethr_atomic32_inc_read
+#define erts_atomic32_dec_read_nob ethr_atomic32_dec_read
+#define erts_atomic32_inc_nob ethr_atomic32_inc
+#define erts_atomic32_dec_nob ethr_atomic32_dec
+#define erts_atomic32_add_read_nob ethr_atomic32_add_read
+#define erts_atomic32_add_nob ethr_atomic32_add
+#define erts_atomic32_read_bor_nob ethr_atomic32_read_bor
+#define erts_atomic32_read_band_nob ethr_atomic32_read_band
+#define erts_atomic32_xchg_nob ethr_atomic32_xchg
+#define erts_atomic32_cmpxchg_nob ethr_atomic32_cmpxchg
+
+#define erts_atomic32_init_mb ethr_atomic32_init_mb
+#define erts_atomic32_set_mb ethr_atomic32_set_mb
+#define erts_atomic32_read_mb ethr_atomic32_read_mb
+#define erts_atomic32_inc_read_mb ethr_atomic32_inc_read_mb
+#define erts_atomic32_dec_read_mb ethr_atomic32_dec_read_mb
+#define erts_atomic32_inc_mb ethr_atomic32_inc_mb
+#define erts_atomic32_dec_mb ethr_atomic32_dec_mb
+#define erts_atomic32_add_read_mb ethr_atomic32_add_read_mb
+#define erts_atomic32_add_mb ethr_atomic32_add_mb
+#define erts_atomic32_read_bor_mb ethr_atomic32_read_bor_mb
+#define erts_atomic32_read_band_mb ethr_atomic32_read_band_mb
+#define erts_atomic32_xchg_mb ethr_atomic32_xchg_mb
+#define erts_atomic32_cmpxchg_mb ethr_atomic32_cmpxchg_mb
+
+#define erts_atomic32_init_acqb ethr_atomic32_init_acqb
+#define erts_atomic32_set_acqb ethr_atomic32_set_acqb
+#define erts_atomic32_read_acqb ethr_atomic32_read_acqb
+#define erts_atomic32_inc_read_acqb ethr_atomic32_inc_read_acqb
+#define erts_atomic32_dec_read_acqb ethr_atomic32_dec_read_acqb
+#define erts_atomic32_inc_acqb ethr_atomic32_inc_acqb
+#define erts_atomic32_dec_acqb ethr_atomic32_dec_acqb
+#define erts_atomic32_add_read_acqb ethr_atomic32_add_read_acqb
+#define erts_atomic32_add_acqb ethr_atomic32_add_acqb
+#define erts_atomic32_read_bor_acqb ethr_atomic32_read_bor_acqb
+#define erts_atomic32_read_band_acqb ethr_atomic32_read_band_acqb
+#define erts_atomic32_xchg_acqb ethr_atomic32_xchg_acqb
+#define erts_atomic32_cmpxchg_acqb ethr_atomic32_cmpxchg_acqb
+
+#define erts_atomic32_init_relb ethr_atomic32_init_relb
+#define erts_atomic32_set_relb ethr_atomic32_set_relb
+#define erts_atomic32_read_relb ethr_atomic32_read_relb
+#define erts_atomic32_inc_read_relb ethr_atomic32_inc_read_relb
+#define erts_atomic32_dec_read_relb ethr_atomic32_dec_read_relb
+#define erts_atomic32_inc_relb ethr_atomic32_inc_relb
+#define erts_atomic32_dec_relb ethr_atomic32_dec_relb
+#define erts_atomic32_add_read_relb ethr_atomic32_add_read_relb
+#define erts_atomic32_add_relb ethr_atomic32_add_relb
+#define erts_atomic32_read_bor_relb ethr_atomic32_read_bor_relb
+#define erts_atomic32_read_band_relb ethr_atomic32_read_band_relb
+#define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb
+#define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb
+
+#define erts_atomic32_init_ddrb ethr_atomic32_init_ddrb
+#define erts_atomic32_set_ddrb ethr_atomic32_set_ddrb
+#define erts_atomic32_read_ddrb ethr_atomic32_read_ddrb
+#define erts_atomic32_inc_read_ddrb ethr_atomic32_inc_read_ddrb
+#define erts_atomic32_dec_read_ddrb ethr_atomic32_dec_read_ddrb
+#define erts_atomic32_inc_ddrb ethr_atomic32_inc_ddrb
+#define erts_atomic32_dec_ddrb ethr_atomic32_dec_ddrb
+#define erts_atomic32_add_read_ddrb ethr_atomic32_add_read_ddrb
+#define erts_atomic32_add_ddrb ethr_atomic32_add_ddrb
+#define erts_atomic32_read_bor_ddrb ethr_atomic32_read_bor_ddrb
+#define erts_atomic32_read_band_ddrb ethr_atomic32_read_band_ddrb
+#define erts_atomic32_xchg_ddrb ethr_atomic32_xchg_ddrb
+#define erts_atomic32_cmpxchg_ddrb ethr_atomic32_cmpxchg_ddrb
+
+#define erts_atomic32_init_rb ethr_atomic32_init_rb
+#define erts_atomic32_set_rb ethr_atomic32_set_rb
+#define erts_atomic32_read_rb ethr_atomic32_read_rb
+#define erts_atomic32_inc_read_rb ethr_atomic32_inc_read_rb
+#define erts_atomic32_dec_read_rb ethr_atomic32_dec_read_rb
+#define erts_atomic32_inc_rb ethr_atomic32_inc_rb
+#define erts_atomic32_dec_rb ethr_atomic32_dec_rb
+#define erts_atomic32_add_read_rb ethr_atomic32_add_read_rb
+#define erts_atomic32_add_rb ethr_atomic32_add_rb
+#define erts_atomic32_read_bor_rb ethr_atomic32_read_bor_rb
+#define erts_atomic32_read_band_rb ethr_atomic32_read_band_rb
+#define erts_atomic32_xchg_rb ethr_atomic32_xchg_rb
+#define erts_atomic32_cmpxchg_rb ethr_atomic32_cmpxchg_rb
+
+#define erts_atomic32_init_wb ethr_atomic32_init_wb
+#define erts_atomic32_set_wb ethr_atomic32_set_wb
+#define erts_atomic32_read_wb ethr_atomic32_read_wb
+#define erts_atomic32_inc_read_wb ethr_atomic32_inc_read_wb
+#define erts_atomic32_dec_read_wb ethr_atomic32_dec_read_wb
+#define erts_atomic32_inc_wb ethr_atomic32_inc_wb
+#define erts_atomic32_dec_wb ethr_atomic32_dec_wb
+#define erts_atomic32_add_read_wb ethr_atomic32_add_read_wb
+#define erts_atomic32_add_wb ethr_atomic32_add_wb
+#define erts_atomic32_read_bor_wb ethr_atomic32_read_bor_wb
+#define erts_atomic32_read_band_wb ethr_atomic32_read_band_wb
+#define erts_atomic32_xchg_wb ethr_atomic32_xchg_wb
+#define erts_atomic32_cmpxchg_wb ethr_atomic32_cmpxchg_wb
+
+#else /* !USE_THREADS */
+
+/* Double word size atomics */
+
+#define erts_dw_atomic_init_nob erts_no_dw_atomic_set
+#define erts_dw_atomic_set_nob erts_no_dw_atomic_set
+#define erts_dw_atomic_read_nob erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg
+
+#define erts_dw_atomic_init_mb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_mb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_mb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg
+
+#define erts_dw_atomic_init_acqb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_acqb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_acqb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg
+
+#define erts_dw_atomic_init_relb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_relb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_relb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg
+
+#define erts_dw_atomic_init_ddrb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_ddrb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_ddrb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg
+
+#define erts_dw_atomic_init_rb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_rb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_rb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg
+
+#define erts_dw_atomic_init_wb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_wb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_wb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg
+
+/* Word size atomics */
+
+#define erts_atomic_init_nob erts_no_atomic_set
+#define erts_atomic_set_nob erts_no_atomic_set
+#define erts_atomic_read_nob erts_no_atomic_read
+#define erts_atomic_inc_read_nob erts_no_atomic_inc_read
+#define erts_atomic_dec_read_nob erts_no_atomic_dec_read
+#define erts_atomic_inc_nob erts_no_atomic_inc
+#define erts_atomic_dec_nob erts_no_atomic_dec
+#define erts_atomic_add_read_nob erts_no_atomic_add_read
+#define erts_atomic_add_nob erts_no_atomic_add
+#define erts_atomic_read_bor_nob erts_no_atomic_read_bor
+#define erts_atomic_read_band_nob erts_no_atomic_read_band
+#define erts_atomic_xchg_nob erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
+
+#define erts_atomic_init_mb erts_no_atomic_set
+#define erts_atomic_set_mb erts_no_atomic_set
+#define erts_atomic_read_mb erts_no_atomic_read
+#define erts_atomic_inc_read_mb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_mb erts_no_atomic_dec_read
+#define erts_atomic_inc_mb erts_no_atomic_inc
+#define erts_atomic_dec_mb erts_no_atomic_dec
+#define erts_atomic_add_read_mb erts_no_atomic_add_read
+#define erts_atomic_add_mb erts_no_atomic_add
+#define erts_atomic_read_bor_mb erts_no_atomic_read_bor
+#define erts_atomic_read_band_mb erts_no_atomic_read_band
+#define erts_atomic_xchg_mb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
+
+#define erts_atomic_init_acqb erts_no_atomic_set
+#define erts_atomic_set_acqb erts_no_atomic_set
+#define erts_atomic_read_acqb erts_no_atomic_read
+#define erts_atomic_inc_read_acqb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_acqb erts_no_atomic_dec_read
+#define erts_atomic_inc_acqb erts_no_atomic_inc
+#define erts_atomic_dec_acqb erts_no_atomic_dec
+#define erts_atomic_add_read_acqb erts_no_atomic_add_read
+#define erts_atomic_add_acqb erts_no_atomic_add
+#define erts_atomic_read_bor_acqb erts_no_atomic_read_bor
+#define erts_atomic_read_band_acqb erts_no_atomic_read_band
+#define erts_atomic_xchg_acqb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
+
+#define erts_atomic_init_relb erts_no_atomic_set
+#define erts_atomic_set_relb erts_no_atomic_set
+#define erts_atomic_read_relb erts_no_atomic_read
+#define erts_atomic_inc_read_relb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_relb erts_no_atomic_dec_read
+#define erts_atomic_inc_relb erts_no_atomic_inc
+#define erts_atomic_dec_relb erts_no_atomic_dec
+#define erts_atomic_add_read_relb erts_no_atomic_add_read
+#define erts_atomic_add_relb erts_no_atomic_add
+#define erts_atomic_read_bor_relb erts_no_atomic_read_bor
+#define erts_atomic_read_band_relb erts_no_atomic_read_band
+#define erts_atomic_xchg_relb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
+
+#define erts_atomic_init_ddrb erts_no_atomic_set
+#define erts_atomic_set_ddrb erts_no_atomic_set
+#define erts_atomic_read_ddrb erts_no_atomic_read
+#define erts_atomic_inc_read_ddrb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_ddrb erts_no_atomic_dec_read
+#define erts_atomic_inc_ddrb erts_no_atomic_inc
+#define erts_atomic_dec_ddrb erts_no_atomic_dec
+#define erts_atomic_add_read_ddrb erts_no_atomic_add_read
+#define erts_atomic_add_ddrb erts_no_atomic_add
+#define erts_atomic_read_bor_ddrb erts_no_atomic_read_bor
+#define erts_atomic_read_band_ddrb erts_no_atomic_read_band
+#define erts_atomic_xchg_ddrb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
+
+#define erts_atomic_init_rb erts_no_atomic_set
+#define erts_atomic_set_rb erts_no_atomic_set
+#define erts_atomic_read_rb erts_no_atomic_read
+#define erts_atomic_inc_read_rb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_rb erts_no_atomic_dec_read
+#define erts_atomic_inc_rb erts_no_atomic_inc
+#define erts_atomic_dec_rb erts_no_atomic_dec
+#define erts_atomic_add_read_rb erts_no_atomic_add_read
+#define erts_atomic_add_rb erts_no_atomic_add
+#define erts_atomic_read_bor_rb erts_no_atomic_read_bor
+#define erts_atomic_read_band_rb erts_no_atomic_read_band
+#define erts_atomic_xchg_rb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
+
+#define erts_atomic_init_wb erts_no_atomic_set
+#define erts_atomic_set_wb erts_no_atomic_set
+#define erts_atomic_read_wb erts_no_atomic_read
+#define erts_atomic_inc_read_wb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_wb erts_no_atomic_dec_read
+#define erts_atomic_inc_wb erts_no_atomic_inc
+#define erts_atomic_dec_wb erts_no_atomic_dec
+#define erts_atomic_add_read_wb erts_no_atomic_add_read
+#define erts_atomic_add_wb erts_no_atomic_add
+#define erts_atomic_read_bor_wb erts_no_atomic_read_bor
+#define erts_atomic_read_band_wb erts_no_atomic_read_band
+#define erts_atomic_xchg_wb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
+
+/* 32-bit atomics */
+
+#define erts_atomic32_init_nob erts_no_atomic32_set
+#define erts_atomic32_set_nob erts_no_atomic32_set
+#define erts_atomic32_read_nob erts_no_atomic32_read
+#define erts_atomic32_inc_read_nob erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_nob erts_no_atomic32_dec_read
+#define erts_atomic32_inc_nob erts_no_atomic32_inc
+#define erts_atomic32_dec_nob erts_no_atomic32_dec
+#define erts_atomic32_add_read_nob erts_no_atomic32_add_read
+#define erts_atomic32_add_nob erts_no_atomic32_add
+#define erts_atomic32_read_bor_nob erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_nob erts_no_atomic32_read_band
+#define erts_atomic32_xchg_nob erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
+
+#define erts_atomic32_init_mb erts_no_atomic32_set
+#define erts_atomic32_set_mb erts_no_atomic32_set
+#define erts_atomic32_read_mb erts_no_atomic32_read
+#define erts_atomic32_inc_read_mb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_mb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_mb erts_no_atomic32_inc
+#define erts_atomic32_dec_mb erts_no_atomic32_dec
+#define erts_atomic32_add_read_mb erts_no_atomic32_add_read
+#define erts_atomic32_add_mb erts_no_atomic32_add
+#define erts_atomic32_read_bor_mb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_mb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_mb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
+
+#define erts_atomic32_init_acqb erts_no_atomic32_set
+#define erts_atomic32_set_acqb erts_no_atomic32_set
+#define erts_atomic32_read_acqb erts_no_atomic32_read
+#define erts_atomic32_inc_read_acqb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_acqb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_acqb erts_no_atomic32_inc
+#define erts_atomic32_dec_acqb erts_no_atomic32_dec
+#define erts_atomic32_add_read_acqb erts_no_atomic32_add_read
+#define erts_atomic32_add_acqb erts_no_atomic32_add
+#define erts_atomic32_read_bor_acqb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_acqb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_acqb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
+
+#define erts_atomic32_init_relb erts_no_atomic32_set
+#define erts_atomic32_set_relb erts_no_atomic32_set
+#define erts_atomic32_read_relb erts_no_atomic32_read
+#define erts_atomic32_inc_read_relb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_relb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_relb erts_no_atomic32_inc
+#define erts_atomic32_dec_relb erts_no_atomic32_dec
+#define erts_atomic32_add_read_relb erts_no_atomic32_add_read
+#define erts_atomic32_add_relb erts_no_atomic32_add
+#define erts_atomic32_read_bor_relb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_relb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_relb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
+
+#define erts_atomic32_init_ddrb erts_no_atomic32_set
+#define erts_atomic32_set_ddrb erts_no_atomic32_set
+#define erts_atomic32_read_ddrb erts_no_atomic32_read
+#define erts_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_ddrb erts_no_atomic32_inc
+#define erts_atomic32_dec_ddrb erts_no_atomic32_dec
+#define erts_atomic32_add_read_ddrb erts_no_atomic32_add_read
+#define erts_atomic32_add_ddrb erts_no_atomic32_add
+#define erts_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_ddrb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_ddrb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
+
+#define erts_atomic32_init_rb erts_no_atomic32_set
+#define erts_atomic32_set_rb erts_no_atomic32_set
+#define erts_atomic32_read_rb erts_no_atomic32_read
+#define erts_atomic32_inc_read_rb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_rb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_rb erts_no_atomic32_inc
+#define erts_atomic32_dec_rb erts_no_atomic32_dec
+#define erts_atomic32_add_read_rb erts_no_atomic32_add_read
+#define erts_atomic32_add_rb erts_no_atomic32_add
+#define erts_atomic32_read_bor_rb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_rb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_rb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
+
+#define erts_atomic32_init_wb erts_no_atomic32_set
+#define erts_atomic32_set_wb erts_no_atomic32_set
+#define erts_atomic32_read_wb erts_no_atomic32_read
+#define erts_atomic32_inc_read_wb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_wb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_wb erts_no_atomic32_inc
+#define erts_atomic32_dec_wb erts_no_atomic32_dec
+#define erts_atomic32_add_read_wb erts_no_atomic32_add_read
+#define erts_atomic32_add_wb erts_no_atomic32_add
+#define erts_atomic32_read_bor_wb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_wb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_wb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
+
+#endif /* !USE_THREADS */
+
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
 ERTS_GLB_INLINE void
@@ -286,15 +1097,21 @@ erts_thr_init(erts_thr_init_data_t *id)
 }
 
 ERTS_GLB_INLINE void
+erts_thr_late_init(erts_thr_late_init_data_t *id)
+{
+#ifdef USE_THREADS
+    int res = ethr_late_init(id);
+    if (res)
+	erts_thr_fatal_error(res, "complete initialization of thread library");
+#endif
+}
+
+ERTS_GLB_INLINE void
 erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg,
 		erts_thr_opts_t *opts)
 {
 #ifdef USE_THREADS
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    int res = erts_lcnt_thr_create(tid, func, arg, opts);
-#else
     int res = ethr_thr_create(tid, func, arg, opts);
-#endif
     if (res)
 	erts_thr_fatal_error(res, "create thread");
 #endif
@@ -362,20 +1179,6 @@ erts_equal_tids(erts_tid_t x, erts_tid_t y)
 #endif
 }
 
-
-#ifdef ERTS_HAVE_REC_MTX_INIT
-ERTS_GLB_INLINE void
-erts_rec_mtx_init(erts_mtx_t *mtx)
-{
-#ifdef USE_THREADS
-    int res = ethr_rec_mutex_init(&mtx->mtx);
-    if (res)
-	erts_thr_fatal_error(res, "initialize recursive mutex");
-#endif
-}
-#endif
-
-
 ERTS_GLB_INLINE void
 erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra)
 {
@@ -422,9 +1225,7 @@ erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra)
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
 #endif
-    res = ethr_mutex_lock(&mtx->mtx);
-    if (res)
-	erts_thr_fatal_error(res, "lock mutex");
+    ethr_mutex_lock(&mtx->mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &mtx->lc);
 #endif
@@ -463,9 +1264,7 @@ erts_mtx_init_locked(erts_mtx_t *mtx, char *name)
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX);
 #endif
-    res = ethr_mutex_lock(&mtx->mtx);
-    if (res)
-	erts_thr_fatal_error(res, "lock mutex");
+    ethr_mutex_lock(&mtx->mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &mtx->lc);
 #endif
@@ -487,28 +1286,17 @@ erts_mtx_destroy(erts_mtx_t *mtx)
     erts_lcnt_destroy_lock(&mtx->lcnt);
 #endif
     res = ethr_mutex_destroy(&mtx->mtx);
-    if (res)
-	erts_thr_fatal_error(res, "destroy mutex");
+    if (res != 0) {
+#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
+	if (res == EBUSY) {
+	    char *warn = "Ignoring busy mutex destroy. "
+		"Most likely a bug in pthread implementation.";
+	    erts_send_warning_to_logger_str_nogl(warn);
+	}
+	else
 #endif
-}
-
-ERTS_GLB_INLINE void
-erts_mtx_set_forksafe(erts_mtx_t *mtx)
-{
-#ifdef USE_THREADS
-    int res = ethr_mutex_set_forksafe(&mtx->mtx);
-    if (res != 0 && res != ENOTSUP)
-	erts_thr_fatal_error(res, "set mutex forksafe");
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_mtx_unset_forksafe(erts_mtx_t *mtx)
-{
-#ifdef USE_THREADS
-    int res = ethr_mutex_unset_forksafe(&mtx->mtx);
-    if (res != 0 && res != ENOTSUP)
-	erts_thr_fatal_error(res, "unset mutex forksafe");
+	    erts_thr_fatal_error(res, "destroy mutex");
+    }
 #endif
 }
 
@@ -531,11 +1319,7 @@ erts_mtx_trylock(erts_mtx_t *mtx)
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_trylock(&mtx->lcnt, res);
-#endif
-
-    if (res != 0 && res != EBUSY)
-	erts_thr_fatal_error(res, "try lock mutex");
-    
+#endif    
     return res;
 #else
     return 0;
@@ -551,19 +1335,16 @@ erts_mtx_lock(erts_mtx_t *mtx)
 #endif
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_lock(&mtx->lc);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock(&mtx->lcnt);
 #endif
-    res = ethr_mutex_lock(&mtx->mtx);
+    ethr_mutex_lock(&mtx->mtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post_x(&mtx->lcnt, file, line);
 #endif
-    if (res)
-	erts_thr_fatal_error(res, "lock mutex");
 #endif
 }
 
@@ -571,16 +1352,13 @@ ERTS_GLB_INLINE void
 erts_mtx_unlock(erts_mtx_t *mtx)
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_unlock(&mtx->lc);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_unlock(&mtx->lcnt);
 #endif
-    res = ethr_mutex_unlock(&mtx->mtx);
-    if (res)
-	erts_thr_fatal_error(res, "unlock mutex");
+    ethr_mutex_unlock(&mtx->mtx);
 #endif
 }
 
@@ -613,8 +1391,17 @@ erts_cnd_destroy(erts_cnd_t *cnd)
 {
 #ifdef USE_THREADS
     int res = ethr_cond_destroy(cnd);
-    if (res)
-	erts_thr_fatal_error(res, "destroy condition variable");
+    if (res != 0) {
+#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
+	if (res == EBUSY) {
+	    char *warn = "Ignoring busy cond destroy. "
+		"Most likely a bug in pthread implementation.";
+	    erts_send_warning_to_logger_str_nogl(warn);
+	}
+	else
+#endif
+	    erts_thr_fatal_error(res, "destroy condition variable");
+    }
 #endif
 }
 
@@ -644,13 +1431,21 @@ erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx)
 #endif
 }
 
+/*
+ * IMPORTANT note about erts_cnd_signal() and erts_cnd_broadcast()
+ *
+ * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast'
+ * even though the associated mutex/mutexes isn't/aren't locked by the
+ * caller. Our implementation do not allow that in order to avoid a
+ * performance penalty. That is, all associated mutexes *need* to be
+ * locked by the caller of erts_cnd_signal()/erts_cnd_broadcast()!
+ */
+
 ERTS_GLB_INLINE void
 erts_cnd_signal(erts_cnd_t *cnd)
 {
 #ifdef USE_THREADS
-    int res = ethr_cond_signal(cnd);
-    if (res)
-	erts_thr_fatal_error(res, "signal on condition variable");
+    ethr_cond_signal(cnd);
 #endif
 }
 
@@ -659,19 +1454,34 @@ ERTS_GLB_INLINE void
 erts_cnd_broadcast(erts_cnd_t *cnd)
 {
 #ifdef USE_THREADS
-    int res = ethr_cond_broadcast(cnd);
-    if (res)
-	erts_thr_fatal_error(res, "broadcast on condition variable");
+    ethr_cond_broadcast(cnd);
 #endif
 }
 
 /* rwmutex */
 
 ERTS_GLB_INLINE void
-erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, char *name, Eterm extra)
+erts_rwmtx_set_reader_group(int no)
+{
+#ifdef USE_THREADS
+    int res;
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    erts_lc_check_no_locked_of_type(ERTS_LC_FLG_LT_RWMUTEX);
+#endif
+    res = ethr_rwmutex_set_reader_group(no);
+    if (res != 0)
+	erts_thr_fatal_error(res, "set reader group");
+#endif
+}
+
+ERTS_GLB_INLINE void
+erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx,
+		      erts_rwmtx_opt_t *opt,
+		      char *name,
+		      Eterm extra)
 {
 #ifdef USE_THREADS
-    int res = ethr_rwmutex_init(&rwmtx->rwmtx);
+    int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt);
     if (res != 0)
 	erts_thr_fatal_error(res, "initialize rwmutex");
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -684,10 +1494,20 @@ erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, char *name, Eterm extra)
 }
 
 ERTS_GLB_INLINE void
-erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name)
+erts_rwmtx_init_x(erts_rwmtx_t *rwmtx,
+		  char *name,
+		  Eterm extra)
+{
+    erts_rwmtx_init_opt_x(rwmtx, NULL, name, extra);
+}
+
+ERTS_GLB_INLINE void
+erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx,
+		    erts_rwmtx_opt_t *opt,
+		    char *name)
 {
 #ifdef USE_THREADS
-    int res = ethr_rwmutex_init(&rwmtx->rwmtx);
+    int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt);
     if (res != 0)
 	erts_thr_fatal_error(res, "initialize rwmutex");
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -700,6 +1520,12 @@ erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name)
 }
 
 ERTS_GLB_INLINE void
+erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name)
+{
+    erts_rwmtx_init_opt(rwmtx, NULL, name);
+}
+
+ERTS_GLB_INLINE void
 erts_rwmtx_destroy(erts_rwmtx_t *rwmtx)
 {
 #ifdef USE_THREADS
@@ -711,8 +1537,17 @@ erts_rwmtx_destroy(erts_rwmtx_t *rwmtx)
     erts_lcnt_destroy_lock(&rwmtx->lcnt);
 #endif
     res = ethr_rwmutex_destroy(&rwmtx->rwmtx);
-    if (res != 0)
-	erts_thr_fatal_error(res, "destroy rwmutex");
+    if (res != 0) {
+#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
+	if (res == EBUSY) {
+	    char *warn = "Ignoring busy rwmutex destroy. "
+		"Most likely a bug in pthread implementation.";
+	    erts_send_warning_to_logger_str_nogl(warn);
+	}
+	else
+#endif
+	    erts_thr_fatal_error(res, "destroy rwmutex");
+    }
 #endif
 }
 
@@ -736,9 +1571,6 @@ erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx)
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ);
 #endif
-
-    if (res != 0 && res != EBUSY)
-	erts_thr_fatal_error(res, "try read lock rwmutex");
     
     return res;
 #else
@@ -754,19 +1586,16 @@ erts_rwmtx_rlock(erts_rwmtx_t *rwmtx)
 #endif
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ);
 #endif
-    res = ethr_rwmutex_rlock(&rwmtx->rwmtx);
+    ethr_rwmutex_rlock(&rwmtx->rwmtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line);
 #endif
-    if (res != 0)
-	erts_thr_fatal_error(res, "read lock rwmutex");
 #endif
 }
 
@@ -774,16 +1603,13 @@ ERTS_GLB_INLINE void
 erts_rwmtx_runlock(erts_rwmtx_t *rwmtx)
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ);
 #endif
-    res = ethr_rwmutex_runlock(&rwmtx->rwmtx);
-    if (res != 0)
-	erts_thr_fatal_error(res, "read unlock rwmutex");
+    ethr_rwmutex_runlock(&rwmtx->rwmtx);
 #endif
 }
 
@@ -808,9 +1634,6 @@ erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx)
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ_WRITE);
 #endif
-
-    if (res != 0 && res != EBUSY)
-	erts_thr_fatal_error(res, "try write lock rwmutex");
     
     return res;
 #else
@@ -826,19 +1649,16 @@ erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx)
 #endif
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE);
 #endif
-    res = ethr_rwmutex_rwlock(&rwmtx->rwmtx);
+    ethr_rwmutex_rwlock(&rwmtx->rwmtx);
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line);
 #endif
-    if (res != 0)
-	erts_thr_fatal_error(res, "write lock rwmutex");
 #endif
 }
 
@@ -846,16 +1666,13 @@ ERTS_GLB_INLINE void
 erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx)
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE);
 #endif
-    res = ethr_rwmutex_rwunlock(&rwmtx->rwmtx);
-    if (res != 0)
-	erts_thr_fatal_error(res, "write unlock rwmutex");
+    ethr_rwmutex_rwunlock(&rwmtx->rwmtx);
 #endif
 }
 
@@ -913,181 +1730,205 @@ erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx)
 #endif
 }
 
+/* No atomic ops */
+
 ERTS_GLB_INLINE void
-erts_atomic_init(erts_atomic_t *var, long i)
+erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val)
 {
-#ifdef USE_THREADS
-    int res = ethr_atomic_init(var, i);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic init");
-#else
-    *var = i;
-#endif
+    var->sint[0] = val->sint[0];
+    var->sint[1] = val->sint[1];
 }
 
 ERTS_GLB_INLINE void
-erts_atomic_set(erts_atomic_t *var, long i)
+erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val)
+{
+    val->sint[0] = var->sint[0];
+    val->sint[1] = var->sint[1];
+}
+
+ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var,
+					      erts_no_dw_atomic_t *new_val,
+					      erts_no_dw_atomic_t *old_val)
+{
+    if (var->sint[0] != old_val->sint[0] || var->sint[1] != old_val->sint[1]) {
+	erts_no_dw_atomic_read(var, old_val);
+	return 0;
+    }
+    else {
+	erts_no_dw_atomic_set(var, new_val);
+	return !0;
+    }
+}
+
+ERTS_GLB_INLINE void
+erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i)
 {
-#ifdef USE_THREADS
-    int res = ethr_atomic_set(var, i);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic set");
-#else
     *var = i;
-#endif
 }
 
-ERTS_GLB_INLINE long
-erts_atomic_read(erts_atomic_t *var)
+ERTS_GLB_INLINE erts_aint_t
+erts_no_atomic_read(erts_no_atomic_t *var)
 {
-#ifdef USE_THREADS
-    long i;
-    int res = ethr_atomic_read(var, &i);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic read");
-    return i;
-#else
     return *var;
-#endif
 }
 
-ERTS_GLB_INLINE long
-erts_atomic_inctest(erts_atomic_t *incp)
+ERTS_GLB_INLINE erts_aint_t
+erts_no_atomic_inc_read(erts_no_atomic_t *incp)
 {
-#ifdef USE_THREADS
-    long test;
-    int res = ethr_atomic_inctest(incp, &test);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic increment and test");
-    return test;
-#else
     return ++(*incp);
-#endif
 }
 
-ERTS_GLB_INLINE long
-erts_atomic_dectest(erts_atomic_t *decp)
+ERTS_GLB_INLINE erts_aint_t
+erts_no_atomic_dec_read(erts_no_atomic_t *decp)
 {
-#ifdef USE_THREADS
-    long test;
-    int res = ethr_atomic_dectest(decp, &test);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic decrement and test");
-    return test;
-#else
     return --(*decp);
-#endif
 }
 
 ERTS_GLB_INLINE void
-erts_atomic_inc(erts_atomic_t *incp)
+erts_no_atomic_inc(erts_no_atomic_t *incp)
 {
-#ifdef USE_THREADS
-    int res = ethr_atomic_inc(incp);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic increment");
-#else
     ++(*incp);
-#endif
 }
 
 ERTS_GLB_INLINE void
-erts_atomic_dec(erts_atomic_t *decp)
+erts_no_atomic_dec(erts_no_atomic_t *decp)
 {
-#ifdef USE_THREADS
-    int res = ethr_atomic_dec(decp);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic decrement");
-#else
     --(*decp);
-#endif
 }
 
-ERTS_GLB_INLINE long
-erts_atomic_addtest(erts_atomic_t *addp, long i)
+ERTS_GLB_INLINE erts_aint_t
+erts_no_atomic_add_read(erts_no_atomic_t *addp, erts_aint_t i)
 {
-#ifdef USE_THREADS
-    long test;
-    int res = ethr_atomic_addtest(addp, i, &test);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic addition and test");
-    return test;
-#else
     return *addp += i;
-#endif
 }
 
 ERTS_GLB_INLINE void
-erts_atomic_add(erts_atomic_t *addp, long i)
+erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i)
 {
-#ifdef USE_THREADS
-    int res = ethr_atomic_add(addp, i);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic addition");
-#else
     *addp += i;
-#endif
 }
 
-ERTS_GLB_INLINE long
-erts_atomic_xchg(erts_atomic_t *xchgp, long new)
+ERTS_GLB_INLINE erts_aint_t
+erts_no_atomic_read_bor(erts_no_atomic_t *var, erts_aint_t mask)
 {
-    long old;
-#ifdef USE_THREADS
-    int res = ethr_atomic_xchg(xchgp, new, &old);
-    if (res)
-	erts_thr_fatal_error(res, "perform atomic exchange");
-#else
-    old = *xchgp;
-    *xchgp = new;
-#endif
+    erts_aint_t old;
+    old = *var;
+    *var |= mask;
     return old;
 }
 
-ERTS_GLB_INLINE long
-erts_atomic_cmpxchg(erts_atomic_t *xchgp, long new, long expected)
+ERTS_GLB_INLINE erts_aint_t
+erts_no_atomic_read_band(erts_no_atomic_t *var, erts_aint_t mask)
 {
-#ifdef USE_THREADS
-    long old;
-    int res = ethr_atomic_cmpxchg(xchgp, new, expected, &old);
-    if (ERTS_UNLIKELY(res != 0))
-	erts_thr_fatal_error(res, "perform atomic exchange");
+    erts_aint_t old;
+    old = *var;
+    *var &= mask;
     return old;
-#else
-    long old = *xchgp;
+}
+
+ERTS_GLB_INLINE erts_aint_t
+erts_no_atomic_xchg(erts_no_atomic_t *xchgp, erts_aint_t new)
+{
+    erts_aint_t old = *xchgp;
+    *xchgp = new;
+    return old;
+}
+
+ERTS_GLB_INLINE erts_aint_t
+erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp,
+		       erts_aint_t new,
+		       erts_aint_t expected)
+{
+    erts_aint_t old = *xchgp;
     if (old == expected)
         *xchgp = new;
     return old;
-#endif
 }
 
-ERTS_GLB_INLINE long
-erts_atomic_bor(erts_atomic_t *var, long mask)
+/* atomic32 */
+
+ERTS_GLB_INLINE void
+erts_no_atomic32_set(erts_no_atomic32_t *var, erts_aint32_t i)
 {
-    long old;
-#ifdef USE_THREADS
-    int res = ethr_atomic_or_old(var, mask, &old);
-    if (res != 0)
-	erts_thr_fatal_error(res, "perform atomic bitwise or");
-#else
+    *var = i;
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_no_atomic32_read(erts_no_atomic32_t *var)
+{
+    return *var;
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_no_atomic32_inc_read(erts_no_atomic32_t *incp)
+{
+    return ++(*incp);
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_no_atomic32_dec_read(erts_no_atomic32_t *decp)
+{
+    return --(*decp);
+}
+
+ERTS_GLB_INLINE void
+erts_no_atomic32_inc(erts_no_atomic32_t *incp)
+{
+    ++(*incp);
+}
+
+ERTS_GLB_INLINE void
+erts_no_atomic32_dec(erts_no_atomic32_t *decp)
+{
+    --(*decp);
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_no_atomic32_add_read(erts_no_atomic32_t *addp, erts_aint32_t i)
+{
+    return *addp += i;
+}
+
+ERTS_GLB_INLINE void
+erts_no_atomic32_add(erts_no_atomic32_t *addp, erts_aint32_t i)
+{
+    *addp += i;
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_no_atomic32_read_bor(erts_no_atomic32_t *var, erts_aint32_t mask)
+{
+    erts_aint32_t old;
     old = *var;
     *var |= mask;
-#endif
     return old;
 }
 
-ERTS_GLB_INLINE long
-erts_atomic_band(erts_atomic_t *var, long mask)
+ERTS_GLB_INLINE erts_aint32_t
+erts_no_atomic32_read_band(erts_no_atomic32_t *var, erts_aint32_t mask)
 {
-    long old;
-#ifdef USE_THREADS
-    int res = ethr_atomic_and_old(var, mask, &old);
-    if (res != 0)
-	erts_thr_fatal_error(res, "perform atomic bitwise and");
-#else
+    erts_aint32_t old;
     old = *var;
     *var &= mask;
-#endif
+    return old;
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp, erts_aint32_t new)
+{
+    erts_aint32_t old = *xchgp;
+    *xchgp = new;
+    return old;
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp,
+			 erts_aint32_t new,
+			 erts_aint32_t expected)
+{
+    erts_aint32_t old = *xchgp;
+    if (old == expected)
+        *xchgp = new;
     return old;
 }
 
@@ -1112,6 +1953,26 @@ erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra)
 }
 
 ERTS_GLB_INLINE void
+erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, Eterm extra,
+			 Uint16 opt)
+{
+#ifdef USE_THREADS
+    int res = ethr_spinlock_init(&lock->slck);
+    if (res)
+	erts_thr_fatal_error(res, "init spinlock");
+#ifdef ERTS_ENABLE_LOCK_CHECK
+    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra);
+#endif
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK|opt, extra);
+#endif
+#else
+    (void)lock;
+#endif
+}
+
+
+ERTS_GLB_INLINE void
 erts_spinlock_init(erts_spinlock_t *lock, char *name)
 {
 #ifdef USE_THREADS
@@ -1141,8 +2002,17 @@ erts_spinlock_destroy(erts_spinlock_t *lock)
     erts_lcnt_destroy_lock(&lock->lcnt);
 #endif
     res = ethr_spinlock_destroy(&lock->slck);
-    if (res)
-	erts_thr_fatal_error(res, "destroy spinlock");
+    if (res != 0) {
+#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
+	if (res == EBUSY) {
+	    char *warn = "Ignoring busy spinlock destroy. "
+		"Most likely a bug in pthread implementation.";
+	    erts_send_warning_to_logger_str_nogl(warn);
+	}
+	else
+#endif
+	    erts_thr_fatal_error(res, "destroy rwlock");
+    }
 #else
     (void)lock;
 #endif
@@ -1152,16 +2022,13 @@ ERTS_GLB_INLINE void
 erts_spin_unlock(erts_spinlock_t *lock)
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_unlock(&lock->lc);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_unlock(&lock->lcnt);
 #endif
-    res = ethr_spin_unlock(&lock->slck);
-    if (res)
-	erts_thr_fatal_error(res, "release spin lock");
+    ethr_spin_unlock(&lock->slck);
 #else
     (void)lock;
 #endif
@@ -1175,19 +2042,16 @@ erts_spin_lock(erts_spinlock_t *lock)
 #endif
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_lock(&lock->lc);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock(&lock->lcnt);
 #endif
-    res = ethr_spin_lock(&lock->slck);
+    ethr_spin_lock(&lock->slck);
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post_x(&lock->lcnt, file, line);
 #endif
-    if (res)
-	erts_thr_fatal_error(res, "take spin lock");
 #else
     (void)lock;
 #endif
@@ -1257,8 +2121,17 @@ erts_rwlock_destroy(erts_rwlock_t *lock)
     erts_lcnt_destroy_lock(&lock->lcnt);
 #endif
     res = ethr_rwlock_destroy(&lock->rwlck);
-    if (res)
-	erts_thr_fatal_error(res, "destroy rwlock");
+    if (res != 0) {
+#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
+	if (res == EBUSY) {
+	    char *warn = "Ignoring busy rwlock destroy. "
+		"Most likely a bug in pthread implementation.";
+	    erts_send_warning_to_logger_str_nogl(warn);
+	}
+	else
+#endif
+	    erts_thr_fatal_error(res, "destroy rwlock");
+    }
 #else
     (void)lock;
 #endif
@@ -1268,16 +2141,13 @@ ERTS_GLB_INLINE void
 erts_read_unlock(erts_rwlock_t *lock)
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ);
 #endif
-    res = ethr_read_unlock(&lock->rwlck);
-    if (res)
-	erts_thr_fatal_error(res, "release read lock");
+    ethr_read_unlock(&lock->rwlck);
 #else
     (void)lock;
 #endif
@@ -1291,19 +2161,16 @@ erts_read_lock(erts_rwlock_t *lock)
 #endif
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ);
 #endif
-    res = ethr_read_lock(&lock->rwlck);
+    ethr_read_lock(&lock->rwlck);
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post_x(&lock->lcnt, file, line);
 #endif
-    if (res)
-	erts_thr_fatal_error(res, "take read lock");
 #else
     (void)lock;
 #endif
@@ -1313,16 +2180,13 @@ ERTS_GLB_INLINE void
 erts_write_unlock(erts_rwlock_t *lock)
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE);
 #endif
-    res = ethr_write_unlock(&lock->rwlck);
-    if (res)
-	erts_thr_fatal_error(res, "release write lock");
+    ethr_write_unlock(&lock->rwlck);
 #else
     (void)lock;
 #endif
@@ -1336,19 +2200,16 @@ erts_write_lock(erts_rwlock_t *lock)
 #endif
 {
 #ifdef USE_THREADS
-    int res;
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE);
 #endif
-    res = ethr_write_lock(&lock->rwlck);
+    ethr_write_lock(&lock->rwlck);
 #ifdef ERTS_ENABLE_LOCK_COUNT
     erts_lcnt_lock_post_x(&lock->lcnt, file, line);
 #endif
-    if (res)
-	erts_thr_fatal_error(res, "take write lock");
 #else
     (void)lock;
 #endif
@@ -1383,16 +2244,6 @@ erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock)
 }
 
 ERTS_GLB_INLINE void
-erts_thr_time_now(erts_thr_timeval_t *time)
-{
-#ifdef USE_THREADS
-    int res = ethr_time_now(time);
-    if (res)
-	erts_thr_fatal_error(res, "get current time");
-#endif
-}
-
-ERTS_GLB_INLINE void
 erts_tsd_key_create(erts_tsd_key_t *keyp)
 {
 #ifdef USE_THREADS
@@ -1432,66 +2283,95 @@ erts_tsd_get(erts_tsd_key_t key)
 #endif
 }
 
-ERTS_GLB_INLINE void
-erts_gate_init(erts_gate_t *gp)
+ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void)
 {
 #ifdef USE_THREADS
-    int res = ethr_gate_init((ethr_gate *) gp);
-    if (res != 0)
-	erts_thr_fatal_error(res, "initialize gate");
+    return (erts_tse_t *) ethr_get_ts_event();
+#else
+    return (erts_tse_t *) NULL;
 #endif
 }
 
-ERTS_GLB_INLINE void
-erts_gate_destroy(erts_gate_t *gp)
+ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep)
 {
 #ifdef USE_THREADS
-    int res = ethr_gate_destroy((ethr_gate *) gp);
-    if (res != 0)
-	erts_thr_fatal_error(res, "destroy gate");
+    ethr_leave_ts_event(ep);
 #endif
 }
 
-ERTS_GLB_INLINE void
-erts_gate_close(erts_gate_t *gp)
+ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep)
 {
 #ifdef USE_THREADS
-    int res = ethr_gate_close((ethr_gate *) gp);
-    if (res != 0)
-	erts_thr_fatal_error(res, "close gate");
+    ethr_event_set(&((ethr_ts_event *) ep)->event);
 #endif
 }
 
-ERTS_GLB_INLINE void
-erts_gate_let_through(erts_gate_t *gp, unsigned no)
+ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep)
 {
 #ifdef USE_THREADS
-    int res = ethr_gate_let_through((ethr_gate *) gp, no);
-    if (res != 0)
-	erts_thr_fatal_error(res, "let through gate");
+    ethr_event_reset(&((ethr_ts_event *) ep)->event);
 #endif
 }
 
-ERTS_GLB_INLINE void
-erts_gate_wait(erts_gate_t *gp)
+ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep)
+{
+#ifdef USE_THREADS
+    return ethr_event_wait(&((ethr_ts_event *) ep)->event);
+#else
+    return ENOTSUP;
+#endif
+}
+
+ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount)
+{
+#ifdef USE_THREADS
+    return ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount);
+#else
+    return ENOTSUP;
+#endif
+}
+
+ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep)
 {
 #ifdef USE_THREADS
-    int res = ethr_gate_wait((ethr_gate *) gp);
+    return (ep->iflgs & ETHR_TS_EV_TMP) == ETHR_TS_EV_TMP;
+#else
+    return 0;
+#endif
+}
+
+ERTS_GLB_INLINE void erts_thr_set_main_status(int on, int no)
+{
+#ifdef USE_THREADS
+    int res = ethr_set_main_thr_status(on, no);
     if (res != 0)
-	erts_thr_fatal_error(res, "wait on gate");
+	erts_thr_fatal_error(res, "set thread main status");
 #endif
 }
 
-ERTS_GLB_INLINE void
-erts_gate_swait(erts_gate_t *gp, int spincount)
+ERTS_GLB_INLINE int erts_thr_get_main_status(void)
 {
 #ifdef USE_THREADS
-    int res = ethr_gate_swait((ethr_gate *) gp, spincount);
+    int main_status;
+    int res = ethr_get_main_thr_status(&main_status);
     if (res != 0)
-	erts_thr_fatal_error(res, "swait on gate");
+	erts_thr_fatal_error(res, "get thread main status");
+    return main_status;
+#else
+    return 1;
 #endif
 }
 
+ERTS_GLB_INLINE void erts_thr_yield(void)
+{
+#ifdef USE_THREADS
+    int res = ETHR_YIELD();
+    if (res != 0)
+	erts_thr_fatal_error(res, "yield");
+#endif    
+}
+
+
 #ifdef ETHR_HAVE_ETHR_SIG_FUNCS
 
 ERTS_GLB_INLINE void
@@ -1522,3 +2402,37 @@ erts_thr_sigwait(const sigset_t *set, int *sig)
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
 #endif /* #ifndef ERL_THREAD_H__ */
+
+#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS
+
+/* Deprecated functions to replace */
+
+#undef erts_atomic_init
+#undef erts_atomic_set
+#undef erts_atomic_read
+#undef erts_atomic_inctest
+#undef erts_atomic_dectest
+#undef erts_atomic_inc
+#undef erts_atomic_dec
+#undef erts_atomic_addtest
+#undef erts_atomic_add
+#undef erts_atomic_xchg
+#undef erts_atomic_cmpxchg
+#undef erts_atomic_bor
+#undef erts_atomic_band
+
+#undef erts_atomic32_init
+#undef erts_atomic32_set
+#undef erts_atomic32_read
+#undef erts_atomic32_inctest
+#undef erts_atomic32_dectest
+#undef erts_atomic32_inc
+#undef erts_atomic32_dec
+#undef erts_atomic32_addtest
+#undef erts_atomic32_add
+#undef erts_atomic32_xchg
+#undef erts_atomic32_cmpxchg
+#undef erts_atomic32_bor
+#undef erts_atomic32_band
+
+#endif
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 6f6b971d34..6c6e193818 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -20,11 +20,19 @@
 #ifndef ERL_TIME_H__
 #define ERL_TIME_H__
 
+#define ERTS_SHORT_TIME_T_MAX ERTS_AINT32_T_MAX
+#define ERTS_SHORT_TIME_T_MIN ERTS_AINT32_T_MIN
+typedef erts_aint32_t erts_short_time_t;
+
+extern erts_smp_atomic32_t do_time;	/* set at clock interrupt */
+extern SysTimeval erts_first_emu_time;
+
 /*
 ** Timer entry:
 */
 typedef struct erl_timer {
     struct erl_timer* next;	/* next entry tiw slot or chain */
+    struct erl_timer* prev;	/* prev entry tiw slot or chain */
     Uint slot;			/* slot in timer wheel */
     Uint count;			/* number of loops remaining */
     int    active;		/* 1=activated, 0=deactivated */
@@ -39,7 +47,6 @@ typedef void (*ErlTimeoutProc)(void*);
 typedef void (*ErlCancelProc)(void*);
 
 #ifdef ERTS_SMP
-
 /*
  * Process and port timer
  */
@@ -61,7 +68,76 @@ void erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref,
 			    ErlTimeoutProc timeout_func,
 			    Uint timeout);
 void erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer);
+#endif
+
+/* timer-wheel api */
 
+void erts_init_time(void);
+void erts_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint);
+void erts_cancel_timer(ErlTimer*);
+void erts_bump_timer(erts_short_time_t);
+Uint erts_timer_wheel_memory_size(void);
+Uint erts_time_left(ErlTimer *);
+erts_short_time_t erts_next_time(void);
+
+#ifdef DEBUG
+void erts_p_slpq(void);
 #endif
 
+ERTS_GLB_INLINE erts_short_time_t erts_do_time_read_and_reset(void);
+ERTS_GLB_INLINE void erts_do_time_add(erts_short_time_t);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE erts_short_time_t erts_do_time_read_and_reset(void)
+{
+    erts_short_time_t time = erts_smp_atomic32_xchg_acqb(&do_time, 0);
+    if (time < 0)
+	erl_exit(ERTS_ABORT_EXIT, "Internal time management error\n");
+    return time;
+}
+
+ERTS_GLB_INLINE void erts_do_time_add(erts_short_time_t elapsed)
+{
+    erts_smp_atomic32_add_relb(&do_time, elapsed);
+}
+
+#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+
+/* time_sup */
+
+#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME))
+#  ifndef HAVE_ERTS_NOW_CPU
+#    define HAVE_ERTS_NOW_CPU
+#    ifdef HAVE_GETHRVTIME
+#      define erts_start_now_cpu() sys_start_hrvtime()
+#      define erts_stop_now_cpu()  sys_stop_hrvtime()
+#    endif
+#  endif
+void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec);
 #endif
+
+void erts_get_timeval(SysTimeval *tv);
+erts_time_t erts_get_time(void);
+void erts_get_emu_time(SysTimeval *);
+
+ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE int
+erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p)
+{
+    if (t1p->tv_sec == t2p->tv_sec) {
+	if (t1p->tv_usec < t2p->tv_usec)
+	    return -1;
+	else if (t1p->tv_usec > t2p->tv_usec)
+	    return 1;
+	return 0;
+    }
+    return t1p->tv_sec < t2p->tv_sec ? -1 : 1;
+}
+
+#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
+#endif /* ERL_TIME_H__ */
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index c15f85f8f1..1d0735aa99 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -358,10 +358,6 @@ static int clock_resolution;
 ** instead of something like select.
 */
 
-#if defined(ERTS_TIMER_THREAD)
-static ERTS_INLINE void init_erts_deliver_time(const SysTimeval *inittv) { }
-static ERTS_INLINE void do_erts_deliver_time(const SysTimeval *current) { }
-#else
 static SysTimeval last_delivered; 
 
 static void init_erts_deliver_time(const SysTimeval *inittv)
@@ -375,7 +371,7 @@ static void init_erts_deliver_time(const SysTimeval *inittv)
 static void do_erts_deliver_time(const SysTimeval *current)
 {
     SysTimeval cur_time;
-    long elapsed;
+    erts_time_t elapsed;
     
     /* calculate and deliver appropriate number of ticks */
     cur_time = *current;
@@ -389,11 +385,13 @@ static void do_erts_deliver_time(const SysTimeval *current)
        this by simply pretend as if the time stood still. :) */
 
     if (elapsed > 0) {
-	do_time_add(elapsed);
+
+	ASSERT(elapsed < ((erts_time_t) ERTS_SHORT_TIME_T_MAX));
+
+	erts_do_time_add((erts_short_time_t) elapsed);
 	last_delivered = cur_time;
     }
 }
-#endif
 
 int 
 erts_init_time_sup(void)
@@ -426,11 +424,11 @@ erts_init_time_sup(void)
 /* info functions */
 
 void 
-elapsed_time_both(unsigned long *ms_user, unsigned long *ms_sys, 
-		  unsigned long *ms_user_diff, unsigned long *ms_sys_diff)
+elapsed_time_both(UWord *ms_user, UWord *ms_sys, 
+		  UWord *ms_user_diff, UWord *ms_sys_diff)
 {
-    unsigned long prev_total_user, prev_total_sys;
-    unsigned long total_user, total_sys;
+    UWord prev_total_user, prev_total_sys;
+    UWord total_user, total_sys;
     SysTimes now;
 
     sys_times(&now);
@@ -461,9 +459,9 @@ elapsed_time_both(unsigned long *ms_user, unsigned long *ms_sys,
 /* wall clock routines */
 
 void 
-wall_clock_elapsed_time_both(unsigned long *ms_total, unsigned long *ms_diff)
+wall_clock_elapsed_time_both(UWord *ms_total, UWord *ms_diff)
 {
-    unsigned long prev_total;
+    UWord prev_total;
     SysTimeval tv;
 
     erts_smp_mtx_lock(&erts_timeofday_mtx);
@@ -496,7 +494,7 @@ get_time(int *hour, int *minute, int *second)
     
     the_clock = time((time_t *)0);
 #ifdef HAVE_LOCALTIME_R
-    localtime_r(&the_clock, (tm = &tmbuf));
+    tm = localtime_r(&the_clock, &tmbuf);
 #else
     tm = localtime(&the_clock);
 #endif
@@ -518,7 +516,7 @@ get_date(int *year, int *month, int *day)
 
     the_clock = time((time_t *)0);
 #ifdef HAVE_LOCALTIME_R
-    localtime_r(&the_clock, (tm = &tmbuf));
+    tm = localtime_r(&the_clock, &tmbuf);
 #else
     tm = localtime(&the_clock);
 #endif
@@ -588,7 +586,44 @@ static const int mdays[14] = {0, 31, 28, 31, 30, 31, 30,
                             (((y) % 100) != 0)) || \
                            (((y) % 400) == 0))
 
-#define  BASEYEAR       1970
+/* This is the earliest year we are sure to be able to handle
+   on all platforms w/o problems */
+#define  BASEYEAR       1902 
+
+/* A more "clever" mktime
+ * return  1, if successful
+ * return -1, if not successful
+ */
+
+static int erl_mktime(time_t *c, struct tm *tm) {
+    time_t clock;
+
+    clock = mktime(tm);
+
+    if (clock != -1) {
+	*c = clock;
+	return 1;
+    }
+
+    /* in rare occasions mktime returns -1
+     * when a correct value has been entered
+     *
+     * decrease seconds with one second
+     * if the result is -2, epochs should be -1
+     */
+
+    tm->tm_sec = tm->tm_sec - 1;
+    clock = mktime(tm);
+    tm->tm_sec = tm->tm_sec + 1;
+
+    *c = -1;
+
+    if (clock == -2) {
+	return 1;
+    }
+
+    return -1;
+}
 
 /*
  * gregday
@@ -597,10 +632,10 @@ static const int mdays[14] = {0, 31, 28, 31, 30, 31, 30,
  * greater of equal to 1600 , and month [1-12] and day [1-31] 
  * are within range. Otherwise it returns -1.
  */
-static int long gregday(int year, int month, int day)
+static time_t gregday(int year, int month, int day)
 {
-  int long ndays = 0;
-  int gyear, pyear, m;
+  Sint ndays = 0;
+  Sint gyear, pyear, m;
   
   /* number of days in previous years */
   gyear = year - 1600;
@@ -615,10 +650,72 @@ static int long gregday(int year, int month, int day)
   if (is_leap_year(year) && (month > 2))
     ndays++;
   ndays += day - 1;
-  return ndays - 135140;        /* 135140 = Jan 1, 1970 */
+  return (time_t) (ndays - 135140);        /* 135140 = Jan 1, 1970 */
 }
 
+#define SECONDS_PER_MINUTE  (60)
+#define SECONDS_PER_HOUR    (60 * SECONDS_PER_MINUTE)
+#define SECONDS_PER_DAY     (24 * SECONDS_PER_HOUR)
+
+int seconds_to_univ(Sint64 time, Sint *year, Sint *month, Sint *day, 
+	Sint *hour, Sint *minute, Sint *second) {
+
+    Sint y,mi;
+    Sint days = time / SECONDS_PER_DAY;
+    Sint secs = time % SECONDS_PER_DAY;
+    Sint tmp;
+
+    if (secs < 0) {
+	days--;
+	secs += SECONDS_PER_DAY;
+    }
+    
+    tmp     = secs % SECONDS_PER_HOUR;
+
+    *hour   = secs / SECONDS_PER_HOUR;
+    *minute = tmp  / SECONDS_PER_MINUTE;
+    *second = tmp  % SECONDS_PER_MINUTE;
+
+    days   += 719468;
+    y       = (10000*((Sint64)days) + 14780) / 3652425; 
+    tmp     = days - (365 * y + y/4 - y/100 + y/400);
 
+    if (tmp < 0) {
+	y--;
+	tmp = days - (365*y + y/4 - y/100 + y/400);
+    }
+    mi = (100 * tmp + 52)/3060;
+    *month = (mi + 2) % 12 + 1;
+    *year  = y + (mi + 2) / 12;
+    *day   = tmp - (mi * 306 + 5)/10 + 1;
+
+    return 1;
+}
+
+int univ_to_seconds(Sint year, Sint month, Sint day, Sint hour, Sint minute, Sint second, Sint64 *time) {
+    Sint days;
+
+    if (!(IN_RANGE(1600, year, INT_MAX - 1) &&
+          IN_RANGE(1, month, 12) &&
+          IN_RANGE(1, day, (mdays[month] + 
+                             (month == 2 
+                              && (year % 4 == 0) 
+                              && (year % 100 != 0 || year % 400 == 0)))) &&
+          IN_RANGE(0, hour, 23) &&
+          IN_RANGE(0, minute, 59) &&
+          IN_RANGE(0, second, 59))) {
+      return 0;
+    }
+ 
+    days   = gregday(year, month, day);
+    *time  = SECONDS_PER_DAY;
+    *time *= days;             /* don't try overflow it, it hurts */
+    *time += SECONDS_PER_HOUR * hour;
+    *time += SECONDS_PER_MINUTE * minute;
+    *time += second;
+
+    return 1;
+}
 
 int 
 local_to_univ(Sint *year, Sint *month, Sint *day, 
@@ -649,12 +746,34 @@ local_to_univ(Sint *year, Sint *month, Sint *day,
     t.tm_min = *minute;
     t.tm_sec = *second;
     t.tm_isdst = isdst;
-    the_clock = mktime(&t);
+
+    /* the nature of mktime makes this a bit interesting,
+     * up to four mktime calls could happen here
+     */
+
+    if (erl_mktime(&the_clock, &t) < 0) {
+	if (isdst) {
+	    /* If this is a timezone without DST and the OS (correctly)
+	       refuses to give us a DST time, we simulate the Linux/Solaris
+	       behaviour of giving the same data as if is_dst was not set. */
+	    t.tm_isdst = 0;
+	    if (erl_mktime(&the_clock, &t)) {
+		/* Failed anyway, something else is bad - will be a badarg */
+		return 0;
+	    }
+	} else {
+	    /* Something else is the matter, badarg. */
+	    return 0;
+	}
+    }
 #ifdef HAVE_GMTIME_R
-    gmtime_r(&the_clock, (tm = &tmbuf));
+    tm = gmtime_r(&the_clock, &tmbuf);
 #else
     tm = gmtime(&the_clock);
 #endif
+    if (!tm) {
+      return 0;
+    }
     *year = tm->tm_year + 1900;
     *month = tm->tm_mon +1;
     *day = tm->tm_mday;
@@ -663,6 +782,10 @@ local_to_univ(Sint *year, Sint *month, Sint *day,
     *second = tm->tm_sec;
     return 1;
 }
+#if defined(HAVE_POSIX2TIME) && defined(HAVE_DECL_POSIX2TIME) && \
+    !HAVE_DECL_POSIX2TIME
+extern time_t posix2time(time_t);
+#endif
 
 int 
 univ_to_local(Sint *year, Sint *month, Sint *day, 
@@ -704,17 +827,20 @@ univ_to_local(Sint *year, Sint *month, Sint *day,
 #endif
 
 #ifdef HAVE_LOCALTIME_R
-    localtime_r(&the_clock, (tm = &tmbuf));
+    tm = localtime_r(&the_clock, &tmbuf);
 #else
     tm = localtime(&the_clock);
 #endif
-    *year = tm->tm_year + 1900;
-    *month = tm->tm_mon +1;
-    *day = tm->tm_mday;
-    *hour = tm->tm_hour;
-    *minute = tm->tm_min;
-    *second = tm->tm_sec;
-    return 1;
+    if (tm) {
+	*year   = tm->tm_year + 1900;
+	*month  = tm->tm_mon +1;
+	*day    = tm->tm_mday;
+	*hour   = tm->tm_hour;
+	*minute = tm->tm_min;
+	*second = tm->tm_sec;
+	return 1;
+    }
+    return 0;
 }
 
 
@@ -766,7 +892,6 @@ get_sys_now(Uint* megasec, Uint* sec, Uint* microsec)
    to a struct timeval representing current time (to save
    a gettimeofday() where possible) or NULL */
 
-#if !defined(ERTS_TIMER_THREAD)
 void erts_deliver_time(void) {
     SysTimeval now;
     
@@ -777,7 +902,6 @@ void erts_deliver_time(void) {
     
     erts_smp_mtx_unlock(&erts_timeofday_mtx);
 }
-#endif
 
 /* get *real* time (not ticks) remaining until next timeout - if there
    isn't one, give a "long" time, that is guaranteed
@@ -785,15 +909,14 @@ void erts_deliver_time(void) {
 
 void erts_time_remaining(SysTimeval *rem_time)
 {
-    int ticks;
-#if !defined(ERTS_TIMER_THREAD)
+    erts_time_t ticks;
     SysTimeval cur_time;
-#endif
-    long elapsed;
+    erts_time_t elapsed;
 
-    /* next_time() returns no of ticks to next timeout or -1 if none */
+    /* erts_next_time() returns no of ticks to next timeout or -1 if none */
 
-    if ((ticks = next_time()) == -1) {
+    ticks = (erts_time_t) erts_next_time();
+    if (ticks == (erts_time_t) -1) {
 	/* timer queue empty */
 	/* this will cause at most 100000000 ticks */
 	rem_time->tv_sec = 100000;
@@ -802,9 +925,6 @@ void erts_time_remaining(SysTimeval *rem_time)
 	/* next timeout after ticks ticks */
 	ticks *= CLOCK_RESOLUTION;
 	
-#if defined(ERTS_TIMER_THREAD)
-	elapsed = 0;
-#else
 	erts_smp_mtx_lock(&erts_timeofday_mtx);
 	
 	get_tolerant_timeofday(&cur_time);
@@ -819,7 +939,6 @@ void erts_time_remaining(SysTimeval *rem_time)
 	    rem_time->tv_sec = rem_time->tv_usec = 0;
 	    return;
 	}
-#endif
 	rem_time->tv_sec = (ticks - elapsed) / 1000;
 	rem_time->tv_usec = 1000 * ((ticks - elapsed) % 1000);
     }
@@ -832,7 +951,7 @@ void erts_get_timeval(SysTimeval *tv)
     erts_smp_mtx_unlock(&erts_timeofday_mtx);
 }
 
-long
+erts_time_t
 erts_get_time(void)
 {
     SysTimeval sys_tv;
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 2842c2361a..009ca1eb52 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1999-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1999-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -36,6 +36,7 @@
 #include "error.h"
 #include "erl_binary.h"
 #include "erl_bits.h"
+#include "erl_thr_progress.h"
 
 #if 0
 #define DEBUG_PRINTOUTS
@@ -43,8 +44,9 @@
 #undef DEBUG_PRINTOUTS
 #endif
 
-extern Eterm beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */
-extern Eterm beam_return_trace[1];    /* OpCode(i_return_trace) */
+extern Eterm beam_return_to_trace[1];   /* OpCode(i_return_to_trace) */
+extern Eterm beam_return_trace[1];      /* OpCode(i_return_trace) */
+extern Eterm beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */
 
 /* Pseudo export entries. Never filled in with data, only used to
    yield unique pointers of the correct type. */
@@ -123,8 +125,13 @@ do { \
     enqueue_sys_msg_unlocked(SYS_MSG_TYPE_TRACE, (FPID), (TPID), (MSG), (BP)); \
 } while(0)
 #else
+#ifdef USE_VM_PROBES
 #define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP) \
-  erts_queue_message((TPROC), NULL, (BP), (MSG), NIL)
+    erts_queue_message((TPROC), NULL, (BP), (MSG), NIL, NIL)
+#else
+#define ERTS_ENQ_TRACE_MSG(FPID, TPROC, MSG, BP) \
+    erts_queue_message((TPROC), NULL, (BP), (MSG), NIL)
+#endif
 #endif
 
 /*
@@ -158,7 +165,7 @@ static Uint active_sched;
 void
 erts_system_profile_setup_active_schedulers(void)
 {
-    ERTS_SMP_LC_ASSERT(erts_is_system_blocked(0));
+    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_blocking());
     active_sched = erts_active_schedulers();
 }
 
@@ -397,11 +404,13 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to,
  */
 static void 
 do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) {
-    Eterm local_heap[4+5+5];
+#define LOCAL_HEAP_SIZE (4+5+5)
+    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
     Eterm message;
     Eterm *hp;
     Eterm mfarity;
 
+    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
     ASSERT(is_pid(pid));
     ASSERT(is_tuple(timestamp));
     ASSERT(*tuple_val(timestamp) == make_arityval(3));
@@ -426,6 +435,8 @@ do_send_schedfix_to_port(Port *trace_port, Eterm pid, Eterm timestamp) {
 		    pid,
 		    SYS_MSG_TYPE_UNDEFINED,
 		    message);
+    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 }
 #endif
 
@@ -442,7 +453,9 @@ send_to_port(Process *c_p, Eterm message,
 	     Eterm *tracer_pid, Uint *tracee_flags) {
     Port* trace_port;
 #ifndef ERTS_SMP
-    Eterm ts, local_heap[4], *hp;
+#define LOCAL_HEAP_SIZE (4)
+    Eterm ts, *hp;
+    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
 #endif
 
     ASSERT(is_internal_port(*tracer_pid));
@@ -486,6 +499,8 @@ send_to_port(Process *c_p, Eterm message,
      * (e.g. getting_linked) need not be the current process. That other
      * process might not have timestamps enabled.
      */
+    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
     if (*tracee_flags & F_TIMESTAMP) {
 	ASSERT(is_tuple(message));
 	hp = tuple_val(message);
@@ -522,6 +537,8 @@ send_to_port(Process *c_p, Eterm message,
 	 */
 	do_send_schedfix_to_port(trace_port, c_p->id, ts);
     }
+    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 #endif
 }
 
@@ -532,7 +549,7 @@ send_to_port(Process *c_p, Eterm message,
  */
 
 static void 
-profile_send(Eterm message) {
+profile_send(Eterm from, Eterm message) {
     Uint sz = 0;
     ErlHeapFragment *bp = NULL;
     Uint *hp = NULL;
@@ -542,6 +559,9 @@ profile_send(Eterm message) {
 
     Eterm profiler = erts_get_system_profile();
 
+    /* do not profile profiler pid */
+    if (from == profiler) return;
+
     if (is_internal_port(profiler)) {
     	Port *profiler_port = NULL;
 
@@ -568,7 +588,11 @@ profile_send(Eterm message) {
 	hp = erts_alloc_message_heap(sz, &bp, &off_heap, profile_p, 0);
 	msg = copy_struct(message, sz, &hp, &bp->off_heap);
 	
-    	erts_queue_message(profile_p, NULL, bp, msg, NIL);
+    	erts_queue_message(profile_p, NULL, bp, msg, NIL
+#ifdef USE_VM_PROBES
+			   , NIL
+#endif
+			   );
     }
 }
 
@@ -589,7 +613,10 @@ seq_trace_send_to_port(Process *c_p,
 {
     Port* trace_port;
 #ifndef ERTS_SMP
-    Eterm ts, local_heap[4], *hp;
+    Eterm ts, *hp;
+#define LOCAL_HEAP_SIZE (4)
+    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 #endif
 
     ASSERT(is_internal_port(seq_tracer));
@@ -607,6 +634,9 @@ seq_trace_send_to_port(Process *c_p,
     if (INVALID_TRACER_PORT(trace_port, seq_tracer)) {
     invalid_tracer_port:
 	system_seq_tracer = am_false;
+#ifndef ERTS_SMP
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#endif
 	return;
     }
 
@@ -620,6 +650,7 @@ seq_trace_send_to_port(Process *c_p,
 			message);
 
 #ifndef ERTS_SMP
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 	return;
     }
     /* Make a fake schedule only if the current process is traced
@@ -660,6 +691,8 @@ seq_trace_send_to_port(Process *c_p,
 	 */
 	do_send_schedfix_to_port(trace_port, c_p->id, ts);
     }
+    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 #endif
 }
 
@@ -719,7 +752,8 @@ send_to_tracer(Process *tracee,
 static void
 trace_sched_aux(Process *p, Eterm what, int never_fake_sched)
 {
-    Eterm local_heap[5+4+1+TS_HEAP_WORDS];
+#define LOCAL_HEAP_SIZE (5+4+1+TS_HEAP_WORDS)
+    DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p);
     Eterm tmp, mess, *hp;
     ErlHeapFragment *bp = NULL;
     ErlOffHeap *off_heap;
@@ -768,8 +802,10 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched)
 	curr_func = p->current != NULL;
     }
 
+    UseTmpHeap(LOCAL_HEAP_SIZE,p);
+
     if (to_port)
-	hp = &local_heap[0];
+	hp = local_heap;
     else {
 	Uint size = 5;
 	if (curr_func)
@@ -802,6 +838,8 @@ trace_sched_aux(Process *p, Eterm what, int never_fake_sched)
     }
 
     send_to_tracer(p, tracer_ref, mess, &hp, bp, no_fake_sched);
+    UnUseTmpHeap(LOCAL_HEAP_SIZE,p);
+#undef LOCAL_HEAP_SIZE
 }
 
 /* Send {trace_ts, Pid, What, {Mod, Func, Arity}, Timestamp}
@@ -848,7 +886,10 @@ trace_send(Process *p, Eterm to, Eterm msg)
     }
 
     if (is_internal_port(p->tracer_proc)) {
-	Eterm local_heap[11];
+#define LOCAL_HEAP_SIZE (11)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
 	hp = local_heap;
 	mess = TUPLE5(hp, am_trace, p->id, operation, msg, to);
 	hp += 6;
@@ -857,6 +898,8 @@ trace_send(Process *p, Eterm to, Eterm msg)
 	    hp = patch_ts(mess, hp);
 	}
 	send_to_port(p, mess, &p->tracer_proc, &p->trace_flags);
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else {
 	Uint need;
@@ -908,7 +951,10 @@ trace_receive(Process *rp, Eterm msg)
     Eterm* hp;
 
     if (is_internal_port(rp->tracer_proc)) {
-	Eterm local_heap[10];
+#define LOCAL_HEAP_SIZE (10)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
 	hp = local_heap;
 	mess = TUPLE4(hp, am_trace, rp->id, am_receive, msg);
 	hp += 5;
@@ -917,6 +963,8 @@ trace_receive(Process *rp, Eterm msg)
 	    hp = patch_ts(mess, hp);
 	}
 	send_to_port(rp, mess, &rp->tracer_proc, &rp->trace_flags);
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else {
 	Uint hsz;
@@ -955,9 +1003,13 @@ seq_trace_update_send(Process *p)
 {
     Eterm seq_tracer = erts_get_system_seq_tracer();
     ASSERT((is_tuple(SEQ_TRACE_TOKEN(p)) || is_nil(SEQ_TRACE_TOKEN(p))));
-    if ( (p->id == seq_tracer) || (SEQ_TRACE_TOKEN(p) == NIL))
+    if ( (p->id == seq_tracer) || (SEQ_TRACE_TOKEN(p) == NIL)
+#ifdef USE_VM_PROBES
+	 || (SEQ_TRACE_TOKEN(p) == am_have_dt_utag)
+#endif
+	 ) {
 	return 0;
-
+    }
     SEQ_TRACE_TOKEN_SENDER(p) = p->id; /* Internal pid */
     SEQ_TRACE_TOKEN_SERIAL(p) = 
 	make_small(++(p -> seq_trace_clock));
@@ -1018,7 +1070,10 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
     }
 
     if (is_internal_port(seq_tracer)) {
-	Eterm local_heap[64];
+#define LOCAL_HEAP_SIZE (64)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
 	hp = local_heap;
 	label = SEQ_TRACE_T_LABEL(token);
 	lastcnt_serial = TUPLE2(hp, SEQ_TRACE_T_LASTCNT(token),
@@ -1043,6 +1098,8 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
 	    mess = TUPLE4(hp, am_seq_trace, label, mess, ts);
 	    seq_trace_send_to_port(process, seq_tracer, mess, ts);
 	}
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else {
 #ifndef ERTS_SMP
@@ -1134,7 +1191,11 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
 	enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SEQTRACE, NIL, NIL, mess, bp);
 	erts_smp_mtx_unlock(&smq_mtx);
 #else
-	erts_queue_message(tracer, NULL, bp, mess, NIL); /* trace_token must be NIL here */
+	erts_queue_message(tracer, NULL, bp, mess, NIL
+#ifdef USE_VM_PROBES
+			   , NIL
+#endif
+			   ); /* trace_token must be NIL here */
 #endif
     }
 }
@@ -1143,14 +1204,18 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type,
  * or   {trace, Pid, return_to, {Mod, Func, Arity}}
  */
 void 
-erts_trace_return_to(Process *p, Uint *pc)
+erts_trace_return_to(Process *p, BeamInstr *pc)
 {
+#define LOCAL_HEAP_SIZE (4+5+5)
     Eterm* hp;
     Eterm mfa;
     Eterm mess;
-    Eterm local_heap[4+5+5];
+    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+
+    BeamInstr *code_ptr = find_function_from_pc(pc);
+
 
-    Eterm *code_ptr = find_function_from_pc(pc);
+    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 
     hp = local_heap;
 
@@ -1196,6 +1261,8 @@ erts_trace_return_to(Process *p, Uint *pc)
 	mess = copy_struct(mess, size, &hp, off_heap);
 	ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, mess, bp);
     }
+    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
     erts_smp_mtx_unlock(&smq_mtx);
 }
 
@@ -1204,7 +1271,7 @@ erts_trace_return_to(Process *p, Uint *pc)
  * or   {trace, Pid, return_from, {Mod, Name, Arity}, Retval}
  */
 void
-erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid)
+erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid)
 {
     Eterm* hp;
     Eterm mfa;
@@ -1262,7 +1329,9 @@ erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid)
     arity = fi[2];
     
     if (is_internal_port(*tracer_pid)) {
-	Eterm local_heap[4+6+5];
+#define LOCAL_HEAP_SIZE (4+6+5)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
 	hp = local_heap;
 	mfa = TUPLE3(hp, mod, name, make_small(arity));
 	hp += 4;
@@ -1273,6 +1342,8 @@ erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid)
 	    hp = patch_ts(mess, hp);
 	}
 	send_to_port(p, mess, tracer_pid, tracee_flags);
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else {
 	ErlHeapFragment *bp;
@@ -1331,7 +1402,7 @@ erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid)
  * Where Class is atomic but Value is any term.
  */
 void
-erts_trace_exception(Process* p, Eterm mfa[3], Eterm class, Eterm value, 
+erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value,
 		     Eterm *tracer_pid)
 {
     Eterm* hp;
@@ -1385,21 +1456,26 @@ erts_trace_exception(Process* p, Eterm mfa[3], Eterm class, Eterm value,
     }
     
     if (is_internal_port(*tracer_pid)) {
-	Eterm local_heap[4+3+6+5];
+#define LOCAL_HEAP_SIZE (4+3+6+5)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
 	hp = local_heap;
-	mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], make_small(mfa[2]));
+	mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], make_small((Eterm)mfa[2]));
 	hp += 4;
 	cv = TUPLE2(hp, class, value);
 	hp += 3;
 	mess = TUPLE5(hp, am_trace, p->id, am_exception_from, mfa_tuple, cv);
 	hp += 6;
-	ASSERT((hp - local_heap)*sizeof(*hp) <= sizeof(local_heap));
+	ASSERT((hp - local_heap) <= LOCAL_HEAP_SIZE);
 	erts_smp_mtx_lock(&smq_mtx);
 	if (*tracee_flags & F_TIMESTAMP) {
 	    hp = patch_ts(mess, hp); /* hp += 5 */
-	    ASSERT((hp - local_heap)*sizeof(*hp) == sizeof(local_heap));
+	    ASSERT((hp - local_heap) == LOCAL_HEAP_SIZE);
 	}
 	send_to_port(p, mess, tracer_pid, tracee_flags);
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else {
 	ErlHeapFragment *bp;
@@ -1431,7 +1507,7 @@ erts_trace_exception(Process* p, Eterm mfa[3], Eterm class, Eterm value,
 	 * Build the trace tuple and put it into receive queue of the tracer process.
 	 */
 	
-	mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], make_small(mfa[2]));
+	mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], make_small((Eterm) mfa[2]));
 	hp += 4;
 	value = copy_struct(value, value_size, &hp, off_heap);
 	cv = TUPLE2(hp, class, value);
@@ -1468,7 +1544,7 @@ erts_trace_exception(Process* p, Eterm mfa[3], Eterm class, Eterm value,
  * if it is a pid or port we do a meta trace.
  */
 Uint32
-erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec, 
+erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec,
 		Eterm* args, int local, Eterm *tracer_pid)
 {
     Eterm* hp;
@@ -1483,7 +1559,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
     Eterm tracee;
 #endif
     Eterm transformed_args[MAX_ARG];
-    ErlSubBin sub_bin_heap;
+    DeclareTypedTmpHeap(ErlSubBin,sub_bin_heap,p);
 
     ASSERT(tracer_pid);
     if (*tracer_pid == am_true) {
@@ -1534,37 +1610,42 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
      * such as size_object() and copy_struct(), we must make sure that we
      * temporarily convert any match contexts to sub binaries.
      */
-    arity = mfa[2];
+    arity = (Eterm) mfa[2];
+    UseTmpHeap(ERL_SUB_BIN_SIZE,p);
 #ifdef DEBUG
-    sub_bin_heap.thing_word = 0;
+    sub_bin_heap->thing_word = 0;
 #endif
     for (i = 0; i < arity; i++) {
 	Eterm arg = args[i];
 	if (is_boxed(arg) && header_is_bin_matchstate(*boxed_val(arg))) {
 	    ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(arg);
 	    ErlBinMatchBuffer* mb = &ms->mb;
-	    ErlSubBin* sb = &sub_bin_heap;
 	    Uint bit_size;
 
-	    ASSERT(sub_bin_heap.thing_word == 0); /* At most one of match context */
+	    ASSERT(sub_bin_heap->thing_word == 0); /* At most one of match context */
 
 	    bit_size = mb->size - mb->offset;
-	    sb->thing_word = HEADER_SUB_BIN;
-	    sb->size = BYTE_OFFSET(bit_size);
-	    sb->bitsize = BIT_OFFSET(bit_size);
-	    sb->offs = BYTE_OFFSET(mb->offset);
-	    sb->bitoffs = BIT_OFFSET(mb->offset);
-	    sb->is_writable = 0;
-	    sb->orig = mb->orig;
-
-	    arg = make_binary(sb);
+	    sub_bin_heap->thing_word = HEADER_SUB_BIN;
+	    sub_bin_heap->size = BYTE_OFFSET(bit_size);
+	    sub_bin_heap->bitsize = BIT_OFFSET(bit_size);
+	    sub_bin_heap->offs = BYTE_OFFSET(mb->offset);
+	    sub_bin_heap->bitoffs = BIT_OFFSET(mb->offset);
+	    sub_bin_heap->is_writable = 0;
+	    sub_bin_heap->orig = mb->orig;
+
+	    arg = make_binary(sub_bin_heap);
 	}
 	transformed_args[i] = arg;
     }
     args = transformed_args;
 
     if (is_internal_port(*tracer_pid)) {
+#if HEAP_ON_C_STACK
 	Eterm local_heap[64+MAX_ARG];
+#else
+	Eterm *local_heap = erts_alloc(ERTS_ALC_T_TEMP_TERM,
+				       sizeof(Eterm)*(64+MAX_ARG));
+#endif
 	hp = local_heap;
 
 	if (!erts_is_valid_tracer_port(*tracer_pid)) {
@@ -1579,6 +1660,10 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 	    if (is_not_nil(tracee))
 		erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
 #endif
+#if !HEAP_ON_C_STACK
+	    erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
+#endif
+	    UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 	    return 0;
 	}
 	
@@ -1602,9 +1687,13 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 	return_flags = 0;
 	if (match_spec) {
 	    pam_result = erts_match_set_run(p, match_spec, args, arity,
-					    &return_flags);
+					    ERTS_PAM_TMP_RESULT, &return_flags);
 	    if (is_non_value(pam_result)) {
 		erts_match_set_release_result(p);
+#if !HEAP_ON_C_STACK
+		erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
+#endif
+		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return 0;
 	    }
 	}
@@ -1612,16 +1701,28 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 	    /* Meta trace */
 	    if (pam_result == am_false) {
 		erts_match_set_release_result(p);
+#if !HEAP_ON_C_STACK
+		erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
+#endif
+		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return return_flags;
 	    }
 	} else {
 	    /* Non-meta trace */
 	    if (*tracee_flags & F_TRACE_SILENT) { 
 		erts_match_set_release_result(p);
+#if !HEAP_ON_C_STACK
+		erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
+#endif
+		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return 0;
 	    }
 	    if (pam_result == am_false) {
 		erts_match_set_release_result(p);
+#if !HEAP_ON_C_STACK
+		erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
+#endif
+		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return return_flags;
 	    }
 	    if (local && (*tracee_flags & F_TRACE_RETURN_TO)) {
@@ -1644,7 +1745,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 		hp += 2;
 	    }
 	}
-	mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], mfa_tuple);
+	mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], mfa_tuple);
 	hp += 4;
 	
 	/*
@@ -1664,6 +1765,10 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 	send_to_port(p, mess, tracer_pid, tracee_flags);
 	erts_smp_mtx_unlock(&smq_mtx);
 	erts_match_set_release_result(p);
+#if !HEAP_ON_C_STACK
+	erts_free(ERTS_ALC_T_TEMP_TERM,local_heap);
+#endif
+	UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 	return *tracer_pid == NIL ? 0 : return_flags;
 
     } else {
@@ -1706,6 +1811,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 	    if (is_not_nil(tracee))
 		erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
 #endif
+	    UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 	    return 0;
 	}
 
@@ -1728,9 +1834,10 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 	return_flags = 0;
 	if (match_spec) {
 	    pam_result = erts_match_set_run(p, match_spec, args, arity,
-					    &return_flags);
+					    ERTS_PAM_TMP_RESULT, &return_flags);
 	    if (is_non_value(pam_result)) {
 		erts_match_set_release_result(p);
+		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return 0;
 	    }
 	}
@@ -1738,16 +1845,19 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 	    /* Meta trace */
 	    if (pam_result == am_false) {
 		erts_match_set_release_result(p);
+		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return return_flags;
 	    }
 	} else {
 	    /* Non-meta trace */
 	    if (*tracee_flags & F_TRACE_SILENT) { 
 		erts_match_set_release_result(p);
+		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return 0;
 	    }
 	    if (pam_result == am_false) {
 		erts_match_set_release_result(p);
+		UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 		return return_flags;
 	    }
 	    if (local && (*tracee_flags & F_TRACE_RETURN_TO)) {
@@ -1798,7 +1908,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 		hp += 2;
 	    }
 	}
-	mfa_tuple = TUPLE3(hp, mfa[0], mfa[1], mfa_tuple);
+	mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], mfa_tuple);
 	hp += 4;
 	
 	/*
@@ -1831,6 +1941,7 @@ erts_call_trace(Process* p, Eterm mfa[3], Binary *match_spec,
 	ASSERT(hp == limit);
 	ERTS_ENQ_TRACE_MSG(tracee, tracer_ref, mess, bp);
 	erts_smp_mtx_unlock(&smq_mtx);
+	UnUseTmpHeap(ERL_SUB_BIN_SIZE,p);
 	return return_flags;
     }
 }
@@ -1850,8 +1961,14 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data)
     Eterm* hp;
     int need;
 
+    ERTS_SMP_LC_ASSERT((erts_proc_lc_my_proc_locks(t_p) != 0)
+		       || erts_thr_progress_is_blocking());
     if (is_internal_port(t_p->tracer_proc)) {
-	Eterm local_heap[5+5];
+#define LOCAL_HEAP_SIZE (5+5)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
+
 	hp = local_heap;
 	mess = TUPLE4(hp, am_trace, t_p->id, what, data);
 	hp += 5;
@@ -1868,6 +1985,8 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data)
 	    c_p,
 #endif
 	    mess, &t_p->tracer_proc, &t_p->trace_flags);
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else {
 	Eterm tmp;
@@ -1919,7 +2038,10 @@ trace_proc_spawn(Process *p, Eterm pid,
     Eterm* hp;
 
     if (is_internal_port(p->tracer_proc)) {
-	Eterm local_heap[4+6+5];
+#define LOCAL_HEAP_SIZE (4+6+5)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
 	hp = local_heap;
 	mfa = TUPLE3(hp, mod, func, args);
 	hp += 4;
@@ -1930,6 +2052,8 @@ trace_proc_spawn(Process *p, Eterm pid,
 	    hp = patch_ts(mess, hp);
 	}
 	send_to_port(p, mess, &p->tracer_proc, &p->trace_flags);
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else {
 	Eterm tmp;
@@ -1990,8 +2114,7 @@ void save_calls(Process *p, Export *e)
  * entries instead of the original BIF functions.
  */
 Eterm
-erts_bif_trace(int bif_index, Process* p, 
-	       Eterm arg1, Eterm arg2, Eterm arg3, Uint *I)
+erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
 {
     Eterm result;
     int meta = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_META);
@@ -2005,28 +2128,20 @@ erts_bif_trace(int bif_index, Process* p,
 	 * no tracing will occur. Doing the whole else branch will 
 	 * also do nothing, only slower.
 	 */
-	Eterm (*func)(Process*, Eterm, Eterm, Eterm, Uint*) = bif_table[bif_index].f;
-	result = func(p, arg1, arg2, arg3, I);
+	Eterm (*func)(Process*, Eterm*, BeamInstr*) = bif_table[bif_index].f;
+	result = func(p, args, I);
     } else {
-	Eterm (*func)(Process*, Eterm, Eterm, Eterm, Uint*);
+	Eterm (*func)(Process*, Eterm*, BeamInstr*);
 	Export* ep = bif_export[bif_index];
 	Uint32 flags = 0, flags_meta = 0;
 	int global = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_GLOBAL);
 	int local  = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_LOCAL);
+	int time   = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_CALL_TIME);
 	Eterm meta_tracer_pid = NIL;
 	int applying = (I == &(ep->code[3])); /* Yup, the apply code for a bif
 					       * is actually in the 
 					       * export entry */
-	Eterm *cp = p->cp;
-	
-#ifndef _OSE_
-	Eterm args[3] = {arg1, arg2, arg3};
-#else
-	Eterm args[3];
-	args[0] = arg1;
-	args[1] = arg2;
-	args[2] = arg3;
-#endif
+	BeamInstr *cp = p->cp;
 	
 	/* 
 	 * Make continuation pointer OK, it is not during direct BIF calls,
@@ -2043,25 +2158,40 @@ erts_bif_trace(int bif_index, Process* p,
 	    flags_meta = erts_bif_mtrace(p, ep->code+3, args, local, 
 					 &meta_tracer_pid);
 	}
+	if (time) {
+	    BpDataTime *bdt = NULL;
+	    BeamInstr *pc = (BeamInstr *)ep->code+3;
+
+	    bdt = (BpDataTime *) erts_get_time_break(p, pc);
+	    ASSERT(bdt);
+
+	    if (!bdt->pause) {
+		erts_trace_time_break(p, pc, bdt, ERTS_BP_CALL_TIME_CALL);
+	    }
+	}
 	/* Restore original continuation pointer (if changed). */
 	p->cp = cp;
 	
 	func = bif_table[bif_index].f;
 
-	result = func(p, arg1, arg2, arg3, I);
+	result = func(p, args, I);
 	
 	if (applying && (flags & MATCH_SET_RETURN_TO_TRACE)) {
-	    Uint i_return_trace = beam_return_trace[0];
-	    Uint i_return_to_trace = beam_return_to_trace[0];
+	    BeamInstr i_return_trace      = beam_return_trace[0];
+	    BeamInstr i_return_to_trace   = beam_return_to_trace[0];
+	    BeamInstr i_return_time_trace = beam_return_time_trace[0];
 	    Eterm *cpp;
 	    /* Maybe advance cp to skip trace stack frames */
 	    for (cpp = p->stop;  ;  cp = cp_val(*cpp++)) {
-		ASSERT(is_CP((Eterm) cp));
-		if (*cp_val((Eterm) cp) == i_return_trace) {
+		if (*cp == i_return_trace) {
 		    /* Skip stack frame variables */
 		    while (is_not_CP(*cpp)) cpp++;
 		    cpp += 2; /* Skip return_trace parameters */
-		} else if (*cp_val((Eterm) cp) == i_return_to_trace) {
+		} else if (*cp == i_return_time_trace) {
+		    /* Skip stack frame variables */
+		    while (is_not_CP(*cpp)) cpp++;
+		    cpp += 1; /* Skip return_time_trace parameters */
+		} else if (*cp == i_return_to_trace) {
 		    /* A return_to trace message is going to be generated
 		     * by normal means, so we do not have to.
 		     */
@@ -2078,7 +2208,8 @@ erts_bif_trace(int bif_index, Process* p,
 	    if (reason != TRAP) {
 		Eterm class;
 		Eterm value = p->fvalue;
-		Eterm nocatch[3];
+		DeclareTmpHeapNoproc(nocatch,3);
+		UseTmpHeapNoproc(3);
 		/* Expand error value like in handle_error() */
 		if (reason & EXF_ARGLIST) {
 		    Eterm *tp;
@@ -2126,6 +2257,7 @@ erts_bif_trace(int bif_index, Process* p,
 			}
 		    }
 		}
+		UnUseTmpHeapNoproc(3);
 		if ((flags_meta|flags) & MATCH_SET_EXCEPTION_TRACE) {
 		    erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
 		    p->trace_flags |= F_EXCEPTION_TRACE;
@@ -2213,15 +2345,19 @@ trace_gc(Process *p, Eterm what)
 	BIN_OLD_VHEAP(p),
 	BIN_OLD_VHEAP_SZ(p)
     };
-    Eterm local_heap[(sizeof(values)/sizeof(Uint))
-		     *(2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE)
-		     + 5/*4-tuple */ + TS_HEAP_WORDS];
+#define LOCAL_HEAP_SIZE						\
+    (sizeof(values)/sizeof(Eterm)) *				\
+	(2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE) +	\
+	5/*4-tuple */ + TS_HEAP_WORDS
+    DeclareTmpHeap(local_heap,LOCAL_HEAP_SIZE,p);
 #ifdef DEBUG
     Eterm* limit;
 #endif
 
     ASSERT(sizeof(values)/sizeof(Uint) == sizeof(tags)/sizeof(Eterm));
 
+    UseTmpHeap(LOCAL_HEAP_SIZE,p);
+
     if (is_internal_port(p->tracer_proc)) {
 	hp = local_heap;
 #ifdef DEBUG
@@ -2252,7 +2388,7 @@ trace_gc(Process *p, Eterm what)
 
 #ifdef DEBUG
     limit = hp + size;
-    ASSERT(size <= sizeof(local_heap)/sizeof(Eterm));
+    ASSERT(size <= LOCAL_HEAP_SIZE);
 #endif
 
     msg = erts_bld_atom_uint_2tup_list(&hp,
@@ -2275,6 +2411,8 @@ trace_gc(Process *p, Eterm what)
     else
 	ERTS_ENQ_TRACE_MSG(p->id, tracer_ref, msg, bp);
     erts_smp_mtx_unlock(&smq_mtx);
+    UnUseTmpHeap(LOCAL_HEAP_SIZE,p);
+#undef LOCAL_HEAP_SIZE
 }
 
 
@@ -2348,7 +2486,11 @@ monitor_long_gc(Process *p, Uint time) {
 #ifdef ERTS_SMP
     enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->id, NIL, msg, bp);
 #else
-    erts_queue_message(monitor_p, NULL, bp, msg, NIL);
+    erts_queue_message(monitor_p, NULL, bp, msg, NIL
+#ifdef USE_VM_PROBES
+			   , NIL
+#endif
+		       );
 #endif
 }
 
@@ -2420,7 +2562,11 @@ monitor_large_heap(Process *p) {
 #ifdef ERTS_SMP
     enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->id, NIL, msg, bp);
 #else
-    erts_queue_message(monitor_p, NULL, bp, msg, NIL);
+    erts_queue_message(monitor_p, NULL, bp, msg, NIL
+#ifdef USE_VM_PROBES
+		       , NIL
+#endif
+		       );
 #endif
 }
 
@@ -2450,7 +2596,11 @@ monitor_generic(Process *p, Eterm type, Eterm spec) {
 #ifdef ERTS_SMP
     enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->id, NIL, msg, bp);
 #else
-    erts_queue_message(monitor_p, NULL, bp, msg, NIL);
+    erts_queue_message(monitor_p, NULL, bp, msg, NIL
+#ifdef USE_VM_PROBES
+		       , NIL
+#endif
+		       );
 #endif
 
 }
@@ -2465,7 +2615,9 @@ profile_scheduler(Eterm scheduler_id, Eterm state) {
     Uint Ms, s, us;
 
 #ifndef ERTS_SMP
-    Eterm local_heap[4 + 7];
+#define LOCAL_HEAP_SIZE (4 + 7)
+    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
     hp = local_heap;
 #else
     ErlHeapFragment *bp;
@@ -2497,7 +2649,9 @@ profile_scheduler(Eterm scheduler_id, Eterm state) {
 		 make_small(active_sched), timestamp); hp += 7;
 
 #ifndef ERTS_SMP
-    profile_send(msg);
+    profile_send(NIL, msg);
+    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 #else
     enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp);
 #endif
@@ -2510,7 +2664,10 @@ profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint M
     Eterm *hp, msg, timestamp;
     
 #ifndef ERTS_SMP	
-    Eterm local_heap[4 + 7];
+#define LOCAL_HEAP_SIZE (4 + 7)
+    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
     hp = local_heap;
 #else    
     ErlHeapFragment *bp;
@@ -2527,7 +2684,9 @@ profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint M
     timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4;
     msg = TUPLE6(hp, am_profile, am_scheduler, scheduler_id, state, no_schedulers, timestamp); hp += 7;
 #ifndef ERTS_SMP
-    profile_send(msg);
+    profile_send(NIL, msg);
+    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 #else
     enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp);
 #endif
@@ -2558,7 +2717,10 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) {
     Eterm* hp;
 
     if (is_internal_port(p->tracer_proc)) {
-	Eterm local_heap[5+6];
+#define LOCAL_HEAP_SIZE (5+6)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
 	hp = local_heap;
 
 	mess = TUPLE5(hp, am_trace, calling_pid, am_open, p->id, drv_name);
@@ -2569,6 +2731,8 @@ trace_port_open(Port *p, Eterm calling_pid, Eterm drv_name) {
 	}
 	/* No fake schedule */
 	send_to_port(NULL, mess, &p->tracer_proc, &p->trace_flags);
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else {
 	ErlHeapFragment *bp;
@@ -2612,8 +2776,14 @@ trace_port(Port *t_p, Eterm what, Eterm data) {
     Eterm mess;
     Eterm* hp;
 
+    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p)
+		       || erts_thr_progress_is_blocking());
+
     if (is_internal_port(t_p->tracer_proc)) {
-	Eterm local_heap[5+5];
+#define LOCAL_HEAP_SIZE (5+5)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
 	hp = local_heap;
 	mess = TUPLE4(hp, am_trace, t_p->id, what, data);
 	hp += 5;
@@ -2623,6 +2793,8 @@ trace_port(Port *t_p, Eterm what, Eterm data) {
 	}
 	/* No fake schedule */
 	send_to_port(NULL, mess, &t_p->tracer_proc, &t_p->trace_flags);
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else {
 	ErlHeapFragment *bp;
@@ -2674,7 +2846,10 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) {
     Eterm sched_id = am_undefined;
     
     if (is_internal_port(p->tracer_proc)) {
-	Eterm local_heap[5+6];
+#define LOCAL_HEAP_SIZE (5+6)
+	DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+	UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
 	hp = local_heap;
 	
 	if (IS_TRACED_FL(p, F_TRACE_SCHED_NO)) {
@@ -2700,6 +2875,8 @@ trace_sched_ports_where(Port *p, Eterm what, Eterm where) {
 	
 	/* No fake scheduling */
 	send_to_port(NULL, mess, &p->tracer_proc, &p->trace_flags);
+	UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 	erts_smp_mtx_unlock(&smq_mtx);
     } else  {
 	ErlHeapFragment *bp;
@@ -2750,7 +2927,11 @@ profile_runnable_port(Port *p, Eterm status) {
     Eterm count = make_small(0);
 
 #ifndef ERTS_SMP
-    Eterm local_heap[4 + 6];
+#define LOCAL_HEAP_SIZE (4 + 6)
+
+    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
     hp = local_heap;
     
 #else
@@ -2770,9 +2951,11 @@ profile_runnable_port(Port *p, Eterm status) {
     msg = TUPLE5(hp, am_profile, p->id, status, count, timestamp); hp += 6;
 
 #ifndef ERTS_SMP
-    profile_send(msg);
+    profile_send(p->id, msg);
+    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 #else
-    enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp);
+    enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, p->id, NIL, msg, bp);
 #endif
     erts_smp_mtx_unlock(&smq_mtx);
 }
@@ -2785,7 +2968,11 @@ profile_runnable_proc(Process *p, Eterm status){
     Eterm where = am_undefined;
 
 #ifndef ERTS_SMP
-    Eterm local_heap[4 + 6 + 4];
+#define LOCAL_HEAP_SIZE (4 + 6 + 4)
+
+    DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE);
+    UseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+
     hp = local_heap;
 #else
     ErlHeapFragment *bp;
@@ -2817,9 +3004,11 @@ profile_runnable_proc(Process *p, Eterm status){
     timestamp = TUPLE3(hp, make_small(Ms), make_small(s), make_small(us)); hp += 4;
     msg = TUPLE5(hp, am_profile, p->id, status, where, timestamp); hp += 6;
 #ifndef ERTS_SMP
-    profile_send(msg);
+    profile_send(p->id, msg);
+    UnUseTmpHeapNoproc(LOCAL_HEAP_SIZE);
+#undef LOCAL_HEAP_SIZE
 #else
-    enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, NIL, NIL, msg, bp);
+    enqueue_sys_msg_unlocked(SYS_MSG_TYPE_SYSPROF, p->id, NIL, msg, bp);
 #endif
     erts_smp_mtx_unlock(&smq_mtx);
 }
@@ -2864,8 +3053,6 @@ static ErtsSysMsgQ *sys_message_queue_end;
 static erts_tid_t sys_msg_dispatcher_tid;
 static erts_cnd_t smq_cnd;
 
-static int dispatcher_waiting;
-
 ERTS_QUALLOC_IMPL(smq_element, ErtsSysMsgQ, 20, ERTS_ALC_T_SYS_MSG_Q)
 
 static void
@@ -2909,18 +3096,6 @@ enqueue_sys_msg(enum ErtsSysMsgType type,
     erts_smp_mtx_unlock(&smq_mtx);
 }
 
-static void
-prepare_for_block(void *unused)
-{
-    erts_smp_mtx_unlock(&smq_mtx);
-}
-
-static void
-resume_after_block(void *unused)
-{
-    erts_smp_mtx_lock(&smq_mtx);
-}
-
 void
 erts_queue_error_logger_message(Eterm from, Eterm msg, ErlHeapFragment *bp)
 {
@@ -2986,10 +3161,10 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver)
 	    && !erts_system_monitor_flags.busy_port
 	    && !erts_system_monitor_flags.busy_dist_port)
 	    break; /* Everything is disabled */
-	erts_smp_block_system(ERTS_BS_FLG_ALLOW_GC);
+	erts_smp_thr_progress_block();
 	if (system_monitor == receiver || receiver == NIL)
 	    erts_system_monitor_clear(NULL);
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 	break;
 	 case SYS_MSG_TYPE_SYSPROF:
 	if (receiver == NIL
@@ -2999,11 +3174,11 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver)
 	    && !erts_system_profile_flags.scheduler)
 		 break;
 	/* Block system to clear flags */
-	erts_smp_block_system(0);
+	erts_smp_thr_progress_block();
 	if (system_profile == receiver || receiver == NIL) { 
 		erts_system_profile_clear(NULL);
 	}
-	erts_smp_release_system();
+	erts_smp_thr_progress_unblock();
 	break;
     case SYS_MSG_TYPE_ERRLGR: {
 	char *no_elgger = "(no error logger present)";
@@ -3044,22 +3219,68 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver)
     }
 }
 
+static void
+sys_msg_dispatcher_wakeup(void *vwait_p)
+{
+    int *wait_p = (int *) vwait_p;
+    erts_smp_mtx_lock(&smq_mtx);
+    *wait_p = 0;
+    erts_smp_cnd_signal(&smq_cnd);
+    erts_smp_mtx_unlock(&smq_mtx);
+}
+
+static void
+sys_msg_dispatcher_prep_wait(void *vwait_p)
+{
+    int *wait_p = (int *) vwait_p;
+    erts_smp_mtx_lock(&smq_mtx);
+    *wait_p = 1;
+    erts_smp_mtx_unlock(&smq_mtx);
+}
+
+static void
+sys_msg_dispatcher_fin_wait(void *vwait_p)
+{
+    int *wait_p = (int *) vwait_p;
+    erts_smp_mtx_lock(&smq_mtx);
+    *wait_p = 0;
+    erts_smp_mtx_unlock(&smq_mtx);
+}
+
+static void
+sys_msg_dispatcher_wait(void *vwait_p)
+{
+    int *wait_p = (int *) vwait_p;
+    erts_smp_mtx_lock(&smq_mtx);
+    while (*wait_p)
+	erts_smp_cnd_wait(&smq_cnd, &smq_mtx);
+    erts_smp_mtx_unlock(&smq_mtx);
+}
+
 static void *
 sys_msg_dispatcher_func(void *unused)
 {
+    ErtsThrPrgrCallbacks callbacks;
     ErtsSysMsgQ *local_sys_message_queue = NULL;
+    int wait = 0;
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_set_thread_name("system message dispatcher");
 #endif
 
-    erts_register_blockable_thread();
-    erts_smp_activity_begin(ERTS_ACTIVITY_IO, NULL, NULL, NULL);
+    callbacks.arg = (void *) &wait;
+    callbacks.wakeup = sys_msg_dispatcher_wakeup;
+    callbacks.prepare_wait = sys_msg_dispatcher_prep_wait;
+    callbacks.wait = sys_msg_dispatcher_wait;
+    callbacks.finalize_wait = sys_msg_dispatcher_fin_wait;
+
+    erts_thr_progress_register_managed_thread(NULL, &callbacks, 0);
 
     while (1) {
+	int end_wait = 0;
 	ErtsSysMsgQ *smqp;
 
-	ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING);
+	ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
 
 	erts_smp_mtx_lock(&smq_mtx);
 
@@ -3071,20 +3292,16 @@ sys_msg_dispatcher_func(void *unused)
 	}
 
 	/* Fetch current trace message queue ... */
-	erts_smp_activity_change(ERTS_ACTIVITY_IO,
-				 ERTS_ACTIVITY_WAIT,
-				 prepare_for_block,
-				 resume_after_block,
-				 NULL);
-	dispatcher_waiting = 1;
+	if (!sys_message_queue) {
+	    erts_smp_mtx_unlock(&smq_mtx);
+	    end_wait = 1;
+	    erts_thr_progress_active(NULL, 0);
+	    erts_thr_progress_prepare_wait(NULL);
+	    erts_smp_mtx_lock(&smq_mtx);
+	}
+
 	while (!sys_message_queue)
 	    erts_smp_cnd_wait(&smq_cnd, &smq_mtx);
-	dispatcher_waiting = 0;
-	erts_smp_activity_change(ERTS_ACTIVITY_WAIT,
-				 ERTS_ACTIVITY_IO,
-				 prepare_for_block,
-				 resume_after_block,
-				 NULL);
 
 	local_sys_message_queue = sys_message_queue;
 	sys_message_queue = NULL;
@@ -3092,6 +3309,11 @@ sys_msg_dispatcher_func(void *unused)
 
 	erts_smp_mtx_unlock(&smq_mtx);
 
+	if (end_wait) {
+	    erts_thr_progress_finalize_wait(NULL);
+	    erts_thr_progress_active(NULL, 1);
+	}
+
 	/* Send trace messages ... */
 
 	ASSERT(local_sys_message_queue);
@@ -3102,6 +3324,9 @@ sys_msg_dispatcher_func(void *unused)
 	    Process *proc = NULL;
 	    Port *port = NULL;
 
+	    if (erts_thr_progress_update(NULL))
+		erts_thr_progress_leader_update(NULL);
+
 #ifdef DEBUG_PRINTOUTS
 	    print_msg_type(smqp);
 #endif
@@ -3161,7 +3386,11 @@ sys_msg_dispatcher_func(void *unused)
 		}
 		else {
 		queue_proc_msg:
-		    erts_queue_message(proc,&proc_locks,smqp->bp,smqp->msg,NIL);
+		    erts_queue_message(proc,&proc_locks,smqp->bp,smqp->msg,NIL
+#ifdef USE_VM_PROBES
+				       , NIL
+#endif
+				       );
 #ifdef DEBUG_PRINTOUTS
 		    erts_fprintf(stderr, "delivered\n");
 #endif
@@ -3215,7 +3444,6 @@ sys_msg_dispatcher_func(void *unused)
 	}
     }
 
-    erts_smp_activity_end(ERTS_ACTIVITY_IO, NULL, NULL, NULL);
     return NULL;
 }
 
@@ -3265,7 +3493,6 @@ init_sys_msg_dispatcher(void)
     sys_message_queue_end = NULL;
     erts_smp_cnd_init(&smq_cnd);
     erts_smp_mtx_init(&smq_mtx, "sys_msg_q");
-    dispatcher_waiting = 0;
     erts_smp_thr_create(&sys_msg_dispatcher_tid,
 			sys_msg_dispatcher_func,
 			NULL,
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index ab5811c70f..6d5eae73b0 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2008-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -30,6 +30,8 @@
 #include "big.h"
 
 #include "erl_unicode.h"
+#include "erl_unicode_normalize.h"
+
 
 typedef struct _restart_context {
     byte *bytes;
@@ -45,7 +47,7 @@ typedef struct _restart_context {
 
 static Uint max_loop_limit;
 
-static BIF_RETTYPE utf8_to_list(BIF_ALIST_1);
+static BIF_RETTYPE utf8_to_list(Process *p, Eterm arg1);
 static BIF_RETTYPE finalize_list_to_list(Process *p, 
 					 byte *bytes,
 					 Eterm rest,
@@ -54,13 +56,6 @@ static BIF_RETTYPE finalize_list_to_list(Process *p,
 					 Uint num_resulting_chars, 
 					 int state, int left,
 					 Eterm tail);
-static int analyze_utf8(byte *source, Uint size, 
-			byte **err_pos, Uint *num_chars, int *left);
-#define UTF8_OK 0
-#define UTF8_INCOMPLETE 1
-#define UTF8_ERROR 2
-#define UTF8_ANALYZE_MORE 3
-
 static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3);
 static BIF_RETTYPE characters_to_list_trap_1(BIF_ALIST_3);
 static BIF_RETTYPE characters_to_list_trap_2(BIF_ALIST_3);
@@ -90,9 +85,9 @@ void erts_init_unicode(void)
 	am_atom_put("characters_to_utf8_trap",23);
     characters_to_utf8_trap_exp.code[2] = 3;
     characters_to_utf8_trap_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     characters_to_utf8_trap_exp.code[4] = 
-	(Eterm) &characters_to_utf8_trap;
+	(BeamInstr) &characters_to_utf8_trap;
 
     memset(&characters_to_list_trap_1_exp, 0, sizeof(Export));
     characters_to_list_trap_1_exp.address = 
@@ -102,9 +97,9 @@ void erts_init_unicode(void)
 	am_atom_put("characters_to_list_trap_1",25);
     characters_to_list_trap_1_exp.code[2] = 3;
     characters_to_list_trap_1_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     characters_to_list_trap_1_exp.code[4] = 
-	(Eterm) &characters_to_list_trap_1;
+	(BeamInstr) &characters_to_list_trap_1;
 
     memset(&characters_to_list_trap_2_exp, 0, sizeof(Export));
     characters_to_list_trap_2_exp.address = 
@@ -114,9 +109,9 @@ void erts_init_unicode(void)
 	am_atom_put("characters_to_list_trap_2",25);
     characters_to_list_trap_2_exp.code[2] = 3;
     characters_to_list_trap_2_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     characters_to_list_trap_2_exp.code[4] = 
-	(Eterm) &characters_to_list_trap_2;
+	(BeamInstr) &characters_to_list_trap_2;
 
 
     memset(&characters_to_list_trap_3_exp, 0, sizeof(Export));
@@ -127,9 +122,9 @@ void erts_init_unicode(void)
 	am_atom_put("characters_to_list_trap_3",25);
     characters_to_list_trap_3_exp.code[2] = 3;
     characters_to_list_trap_3_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     characters_to_list_trap_3_exp.code[4] = 
-	(Eterm) &characters_to_list_trap_3;
+	(BeamInstr) &characters_to_list_trap_3;
 
     memset(&characters_to_list_trap_4_exp, 0, sizeof(Export));
     characters_to_list_trap_4_exp.address = 
@@ -139,9 +134,9 @@ void erts_init_unicode(void)
 	am_atom_put("characters_to_list_trap_4",25);
     characters_to_list_trap_4_exp.code[2] = 1;
     characters_to_list_trap_4_exp.code[3] =
-	(Eterm) em_apply_bif;
+	(BeamInstr) em_apply_bif;
     characters_to_list_trap_4_exp.code[4] = 
-	(Eterm) &characters_to_list_trap_4;
+	(BeamInstr) &characters_to_list_trap_4;
 
     c_to_b_int_trap_exportp =  erts_export_put(am_unicode,am_characters_to_binary_int,2);
     c_to_l_int_trap_exportp =  erts_export_put(am_unicode,am_characters_to_list_int,2);
@@ -232,8 +227,8 @@ static ERTS_INLINE int simple_loops_to_common(int cost)
 
 static Sint aligned_binary_size(Eterm binary)
 {
-    unsigned char *bytes;
-    Uint bitoffs;
+    ERTS_DECLARE_DUMMY(unsigned char *bytes);
+    ERTS_DECLARE_DUMMY(Uint bitoffs);
     Uint bitsize;
     
     ERTS_GET_BINARY_BYTES(binary, bytes, bitoffs, bitsize);
@@ -353,12 +348,6 @@ static int copy_utf8_bin(byte *target, byte *source, Uint size,
 		return copied;
 	    }
 
-	    if (((*source) == 0xEF) && (source[1] == 0xBF) &&
-		((source[2] == 0xBE) || (source[2] == 0xBF))) {
-		*err_pos = source;
-		return copied;
-	    }
-		
 	    *(target++) = *(source++);
 	    *(target++) = *(source++);
 	    *(target++) = *(source++);
@@ -463,7 +452,7 @@ L_Again:   /* Restart with sublist, old listend was pushed on stack */
 			}
 			objp = list_val(ioterm);
 			obj = CAR(objp);
-			if (!is_byte(obj))
+			if (!is_small(obj))
 			    break;
 		    }
 		} else if (is_nil(obj)) {
@@ -719,9 +708,8 @@ L_Again:   /* Restart with sublist, old listend was pushed on stack */
 			    target[(*pos)++] = (((byte) (x & 0x3F)) | 
 						((byte) 0x80));
 			} else if (x < 0x10000) {
-			    if ((x >= 0xD800 && x <= 0xDFFF) ||
-				(x == 0xFFFE) ||
-				(x == 0xFFFF)) { /* Invalid unicode range */
+			    if (x >= 0xD800 && x <= 0xDFFF) {
+				/* Invalid unicode range */
 				*err = 1;
 				goto done;
 			    }
@@ -906,8 +894,9 @@ static BIF_RETTYPE build_utf8_return(Process *p,Eterm bin,int pos,
 
 static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3)
 {
+#ifdef DEBUG
     Eterm *real_bin;
-    Sint need;
+#endif
     byte* bytes;
     Eterm rest_term;
     int left, sleft;
@@ -921,9 +910,10 @@ static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3)
     
     /*erts_printf("Trap %T!\r\n",BIF_ARG_2);*/
     ASSERT(is_binary(BIF_ARG_1));
+#ifdef DEBUG
     real_bin = binary_val(BIF_ARG_1);
     ASSERT(*real_bin == HEADER_PROC_BIN);
-    need = ((ProcBin *) real_bin)->val->orig_size;
+#endif
     pos = (int) binary_size(BIF_ARG_1);
     bytes = binary_bytes(BIF_ARG_1);
     sleft = left = allowed_iterations(BIF_P);
@@ -970,11 +960,11 @@ static int is_valid_utf8(Eterm orig_bin)
 	bytes = erts_get_aligned_binary_bytes(orig_bin, &temp_alloc);
     }
     size = binary_size(orig_bin);
-    ret = analyze_utf8(bytes,
+    ret = erts_analyze_utf8(bytes,
 		       size,
 		       &endpos,&numchar,NULL);
     erts_free_aligned_binary_bytes(temp_alloc);
-    return (ret == UTF8_OK);
+    return (ret == ERTS_UTF8_OK);
 }
 
 BIF_RETTYPE unicode_characters_to_binary_2(BIF_ALIST_2)
@@ -1084,14 +1074,14 @@ static BIF_RETTYPE build_list_return(Process *p, byte *bytes, int pos, Uint char
 	    hp += 2;
 	    rest_term = CONS(hp,leftover_bin,rest_term);
 	}
-	BIF_RET(finalize_list_to_list(p, bytes, rest_term, 0U, pos, characters, UTF8_ERROR, left, NIL));
+	BIF_RET(finalize_list_to_list(p, bytes, rest_term, 0U, pos, characters, ERTS_UTF8_ERROR, left, NIL));
     } else if (rest_term == NIL && num_leftovers != 0) {
 	Eterm leftover_bin = new_binary(p, leftover, num_leftovers);
 	if (check_leftovers(leftover,num_leftovers) != 0) {
-	    BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, UTF8_ERROR, 
+	    BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, ERTS_UTF8_ERROR, 
 					  left, NIL));
 	} else {
-	    BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, UTF8_INCOMPLETE, 
+	    BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, ERTS_UTF8_INCOMPLETE, 
 					  left, NIL));
 	}
     } else { /* All OK */	    
@@ -1107,11 +1097,11 @@ static BIF_RETTYPE build_list_return(Process *p, byte *bytes, int pos, Uint char
 	    rc.num_processed_bytes = 0; /* not used */
 	    rc.num_bytes_to_process = pos;
 	    rc.num_resulting_chars = characters;
-	    rc.state = UTF8_OK; /* not used */
+	    rc.state = ERTS_UTF8_OK; /* not used */
 	    BIF_TRAP3(&characters_to_list_trap_1_exp, p, make_magic_bin_for_restart(p,&rc), 
 		      rest_term, latin1);
 	} else { /* Success */
-	    BIF_RET(finalize_list_to_list(p, bytes, NIL, 0U, pos, characters, UTF8_OK, left, NIL));
+	    BIF_RET(finalize_list_to_list(p, bytes, NIL, 0U, pos, characters, ERTS_UTF8_OK, left, NIL));
 	}
     }
 }
@@ -1205,7 +1195,7 @@ BIF_RETTYPE unicode_characters_to_list_2(BIF_ALIST_2)
  * When input to characters_to_list is a plain binary and the format is 'unicode', we do
  * a faster analyze and size count with this function.
  */
-static int analyze_utf8(byte *source, Uint size, 
+int erts_analyze_utf8(byte *source, Uint size, 
 			byte **err_pos, Uint *num_chars, int *left)
 {
     *err_pos = source;
@@ -1216,60 +1206,56 @@ static int analyze_utf8(byte *source, Uint size,
 	    --size; 
 	} else if (((*source) & ((byte) 0xE0)) == 0xC0) {
 	    if (size < 2) {
-		return UTF8_INCOMPLETE;
+		return ERTS_UTF8_INCOMPLETE;
 	    }
 	    if (((source[1] & ((byte) 0xC0)) != 0x80) ||
 		((*source) < 0xC2) /* overlong */) {
-		return UTF8_ERROR;
+		return ERTS_UTF8_ERROR;
 	    }
 	    source += 2;
 	    size -= 2;
 	} else if (((*source) & ((byte) 0xF0)) == 0xE0) {
 	    if (size < 3) {
-		return UTF8_INCOMPLETE;
+		return ERTS_UTF8_INCOMPLETE;
 	    }
 	    if (((source[1] & ((byte) 0xC0)) != 0x80) ||
 		((source[2] & ((byte) 0xC0)) != 0x80) ||
 		(((*source) == 0xE0) && (source[1] < 0xA0)) /* overlong */ ) {
-		return UTF8_ERROR;
+		return ERTS_UTF8_ERROR;
 	    }
 	    if ((((*source) & ((byte) 0xF)) == 0xD) && 
 		((source[1] & 0x20) != 0)) {
-		return UTF8_ERROR;
-	    }
-	    if (((*source) == 0xEF) && (source[1] == 0xBF) &&
-		((source[2] == 0xBE) || (source[2] == 0xBF))) {
-		return UTF8_ERROR;
+		return ERTS_UTF8_ERROR;
 	    }
 	    source += 3;
 	    size -= 3;
 	} else if (((*source) & ((byte) 0xF8)) == 0xF0) {
 	    if (size < 4) {
-		return UTF8_INCOMPLETE;
+		return ERTS_UTF8_INCOMPLETE;
 	    }
 	    if (((source[1] & ((byte) 0xC0)) != 0x80) ||
 		((source[2] & ((byte) 0xC0)) != 0x80) ||
 		((source[3] & ((byte) 0xC0)) != 0x80) ||
 		(((*source) == 0xF0) && (source[1] < 0x90)) /* overlong */) {
-		return UTF8_ERROR;
+		return ERTS_UTF8_ERROR;
 	    }
 	    if ((((*source) & ((byte)0x7)) > 0x4U) ||
 		((((*source) & ((byte)0x7)) == 0x4U) && 
 		 ((source[1] & ((byte)0x3F)) > 0xFU))) {
-		return UTF8_ERROR;
+		return ERTS_UTF8_ERROR;
 	    }
 	    source += 4;
 	    size -= 4; 
 	} else {
-	    return UTF8_ERROR;
+	    return ERTS_UTF8_ERROR;
 	}
 	++(*num_chars);
 	*err_pos = source;
 	if (left && --(*left) <= 0) {
-	    return UTF8_ANALYZE_MORE;
+	    return ERTS_UTF8_ANALYZE_MORE;
 	}
     }
-    return UTF8_OK;
+    return ERTS_UTF8_OK;
 }
 
 /*
@@ -1304,7 +1290,7 @@ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz,
 	} else if (((*source) & ((byte) 0xE0)) == 0xC0) {
 	    unipoint = 
 		(((Uint) ((*source) & ((byte) 0x1F))) << 6) |
-		((Uint) (source[1] & ((byte) 0x3F))); 	
+		((Uint) (source[1] & ((byte) 0x3F)));
 	} else if (((*source) & ((byte) 0xF0)) == 0xE0) {
 	    unipoint = 
 		(((Uint) ((*source) & ((byte) 0xF))) << 12) |
@@ -1330,6 +1316,216 @@ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz,
     return ret;
 }
 
+static int is_candidate(Uint cp)
+{
+    int index,pos;
+    if (cp < 768) return 0;
+    if (cp > 4023) {
+	if (cp == 12441 || cp == 12442) return 1;
+	return 0;
+    }
+    index = cp / 32 - COMP_CANDIDATE_MAP_OFFSET;
+    pos = cp % 32;
+    return !!(comp_candidate_map[index] & (1UL << pos));
+}
+
+static int hashsearch(int *htab, int htab_size, CompEntry *cv, Uint16 c)
+{
+	int bucket = c % htab_size;
+	while (htab[bucket] != -1 && cv[htab[bucket]].c != c)
+	    bucket = (bucket + 1) % htab_size;
+	return htab[bucket];
+}
+
+#define TRANSLATE_NO 0
+#define TRANSLATE_MAYBE -1
+
+/* The s array is reversed */
+static int translate(Uint16 *s, int slen, Uint16 *res)
+{
+    /* Go backwards through buffer and match against tree */
+    int pos = 0;
+    CompEntry *cv = compose_tab;
+    int *hc = hash_compose_tab;
+    int cvs = compose_tab_size;
+    int x;
+    while (pos < slen) {
+	x = hashsearch(hc,cvs*HASH_SIZE_FACTOR,cv,s[pos]);
+	if (x < 0) {
+	    return TRANSLATE_NO;
+	} 
+	if (cv[x].res) {
+	    *res = cv[x].res;
+	    return pos;
+	}
+	cvs = cv[x].num_subs;
+	hc = cv[x].hash;
+	cv = cv[x].subs;
+	++pos;
+    }
+    return TRANSLATE_MAYBE;
+}
+
+static void handle_first_norm(Uint16 *savepoints, int *numpointsp, Uint unipoint)
+{
+    /*erts_fprintf(stderr,"CP = %d, numpoints = %d\n",(int) unipoint,(int) *numpointsp);*/
+    *numpointsp = 1;
+    savepoints[0] = (Uint16) unipoint;
+}
+
+static void cleanup_norm(Eterm **hpp, Uint16 *savepoints, int numpoints, Eterm *retp)
+{
+    Eterm *hp = *hpp;
+    int res,i;
+    Uint16 newpoint;
+    Eterm ret = *retp;
+    
+    ret = CONS(hp,make_small((Uint) savepoints[0]),ret);
+    hp += 2;
+    
+    for (i = 1;i < numpoints;) {
+	if(!is_candidate(savepoints[i]) || 
+	   ((res = translate(savepoints+i,numpoints - i, &newpoint)) <= 0)) {
+	    ret = CONS(hp,make_small((Uint) savepoints[i]),ret);
+	    hp += 2;
+	    ++i;
+	} else {
+	    ret = CONS(hp,make_small((Uint) newpoint),ret);
+	    hp += 2;
+	    i += res;
+	}
+    }
+    *retp = ret;
+}
+    
+static void handle_potential_norm(Eterm **hpp, Uint16 *savepoints, int *numpointsp, Uint unipoint, Eterm *retp)
+{
+    Eterm *hp = *hpp;
+    int numpoints = *numpointsp;
+    int res,i;
+    Uint16 newpoint;
+    Eterm ret = *retp;
+
+    /* erts_fprintf(stderr,"CP = %d, numpoints = %d\n",(int) unipoint,(int) numpoints);*/
+    if ((unipoint >> 16) == 0) { /* otherwise we're done here */ 
+	savepoints[numpoints++] = (Uint16) unipoint;
+	res = translate(savepoints,numpoints,&newpoint);
+	if (res == TRANSLATE_NO) {
+	    ret = CONS(hp,make_small((Uint) savepoints[0]),ret);
+	    hp += 2;
+	    for (i = 1;i < numpoints;) {
+		if(!is_candidate(savepoints[i]) ||
+		   ((res = translate(savepoints+i,numpoints - i, &newpoint)) == 0)) {
+		    ret = CONS(hp,make_small((Uint) savepoints[i]),ret);
+		    hp += 2;
+		    ++i;
+		} else if (res > 0) {
+		    ret = CONS(hp,make_small((Uint) newpoint),ret);
+		    hp += 2;
+		    i += res;
+		} else { /* res < 0 */
+		    /* A "maybe", means we are not done yet */
+		    int j = 0;
+		    while (i < numpoints) {
+			savepoints[j++] = savepoints[i++];
+		    }
+		    numpoints = j;
+		    goto breakaway;
+		}
+	    }
+	    numpoints = 0;
+	breakaway:
+	    ;
+	} else if (res > 0) {
+	    numpoints = 0;
+	    ret = CONS(hp,make_small((Uint) newpoint),ret);
+	    hp += 2;
+	} /* < 0 means go on */
+    } else {
+	/* Unconditional rollup, this character is larger than 16 bit */
+	ret = CONS(hp,make_small((Uint) savepoints[0]),ret);
+	hp += 2;
+	
+	for (i = 1;i < numpoints;) {
+	    if(!is_candidate(savepoints[i]) || 
+	       ((res = translate(savepoints+i,numpoints - i, &newpoint)) <= 0)) {
+		ret = CONS(hp,make_small((Uint) savepoints[i]),ret);
+		hp += 2;
+		++i;
+	    } else {
+		ret = CONS(hp,make_small((Uint) newpoint),ret);
+		hp += 2;
+		i += res;
+	    }
+	}
+	ret = CONS(hp,make_small(unipoint),ret);
+	hp += 2;
+	numpoints = 0;
+    }	
+    *hpp = hp;
+    *numpointsp = numpoints;
+    *retp = ret;
+} 
+
+static Eterm do_utf8_to_list_normalize(Process *p, Uint num, byte *bytes, Uint sz)
+{
+    Eterm *hp,*hp_end;
+    Eterm ret;
+    byte *source;
+    Uint unipoint;
+    Uint16 savepoints[4];
+    int numpoints = 0;
+
+    ASSERT(num > 0);
+
+    hp = HAlloc(p,num * 2); /* May be to much */
+    hp_end = hp + num * 2;
+    ret = NIL;
+    source = bytes + sz;
+    while(--source >= bytes) {
+	if (((*source) & ((byte) 0x80)) == 0) {
+	    unipoint = (Uint) *source;
+	} else if (((*source) & ((byte) 0xE0)) == 0xC0) {
+	    unipoint = 
+		(((Uint) ((*source) & ((byte) 0x1F))) << 6) |
+		((Uint) (source[1] & ((byte) 0x3F))); 	
+	} else if (((*source) & ((byte) 0xF0)) == 0xE0) {
+	    unipoint = 
+		(((Uint) ((*source) & ((byte) 0xF))) << 12) |
+		(((Uint) (source[1] & ((byte) 0x3F))) << 6) |
+		((Uint) (source[2] & ((byte) 0x3F))); 	 	
+	} else if (((*source) & ((byte) 0xF8)) == 0xF0) {
+	    unipoint = 
+		(((Uint) ((*source) & ((byte) 0x7))) << 18) |
+		(((Uint) (source[1] & ((byte) 0x3F))) << 12) |
+		(((Uint) (source[2] & ((byte) 0x3F))) << 6) |
+		((Uint) (source[3] & ((byte) 0x3F))); 	 	
+	} else {
+	    /* ignore 2#10XXXXXX */
+	    continue;
+	}
+	if (numpoints) {
+	    handle_potential_norm(&hp,savepoints,&numpoints,unipoint,&ret);
+	    continue;
+	}
+	/* We are not building up any normalizations yet, look that we shouldn't start... */
+	if (is_candidate(unipoint)) {
+	    handle_first_norm(savepoints,&numpoints,unipoint);
+	    continue;
+	} 
+	ret = CONS(hp,make_small(unipoint),ret);
+	hp += 2;
+    }
+    /* so, we'we looped to the beginning, do we have anything saved? */
+    if (numpoints) {
+	cleanup_norm(&hp,savepoints,numpoints,&ret);
+    }
+    if (hp_end != hp) {
+	HRelease(p,hp_end,hp);
+    }
+    return ret;
+}
+
 /*
  * The last step of characters_to_list, build a list from the buffer 'bytes' (created in the same way
  * as for characters_to_utf8). All sizes are known in advance and most data will be held in a 
@@ -1378,10 +1574,10 @@ static BIF_RETTYPE finalize_list_to_list(Process *p,
      */
 
     free_restart(bytes);
-    if (state == UTF8_INCOMPLETE) {
+    if (state == ERTS_UTF8_INCOMPLETE) {
 	hp = HAlloc(p,4);
 	ret = TUPLE3(hp,am_incomplete,converted,rest);
-    } else if (state == UTF8_ERROR) {
+    } else if (state == ERTS_UTF8_ERROR) {
 	hp = HAlloc(p,4);
 	ret = TUPLE3(hp,am_error,converted,rest);
     } else {
@@ -1408,7 +1604,7 @@ static BIF_RETTYPE characters_to_list_trap_2(BIF_ALIST_3)
 
 /*
  * Hooks into the process of decoding a binary depending on state.
- * If last_state is UTF8_ANALYZE_MORE, num_bytes_to_process 
+ * If last_state is ERTS_UTF8_ANALYZE_MORE, num_bytes_to_process 
  * and num_resulting_chars will grow
  * until we're done analyzing the binary. Then we'll eat 
  * the bytes to process, lowering num_bytes_to_process and num_resulting_chars,
@@ -1465,14 +1661,14 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p,
 
     left = allowed_iterations(p);
     
-    if (state == UTF8_ANALYZE_MORE) {
-	state = analyze_utf8(bytes + num_bytes_to_process,
+    if (state == ERTS_UTF8_ANALYZE_MORE) {
+	state = erts_analyze_utf8(bytes + num_bytes_to_process,
 			     size - num_bytes_to_process,
 			     &endpos,&numchar,&left);
 	cost_to_proc(p,numchar);
 	num_resulting_chars += numchar;
 	num_bytes_to_process = endpos - bytes;
-	if (state == UTF8_ANALYZE_MORE) {
+	if (state == ERTS_UTF8_ANALYZE_MORE) {
 	    Eterm epos = erts_make_integer(num_bytes_to_process,p);
 	    Eterm enumchar = erts_make_integer(num_resulting_chars,p);
 	    erts_free_aligned_binary_bytes(temp_alloc);
@@ -1527,8 +1723,8 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p,
     if (b_sz) {
 	ErlSubBin *sb;
 	Eterm orig;
-	Uint offset;
-	ASSERT(state != UTF8_OK);
+	ERTS_DECLARE_DUMMY(Uint offset);
+	ASSERT(state != ERTS_UTF8_OK);
 	hp = HAlloc(p, ERL_SUB_BIN_SIZE);
 	sb = (ErlSubBin *) hp;
 	ERTS_GET_REAL_BIN(orig_bin, orig, offset, bitoffs, bitsize);
@@ -1544,14 +1740,14 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p,
 
     /* Done */
 
-    if (state == UTF8_INCOMPLETE) {
+    if (state == ERTS_UTF8_INCOMPLETE) {
 	if (check_leftovers(bytes + num_bytes_to_process + num_processed_bytes,
 			    b_sz) != 0) {
 	    goto error_return;
 	}
 	hp = HAlloc(p,4);
 	ret = TUPLE3(hp,am_incomplete,converted,rest);
-    } else if (state == UTF8_ERROR) {
+    } else if (state == ERTS_UTF8_ERROR) {
  error_return:
 	hp = HAlloc(p,4);
 	ret = TUPLE3(hp,am_error,converted,rest);
@@ -1589,7 +1785,7 @@ static BIF_RETTYPE characters_to_list_trap_3(BIF_ALIST_3)
 			       0U, /* nothing processed yet */
 			       num_bytes_to_process, 
 			       num_resulting_chars,
-			       UTF8_ANALYZE_MORE, /* always this state here */
+			       ERTS_UTF8_ANALYZE_MORE, /* always this state here */
 			       NIL); /* Nothing built -> no tail yet */
 	
 }
@@ -1636,13 +1832,13 @@ static BIF_RETTYPE characters_to_list_trap_4(BIF_ALIST_1)
  * Instead of building an utf8 buffer, we analyze the binary given and use that.
  */
 
-static BIF_RETTYPE utf8_to_list(BIF_ALIST_1)
+static BIF_RETTYPE utf8_to_list(Process* p, Eterm arg)
 {
-    if (!is_binary(BIF_ARG_1) || aligned_binary_size(BIF_ARG_1) < 0) {
-	BIF_ERROR(BIF_P,BADARG);
+    if (!is_binary(arg) || aligned_binary_size(arg) < 0) {
+	BIF_ERROR(p, BADARG);
     }
-    return do_bif_utf8_to_list(BIF_P, BIF_ARG_1, 0U, 0U, 0U, 
-			       UTF8_ANALYZE_MORE,NIL);
+    return do_bif_utf8_to_list(p, arg, 0U, 0U, 0U,
+			       ERTS_UTF8_ANALYZE_MORE, NIL);
 }
 
 
@@ -1728,8 +1924,8 @@ binary_to_atom(Process* p, Eterm bin, Eterm enc, int must_exist)
 	    Uint n;
 	    int reds_left = bin_size+1; /* Number of reductions left. */
 
-	    if (analyze_utf8(bytes, bin_size, &err_pos,
-			     &n, &reds_left) == UTF8_OK) {
+	    if (erts_analyze_utf8(bytes, bin_size, &err_pos,
+			     &n, &reds_left) == ERTS_UTF8_OK) {
 		/* 
 		 * Correct UTF-8 encoding, but too many characters to
 		 * fit in an atom.
@@ -1813,3 +2009,613 @@ BIF_RETTYPE binary_to_existing_atom_2(BIF_ALIST_2)
 {
     return binary_to_atom(BIF_P, BIF_ARG_1, BIF_ARG_2, 1);
 }
+
+/**********************************************************
+ * Simpler non-interruptable routines for UTF-8 and 
+ * Windowish UTF-16 (restricted)
+ **********************************************************/
+/*
+ * This function is the heart of the Unicode support for 
+ * open_port - spawn_executable. It converts both the name
+ * of the executable and the arguments according to the same rules
+ * as for filename conversion. That means as if your arguments are
+ * to be raw, you supply binaries, else unicode characters are allowed up to
+ * the encoding maximum (256 of the unicode max).
+ * Depending on the filename encoding standard, the vector is then
+ * converted to whatever is used, which might mean win_utf16 if on windows.
+ * Do not peek into the argument vector or filenam with ordinary
+ * string routines, that will certainly fail on some OS.
+ */
+
+char *erts_convert_filename_to_native(Eterm name, ErtsAlcType_t alloc_type, int allow_empty)
+{
+    int encoding = erts_get_native_filename_encoding();
+    char* name_buf = NULL;
+
+    if (is_atom(name) || is_list(name) || (allow_empty && is_nil(name))) {
+	Sint need;
+	if ((need = erts_native_filename_need(name,encoding)) < 0) {
+	    return NULL;
+	}
+	if (encoding == ERL_FILENAME_WIN_WCHAR) {
+	    need += 2;
+	} else {
+	    ++need;
+	}
+	name_buf = (char *) erts_alloc(alloc_type, need);
+	erts_native_filename_put(name,encoding,(byte *)name_buf); 
+	name_buf[need-1] = 0;
+	if (encoding == ERL_FILENAME_WIN_WCHAR) {
+	    name_buf[need-2] = 0;
+	}
+    } else if (is_binary(name)) {
+	byte *temp_alloc = NULL;
+	byte *bytes;
+	byte *err_pos;
+	Uint size,num_chars;
+	
+	size = binary_size(name);
+	bytes = erts_get_aligned_binary_bytes(name, &temp_alloc);
+	if (encoding != ERL_FILENAME_WIN_WCHAR) {
+	    /*Add 0 termination only*/
+	    name_buf = (char *) erts_alloc(alloc_type, size+1);
+	    memcpy(name_buf,bytes,size);
+	    name_buf[size]=0;
+	} else if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK || 
+		   erts_get_user_requested_filename_encoding() ==  ERL_FILENAME_LATIN1) {
+	    byte *p;
+	    /* What to do now? Maybe latin1, so just take byte for byte instead */
+	    name_buf = (char *) erts_alloc(alloc_type, (size+1)*2);
+	    p = (byte *) name_buf;
+	    while (size--) {
+		*p++ = *bytes++;
+		*p++ = 0;
+	    }
+	    *p++ = 0;
+	    *p++ = 0;
+	} else { /* WIN_WCHAR and valid UTF8 */
+	    name_buf = (char *) erts_alloc(alloc_type, (num_chars+1)*2);
+	    erts_copy_utf8_to_utf16_little((byte *) name_buf, bytes, num_chars);
+	    name_buf[num_chars*2] = 0;
+	    name_buf[num_chars*2+1] = 0;
+	}
+	erts_free_aligned_binary_bytes(temp_alloc);
+    } else {
+	return NULL;
+    }
+    return name_buf;
+}
+
+
+Sint erts_native_filename_need(Eterm ioterm, int encoding) 
+{
+    Eterm *objp;
+    Eterm obj;
+    DECLARE_ESTACK(stack);
+    Sint need = 0;
+
+    if (is_atom(ioterm)) {
+	Atom* ap;
+	int i;
+	ap = atom_tab(atom_val(ioterm));
+	switch (encoding) {
+	case ERL_FILENAME_LATIN1:
+	    need = ap->len;
+	    break;
+	case ERL_FILENAME_UTF8_MAC:
+	case ERL_FILENAME_UTF8:
+	    for (i = 0; i < ap->len; i++) {
+		need += (ap->name[i] >= 0x80) ? 2 : 1;
+	    }
+	    break;
+	case ERL_FILENAME_WIN_WCHAR:
+	    need = 2*(ap->len);
+	    break;
+	default:
+	    need = -1;
+	}
+	DESTROY_ESTACK(stack);
+	return need;
+    }
+
+    if (is_nil(ioterm)) {
+	DESTROY_ESTACK(stack);
+	return need;
+    }
+    if (!is_list(ioterm)) {
+	DESTROY_ESTACK(stack);
+	return (Sint) -1;
+    }
+    /* OK a list, needs to be processed in order, handling each flat list-level
+       as they occur, just like io_list_to_binary would */
+    ESTACK_PUSH(stack,ioterm);
+    while (!ESTACK_ISEMPTY(stack)) {
+	ioterm = ESTACK_POP(stack);	
+	if (is_nil(ioterm)) {
+	    /* ignore empty lists */
+	    continue;
+	}
+	if(is_list(ioterm)) {
+L_Again:   /* Restart with sublist, old listend was pushed on stack */
+	    objp = list_val(ioterm);
+	    obj = CAR(objp);
+	    for(;;) { /* loop over one flat list of bytes and binaries
+		         until sublist or list end is encountered */
+		if (is_small(obj)) { /* Always small */
+		    for(;;) {
+			Uint x = unsigned_val(obj);
+			switch (encoding) {
+			case ERL_FILENAME_LATIN1:
+			    if (x > 255) {
+				DESTROY_ESTACK(stack);
+				return ((Sint) -1);
+			    }
+			    need += 1;
+			    break;
+			case ERL_FILENAME_UTF8_MAC:
+			case ERL_FILENAME_UTF8:
+			    if (x < 0x80) {
+				need +=1;
+			    } else if (x < 0x800) {
+				need += 2;
+			    } else if (x < 0x10000) {
+				if (x >= 0xD800 && x <= 0xDFFF) {
+				    /* Invalid unicode range */
+				    DESTROY_ESTACK(stack);
+				    return ((Sint) -1);
+				}
+				need += 3;
+			    } else  if (x < 0x110000) {
+				need += 4; 
+			    } else {
+				DESTROY_ESTACK(stack);
+				return ((Sint) -1);
+			    }
+			    break;
+			case ERL_FILENAME_WIN_WCHAR:
+			    if (x <= 0xffff) { 
+				need += 2;
+				break;
+			    } /* else fall throug to error */
+			default:
+			    DESTROY_ESTACK(stack);
+			    return ((Sint) -1);
+			}
+			    
+			/* everything else will give badarg later 
+			   in the process, so we dont check */
+			ioterm = CDR(objp);
+			if (!is_list(ioterm)) {
+			    break;
+			}
+			objp = list_val(ioterm);
+			obj = CAR(objp);
+			if (!is_small(obj))
+			    break;
+		    }
+		} else if (is_nil(obj)) {
+		    ioterm = CDR(objp);
+		    if (!is_list(ioterm)) {
+			break;
+		    }
+		    objp = list_val(ioterm);
+		    obj = CAR(objp);
+		} else if (is_list(obj)) {
+		    /* push rest of list for later processing, start 
+		       again with sublist */
+		    ESTACK_PUSH(stack,CDR(objp));
+		    ioterm = obj;
+		    goto L_Again;
+		} else {
+		    DESTROY_ESTACK(stack);
+		    return ((Sint) -1);
+		} 
+		if (is_nil(ioterm) || !is_list(ioterm)) {
+		    break;
+		}
+	    } /* for(;;) */
+	} /* is_list(ioterm) */
+	
+	if (!is_list(ioterm) && !is_nil(ioterm)) {
+	    /* inproper list end */
+	    DESTROY_ESTACK(stack);
+	    return ((Sint) -1);
+	}
+    } /* while  not estack empty */
+    DESTROY_ESTACK(stack);
+    return need;
+}
+
+void erts_native_filename_put(Eterm ioterm, int encoding, byte *p) 
+{
+    Eterm *objp;
+    Eterm obj;
+    DECLARE_ESTACK(stack);
+
+    if (is_atom(ioterm)) {
+	Atom* ap;
+	int i;
+	ap = atom_tab(atom_val(ioterm));
+	switch (encoding) {
+	case ERL_FILENAME_LATIN1:
+	    for (i = 0; i < ap->len; i++) {
+		*p++ = ap->name[i];
+	    }
+	    break;
+	case ERL_FILENAME_UTF8_MAC:
+	case ERL_FILENAME_UTF8:
+	    for (i = 0; i < ap->len; i++) {
+		if(ap->name[i] < 0x80) {
+		    *p++ = ap->name[i];
+		} else {
+		    *p++ = (((ap->name[i]) >> 6) | ((byte) 0xC0));
+		    *p++ = (((ap->name[i]) & 0x3F) | ((byte) 0x80));
+		}
+	    }
+	    break;
+	case ERL_FILENAME_WIN_WCHAR:
+	    for (i = 0; i < ap->len; i++) {
+		/* Little endian */
+		*p++ = ap->name[i];
+		*p++ = 0;
+	    }
+	    break;
+	default:
+	    ASSERT(0);
+	}
+	DESTROY_ESTACK(stack);
+	return;
+    }
+
+    if (is_nil(ioterm)) {
+	DESTROY_ESTACK(stack);
+	return;
+    }
+    ASSERT(is_list(ioterm));
+    /* OK a list, needs to be processed in order, handling each flat list-level
+       as they occur, just like io_list_to_binary would */
+    ESTACK_PUSH(stack,ioterm);
+    while (!ESTACK_ISEMPTY(stack)) {
+	ioterm = ESTACK_POP(stack);	
+	if (is_nil(ioterm)) {
+	    /* ignore empty lists */
+	    continue;
+	}
+	if(is_list(ioterm)) {
+L_Again:   /* Restart with sublist, old listend was pushed on stack */
+	    objp = list_val(ioterm);
+	    obj = CAR(objp);
+	    for(;;) { /* loop over one flat list of bytes and binaries
+		         until sublist or list end is encountered */
+		if (is_small(obj)) { /* Always small */
+		    for(;;) {
+			Uint x = unsigned_val(obj);
+			switch (encoding) {
+			case ERL_FILENAME_LATIN1:
+			    ASSERT( x < 256);
+			    *p++ = (byte) x;
+			    break;
+			case ERL_FILENAME_UTF8_MAC:
+			case ERL_FILENAME_UTF8:
+			    if (x < 0x80) {
+				*p++ = (byte) x;
+			    }
+			    else if (x < 0x800) {
+				*p++ = (((byte) (x >> 6)) | 
+					((byte) 0xC0));
+				*p++ = (((byte) (x & 0x3F)) | 
+					((byte) 0x80));
+			    } else if (x < 0x10000) {
+				ASSERT(!(x >= 0xD800 && x <= 0xDFFF));
+				*p++ = (((byte) (x >> 12)) | 
+					((byte) 0xE0));
+				*p++ = ((((byte) (x >> 6)) & 0x3F)  | 
+					((byte) 0x80));
+				*p++ = (((byte) (x & 0x3F)) | 
+					((byte) 0x80));
+			    } else {
+				ASSERT(x < 0x110000);
+				*p++ = (((byte) (x >> 18)) | 
+					((byte) 0xF0));
+				*p++ = ((((byte) (x >> 12)) & 0x3F)  | 
+					((byte) 0x80));
+				*p++ = ((((byte) (x >> 6)) & 0x3F)  | 
+					((byte) 0x80));
+				*p++ = (((byte) (x & 0x3F)) | 
+					((byte) 0x80));
+			    }
+			    break;
+			case ERL_FILENAME_WIN_WCHAR:
+			    ASSERT(x <= 0xFFFF); 
+			    *p++ = (byte) (x & 0xFFU);
+			    *p++ = (byte) ((x >> 8) & 0xFFU);
+			    break;
+			default:
+			    ASSERT(0);
+			}
+			    
+			/* everything else will give badarg later 
+			   in the process, so we dont check */
+			ioterm = CDR(objp);
+			if (!is_list(ioterm)) {
+			    break;
+			}
+			objp = list_val(ioterm);
+			obj = CAR(objp);
+			if (!is_small(obj))
+			    break;
+		    }
+		} else if (is_nil(obj)) {
+		    ioterm = CDR(objp);
+		    if (!is_list(ioterm)) {
+			break;
+		    }
+		    objp = list_val(ioterm);
+		    obj = CAR(objp);
+		} else if (is_list(obj)) {
+		    /* push rest of list for later processing, start 
+		       again with sublist */
+		    ESTACK_PUSH(stack,CDR(objp));
+		    ioterm = obj;
+		    goto L_Again;
+		} else {
+		    ASSERT(0);
+		} 
+		if (is_nil(ioterm) || !is_list(ioterm)) {
+		    break;
+		}
+	    } /* for(;;) */
+	} /* is_list(ioterm) */
+	
+	ASSERT(is_list(ioterm) || is_nil(ioterm));
+    } /* while  not estack empty */
+    DESTROY_ESTACK(stack);
+    return;
+}
+void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars)
+{
+    Uint unipoint;
+    
+    while (num_chars--) {
+	if (((*bytes) & ((byte) 0x80)) == 0) {
+	    unipoint = (Uint) *bytes;
+	    ++bytes;
+	} else if (((*bytes) & ((byte) 0xE0)) == 0xC0) {
+	    unipoint = 
+		(((Uint) ((*bytes) & ((byte) 0x1F))) << 6) |
+		((Uint) (bytes[1] & ((byte) 0x3F))); 	
+	    bytes += 2;
+	} else if (((*bytes) & ((byte) 0xF0)) == 0xE0) {
+	    unipoint = 
+		(((Uint) ((*bytes) & ((byte) 0xF))) << 12) |
+		(((Uint) (bytes[1] & ((byte) 0x3F))) << 6) |
+		((Uint) (bytes[2] & ((byte) 0x3F)));
+	    bytes +=3;
+	} else if (((*bytes) & ((byte) 0xF8)) == 0xF0) {
+	    unipoint = 
+		(((Uint) ((*bytes) & ((byte) 0x7))) << 18) |
+		(((Uint) (bytes[1] & ((byte) 0x3F))) << 12) |
+		(((Uint) (bytes[2] & ((byte) 0x3F))) << 6) |
+		((Uint) (bytes[3] & ((byte) 0x3F)));
+	    bytes += 4;
+	} else {
+	    erl_exit(1,"Internal unicode error in prim_file:internal_name2native/1");
+	}
+	*target++ = (byte) (unipoint & 0xFF);
+	*target++ = (byte) ((unipoint >> 8) & 0xFF);
+    }
+}
+
+/*
+ * This internal bif converts a filename to whatever format is suitable for the file driver
+ * It also adds zero termination so that prim_file needn't bother with the character encoding
+ * of the file driver 
+ */
+BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
+{
+    int encoding = erts_get_native_filename_encoding();
+    Sint need;
+    Eterm bin_term;
+    byte* bin_p;
+    /* Prim file explicitly does not allow atoms, although we could 
+       very well cope with it. Instead of letting 'file' handle them,
+       it would probably be more efficient to handle them here. Subject to 
+       change in R15. */ 
+    if (is_atom(BIF_ARG_1)) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    if (is_binary(BIF_ARG_1)) {
+	byte *temp_alloc = NULL;
+	byte *bytes;
+	byte *err_pos;
+	Uint size,num_chars;
+	/* Uninterpreted encoding except if windows widechar, in case we convert from 
+	   utf8 to win_wchar */
+	size = binary_size(BIF_ARG_1);
+	bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
+	if (encoding != ERL_FILENAME_WIN_WCHAR) {
+	    /*Add 0 termination only*/
+	    bin_term = new_binary(BIF_P, NULL, size+1);
+	    bin_p = binary_bytes(bin_term);
+	    memcpy(bin_p,bytes,size);
+	    bin_p[size]=0;
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    BIF_RET(bin_term);
+	} 
+	/* In a wchar world, the emulator flags only affect how
+	   binaries are interpreted when sent from the user. */
+	/* Determine real length and create a new binary */
+	if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK || 
+	    erts_get_user_requested_filename_encoding() ==  ERL_FILENAME_LATIN1) {
+	    /* What to do now? Maybe latin1, so just take byte for byte instead */
+	    bin_term = new_binary(BIF_P, 0, (size+1)*2);
+	    bin_p = binary_bytes(bin_term);
+	    while (size--) {
+		*bin_p++ = *bytes++;
+		*bin_p++ = 0;
+	    }
+	    *bin_p++ = 0;
+	    *bin_p++ = 0;
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    BIF_RET(bin_term);
+	}
+	/* OK, UTF8 ok, number of characters is in num_chars */
+	bin_term = new_binary(BIF_P, 0, (num_chars+1)*2);
+	bin_p = binary_bytes(bin_term);
+	erts_copy_utf8_to_utf16_little(bin_p, bytes, num_chars);
+	/* zero termination */
+	bin_p[num_chars*2] = 0;
+	bin_p[num_chars*2+1] = 0;
+	erts_free_aligned_binary_bytes(temp_alloc);
+	BIF_RET(bin_term);
+    } /* binary */   
+	    
+
+    if ((need = erts_native_filename_need(BIF_ARG_1,encoding)) < 0) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    if (encoding == ERL_FILENAME_WIN_WCHAR) {
+	need += 2;
+    } else {
+	++need;
+    }
+    
+    bin_term = new_binary(BIF_P, 0, need);
+    bin_p = binary_bytes(bin_term);
+    erts_native_filename_put(BIF_ARG_1,encoding,bin_p); 
+    bin_p[need-1] = 0;
+    if (encoding == ERL_FILENAME_WIN_WCHAR) {
+	bin_p[need-2] = 0;
+    }
+    BIF_RET(bin_term);
+}
+
+BIF_RETTYPE prim_file_internal_native2name_1(BIF_ALIST_1)
+{
+    Eterm real_bin;
+    Uint offset;
+    Uint size,num_chars;
+    Uint bitsize;
+    Uint bitoffs;
+    Eterm *hp;
+    byte *temp_alloc = NULL;
+    byte *bytes;
+    byte *err_pos;
+    Uint num_built; /* characters */
+    Uint num_eaten; /* bytes */
+    Eterm ret;
+    int mac = 0;
+
+    if (is_not_binary(BIF_ARG_1)) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    size = binary_size(BIF_ARG_1);
+    ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize);
+    if (bitsize != 0) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    if (size == 0) {
+	BIF_RET(NIL);
+    }
+    switch (erts_get_native_filename_encoding()) {
+    case ERL_FILENAME_LATIN1:
+	hp = HAlloc(BIF_P, 2 * size);
+	bytes = binary_bytes(real_bin)+offset;
+    
+	BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes, size, bitoffs));
+    case ERL_FILENAME_UTF8_MAC:
+	mac = 1;
+    case ERL_FILENAME_UTF8:
+	bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
+	if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) {
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    goto noconvert;
+	}
+	num_built = 0;
+	num_eaten = 0;
+	if (mac) {
+	    ret = do_utf8_to_list_normalize(BIF_P, num_chars, bytes, size);
+	} else {
+	    ret = do_utf8_to_list(BIF_P, num_chars, bytes, size, num_chars, &num_built, &num_eaten, NIL);
+	} 
+	erts_free_aligned_binary_bytes(temp_alloc);
+	BIF_RET(ret);
+    case ERL_FILENAME_WIN_WCHAR:
+	bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
+	if ((size % 2) != 0) { /* Panic fixup to avoid crashing the emulator */
+	    size--;
+	    hp = HAlloc(BIF_P, size+2);
+	    ret = CONS(hp,make_small((Uint) bytes[size]),NIL);
+	    hp += 2;
+	} else {
+	    hp = HAlloc(BIF_P, size);
+	    ret = NIL;
+	}
+	bytes += size-1;
+	while (size > 0) {
+	    Uint x = ((Uint) *bytes--) << 8;
+	    x |= ((Uint) *bytes--);
+	    size -= 2;
+	    ret = CONS(hp,make_small(x),ret);
+	    hp += 2;
+	}	    
+	erts_free_aligned_binary_bytes(temp_alloc);
+	BIF_RET(ret);
+    default:
+	goto noconvert;
+    }
+ noconvert:
+    BIF_RET(BIF_ARG_1);
+}
+
+BIF_RETTYPE prim_file_internal_normalize_utf8_1(BIF_ALIST_1)
+{
+    ERTS_DECLARE_DUMMY(Eterm real_bin);
+    ERTS_DECLARE_DUMMY(Uint offset);
+    Uint size,num_chars;
+    Uint bitsize;
+    ERTS_DECLARE_DUMMY(Uint bitoffs);
+    Eterm ret;
+    byte *temp_alloc = NULL;
+    byte *bytes;
+    byte *err_pos;
+
+    if (is_not_binary(BIF_ARG_1)) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    size = binary_size(BIF_ARG_1);
+    ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize);
+    if (bitsize != 0) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    if (size == 0) {
+	BIF_RET(NIL);
+    }
+    bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
+    if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) {
+	erts_free_aligned_binary_bytes(temp_alloc);
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    ret = do_utf8_to_list_normalize(BIF_P, num_chars, bytes, size);
+    erts_free_aligned_binary_bytes(temp_alloc);
+    BIF_RET(ret);
+}  
+
+BIF_RETTYPE file_native_name_encoding_0(BIF_ALIST_0)
+{
+    switch (erts_get_native_filename_encoding()) {
+    case ERL_FILENAME_LATIN1:
+	BIF_RET(am_latin1);
+    case ERL_FILENAME_UTF8_MAC:
+    case ERL_FILENAME_UTF8:
+	BIF_RET(am_utf8);
+    case ERL_FILENAME_WIN_WCHAR:
+	if (erts_get_user_requested_filename_encoding() ==  ERL_FILENAME_LATIN1) {
+	    BIF_RET(am_latin1);
+	} else {
+	    BIF_RET(am_utf8);
+	}
+    default:
+	BIF_RET(am_undefined);
+    }
+}
diff --git a/erts/emulator/beam/erl_unicode_normalize.h b/erts/emulator/beam/erl_unicode_normalize.h
new file mode 100644
index 0000000000..fb0a111ca2
--- /dev/null
+++ b/erts/emulator/beam/erl_unicode_normalize.h
@@ -0,0 +1,1687 @@
+/*
+* %CopyrightBegin%
+*
+* Copyright Ericsson AB 1999-2010. All Rights Reserved.
+*
+* The contents of this file are subject to the Erlang Public License,
+* Version 1.1, (the "License"); you may not use this file except in
+* compliance with the License. You should have received a copy of the
+* Erlang Public License along with this software. If not, it can be
+* retrieved online at http://www.erlang.org/.
+*
+* Software distributed under the License is distributed on an "AS IS"
+* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+* the License for the specific language governing rights and limitations
+* under the License.
+*
+* %CopyrightEnd%
+*/
+/*
+* This file is automatically generated by dec.erl, do not edit manually
+*/
+#define HASH_SIZE_FACTOR 2
+typedef struct _compose_entry {
+    Uint16 c;
+    Uint16 res;
+    Uint16 num_subs;
+    struct _compose_entry *subs;
+    int *hash;
+} CompEntry;
+
+static int compose_tab_size = 61;
+static int hash_compose_tab_0_15[12] = 
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_0_15 */ 
+static CompEntry compose_tab_0_15[] = {
+{65, 7846, 0, NULL, NULL},
+{69, 7872, 0, NULL, NULL},
+{79, 7890, 0, NULL, NULL},
+{97, 7847, 0, NULL, NULL},
+{101, 7873, 0, NULL, NULL},
+{111, 7891, 0, NULL, NULL}
+}; /* compose_tab_0_15 */ 
+static int hash_compose_tab_0_16[8] = 
+{3,-1,-1,-1,-1,0,2,1}; /* hash_compose_tab_0_16 */ 
+static CompEntry compose_tab_0_16[] = {
+{69, 7700, 0, NULL, NULL},
+{79, 7760, 0, NULL, NULL},
+{101, 7701, 0, NULL, NULL},
+{111, 7761, 0, NULL, NULL}
+}; /* compose_tab_0_16 */ 
+static int hash_compose_tab_0_17[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_0_17 */ 
+static CompEntry compose_tab_0_17[] = {
+{65, 7856, 0, NULL, NULL},
+{97, 7857, 0, NULL, NULL}
+}; /* compose_tab_0_17 */ 
+static int hash_compose_tab_0_18[8] = 
+{-1,2,-1,-1,-1,0,1,3}; /* hash_compose_tab_0_18 */ 
+static CompEntry compose_tab_0_18[] = {
+{85, 475, 0, NULL, NULL},
+{117, 476, 0, NULL, NULL},
+{953, 8146, 0, NULL, NULL},
+{965, 8162, 0, NULL, NULL}
+}; /* compose_tab_0_18 */ 
+static int hash_compose_tab_0_19_0[12] = 
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_0_19_0 */ 
+static CompEntry compose_tab_0_19_0[] = {
+{913, 8074, 0, NULL, NULL},
+{919, 8090, 0, NULL, NULL},
+{937, 8106, 0, NULL, NULL},
+{945, 8066, 0, NULL, NULL},
+{951, 8082, 0, NULL, NULL},
+{969, 8098, 0, NULL, NULL}
+}; /* compose_tab_0_19_0 */ 
+static int hash_compose_tab_0_19[28] = 
+{9,10,-1,5,-1,-1,-1,11,-1,-1,-1,-1,-1,6,12,-1,-1,1,13,-1,-1,2,7,3,-1,0,4,8}; /* hash_compose_tab_0_19 */ 
+static CompEntry compose_tab_0_19[] = {
+{837, 0, 6, compose_tab_0_19_0, hash_compose_tab_0_19_0},
+{913, 7946, 0, NULL, NULL},
+{917, 7962, 0, NULL, NULL},
+{919, 7978, 0, NULL, NULL},
+{921, 7994, 0, NULL, NULL},
+{927, 8010, 0, NULL, NULL},
+{937, 8042, 0, NULL, NULL},
+{945, 7938, 0, NULL, NULL},
+{949, 7954, 0, NULL, NULL},
+{951, 7970, 0, NULL, NULL},
+{953, 7986, 0, NULL, NULL},
+{959, 8002, 0, NULL, NULL},
+{965, 8018, 0, NULL, NULL},
+{969, 8034, 0, NULL, NULL}
+}; /* compose_tab_0_19 */ 
+static int hash_compose_tab_0_20_0[12] = 
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_0_20_0 */ 
+static CompEntry compose_tab_0_20_0[] = {
+{913, 8075, 0, NULL, NULL},
+{919, 8091, 0, NULL, NULL},
+{937, 8107, 0, NULL, NULL},
+{945, 8067, 0, NULL, NULL},
+{951, 8083, 0, NULL, NULL},
+{969, 8099, 0, NULL, NULL}
+}; /* compose_tab_0_20_0 */ 
+static int hash_compose_tab_0_20[30] = 
+{-1,-1,-1,6,-1,13,-1,7,-1,14,-1,-1,-1,1,-1,8,-1,2,-1,3,9,4,10,11,-1,-1,-1,0,5,
+ 12}; /* hash_compose_tab_0_20 */ 
+static CompEntry compose_tab_0_20[] = {
+{837, 0, 6, compose_tab_0_20_0, hash_compose_tab_0_20_0},
+{913, 7947, 0, NULL, NULL},
+{917, 7963, 0, NULL, NULL},
+{919, 7979, 0, NULL, NULL},
+{921, 7995, 0, NULL, NULL},
+{927, 8011, 0, NULL, NULL},
+{933, 8027, 0, NULL, NULL},
+{937, 8043, 0, NULL, NULL},
+{945, 7939, 0, NULL, NULL},
+{949, 7955, 0, NULL, NULL},
+{951, 7971, 0, NULL, NULL},
+{953, 7987, 0, NULL, NULL},
+{959, 8003, 0, NULL, NULL},
+{965, 8019, 0, NULL, NULL},
+{969, 8035, 0, NULL, NULL}
+}; /* compose_tab_0_20 */ 
+static int hash_compose_tab_0_21[8] = 
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_0_21 */ 
+static CompEntry compose_tab_0_21[] = {
+{79, 7900, 0, NULL, NULL},
+{85, 7914, 0, NULL, NULL},
+{111, 7901, 0, NULL, NULL},
+{117, 7915, 0, NULL, NULL}
+}; /* compose_tab_0_21 */ 
+static int hash_compose_tab_0_22[6] = 
+{-1,-1,-1,0,1,2}; /* hash_compose_tab_0_22 */ 
+static CompEntry compose_tab_0_22[] = {
+{945, 8114, 0, NULL, NULL},
+{951, 8130, 0, NULL, NULL},
+{969, 8178, 0, NULL, NULL}
+}; /* compose_tab_0_22 */ 
+static int hash_compose_tab_0[78] = 
+{38,3,29,-1,-1,-1,-1,4,19,5,20,6,14,30,31,21,32,33,37,7,-1,-1,-1,8,34,-1,-1,9,
+ -1,35,-1,-1,-1,10,36,-1,-1,-1,-1,11,-1,12,-1,13,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,23,-1,22,-1,24,-1,25,-1,26,-1,0,-1,-1,15,1,16,27,17,2,18,28,-1,-1}; /* hash_compose_tab_0 */ 
+static CompEntry compose_tab_0[] = {
+{65, 192, 0, NULL, NULL},
+{69, 200, 0, NULL, NULL},
+{73, 204, 0, NULL, NULL},
+{79, 210, 0, NULL, NULL},
+{85, 217, 0, NULL, NULL},
+{87, 7808, 0, NULL, NULL},
+{89, 7922, 0, NULL, NULL},
+{97, 224, 0, NULL, NULL},
+{101, 232, 0, NULL, NULL},
+{105, 236, 0, NULL, NULL},
+{111, 242, 0, NULL, NULL},
+{117, 249, 0, NULL, NULL},
+{119, 7809, 0, NULL, NULL},
+{121, 7923, 0, NULL, NULL},
+{168, 8173, 0, NULL, NULL},
+{770, 0, 6, compose_tab_0_15, hash_compose_tab_0_15},
+{772, 0, 4, compose_tab_0_16, hash_compose_tab_0_16},
+{774, 0, 2, compose_tab_0_17, hash_compose_tab_0_17},
+{776, 0, 4, compose_tab_0_18, hash_compose_tab_0_18},
+{787, 0, 14, compose_tab_0_19, hash_compose_tab_0_19},
+{788, 0, 15, compose_tab_0_20, hash_compose_tab_0_20},
+{795, 0, 4, compose_tab_0_21, hash_compose_tab_0_21},
+{837, 0, 3, compose_tab_0_22, hash_compose_tab_0_22},
+{913, 8122, 0, NULL, NULL},
+{917, 8136, 0, NULL, NULL},
+{919, 8138, 0, NULL, NULL},
+{921, 8154, 0, NULL, NULL},
+{927, 8184, 0, NULL, NULL},
+{933, 8170, 0, NULL, NULL},
+{937, 8186, 0, NULL, NULL},
+{945, 8048, 0, NULL, NULL},
+{949, 8050, 0, NULL, NULL},
+{951, 8052, 0, NULL, NULL},
+{953, 8054, 0, NULL, NULL},
+{959, 8056, 0, NULL, NULL},
+{965, 8058, 0, NULL, NULL},
+{969, 8060, 0, NULL, NULL},
+{8127, 8141, 0, NULL, NULL},
+{8190, 8157, 0, NULL, NULL}
+}; /* compose_tab_0 */ 
+static int hash_compose_tab_1_39[12] = 
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_1_39 */ 
+static CompEntry compose_tab_1_39[] = {
+{65, 7844, 0, NULL, NULL},
+{69, 7870, 0, NULL, NULL},
+{79, 7888, 0, NULL, NULL},
+{97, 7845, 0, NULL, NULL},
+{101, 7871, 0, NULL, NULL},
+{111, 7889, 0, NULL, NULL}
+}; /* compose_tab_1_39 */ 
+static int hash_compose_tab_1_40[8] = 
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_1_40 */ 
+static CompEntry compose_tab_1_40[] = {
+{79, 7756, 0, NULL, NULL},
+{85, 7800, 0, NULL, NULL},
+{111, 7757, 0, NULL, NULL},
+{117, 7801, 0, NULL, NULL}
+}; /* compose_tab_1_40 */ 
+static int hash_compose_tab_1_41[8] = 
+{3,-1,-1,-1,-1,0,2,1}; /* hash_compose_tab_1_41 */ 
+static CompEntry compose_tab_1_41[] = {
+{69, 7702, 0, NULL, NULL},
+{79, 7762, 0, NULL, NULL},
+{101, 7703, 0, NULL, NULL},
+{111, 7763, 0, NULL, NULL}
+}; /* compose_tab_1_41 */ 
+static int hash_compose_tab_1_42[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_1_42 */ 
+static CompEntry compose_tab_1_42[] = {
+{65, 7854, 0, NULL, NULL},
+{97, 7855, 0, NULL, NULL}
+}; /* compose_tab_1_42 */ 
+static int hash_compose_tab_1_43[12] = 
+{-1,0,1,-1,-1,4,5,-1,-1,2,3,-1}; /* hash_compose_tab_1_43 */ 
+static CompEntry compose_tab_1_43[] = {
+{73, 7726, 0, NULL, NULL},
+{85, 471, 0, NULL, NULL},
+{105, 7727, 0, NULL, NULL},
+{117, 472, 0, NULL, NULL},
+{953, 8147, 0, NULL, NULL},
+{965, 8163, 0, NULL, NULL}
+}; /* compose_tab_1_43 */ 
+static int hash_compose_tab_1_44[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_1_44 */ 
+static CompEntry compose_tab_1_44[] = {
+{65, 506, 0, NULL, NULL},
+{97, 507, 0, NULL, NULL}
+}; /* compose_tab_1_44 */ 
+static int hash_compose_tab_1_45_0[12] = 
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_1_45_0 */ 
+static CompEntry compose_tab_1_45_0[] = {
+{913, 8076, 0, NULL, NULL},
+{919, 8092, 0, NULL, NULL},
+{937, 8108, 0, NULL, NULL},
+{945, 8068, 0, NULL, NULL},
+{951, 8084, 0, NULL, NULL},
+{969, 8100, 0, NULL, NULL}
+}; /* compose_tab_1_45_0 */ 
+static int hash_compose_tab_1_45[28] = 
+{9,10,-1,5,-1,-1,-1,11,-1,-1,-1,-1,-1,6,12,-1,-1,1,13,-1,-1,2,7,3,-1,0,4,8}; /* hash_compose_tab_1_45 */ 
+static CompEntry compose_tab_1_45[] = {
+{837, 0, 6, compose_tab_1_45_0, hash_compose_tab_1_45_0},
+{913, 7948, 0, NULL, NULL},
+{917, 7964, 0, NULL, NULL},
+{919, 7980, 0, NULL, NULL},
+{921, 7996, 0, NULL, NULL},
+{927, 8012, 0, NULL, NULL},
+{937, 8044, 0, NULL, NULL},
+{945, 7940, 0, NULL, NULL},
+{949, 7956, 0, NULL, NULL},
+{951, 7972, 0, NULL, NULL},
+{953, 7988, 0, NULL, NULL},
+{959, 8004, 0, NULL, NULL},
+{965, 8020, 0, NULL, NULL},
+{969, 8036, 0, NULL, NULL}
+}; /* compose_tab_1_45 */ 
+static int hash_compose_tab_1_46_0[12] = 
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_1_46_0 */ 
+static CompEntry compose_tab_1_46_0[] = {
+{913, 8077, 0, NULL, NULL},
+{919, 8093, 0, NULL, NULL},
+{937, 8109, 0, NULL, NULL},
+{945, 8069, 0, NULL, NULL},
+{951, 8085, 0, NULL, NULL},
+{969, 8101, 0, NULL, NULL}
+}; /* compose_tab_1_46_0 */ 
+static int hash_compose_tab_1_46[30] = 
+{-1,-1,-1,6,-1,13,-1,7,-1,14,-1,-1,-1,1,-1,8,-1,2,-1,3,9,4,10,11,-1,-1,-1,0,5,
+ 12}; /* hash_compose_tab_1_46 */ 
+static CompEntry compose_tab_1_46[] = {
+{837, 0, 6, compose_tab_1_46_0, hash_compose_tab_1_46_0},
+{913, 7949, 0, NULL, NULL},
+{917, 7965, 0, NULL, NULL},
+{919, 7981, 0, NULL, NULL},
+{921, 7997, 0, NULL, NULL},
+{927, 8013, 0, NULL, NULL},
+{933, 8029, 0, NULL, NULL},
+{937, 8045, 0, NULL, NULL},
+{945, 7941, 0, NULL, NULL},
+{949, 7957, 0, NULL, NULL},
+{951, 7973, 0, NULL, NULL},
+{953, 7989, 0, NULL, NULL},
+{959, 8005, 0, NULL, NULL},
+{965, 8021, 0, NULL, NULL},
+{969, 8037, 0, NULL, NULL}
+}; /* compose_tab_1_46 */ 
+static int hash_compose_tab_1_47[8] = 
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_1_47 */ 
+static CompEntry compose_tab_1_47[] = {
+{79, 7898, 0, NULL, NULL},
+{85, 7912, 0, NULL, NULL},
+{111, 7899, 0, NULL, NULL},
+{117, 7913, 0, NULL, NULL}
+}; /* compose_tab_1_47 */ 
+static int hash_compose_tab_1_48[4] = 
+{1,-1,-1,0}; /* hash_compose_tab_1_48 */ 
+static CompEntry compose_tab_1_48[] = {
+{67, 7688, 0, NULL, NULL},
+{99, 7689, 0, NULL, NULL}
+}; /* compose_tab_1_48 */ 
+static int hash_compose_tab_1_49[6] = 
+{-1,-1,-1,0,1,2}; /* hash_compose_tab_1_49 */ 
+static CompEntry compose_tab_1_49[] = {
+{945, 8116, 0, NULL, NULL},
+{951, 8132, 0, NULL, NULL},
+{959, 8180, 0, NULL, NULL}
+}; /* compose_tab_1_49 */ 
+static int hash_compose_tab_1[140] = 
+{-1,-1,-1,-1,-1,-1,-1,68,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,34,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,35,-1,-1,-1,-1,64,-1,0,-1,1,-1,2,39,3,40,4,41,5,6,7,
+ 8,9,10,36,11,12,42,13,43,14,44,15,16,37,45,46,50,47,51,17,52,18,53,19,54,20,
+ 55,21,56,22,23,24,25,26,27,38,28,29,48,30,57,31,58,32,33,59,60,61,62,65,66,
+ 63,67,69,-1,-1,-1,-1,-1,49,-1,-1}; /* hash_compose_tab_1 */ 
+static CompEntry compose_tab_1[] = {
+{65, 193, 0, NULL, NULL},
+{67, 262, 0, NULL, NULL},
+{69, 201, 0, NULL, NULL},
+{71, 500, 0, NULL, NULL},
+{73, 205, 0, NULL, NULL},
+{75, 7728, 0, NULL, NULL},
+{76, 313, 0, NULL, NULL},
+{77, 7742, 0, NULL, NULL},
+{78, 323, 0, NULL, NULL},
+{79, 211, 0, NULL, NULL},
+{80, 7764, 0, NULL, NULL},
+{82, 340, 0, NULL, NULL},
+{83, 346, 0, NULL, NULL},
+{85, 218, 0, NULL, NULL},
+{87, 7810, 0, NULL, NULL},
+{89, 221, 0, NULL, NULL},
+{90, 377, 0, NULL, NULL},
+{97, 225, 0, NULL, NULL},
+{99, 263, 0, NULL, NULL},
+{101, 233, 0, NULL, NULL},
+{103, 501, 0, NULL, NULL},
+{105, 237, 0, NULL, NULL},
+{107, 7729, 0, NULL, NULL},
+{108, 314, 0, NULL, NULL},
+{109, 7743, 0, NULL, NULL},
+{110, 324, 0, NULL, NULL},
+{111, 243, 0, NULL, NULL},
+{112, 7765, 0, NULL, NULL},
+{114, 341, 0, NULL, NULL},
+{115, 347, 0, NULL, NULL},
+{117, 250, 0, NULL, NULL},
+{119, 7811, 0, NULL, NULL},
+{121, 253, 0, NULL, NULL},
+{122, 378, 0, NULL, NULL},
+{168, 8174, 0, NULL, NULL},
+{198, 508, 0, NULL, NULL},
+{216, 510, 0, NULL, NULL},
+{230, 509, 0, NULL, NULL},
+{248, 511, 0, NULL, NULL},
+{770, 0, 6, compose_tab_1_39, hash_compose_tab_1_39},
+{771, 0, 4, compose_tab_1_40, hash_compose_tab_1_40},
+{772, 0, 4, compose_tab_1_41, hash_compose_tab_1_41},
+{774, 0, 2, compose_tab_1_42, hash_compose_tab_1_42},
+{776, 0, 6, compose_tab_1_43, hash_compose_tab_1_43},
+{778, 0, 2, compose_tab_1_44, hash_compose_tab_1_44},
+{787, 0, 14, compose_tab_1_45, hash_compose_tab_1_45},
+{788, 0, 15, compose_tab_1_46, hash_compose_tab_1_46},
+{795, 0, 4, compose_tab_1_47, hash_compose_tab_1_47},
+{807, 0, 2, compose_tab_1_48, hash_compose_tab_1_48},
+{837, 0, 3, compose_tab_1_49, hash_compose_tab_1_49},
+{913, 8123, 0, NULL, NULL},
+{917, 8137, 0, NULL, NULL},
+{919, 8139, 0, NULL, NULL},
+{921, 8155, 0, NULL, NULL},
+{927, 8185, 0, NULL, NULL},
+{933, 8171, 0, NULL, NULL},
+{937, 8187, 0, NULL, NULL},
+{945, 8049, 0, NULL, NULL},
+{949, 8051, 0, NULL, NULL},
+{951, 8053, 0, NULL, NULL},
+{953, 8055, 0, NULL, NULL},
+{959, 8057, 0, NULL, NULL},
+{965, 8059, 0, NULL, NULL},
+{969, 8061, 0, NULL, NULL},
+{1043, 1027, 0, NULL, NULL},
+{1050, 1036, 0, NULL, NULL},
+{1075, 1107, 0, NULL, NULL},
+{1082, 1116, 0, NULL, NULL},
+{8127, 8142, 0, NULL, NULL},
+{8190, 8158, 0, NULL, NULL}
+}; /* compose_tab_1 */ 
+static int hash_compose_tab_2_26[12] = 
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_2_26 */ 
+static CompEntry compose_tab_2_26[] = {
+{65, 7852, 0, NULL, NULL},
+{69, 7878, 0, NULL, NULL},
+{79, 7896, 0, NULL, NULL},
+{97, 7853, 0, NULL, NULL},
+{101, 7879, 0, NULL, NULL},
+{111, 7897, 0, NULL, NULL}
+}; /* compose_tab_2_26 */ 
+static int hash_compose_tab_2[54] = 
+{-1,-1,-1,20,-1,-1,-1,21,-1,22,-1,0,23,1,24,2,25,3,4,5,6,-1,-1,-1,-1,7,-1,-1,
+ -1,8,-1,9,-1,10,-1,11,12,-1,-1,-1,-1,-1,-1,13,-1,14,-1,15,26,16,17,18,19,-1}; /* hash_compose_tab_2 */ 
+static CompEntry compose_tab_2[] = {
+{65, 194, 0, NULL, NULL},
+{67, 264, 0, NULL, NULL},
+{69, 202, 0, NULL, NULL},
+{71, 284, 0, NULL, NULL},
+{72, 292, 0, NULL, NULL},
+{73, 206, 0, NULL, NULL},
+{74, 308, 0, NULL, NULL},
+{79, 212, 0, NULL, NULL},
+{83, 348, 0, NULL, NULL},
+{85, 219, 0, NULL, NULL},
+{87, 372, 0, NULL, NULL},
+{89, 374, 0, NULL, NULL},
+{90, 7824, 0, NULL, NULL},
+{97, 226, 0, NULL, NULL},
+{99, 265, 0, NULL, NULL},
+{101, 234, 0, NULL, NULL},
+{103, 285, 0, NULL, NULL},
+{104, 293, 0, NULL, NULL},
+{105, 238, 0, NULL, NULL},
+{106, 309, 0, NULL, NULL},
+{111, 244, 0, NULL, NULL},
+{115, 349, 0, NULL, NULL},
+{117, 251, 0, NULL, NULL},
+{119, 373, 0, NULL, NULL},
+{121, 375, 0, NULL, NULL},
+{122, 7825, 0, NULL, NULL},
+{803, 0, 6, compose_tab_2_26, hash_compose_tab_2_26}
+}; /* compose_tab_2 */ 
+static int hash_compose_tab_3_16[12] = 
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_3_16 */ 
+static CompEntry compose_tab_3_16[] = {
+{65, 7850, 0, NULL, NULL},
+{69, 7876, 0, NULL, NULL},
+{79, 7894, 0, NULL, NULL},
+{97, 7851, 0, NULL, NULL},
+{101, 7877, 0, NULL, NULL},
+{111, 7895, 0, NULL, NULL}
+}; /* compose_tab_3_16 */ 
+static int hash_compose_tab_3_17[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_3_17 */ 
+static CompEntry compose_tab_3_17[] = {
+{65, 7860, 0, NULL, NULL},
+{97, 7861, 0, NULL, NULL}
+}; /* compose_tab_3_17 */ 
+static int hash_compose_tab_3_18[8] = 
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_3_18 */ 
+static CompEntry compose_tab_3_18[] = {
+{79, 7904, 0, NULL, NULL},
+{85, 7918, 0, NULL, NULL},
+{111, 7905, 0, NULL, NULL},
+{117, 7919, 0, NULL, NULL}
+}; /* compose_tab_3_18 */ 
+static int hash_compose_tab_3[38] = 
+{-1,-1,3,4,13,14,-1,15,-1,5,6,16,-1,7,17,-1,-1,-1,-1,-1,-1,8,-1,-1,-1,9,-1,0,
+ -1,10,-1,1,-1,-1,11,2,12,18}; /* hash_compose_tab_3 */ 
+static CompEntry compose_tab_3[] = {
+{65, 195, 0, NULL, NULL},
+{69, 7868, 0, NULL, NULL},
+{73, 296, 0, NULL, NULL},
+{78, 209, 0, NULL, NULL},
+{79, 213, 0, NULL, NULL},
+{85, 360, 0, NULL, NULL},
+{86, 7804, 0, NULL, NULL},
+{89, 7928, 0, NULL, NULL},
+{97, 227, 0, NULL, NULL},
+{101, 7869, 0, NULL, NULL},
+{105, 297, 0, NULL, NULL},
+{110, 241, 0, NULL, NULL},
+{111, 245, 0, NULL, NULL},
+{117, 361, 0, NULL, NULL},
+{118, 7805, 0, NULL, NULL},
+{121, 7929, 0, NULL, NULL},
+{770, 0, 6, compose_tab_3_16, hash_compose_tab_3_16},
+{774, 0, 2, compose_tab_3_17, hash_compose_tab_3_17},
+{795, 0, 4, compose_tab_3_18, hash_compose_tab_3_18}
+}; /* compose_tab_3 */ 
+static int hash_compose_tab_4_14[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_4_14 */ 
+static CompEntry compose_tab_4_14[] = {
+{65, 480, 0, NULL, NULL},
+{97, 481, 0, NULL, NULL}
+}; /* compose_tab_4_14 */ 
+static int hash_compose_tab_4_15[8] = 
+{-1,0,2,-1,-1,1,3,-1}; /* hash_compose_tab_4_15 */ 
+static CompEntry compose_tab_4_15[] = {
+{65, 478, 0, NULL, NULL},
+{85, 469, 0, NULL, NULL},
+{97, 479, 0, NULL, NULL},
+{117, 470, 0, NULL, NULL}
+}; /* compose_tab_4_15 */ 
+static int hash_compose_tab_4_16[8] = 
+{-1,-1,1,3,0,2,-1,-1}; /* hash_compose_tab_4_16 */ 
+static CompEntry compose_tab_4_16[] = {
+{76, 7736, 0, NULL, NULL},
+{82, 7772, 0, NULL, NULL},
+{108, 7737, 0, NULL, NULL},
+{114, 7773, 0, NULL, NULL}
+}; /* compose_tab_4_16 */ 
+static int hash_compose_tab_4_17[4] = 
+{1,-1,-1,0}; /* hash_compose_tab_4_17 */ 
+static CompEntry compose_tab_4_17[] = {
+{79, 492, 0, NULL, NULL},
+{111, 493, 0, NULL, NULL}
+}; /* compose_tab_4_17 */ 
+static int hash_compose_tab_4[56] = 
+{-1,22,-1,-1,-1,11,13,-1,-1,0,-1,-1,-1,1,23,2,26,3,18,16,-1,-1,-1,4,17,19,-1,
+ 27,-1,5,12,-1,-1,-1,-1,-1,-1,20,-1,-1,24,6,-1,-1,-1,7,-1,8,14,9,15,21,25,-1,
+ -1,10}; /* hash_compose_tab_4 */ 
+static CompEntry compose_tab_4[] = {
+{65, 256, 0, NULL, NULL},
+{69, 274, 0, NULL, NULL},
+{71, 7712, 0, NULL, NULL},
+{73, 298, 0, NULL, NULL},
+{79, 332, 0, NULL, NULL},
+{85, 362, 0, NULL, NULL},
+{97, 257, 0, NULL, NULL},
+{101, 275, 0, NULL, NULL},
+{103, 7713, 0, NULL, NULL},
+{105, 299, 0, NULL, NULL},
+{111, 333, 0, NULL, NULL},
+{117, 363, 0, NULL, NULL},
+{198, 482, 0, NULL, NULL},
+{230, 483, 0, NULL, NULL},
+{775, 0, 2, compose_tab_4_14, hash_compose_tab_4_14},
+{776, 0, 4, compose_tab_4_15, hash_compose_tab_4_15},
+{803, 0, 4, compose_tab_4_16, hash_compose_tab_4_16},
+{808, 0, 2, compose_tab_4_17, hash_compose_tab_4_17},
+{913, 8121, 0, NULL, NULL},
+{921, 8153, 0, NULL, NULL},
+{933, 8169, 0, NULL, NULL},
+{945, 8113, 0, NULL, NULL},
+{953, 8145, 0, NULL, NULL},
+{965, 8161, 0, NULL, NULL},
+{1048, 1250, 0, NULL, NULL},
+{1059, 1262, 0, NULL, NULL},
+{1080, 1251, 0, NULL, NULL},
+{1091, 1263, 0, NULL, NULL}
+}; /* compose_tab_4 */ 
+static int hash_compose_tab_5_12[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_5_12 */ 
+static CompEntry compose_tab_5_12[] = {
+{65, 7862, 0, NULL, NULL},
+{97, 7863, 0, NULL, NULL}
+}; /* compose_tab_5_12 */ 
+static int hash_compose_tab_5_13[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_5_13 */ 
+static CompEntry compose_tab_5_13[] = {
+{69, 7708, 0, NULL, NULL},
+{101, 7709, 0, NULL, NULL}
+}; /* compose_tab_5_13 */ 
+static int hash_compose_tab_5[60] = 
+{28,-1,-1,-1,-1,0,19,-1,-1,1,-1,2,29,3,14,-1,-1,-1,-1,4,20,15,-1,12,-1,5,21,
+ 13,22,23,-1,-1,-1,16,-1,-1,-1,6,-1,24,-1,7,-1,8,-1,9,17,-1,-1,-1,-1,10,25,18,
+ -1,-1,-1,11,26,27}; /* hash_compose_tab_5 */ 
+static CompEntry compose_tab_5[] = {
+{65, 258, 0, NULL, NULL},
+{69, 276, 0, NULL, NULL},
+{71, 286, 0, NULL, NULL},
+{73, 300, 0, NULL, NULL},
+{79, 334, 0, NULL, NULL},
+{85, 364, 0, NULL, NULL},
+{97, 259, 0, NULL, NULL},
+{101, 277, 0, NULL, NULL},
+{103, 287, 0, NULL, NULL},
+{105, 301, 0, NULL, NULL},
+{111, 335, 0, NULL, NULL},
+{117, 365, 0, NULL, NULL},
+{803, 0, 2, compose_tab_5_12, hash_compose_tab_5_12},
+{807, 0, 2, compose_tab_5_13, hash_compose_tab_5_13},
+{913, 8120, 0, NULL, NULL},
+{921, 8152, 0, NULL, NULL},
+{933, 8168, 0, NULL, NULL},
+{945, 8112, 0, NULL, NULL},
+{953, 8144, 0, NULL, NULL},
+{965, 8160, 0, NULL, NULL},
+{1040, 1232, 0, NULL, NULL},
+{1045, 1238, 0, NULL, NULL},
+{1046, 1217, 0, NULL, NULL},
+{1048, 1049, 0, NULL, NULL},
+{1059, 1038, 0, NULL, NULL},
+{1072, 1233, 0, NULL, NULL},
+{1077, 1239, 0, NULL, NULL},
+{1078, 1218, 0, NULL, NULL},
+{1080, 1081, 0, NULL, NULL},
+{1091, 1118, 0, NULL, NULL}
+}; /* compose_tab_5 */ 
+static int hash_compose_tab_6_36[4] = 
+{1,-1,-1,0}; /* hash_compose_tab_6_36 */ 
+static CompEntry compose_tab_6_36[] = {
+{83, 7780, 0, NULL, NULL},
+{115, 7781, 0, NULL, NULL}
+}; /* compose_tab_6_36 */ 
+static int hash_compose_tab_6_38[4] = 
+{1,-1,-1,0}; /* hash_compose_tab_6_38 */ 
+static CompEntry compose_tab_6_38[] = {
+{83, 7782, 0, NULL, NULL},
+{115, 7783, 0, NULL, NULL}
+}; /* compose_tab_6_38 */ 
+static int hash_compose_tab_6_39[4] = 
+{1,-1,-1,0}; /* hash_compose_tab_6_39 */ 
+static CompEntry compose_tab_6_39[] = {
+{83, 7784, 0, NULL, NULL},
+{115, 7785, 0, NULL, NULL}
+}; /* compose_tab_6_39 */ 
+static int hash_compose_tab_6[80] = 
+{10,-1,11,12,13,39,-1,14,15,16,17,-1,-1,-1,-1,-1,-1,-1,18,19,20,21,22,23,24,
+ -1,-1,-1,-1,25,26,-1,27,-1,28,29,30,-1,-1,31,32,33,34,-1,-1,-1,-1,-1,-1,36,
+ -1,-1,-1,-1,37,-1,-1,-1,-1,-1,38,-1,-1,35,-1,-1,0,1,2,3,4,5,6,7,-1,-1,-1,8,9,
+ -1}; /* hash_compose_tab_6 */ 
+static CompEntry compose_tab_6[] = {
+{66, 7682, 0, NULL, NULL},
+{67, 266, 0, NULL, NULL},
+{68, 7690, 0, NULL, NULL},
+{69, 278, 0, NULL, NULL},
+{70, 7710, 0, NULL, NULL},
+{71, 288, 0, NULL, NULL},
+{72, 7714, 0, NULL, NULL},
+{73, 304, 0, NULL, NULL},
+{77, 7744, 0, NULL, NULL},
+{78, 7748, 0, NULL, NULL},
+{80, 7766, 0, NULL, NULL},
+{82, 7768, 0, NULL, NULL},
+{83, 7776, 0, NULL, NULL},
+{84, 7786, 0, NULL, NULL},
+{87, 7814, 0, NULL, NULL},
+{88, 7818, 0, NULL, NULL},
+{89, 7822, 0, NULL, NULL},
+{90, 379, 0, NULL, NULL},
+{98, 7683, 0, NULL, NULL},
+{99, 267, 0, NULL, NULL},
+{100, 7691, 0, NULL, NULL},
+{101, 279, 0, NULL, NULL},
+{102, 7711, 0, NULL, NULL},
+{103, 289, 0, NULL, NULL},
+{104, 7715, 0, NULL, NULL},
+{109, 7745, 0, NULL, NULL},
+{110, 7749, 0, NULL, NULL},
+{112, 7767, 0, NULL, NULL},
+{114, 7769, 0, NULL, NULL},
+{115, 7777, 0, NULL, NULL},
+{116, 7787, 0, NULL, NULL},
+{119, 7815, 0, NULL, NULL},
+{120, 7819, 0, NULL, NULL},
+{121, 7823, 0, NULL, NULL},
+{122, 380, 0, NULL, NULL},
+{383, 7835, 0, NULL, NULL},
+{769, 0, 2, compose_tab_6_36, hash_compose_tab_6_36},
+{774, 784, 0, NULL, NULL},
+{780, 0, 2, compose_tab_6_38, hash_compose_tab_6_38},
+{803, 0, 2, compose_tab_6_39, hash_compose_tab_6_39}
+}; /* compose_tab_6 */ 
+static int hash_compose_tab_7_23[4] = 
+{1,-1,-1,0}; /* hash_compose_tab_7_23 */ 
+static CompEntry compose_tab_7_23[] = {
+{79, 7758, 0, NULL, NULL},
+{111, 7759, 0, NULL, NULL}
+}; /* compose_tab_7_23 */ 
+static int hash_compose_tab_7_24[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_7_24 */ 
+static CompEntry compose_tab_7_24[] = {
+{85, 7802, 0, NULL, NULL},
+{117, 7803, 0, NULL, NULL}
+}; /* compose_tab_7_24 */ 
+static int hash_compose_tab_7[100] = 
+{48,10,21,-1,11,12,-1,-1,-1,-1,49,13,-1,-1,-1,20,14,15,-1,16,17,18,25,-1,-1,
+ -1,-1,-1,-1,22,30,-1,-1,26,-1,-1,-1,-1,-1,-1,31,-1,-1,-1,-1,32,33,34,35,-1,
+ -1,-1,-1,27,36,-1,-1,-1,-1,37,-1,-1,-1,38,-1,0,28,39,-1,1,-1,23,2,3,24,40,-1,
+ 41,29,4,42,43,44,-1,-1,5,45,6,7,8,-1,46,-1,-1,-1,47,-1,9,-1,19}; /* hash_compose_tab_7 */ 
+static CompEntry compose_tab_7[] = {
+{65, 196, 0, NULL, NULL},
+{69, 203, 0, NULL, NULL},
+{72, 7718, 0, NULL, NULL},
+{73, 207, 0, NULL, NULL},
+{79, 214, 0, NULL, NULL},
+{85, 220, 0, NULL, NULL},
+{87, 7812, 0, NULL, NULL},
+{88, 7820, 0, NULL, NULL},
+{89, 376, 0, NULL, NULL},
+{97, 228, 0, NULL, NULL},
+{101, 235, 0, NULL, NULL},
+{104, 7719, 0, NULL, NULL},
+{105, 239, 0, NULL, NULL},
+{111, 246, 0, NULL, NULL},
+{116, 7831, 0, NULL, NULL},
+{117, 252, 0, NULL, NULL},
+{119, 7813, 0, NULL, NULL},
+{120, 7821, 0, NULL, NULL},
+{121, 255, 0, NULL, NULL},
+{399, 1242, 0, NULL, NULL},
+{415, 1258, 0, NULL, NULL},
+{601, 1243, 0, NULL, NULL},
+{629, 1259, 0, NULL, NULL},
+{771, 0, 2, compose_tab_7_23, hash_compose_tab_7_23},
+{772, 0, 2, compose_tab_7_24, hash_compose_tab_7_24},
+{921, 938, 0, NULL, NULL},
+{933, 939, 0, NULL, NULL},
+{953, 970, 0, NULL, NULL},
+{965, 971, 0, NULL, NULL},
+{978, 980, 0, NULL, NULL},
+{1030, 1031, 0, NULL, NULL},
+{1040, 1234, 0, NULL, NULL},
+{1045, 1025, 0, NULL, NULL},
+{1046, 1244, 0, NULL, NULL},
+{1047, 1246, 0, NULL, NULL},
+{1048, 1252, 0, NULL, NULL},
+{1054, 1254, 0, NULL, NULL},
+{1059, 1264, 0, NULL, NULL},
+{1063, 1268, 0, NULL, NULL},
+{1067, 1272, 0, NULL, NULL},
+{1072, 1235, 0, NULL, NULL},
+{1077, 1105, 0, NULL, NULL},
+{1078, 1245, 0, NULL, NULL},
+{1079, 1247, 0, NULL, NULL},
+{1080, 1253, 0, NULL, NULL},
+{1086, 1255, 0, NULL, NULL},
+{1091, 1265, 0, NULL, NULL},
+{1095, 1269, 0, NULL, NULL},
+{1099, 1273, 0, NULL, NULL},
+{1110, 1111, 0, NULL, NULL}
+}; /* compose_tab_7 */ 
+static int hash_compose_tab_8_12[12] = 
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_8_12 */ 
+static CompEntry compose_tab_8_12[] = {
+{65, 7848, 0, NULL, NULL},
+{69, 7874, 0, NULL, NULL},
+{79, 7892, 0, NULL, NULL},
+{97, 7849, 0, NULL, NULL},
+{101, 7875, 0, NULL, NULL},
+{111, 7893, 0, NULL, NULL}
+}; /* compose_tab_8_12 */ 
+static int hash_compose_tab_8_13[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_8_13 */ 
+static CompEntry compose_tab_8_13[] = {
+{65, 7858, 0, NULL, NULL},
+{97, 7859, 0, NULL, NULL}
+}; /* compose_tab_8_13 */ 
+static int hash_compose_tab_8_14[8] = 
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_8_14 */ 
+static CompEntry compose_tab_8_14[] = {
+{79, 7902, 0, NULL, NULL},
+{85, 7916, 0, NULL, NULL},
+{111, 7903, 0, NULL, NULL},
+{117, 7917, 0, NULL, NULL}
+}; /* compose_tab_8_14 */ 
+static int hash_compose_tab_8[30] = 
+{-1,11,-1,-1,-1,0,-1,6,-1,1,-1,7,-1,2,-1,8,14,-1,-1,3,12,9,-1,-1,13,4,-1,10,
+ -1,5}; /* hash_compose_tab_8 */ 
+static CompEntry compose_tab_8[] = {
+{65, 7842, 0, NULL, NULL},
+{69, 7866, 0, NULL, NULL},
+{73, 7880, 0, NULL, NULL},
+{79, 7886, 0, NULL, NULL},
+{85, 7910, 0, NULL, NULL},
+{89, 7926, 0, NULL, NULL},
+{97, 7843, 0, NULL, NULL},
+{101, 7867, 0, NULL, NULL},
+{105, 7881, 0, NULL, NULL},
+{111, 7887, 0, NULL, NULL},
+{117, 7911, 0, NULL, NULL},
+{121, 7927, 0, NULL, NULL},
+{770, 0, 6, compose_tab_8_12, hash_compose_tab_8_12},
+{774, 0, 2, compose_tab_8_13, hash_compose_tab_8_13},
+{795, 0, 4, compose_tab_8_14, hash_compose_tab_8_14}
+}; /* compose_tab_8 */ 
+static int hash_compose_tab_9[12] = 
+{-1,1,2,5,-1,0,-1,-1,-1,3,-1,4}; /* hash_compose_tab_9 */ 
+static CompEntry compose_tab_9[] = {
+{65, 197, 0, NULL, NULL},
+{85, 366, 0, NULL, NULL},
+{97, 229, 0, NULL, NULL},
+{117, 367, 0, NULL, NULL},
+{119, 7832, 0, NULL, NULL},
+{121, 7833, 0, NULL, NULL}
+}; /* compose_tab_9 */ 
+static int hash_compose_tab_10[12] = 
+{-1,1,-1,2,4,-1,-1,0,-1,3,-1,5}; /* hash_compose_tab_10 */ 
+static CompEntry compose_tab_10[] = {
+{79, 336, 0, NULL, NULL},
+{85, 368, 0, NULL, NULL},
+{111, 337, 0, NULL, NULL},
+{117, 369, 0, NULL, NULL},
+{1059, 1266, 0, NULL, NULL},
+{1091, 1267, 0, NULL, NULL}
+}; /* compose_tab_10 */ 
+static int hash_compose_tab_11_33[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_11_33 */ 
+static CompEntry compose_tab_11_33[] = {
+{85, 473, 0, NULL, NULL},
+{117, 474, 0, NULL, NULL}
+}; /* compose_tab_11_33 */ 
+static int hash_compose_tab_11[68] = 
+{2,3,-1,4,-1,5,-1,6,7,-1,8,9,-1,-1,10,11,12,13,-1,-1,-1,-1,14,-1,-1,-1,-1,-1,
+ 33,15,-1,16,17,18,31,19,-1,20,21,22,23,-1,24,25,-1,-1,26,27,28,29,32,-1,-1,
+ -1,30,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,-1,1}; /* hash_compose_tab_11 */ 
+static CompEntry compose_tab_11[] = {
+{65, 461, 0, NULL, NULL},
+{67, 268, 0, NULL, NULL},
+{68, 270, 0, NULL, NULL},
+{69, 282, 0, NULL, NULL},
+{71, 486, 0, NULL, NULL},
+{73, 463, 0, NULL, NULL},
+{75, 488, 0, NULL, NULL},
+{76, 317, 0, NULL, NULL},
+{78, 327, 0, NULL, NULL},
+{79, 465, 0, NULL, NULL},
+{82, 344, 0, NULL, NULL},
+{83, 352, 0, NULL, NULL},
+{84, 356, 0, NULL, NULL},
+{85, 467, 0, NULL, NULL},
+{90, 381, 0, NULL, NULL},
+{97, 462, 0, NULL, NULL},
+{99, 269, 0, NULL, NULL},
+{100, 271, 0, NULL, NULL},
+{101, 283, 0, NULL, NULL},
+{103, 487, 0, NULL, NULL},
+{105, 464, 0, NULL, NULL},
+{106, 496, 0, NULL, NULL},
+{107, 489, 0, NULL, NULL},
+{108, 318, 0, NULL, NULL},
+{110, 328, 0, NULL, NULL},
+{111, 466, 0, NULL, NULL},
+{114, 345, 0, NULL, NULL},
+{115, 353, 0, NULL, NULL},
+{116, 357, 0, NULL, NULL},
+{117, 468, 0, NULL, NULL},
+{122, 382, 0, NULL, NULL},
+{439, 494, 0, NULL, NULL},
+{658, 495, 0, NULL, NULL},
+{776, 0, 2, compose_tab_11_33, hash_compose_tab_11_33}
+}; /* compose_tab_11 */ 
+static int hash_compose_tab_12_1[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_12_1 */ 
+static CompEntry compose_tab_12_1[] = {
+{953, 912, 0, NULL, NULL},
+{965, 944, 0, NULL, NULL}
+}; /* compose_tab_12_1 */ 
+static int hash_compose_tab_12[34] = 
+{11,4,12,5,-1,-1,-1,13,-1,6,-1,-1,-1,14,-1,7,-1,15,-1,8,-1,-1,-1,-1,-1,-1,16,
+ 9,1,2,-1,10,0,3}; /* hash_compose_tab_12 */ 
+static CompEntry compose_tab_12[] = {
+{168, 901, 0, NULL, NULL},
+{776, 0, 2, compose_tab_12_1, hash_compose_tab_12_1},
+{913, 902, 0, NULL, NULL},
+{917, 904, 0, NULL, NULL},
+{919, 905, 0, NULL, NULL},
+{921, 906, 0, NULL, NULL},
+{927, 908, 0, NULL, NULL},
+{933, 910, 0, NULL, NULL},
+{937, 911, 0, NULL, NULL},
+{945, 940, 0, NULL, NULL},
+{949, 941, 0, NULL, NULL},
+{951, 942, 0, NULL, NULL},
+{953, 943, 0, NULL, NULL},
+{959, 972, 0, NULL, NULL},
+{965, 973, 0, NULL, NULL},
+{969, 974, 0, NULL, NULL},
+{978, 979, 0, NULL, NULL}
+}; /* compose_tab_12 */ 
+static int hash_compose_tab_13[28] = 
+{-1,5,10,-1,-1,11,-1,-1,-1,0,-1,-1,-1,1,6,-1,-1,2,7,-1,12,8,13,3,-1,-1,4,9}; /* hash_compose_tab_13 */ 
+static CompEntry compose_tab_13[] = {
+{65, 512, 0, NULL, NULL},
+{69, 516, 0, NULL, NULL},
+{73, 520, 0, NULL, NULL},
+{79, 524, 0, NULL, NULL},
+{82, 528, 0, NULL, NULL},
+{85, 532, 0, NULL, NULL},
+{97, 513, 0, NULL, NULL},
+{101, 517, 0, NULL, NULL},
+{105, 521, 0, NULL, NULL},
+{111, 525, 0, NULL, NULL},
+{114, 529, 0, NULL, NULL},
+{117, 533, 0, NULL, NULL},
+{1140, 1142, 0, NULL, NULL},
+{1141, 1143, 0, NULL, NULL}
+}; /* compose_tab_13 */ 
+static int hash_compose_tab_14[24] = 
+{-1,2,6,-1,-1,7,-1,3,-1,8,4,-1,-1,5,-1,9,-1,0,10,-1,-1,1,11,-1}; /* hash_compose_tab_14 */ 
+static CompEntry compose_tab_14[] = {
+{65, 514, 0, NULL, NULL},
+{69, 518, 0, NULL, NULL},
+{73, 522, 0, NULL, NULL},
+{79, 526, 0, NULL, NULL},
+{82, 530, 0, NULL, NULL},
+{85, 534, 0, NULL, NULL},
+{97, 515, 0, NULL, NULL},
+{101, 519, 0, NULL, NULL},
+{105, 523, 0, NULL, NULL},
+{111, 527, 0, NULL, NULL},
+{114, 531, 0, NULL, NULL},
+{117, 535, 0, NULL, NULL}
+}; /* compose_tab_14 */ 
+static int hash_compose_tab_15_0[12] = 
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_15_0 */ 
+static CompEntry compose_tab_15_0[] = {
+{913, 8072, 0, NULL, NULL},
+{919, 8088, 0, NULL, NULL},
+{937, 8104, 0, NULL, NULL},
+{945, 8064, 0, NULL, NULL},
+{951, 8080, 0, NULL, NULL},
+{969, 8096, 0, NULL, NULL}
+}; /* compose_tab_15_0 */ 
+static int hash_compose_tab_15[30] = 
+{-1,12,-1,-1,-1,13,-1,6,-1,14,-1,-1,-1,1,-1,7,-1,2,-1,3,8,4,9,10,-1,-1,-1,0,5,
+ 11}; /* hash_compose_tab_15 */ 
+static CompEntry compose_tab_15[] = {
+{837, 0, 6, compose_tab_15_0, hash_compose_tab_15_0},
+{913, 7944, 0, NULL, NULL},
+{917, 7960, 0, NULL, NULL},
+{919, 7976, 0, NULL, NULL},
+{921, 7992, 0, NULL, NULL},
+{927, 8008, 0, NULL, NULL},
+{937, 8040, 0, NULL, NULL},
+{945, 7936, 0, NULL, NULL},
+{949, 7952, 0, NULL, NULL},
+{951, 7968, 0, NULL, NULL},
+{953, 7984, 0, NULL, NULL},
+{959, 8000, 0, NULL, NULL},
+{961, 8164, 0, NULL, NULL},
+{965, 8016, 0, NULL, NULL},
+{969, 8032, 0, NULL, NULL}
+}; /* compose_tab_15 */ 
+static int hash_compose_tab_16_0[12] = 
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_16_0 */ 
+static CompEntry compose_tab_16_0[] = {
+{913, 8073, 0, NULL, NULL},
+{919, 8089, 0, NULL, NULL},
+{937, 8105, 0, NULL, NULL},
+{945, 8065, 0, NULL, NULL},
+{951, 8081, 0, NULL, NULL},
+{969, 8097, 0, NULL, NULL}
+}; /* compose_tab_16_0 */ 
+static int hash_compose_tab_16[34] = 
+{11,3,12,4,-1,-1,-1,13,-1,5,14,6,-1,15,-1,7,-1,16,-1,8,-1,0,-1,-1,-1,-1,-1,9,
+ -1,1,-1,10,-1,2}; /* hash_compose_tab_16 */ 
+static CompEntry compose_tab_16[] = {
+{837, 0, 6, compose_tab_16_0, hash_compose_tab_16_0},
+{913, 7945, 0, NULL, NULL},
+{917, 7961, 0, NULL, NULL},
+{919, 7977, 0, NULL, NULL},
+{921, 7993, 0, NULL, NULL},
+{927, 8009, 0, NULL, NULL},
+{929, 8172, 0, NULL, NULL},
+{933, 8025, 0, NULL, NULL},
+{937, 8041, 0, NULL, NULL},
+{945, 7937, 0, NULL, NULL},
+{949, 7953, 0, NULL, NULL},
+{951, 7969, 0, NULL, NULL},
+{953, 7985, 0, NULL, NULL},
+{959, 8001, 0, NULL, NULL},
+{961, 8165, 0, NULL, NULL},
+{965, 8017, 0, NULL, NULL},
+{969, 8033, 0, NULL, NULL}
+}; /* compose_tab_16 */ 
+static int hash_compose_tab_17[8] = 
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_17 */ 
+static CompEntry compose_tab_17[] = {
+{79, 416, 0, NULL, NULL},
+{85, 431, 0, NULL, NULL},
+{111, 417, 0, NULL, NULL},
+{117, 432, 0, NULL, NULL}
+}; /* compose_tab_17 */ 
+static int hash_compose_tab_18_38[8] = 
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_18_38 */ 
+static CompEntry compose_tab_18_38[] = {
+{79, 7906, 0, NULL, NULL},
+{85, 7920, 0, NULL, NULL},
+{111, 7907, 0, NULL, NULL},
+{117, 7921, 0, NULL, NULL}
+}; /* compose_tab_18_38 */ 
+static int hash_compose_tab_18[78] = 
+{9,10,-1,-1,11,12,13,14,15,16,-1,17,18,-1,-1,38,-1,-1,-1,19,20,-1,21,22,-1,-1,
+ 23,24,-1,25,26,27,28,29,-1,-1,30,31,32,33,34,35,-1,36,37,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,-1,2,3,-1,-1,4,5,-1,6,7,8}; /* hash_compose_tab_18 */ 
+static CompEntry compose_tab_18[] = {
+{65, 7840, 0, NULL, NULL},
+{66, 7684, 0, NULL, NULL},
+{68, 7692, 0, NULL, NULL},
+{69, 7864, 0, NULL, NULL},
+{72, 7716, 0, NULL, NULL},
+{73, 7882, 0, NULL, NULL},
+{75, 7730, 0, NULL, NULL},
+{76, 7734, 0, NULL, NULL},
+{77, 7746, 0, NULL, NULL},
+{78, 7750, 0, NULL, NULL},
+{79, 7884, 0, NULL, NULL},
+{82, 7770, 0, NULL, NULL},
+{83, 7778, 0, NULL, NULL},
+{84, 7788, 0, NULL, NULL},
+{85, 7908, 0, NULL, NULL},
+{86, 7806, 0, NULL, NULL},
+{87, 7816, 0, NULL, NULL},
+{89, 7924, 0, NULL, NULL},
+{90, 7826, 0, NULL, NULL},
+{97, 7841, 0, NULL, NULL},
+{98, 7685, 0, NULL, NULL},
+{100, 7693, 0, NULL, NULL},
+{101, 7865, 0, NULL, NULL},
+{104, 7717, 0, NULL, NULL},
+{105, 7883, 0, NULL, NULL},
+{107, 7731, 0, NULL, NULL},
+{108, 7735, 0, NULL, NULL},
+{109, 7747, 0, NULL, NULL},
+{110, 7751, 0, NULL, NULL},
+{111, 7885, 0, NULL, NULL},
+{114, 7771, 0, NULL, NULL},
+{115, 7779, 0, NULL, NULL},
+{116, 7789, 0, NULL, NULL},
+{117, 7909, 0, NULL, NULL},
+{118, 7807, 0, NULL, NULL},
+{119, 7817, 0, NULL, NULL},
+{121, 7925, 0, NULL, NULL},
+{122, 7827, 0, NULL, NULL},
+{795, 0, 4, compose_tab_18_38, hash_compose_tab_18_38}
+}; /* compose_tab_18 */ 
+static int hash_compose_tab_19[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_19 */ 
+static CompEntry compose_tab_19[] = {
+{85, 7794, 0, NULL, NULL},
+{117, 7795, 0, NULL, NULL}
+}; /* compose_tab_19 */ 
+static int hash_compose_tab_20[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_20 */ 
+static CompEntry compose_tab_20[] = {
+{65, 7680, 0, NULL, NULL},
+{97, 7681, 0, NULL, NULL}
+}; /* compose_tab_20 */ 
+static int hash_compose_tab_21[40] = 
+{-1,-1,7,8,9,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,10,11,-1,-1,12,13,-1,
+ -1,0,1,14,15,2,3,16,17,4,5,18,6,19}; /* hash_compose_tab_21 */ 
+static CompEntry compose_tab_21[] = {
+{67, 199, 0, NULL, NULL},
+{68, 7696, 0, NULL, NULL},
+{71, 290, 0, NULL, NULL},
+{72, 7720, 0, NULL, NULL},
+{75, 310, 0, NULL, NULL},
+{76, 315, 0, NULL, NULL},
+{78, 325, 0, NULL, NULL},
+{82, 342, 0, NULL, NULL},
+{83, 350, 0, NULL, NULL},
+{84, 354, 0, NULL, NULL},
+{99, 231, 0, NULL, NULL},
+{100, 7697, 0, NULL, NULL},
+{103, 291, 0, NULL, NULL},
+{104, 7721, 0, NULL, NULL},
+{107, 311, 0, NULL, NULL},
+{108, 316, 0, NULL, NULL},
+{110, 326, 0, NULL, NULL},
+{114, 343, 0, NULL, NULL},
+{115, 351, 0, NULL, NULL},
+{116, 355, 0, NULL, NULL}
+}; /* compose_tab_21 */ 
+static int hash_compose_tab_22[20] = 
+{-1,6,-1,-1,-1,0,4,7,-1,1,-1,8,-1,2,-1,-1,-1,5,9,3}; /* hash_compose_tab_22 */ 
+static CompEntry compose_tab_22[] = {
+{65, 260, 0, NULL, NULL},
+{69, 280, 0, NULL, NULL},
+{73, 302, 0, NULL, NULL},
+{79, 490, 0, NULL, NULL},
+{85, 370, 0, NULL, NULL},
+{97, 261, 0, NULL, NULL},
+{101, 281, 0, NULL, NULL},
+{105, 303, 0, NULL, NULL},
+{111, 491, 0, NULL, NULL},
+{117, 371, 0, NULL, NULL}
+}; /* compose_tab_22 */ 
+static int hash_compose_tab_23[24] = 
+{-1,-1,-1,-1,2,6,3,7,-1,-1,-1,-1,4,5,8,9,-1,-1,-1,-1,0,1,10,11}; /* hash_compose_tab_23 */ 
+static CompEntry compose_tab_23[] = {
+{68, 7698, 0, NULL, NULL},
+{69, 7704, 0, NULL, NULL},
+{76, 7740, 0, NULL, NULL},
+{78, 7754, 0, NULL, NULL},
+{84, 7792, 0, NULL, NULL},
+{85, 7798, 0, NULL, NULL},
+{100, 7699, 0, NULL, NULL},
+{101, 7705, 0, NULL, NULL},
+{108, 7741, 0, NULL, NULL},
+{110, 7755, 0, NULL, NULL},
+{116, 7793, 0, NULL, NULL},
+{117, 7799, 0, NULL, NULL}
+}; /* compose_tab_23 */ 
+static int hash_compose_tab_24[4] = 
+{0,1,-1,-1}; /* hash_compose_tab_24 */ 
+static CompEntry compose_tab_24[] = {
+{72, 7722, 0, NULL, NULL},
+{104, 7723, 0, NULL, NULL}
+}; /* compose_tab_24 */ 
+static int hash_compose_tab_25[12] = 
+{-1,1,2,-1,-1,3,-1,-1,-1,0,4,5}; /* hash_compose_tab_25 */ 
+static CompEntry compose_tab_25[] = {
+{69, 7706, 0, NULL, NULL},
+{73, 7724, 0, NULL, NULL},
+{85, 7796, 0, NULL, NULL},
+{101, 7707, 0, NULL, NULL},
+{105, 7725, 0, NULL, NULL},
+{117, 7797, 0, NULL, NULL}
+}; /* compose_tab_25 */ 
+static int hash_compose_tab_26[34] = 
+{1,-1,10,-1,-1,11,12,2,3,13,4,-1,14,-1,5,15,6,-1,-1,-1,16,-1,7,-1,-1,-1,-1,-1,
+ -1,-1,8,-1,0,9}; /* hash_compose_tab_26 */ 
+static CompEntry compose_tab_26[] = {
+{66, 7686, 0, NULL, NULL},
+{68, 7694, 0, NULL, NULL},
+{75, 7732, 0, NULL, NULL},
+{76, 7738, 0, NULL, NULL},
+{78, 7752, 0, NULL, NULL},
+{82, 7774, 0, NULL, NULL},
+{84, 7790, 0, NULL, NULL},
+{90, 7828, 0, NULL, NULL},
+{98, 7687, 0, NULL, NULL},
+{100, 7695, 0, NULL, NULL},
+{104, 7830, 0, NULL, NULL},
+{107, 7733, 0, NULL, NULL},
+{108, 7739, 0, NULL, NULL},
+{110, 7753, 0, NULL, NULL},
+{114, 7775, 0, NULL, NULL},
+{116, 7791, 0, NULL, NULL},
+{122, 7829, 0, NULL, NULL}
+}; /* compose_tab_26 */ 
+static int hash_compose_tab_27_1[4] = 
+{-1,0,1,-1}; /* hash_compose_tab_27_1 */ 
+static CompEntry compose_tab_27_1[] = {
+{953, 8151, 0, NULL, NULL},
+{965, 8167, 0, NULL, NULL}
+}; /* compose_tab_27_1 */ 
+static int hash_compose_tab_27_2_0[12] = 
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_27_2_0 */ 
+static CompEntry compose_tab_27_2_0[] = {
+{913, 8078, 0, NULL, NULL},
+{919, 8094, 0, NULL, NULL},
+{937, 8110, 0, NULL, NULL},
+{945, 8070, 0, NULL, NULL},
+{951, 8086, 0, NULL, NULL},
+{969, 8102, 0, NULL, NULL}
+}; /* compose_tab_27_2_0 */ 
+static int hash_compose_tab_27_2[20] = 
+{-1,3,-1,-1,-1,5,8,-1,-1,9,-1,6,-1,1,7,-1,-1,0,4,2}; /* hash_compose_tab_27_2 */ 
+static CompEntry compose_tab_27_2[] = {
+{837, 0, 6, compose_tab_27_2_0, hash_compose_tab_27_2_0},
+{913, 7950, 0, NULL, NULL},
+{919, 7982, 0, NULL, NULL},
+{921, 7998, 0, NULL, NULL},
+{937, 8046, 0, NULL, NULL},
+{945, 7942, 0, NULL, NULL},
+{951, 7974, 0, NULL, NULL},
+{953, 7990, 0, NULL, NULL},
+{965, 8022, 0, NULL, NULL},
+{969, 8038, 0, NULL, NULL}
+}; /* compose_tab_27_2 */ 
+static int hash_compose_tab_27_3_0[12] = 
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_27_3_0 */ 
+static CompEntry compose_tab_27_3_0[] = {
+{913, 8079, 0, NULL, NULL},
+{919, 8095, 0, NULL, NULL},
+{937, 8111, 0, NULL, NULL},
+{945, 8071, 0, NULL, NULL},
+{951, 8087, 0, NULL, NULL},
+{969, 8103, 0, NULL, NULL}
+}; /* compose_tab_27_3_0 */ 
+static int hash_compose_tab_27_3[22] = 
+{-1,0,10,-1,-1,7,-1,8,-1,4,-1,1,-1,5,-1,-1,-1,2,-1,3,9,6}; /* hash_compose_tab_27_3 */ 
+static CompEntry compose_tab_27_3[] = {
+{837, 0, 6, compose_tab_27_3_0, hash_compose_tab_27_3_0},
+{913, 7951, 0, NULL, NULL},
+{919, 7983, 0, NULL, NULL},
+{921, 7999, 0, NULL, NULL},
+{933, 8031, 0, NULL, NULL},
+{937, 8047, 0, NULL, NULL},
+{945, 7943, 0, NULL, NULL},
+{951, 7975, 0, NULL, NULL},
+{953, 7991, 0, NULL, NULL},
+{965, 8023, 0, NULL, NULL},
+{969, 8039, 0, NULL, NULL}
+}; /* compose_tab_27_3 */ 
+static int hash_compose_tab_27_4[6] = 
+{-1,-1,-1,0,1,2}; /* hash_compose_tab_27_4 */ 
+static CompEntry compose_tab_27_4[] = {
+{945, 8119, 0, NULL, NULL},
+{951, 8135, 0, NULL, NULL},
+{969, 8183, 0, NULL, NULL}
+}; /* compose_tab_27_4 */ 
+static int hash_compose_tab_27[24] = 
+{0,-1,-1,-1,-1,8,11,-1,1,5,9,-1,-1,-1,-1,6,10,7,-1,2,3,4,-1,-1}; /* hash_compose_tab_27 */ 
+static CompEntry compose_tab_27[] = {
+{168, 8129, 0, NULL, NULL},
+{776, 0, 2, compose_tab_27_1, hash_compose_tab_27_1},
+{787, 0, 10, compose_tab_27_2, hash_compose_tab_27_2},
+{788, 0, 11, compose_tab_27_3, hash_compose_tab_27_3},
+{837, 0, 3, compose_tab_27_4, hash_compose_tab_27_4},
+{945, 8118, 0, NULL, NULL},
+{951, 8134, 0, NULL, NULL},
+{953, 8150, 0, NULL, NULL},
+{965, 8166, 0, NULL, NULL},
+{969, 8182, 0, NULL, NULL},
+{8127, 8143, 0, NULL, NULL},
+{8190, 8159, 0, NULL, NULL}
+}; /* compose_tab_27 */ 
+static int hash_compose_tab_28[12] = 
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_28 */ 
+static CompEntry compose_tab_28[] = {
+{913, 8124, 0, NULL, NULL},
+{919, 8140, 0, NULL, NULL},
+{937, 8188, 0, NULL, NULL},
+{945, 8115, 0, NULL, NULL},
+{951, 8131, 0, NULL, NULL},
+{969, 8179, 0, NULL, NULL}
+}; /* compose_tab_28 */ 
+static int hash_compose_tab_29[4] = 
+{0,-1,1,-1}; /* hash_compose_tab_29 */ 
+static CompEntry compose_tab_29[] = {
+{1488, 64302, 0, NULL, NULL},
+{1522, 64287, 0, NULL, NULL}
+}; /* compose_tab_29 */ 
+static int hash_compose_tab_30[2] = 
+{0,-1}; /* hash_compose_tab_30 */ 
+static CompEntry compose_tab_30[] = {
+{1488, 64303, 0, NULL, NULL}
+}; /* compose_tab_30 */ 
+static int hash_compose_tab_31[2] = 
+{-1,0}; /* hash_compose_tab_31 */ 
+static CompEntry compose_tab_31[] = {
+{1493, 64331, 0, NULL, NULL}
+}; /* compose_tab_31 */ 
+static int hash_compose_tab_32[44] = 
+{7,8,9,10,11,-1,12,-1,13,14,-1,15,16,-1,17,18,19,20,21,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,2,3,4,5,6,-1}; /* hash_compose_tab_32 */ 
+static CompEntry compose_tab_32[] = {
+{1488, 64304, 0, NULL, NULL},
+{1489, 64305, 0, NULL, NULL},
+{1490, 64306, 0, NULL, NULL},
+{1491, 64307, 0, NULL, NULL},
+{1492, 64308, 0, NULL, NULL},
+{1493, 64309, 0, NULL, NULL},
+{1494, 64310, 0, NULL, NULL},
+{1496, 64312, 0, NULL, NULL},
+{1497, 64313, 0, NULL, NULL},
+{1498, 64314, 0, NULL, NULL},
+{1499, 64315, 0, NULL, NULL},
+{1500, 64316, 0, NULL, NULL},
+{1502, 64318, 0, NULL, NULL},
+{1504, 64320, 0, NULL, NULL},
+{1505, 64321, 0, NULL, NULL},
+{1507, 64323, 0, NULL, NULL},
+{1508, 64324, 0, NULL, NULL},
+{1510, 64326, 0, NULL, NULL},
+{1511, 64327, 0, NULL, NULL},
+{1512, 64328, 0, NULL, NULL},
+{1513, 64329, 0, NULL, NULL},
+{1514, 64330, 0, NULL, NULL}
+}; /* compose_tab_32 */ 
+static int hash_compose_tab_33[6] = 
+{-1,0,2,-1,-1,1}; /* hash_compose_tab_33 */ 
+static CompEntry compose_tab_33[] = {
+{1489, 64332, 0, NULL, NULL},
+{1499, 64333, 0, NULL, NULL},
+{1508, 64334, 0, NULL, NULL}
+}; /* compose_tab_33 */ 
+static int hash_compose_tab_34_0[2] = 
+{-1,0}; /* hash_compose_tab_34_0 */ 
+static CompEntry compose_tab_34_0[] = {
+{1513, 64300, 0, NULL, NULL}
+}; /* compose_tab_34_0 */ 
+static int hash_compose_tab_34[4] = 
+{0,1,-1,-1}; /* hash_compose_tab_34 */ 
+static CompEntry compose_tab_34[] = {
+{1468, 0, 1, compose_tab_34_0, hash_compose_tab_34_0},
+{1513, 64298, 0, NULL, NULL}
+}; /* compose_tab_34 */ 
+static int hash_compose_tab_35_0[2] = 
+{-1,0}; /* hash_compose_tab_35_0 */ 
+static CompEntry compose_tab_35_0[] = {
+{1513, 64301, 0, NULL, NULL}
+}; /* compose_tab_35_0 */ 
+static int hash_compose_tab_35[4] = 
+{0,1,-1,-1}; /* hash_compose_tab_35 */ 
+static CompEntry compose_tab_35[] = {
+{1468, 0, 1, compose_tab_35_0, hash_compose_tab_35_0},
+{1513, 64299, 0, NULL, NULL}
+}; /* compose_tab_35 */ 
+static int hash_compose_tab_36[22] = 
+{3,10,-1,-1,-1,4,5,-1,-1,-1,-1,-1,6,-1,-1,0,1,2,7,8,9,-1}; /* hash_compose_tab_36 */ 
+static CompEntry compose_tab_36[] = {
+{2325, 2392, 0, NULL, NULL},
+{2326, 2393, 0, NULL, NULL},
+{2327, 2394, 0, NULL, NULL},
+{2332, 2395, 0, NULL, NULL},
+{2337, 2396, 0, NULL, NULL},
+{2338, 2397, 0, NULL, NULL},
+{2344, 2345, 0, NULL, NULL},
+{2347, 2398, 0, NULL, NULL},
+{2351, 2399, 0, NULL, NULL},
+{2352, 2353, 0, NULL, NULL},
+{2355, 2356, 0, NULL, NULL}
+}; /* compose_tab_36 */ 
+static int hash_compose_tab_37[8] = 
+{-1,0,1,-1,2,-1,-1,3}; /* hash_compose_tab_37 */ 
+static CompEntry compose_tab_37[] = {
+{2465, 2524, 0, NULL, NULL},
+{2466, 2525, 0, NULL, NULL},
+{2476, 2480, 0, NULL, NULL},
+{2479, 2527, 0, NULL, NULL}
+}; /* compose_tab_37 */ 
+static int hash_compose_tab_38[2] = 
+{-1,0}; /* hash_compose_tab_38 */ 
+static CompEntry compose_tab_38[] = {
+{2503, 2507, 0, NULL, NULL}
+}; /* compose_tab_38 */ 
+static int hash_compose_tab_39[2] = 
+{-1,0}; /* hash_compose_tab_39 */ 
+static CompEntry compose_tab_39[] = {
+{2503, 2508, 0, NULL, NULL}
+}; /* compose_tab_39 */ 
+static int hash_compose_tab_40[10] = 
+{-1,-1,0,1,3,4,-1,-1,2,-1}; /* hash_compose_tab_40 */ 
+static CompEntry compose_tab_40[] = {
+{2582, 2649, 0, NULL, NULL},
+{2583, 2650, 0, NULL, NULL},
+{2588, 2651, 0, NULL, NULL},
+{2593, 2652, 0, NULL, NULL},
+{2603, 2654, 0, NULL, NULL}
+}; /* compose_tab_40 */ 
+static int hash_compose_tab_41[6] = 
+{1,2,-1,-1,-1,0}; /* hash_compose_tab_41 */ 
+static CompEntry compose_tab_41[] = {
+{2849, 2908, 0, NULL, NULL},
+{2850, 2909, 0, NULL, NULL},
+{2863, 2911, 0, NULL, NULL}
+}; /* compose_tab_41 */ 
+static int hash_compose_tab_42[2] = 
+{-1,0}; /* hash_compose_tab_42 */ 
+static CompEntry compose_tab_42[] = {
+{2887, 2891, 0, NULL, NULL}
+}; /* compose_tab_42 */ 
+static int hash_compose_tab_43[2] = 
+{-1,0}; /* hash_compose_tab_43 */ 
+static CompEntry compose_tab_43[] = {
+{2887, 2888, 0, NULL, NULL}
+}; /* compose_tab_43 */ 
+static int hash_compose_tab_44[2] = 
+{-1,0}; /* hash_compose_tab_44 */ 
+static CompEntry compose_tab_44[] = {
+{2887, 2892, 0, NULL, NULL}
+}; /* compose_tab_44 */ 
+static int hash_compose_tab_45[4] = 
+{-1,-1,0,1}; /* hash_compose_tab_45 */ 
+static CompEntry compose_tab_45[] = {
+{3014, 3018, 0, NULL, NULL},
+{3015, 3019, 0, NULL, NULL}
+}; /* compose_tab_45 */ 
+static int hash_compose_tab_46[4] = 
+{-1,-1,0,1}; /* hash_compose_tab_46 */ 
+static CompEntry compose_tab_46[] = {
+{2962, 2964, 0, NULL, NULL},
+{3014, 3020, 0, NULL, NULL}
+}; /* compose_tab_46 */ 
+static int hash_compose_tab_47[2] = 
+{0,-1}; /* hash_compose_tab_47 */ 
+static CompEntry compose_tab_47[] = {
+{3142, 3144, 0, NULL, NULL}
+}; /* compose_tab_47 */ 
+static int hash_compose_tab_48[2] = 
+{0,-1}; /* hash_compose_tab_48 */ 
+static CompEntry compose_tab_48[] = {
+{3270, 3274, 0, NULL, NULL}
+}; /* compose_tab_48 */ 
+static int hash_compose_tab_49_1[2] = 
+{0,-1}; /* hash_compose_tab_49_1 */ 
+static CompEntry compose_tab_49_1[] = {
+{3270, 3275, 0, NULL, NULL}
+}; /* compose_tab_49_1 */ 
+static int hash_compose_tab_49[6] = 
+{2,-1,1,-1,-1,0}; /* hash_compose_tab_49 */ 
+static CompEntry compose_tab_49[] = {
+{3263, 3264, 0, NULL, NULL},
+{3266, 0, 1, compose_tab_49_1, hash_compose_tab_49_1},
+{3270, 3271, 0, NULL, NULL}
+}; /* compose_tab_49 */ 
+static int hash_compose_tab_50[2] = 
+{0,-1}; /* hash_compose_tab_50 */ 
+static CompEntry compose_tab_50[] = {
+{3270, 3272, 0, NULL, NULL}
+}; /* compose_tab_50 */ 
+static int hash_compose_tab_51[4] = 
+{-1,-1,0,1}; /* hash_compose_tab_51 */ 
+static CompEntry compose_tab_51[] = {
+{3398, 3402, 0, NULL, NULL},
+{3399, 3403, 0, NULL, NULL}
+}; /* compose_tab_51 */ 
+static int hash_compose_tab_52[2] = 
+{0,-1}; /* hash_compose_tab_52 */ 
+static CompEntry compose_tab_52[] = {
+{3398, 3404, 0, NULL, NULL}
+}; /* compose_tab_52 */ 
+static int hash_compose_tab_53[2] = 
+{-1,0}; /* hash_compose_tab_53 */ 
+static CompEntry compose_tab_53[] = {
+{3661, 3635, 0, NULL, NULL}
+}; /* compose_tab_53 */ 
+static int hash_compose_tab_54[2] = 
+{-1,0}; /* hash_compose_tab_54 */ 
+static CompEntry compose_tab_54[] = {
+{3789, 3763, 0, NULL, NULL}
+}; /* compose_tab_54 */ 
+static int hash_compose_tab_55_2[4] = 
+{-1,-1,0,1}; /* hash_compose_tab_55_2 */ 
+static CompEntry compose_tab_55_2[] = {
+{4018, 3959, 0, NULL, NULL},
+{4019, 3961, 0, NULL, NULL}
+}; /* compose_tab_55_2 */ 
+static int hash_compose_tab_55[6] = 
+{0,-1,1,2,-1,-1}; /* hash_compose_tab_55 */ 
+static CompEntry compose_tab_55[] = {
+{3954, 3955, 0, NULL, NULL},
+{3956, 3957, 0, NULL, NULL},
+{3968, 0, 2, compose_tab_55_2, hash_compose_tab_55_2}
+}; /* compose_tab_55 */ 
+static int hash_compose_tab_56[4] = 
+{-1,-1,0,1}; /* hash_compose_tab_56 */ 
+static CompEntry compose_tab_56[] = {
+{4018, 3958, 0, NULL, NULL},
+{4019, 3960, 0, NULL, NULL}
+}; /* compose_tab_56 */ 
+static int hash_compose_tab_57[4] = 
+{0,1,-1,-1}; /* hash_compose_tab_57 */ 
+static CompEntry compose_tab_57[] = {
+{3904, 3945, 0, NULL, NULL},
+{3984, 4025, 0, NULL, NULL}
+}; /* compose_tab_57 */ 
+static int hash_compose_tab_58[20] = 
+{-1,2,7,-1,-1,-1,0,3,5,8,-1,4,9,-1,-1,-1,1,6,-1,-1}; /* hash_compose_tab_58 */ 
+static CompEntry compose_tab_58[] = {
+{3906, 3907, 0, NULL, NULL},
+{3916, 3917, 0, NULL, NULL},
+{3921, 3922, 0, NULL, NULL},
+{3926, 3927, 0, NULL, NULL},
+{3931, 3932, 0, NULL, NULL},
+{3986, 3987, 0, NULL, NULL},
+{3996, 3997, 0, NULL, NULL},
+{4001, 4002, 0, NULL, NULL},
+{4006, 4007, 0, NULL, NULL},
+{4011, 4012, 0, NULL, NULL}
+}; /* compose_tab_58 */ 
+static int hash_compose_tab_59[96] = 
+{33,12,34,-1,13,35,14,36,15,37,-1,-1,-1,-1,-1,16,38,-1,17,39,-1,18,40,-1,19,
+ 41,-1,20,42,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,43,44,45,
+ 46,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,21,47,-1,-1,-1,-1,-1,-1,-1,0,22,-1,-1,-1,1,
+ 23,2,24,3,25,4,26,5,27,6,28,7,29,8,30,9,31,10,32,11}; /* hash_compose_tab_59 */ 
+static CompEntry compose_tab_59[] = {
+{12358, 12436, 0, NULL, NULL},
+{12363, 12364, 0, NULL, NULL},
+{12365, 12366, 0, NULL, NULL},
+{12367, 12368, 0, NULL, NULL},
+{12369, 12370, 0, NULL, NULL},
+{12371, 12372, 0, NULL, NULL},
+{12373, 12374, 0, NULL, NULL},
+{12375, 12376, 0, NULL, NULL},
+{12377, 12378, 0, NULL, NULL},
+{12379, 12380, 0, NULL, NULL},
+{12381, 12382, 0, NULL, NULL},
+{12383, 12384, 0, NULL, NULL},
+{12385, 12386, 0, NULL, NULL},
+{12388, 12389, 0, NULL, NULL},
+{12390, 12391, 0, NULL, NULL},
+{12392, 12393, 0, NULL, NULL},
+{12399, 12400, 0, NULL, NULL},
+{12402, 12403, 0, NULL, NULL},
+{12405, 12406, 0, NULL, NULL},
+{12408, 12409, 0, NULL, NULL},
+{12411, 12412, 0, NULL, NULL},
+{12445, 12446, 0, NULL, NULL},
+{12454, 12532, 0, NULL, NULL},
+{12459, 12460, 0, NULL, NULL},
+{12461, 12462, 0, NULL, NULL},
+{12463, 12464, 0, NULL, NULL},
+{12465, 12466, 0, NULL, NULL},
+{12467, 12468, 0, NULL, NULL},
+{12469, 12470, 0, NULL, NULL},
+{12471, 12472, 0, NULL, NULL},
+{12473, 12474, 0, NULL, NULL},
+{12475, 12476, 0, NULL, NULL},
+{12477, 12478, 0, NULL, NULL},
+{12479, 12480, 0, NULL, NULL},
+{12481, 12482, 0, NULL, NULL},
+{12484, 12485, 0, NULL, NULL},
+{12486, 12487, 0, NULL, NULL},
+{12488, 12489, 0, NULL, NULL},
+{12495, 12496, 0, NULL, NULL},
+{12498, 12499, 0, NULL, NULL},
+{12501, 12502, 0, NULL, NULL},
+{12504, 12505, 0, NULL, NULL},
+{12507, 12508, 0, NULL, NULL},
+{12527, 12535, 0, NULL, NULL},
+{12528, 12536, 0, NULL, NULL},
+{12529, 12537, 0, NULL, NULL},
+{12530, 12538, 0, NULL, NULL},
+{12541, 12542, 0, NULL, NULL}
+}; /* compose_tab_59 */ 
+static int hash_compose_tab_60[20] = 
+{-1,7,1,-1,8,2,-1,9,3,-1,-1,4,-1,-1,-1,5,-1,-1,6,0}; /* hash_compose_tab_60 */ 
+static CompEntry compose_tab_60[] = {
+{12399, 12401, 0, NULL, NULL},
+{12402, 12404, 0, NULL, NULL},
+{12405, 12407, 0, NULL, NULL},
+{12408, 12410, 0, NULL, NULL},
+{12411, 12413, 0, NULL, NULL},
+{12495, 12497, 0, NULL, NULL},
+{12498, 12500, 0, NULL, NULL},
+{12501, 12503, 0, NULL, NULL},
+{12504, 12506, 0, NULL, NULL},
+{12507, 12509, 0, NULL, NULL}
+}; /* compose_tab_60 */ 
+static int hash_compose_tab[122] = 
+{30,31,52,60,32,-1,-1,33,-1,34,35,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,2,3,4,-1,5,6,7,8,9,10,11,12,36,13,37,14,
+ 38,15,16,55,40,-1,-1,-1,-1,17,56,-1,-1,-1,-1,-1,41,18,19,20,42,21,22,-1,45,
+ 39,-1,23,24,-1,25,26,-1,-1,-1,-1,-1,-1,-1,-1,48,-1,43,44,51,53,-1,-1,27,46,
+ 54,28,-1,-1,47,-1,-1,-1,-1,49,50,-1,-1,57,-1,58,59,29}; /* hash_compose_tab */ 
+static CompEntry compose_tab[] = {
+{768, 0, 39, compose_tab_0, hash_compose_tab_0},
+{769, 0, 70, compose_tab_1, hash_compose_tab_1},
+{770, 0, 27, compose_tab_2, hash_compose_tab_2},
+{771, 0, 19, compose_tab_3, hash_compose_tab_3},
+{772, 0, 28, compose_tab_4, hash_compose_tab_4},
+{774, 0, 30, compose_tab_5, hash_compose_tab_5},
+{775, 0, 40, compose_tab_6, hash_compose_tab_6},
+{776, 0, 50, compose_tab_7, hash_compose_tab_7},
+{777, 0, 15, compose_tab_8, hash_compose_tab_8},
+{778, 0, 6, compose_tab_9, hash_compose_tab_9},
+{779, 0, 6, compose_tab_10, hash_compose_tab_10},
+{780, 0, 34, compose_tab_11, hash_compose_tab_11},
+{781, 0, 17, compose_tab_12, hash_compose_tab_12},
+{783, 0, 14, compose_tab_13, hash_compose_tab_13},
+{785, 0, 12, compose_tab_14, hash_compose_tab_14},
+{787, 0, 15, compose_tab_15, hash_compose_tab_15},
+{788, 0, 17, compose_tab_16, hash_compose_tab_16},
+{795, 0, 4, compose_tab_17, hash_compose_tab_17},
+{803, 0, 39, compose_tab_18, hash_compose_tab_18},
+{804, 0, 2, compose_tab_19, hash_compose_tab_19},
+{805, 0, 2, compose_tab_20, hash_compose_tab_20},
+{807, 0, 20, compose_tab_21, hash_compose_tab_21},
+{808, 0, 10, compose_tab_22, hash_compose_tab_22},
+{813, 0, 12, compose_tab_23, hash_compose_tab_23},
+{814, 0, 2, compose_tab_24, hash_compose_tab_24},
+{816, 0, 6, compose_tab_25, hash_compose_tab_25},
+{817, 0, 17, compose_tab_26, hash_compose_tab_26},
+{834, 0, 12, compose_tab_27, hash_compose_tab_27},
+{837, 0, 6, compose_tab_28, hash_compose_tab_28},
+{1463, 0, 2, compose_tab_29, hash_compose_tab_29},
+{1464, 0, 1, compose_tab_30, hash_compose_tab_30},
+{1465, 0, 1, compose_tab_31, hash_compose_tab_31},
+{1468, 0, 22, compose_tab_32, hash_compose_tab_32},
+{1471, 0, 3, compose_tab_33, hash_compose_tab_33},
+{1473, 0, 2, compose_tab_34, hash_compose_tab_34},
+{1474, 0, 2, compose_tab_35, hash_compose_tab_35},
+{2364, 0, 11, compose_tab_36, hash_compose_tab_36},
+{2492, 0, 4, compose_tab_37, hash_compose_tab_37},
+{2494, 0, 1, compose_tab_38, hash_compose_tab_38},
+{2519, 0, 1, compose_tab_39, hash_compose_tab_39},
+{2620, 0, 5, compose_tab_40, hash_compose_tab_40},
+{2876, 0, 3, compose_tab_41, hash_compose_tab_41},
+{2878, 0, 1, compose_tab_42, hash_compose_tab_42},
+{2902, 0, 1, compose_tab_43, hash_compose_tab_43},
+{2903, 0, 1, compose_tab_44, hash_compose_tab_44},
+{3006, 0, 2, compose_tab_45, hash_compose_tab_45},
+{3031, 0, 2, compose_tab_46, hash_compose_tab_46},
+{3158, 0, 1, compose_tab_47, hash_compose_tab_47},
+{3266, 0, 1, compose_tab_48, hash_compose_tab_48},
+{3285, 0, 3, compose_tab_49, hash_compose_tab_49},
+{3286, 0, 1, compose_tab_50, hash_compose_tab_50},
+{3390, 0, 2, compose_tab_51, hash_compose_tab_51},
+{3415, 0, 1, compose_tab_52, hash_compose_tab_52},
+{3634, 0, 1, compose_tab_53, hash_compose_tab_53},
+{3762, 0, 1, compose_tab_54, hash_compose_tab_54},
+{3953, 0, 3, compose_tab_55, hash_compose_tab_55},
+{3968, 0, 2, compose_tab_56, hash_compose_tab_56},
+{4021, 0, 2, compose_tab_57, hash_compose_tab_57},
+{4023, 0, 10, compose_tab_58, hash_compose_tab_58},
+{12441, 0, 48, compose_tab_59, hash_compose_tab_59},
+{12442, 0, 10, compose_tab_60, hash_compose_tab_60}
+}; /* compose_tab */ 
+#define COMP_CANDIDATE_MAP_OFFSET 24
+static Uint32 comp_candidate_map[] = {
+    0x081ABFDFU,
+    0x000361B8U,
+    0x00000024U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x93800000U,
+    0x00000006U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x10000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x50000000U,
+    0x00800000U,
+    0x00000000U,
+    0x00000000U,
+    0x10000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x50000000U,
+    0x00C00000U,
+    0x00000000U,
+    0x00000000U,
+    0x40000000U,
+    0x00800000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00400000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00600004U,
+    0x00000000U,
+    0x00000000U,
+    0x40000000U,
+    0x00800000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00040000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00040000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00000000U,
+    0x00020000U,
+    0x00000001U,
+    0x00A00000U
+};
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 50b3e5b61c..a0e12e57f2 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -29,54 +29,42 @@
 /* #define FORCE_HEAP_FRAGS */
 
 
-#if defined(HYBRID)
-/* # define CHECK_FOR_HOLES */
-#endif
-
 #if defined(DEBUG) && !defined(CHECK_FOR_HOLES) && !defined(__WIN32__)
 # define CHECK_FOR_HOLES
 #endif
 
-#if defined(HYBRID)
-/* #  define INCREMENTAL 1    */ /* Incremental garbage collection */
-/* #  define INC_TIME_BASED 1 */ /* Time-based incremental GC (vs Work-based) */
-#endif
-
 #define BEAM 1
 #define EMULATOR "BEAM"
 #define SEQ_TRACE 1
 
 #define CONTEXT_REDS 2000	/* Swap process out after this number */
-#define MAX_ARG 256	        /* Max number of arguments allowed */
+#define MAX_ARG 255	        /* Max number of arguments allowed */
 #define MAX_REG 1024            /* Max number of x(N) registers used */
 
+/* Scheduler stores data for temporary heaps if
+   !HEAP_ON_C_STACK. Macros (*TmpHeap*) in global.h selects if we put temporary
+   heap data on the C stack or if we use the buffers in the scheduler data. */
+#define TMP_HEAP_SIZE 128            /* Number of Eterm in the schedulers
+				        small heap for transient heap data */
+#define CMP_TMP_HEAP_SIZE       32   /* cmp wants its own tmp-heap... */
+#define ERL_ARITH_TMP_HEAP_SIZE 4    /* as does erl_arith... */
+#define BEAM_EMU_TMP_HEAP_SIZE  2    /* and beam_emu... */
+
 /*
  * The new arithmetic operations need some extra X registers in the register array.
+ * so does the gc_bif's (i_gc_bif3 need 3 extra).
  */
-#define ERTS_X_REGS_ALLOCATED (MAX_REG+2)
+#define ERTS_X_REGS_ALLOCATED (MAX_REG+3)
 
 #define INPUT_REDUCTIONS (2 * CONTEXT_REDS)
 
 #define H_DEFAULT_SIZE  233        /* default (heap + stack) min size */
 #define VH_DEFAULT_SIZE  32768     /* default virtual (bin) heap min size (words) */
 
-#ifdef HYBRID
-#  define SH_DEFAULT_SIZE  2629425 /* default message area min size */
-#endif
-
-#ifdef INCREMENTAL
-#  define INC_NoPAGES       256   /* Number of pages in the old generation */
-#  define INC_PAGESIZE      32768 /* The size of each page */
-#  define INC_STORAGE_SIZE  1024  /* The size of gray stack and similar */
-#endif
-
 #define CP_SIZE 1
 
 #define ErtsHAllocLockCheck(P) \
-  ERTS_SMP_LC_ASSERT((ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks((P))) \
-		     || ((P)->scheduler_data \
-			 && (P) == (P)->scheduler_data->match_pseudo_process) \
-		     || erts_is_system_blocked(0))
+  ERTS_SMP_LC_ASSERT(erts_dbg_check_halloc_lock((P)))
 
 
 #ifdef DEBUG
@@ -109,14 +97,15 @@
  * Allocate heap memory, first on the ordinary heap;
  * failing that, in a heap fragment.
  */
-#define HAlloc(p, sz)			                              \
+#define HAllocX(p, sz, xtra)		                              \
     (ASSERT_EXPR((sz) >= 0),					      \
      ErtsHAllocLockCheck(p),					      \
      (IS_FORCE_HEAP_FRAGS || (((HEAP_LIMIT(p) - HEAP_TOP(p)) < (sz))) \
-      ? erts_heap_alloc((p),(sz))                                     \
+      ? erts_heap_alloc((p),(sz),(xtra))                              \
       : (INIT_HEAP_MEM(p,sz),		                              \
          HEAP_TOP(p) = HEAP_TOP(p) + (sz), HEAP_TOP(p) - (sz))))
 
+#define HAlloc(P, SZ) HAllocX(P,SZ,0)
 
 #define HRelease(p, endp, ptr)					\
   if ((ptr) == (endp)) {					\
@@ -130,8 +119,12 @@
 #define HeapWordsLeft(p) (HEAP_LIMIT(p) - HEAP_TOP(p))
 
 #if defined(DEBUG) || defined(CHECK_FOR_HOLES)
+#if HALFWORD_HEAP
+# define ERTS_HOLE_MARKER (0xaf5e78ccU)
+#else
 # define ERTS_HOLE_MARKER (((0xaf5e78ccUL << 24) << 8) | 0xaf5e78ccUL)
 #endif
+#endif
 
 /*
  * Allocate heap memory on the ordinary heap, NEVER in a heap
@@ -184,6 +177,7 @@ extern int BIN_VH_MIN_SIZE;	/* minimum virtual (bin) heap */
 extern int erts_atom_table_size;/* Atom table size */
 
 #define ORIG_CREATION 0
+#define INTERNAL_CREATION 255
 
 /* macros for extracting bytes from uint16's */
 
diff --git a/erts/emulator/beam/erlang_dtrace.d b/erts/emulator/beam/erlang_dtrace.d
new file mode 100644
index 0000000000..c1024dafc4
--- /dev/null
+++ b/erts/emulator/beam/erlang_dtrace.d
@@ -0,0 +1,726 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2012.
+ * All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * A note on probe naming: if "__" appears in a provider probe
+ * definition, then two things happen during compilation:
+ *
+ *    1. The "__" will turn into a hypen, "-", for the probe name.
+ *    2. The "__" will turn into a single underscore, "_", for the
+ *       macro names and function definitions that the compiler and
+ *       C developers will see.
+ *
+ * We'll try to use the following naming convention.  We're a bit
+ * limited because, as a USDT probe, we can only specify the 4th part
+ * of the probe name, e.g. erlang*:::mumble.  The 2nd part of the
+ * probe name is always going to be "beam" or "beam.smp", and the 3rd
+ * part of the probe name will always be the name of the function
+ * that's calling the probe.
+ *
+ * So, all probes will be have names defined in this file using the
+ * convention category__name or category__sub_category__name.  This
+ * will translate to probe names of category-name or
+ * category-sub_category-name.
+ *
+ * Each of "category", "sub_category", and "name" may have underscores
+ * but may not have hyphens.
+ */
+
+provider erlang {
+    /**
+     * Fired when a message is sent from one local process to another.
+     *
+     * NOTE: The 'size' parameter is in machine-dependent words and
+     *       that the actual size of any binary terms in the message
+     *       are not included.
+     *
+     * @param sender the PID (string form) of the sender
+     * @param receiver the PID (string form) of the receiver
+     * @param size the size of the message being delivered (words)
+     * @param token_label for the sender's sequential trace token
+     * @param token_previous count for the sender's sequential trace token
+     * @param token_current count for the sender's sequential trace token
+     */
+    probe message__send(char *sender, char *receiver, uint32_t size,
+                        int token_label, int token_previous, int token_current);
+
+    /**
+     * Fired when a message is sent from a local process to a remote process.
+     *
+     * NOTE: The 'size' parameter is in machine-dependent words and
+     *       that the actual size of any binary terms in the message
+     *       are not included.
+     *
+     * @param sender the PID (string form) of the sender
+     * @param node_name the Erlang node name (string form) of the receiver
+     * @param receiver the PID/name (string form) of the receiver
+     * @param size the size of the message being delivered (words)
+     * @param token_label for the sender's sequential trace token
+     * @param token_previous count for the sender's sequential trace token
+     * @param token_current count for the sender's sequential trace token
+     */
+    probe message__send__remote(char *sender, char *node_name, char *receiver,
+                                uint32_t size,
+                        int token_label, int token_previous, int token_current);
+
+    /**
+     * Fired when a message is queued to a local process.  This probe
+     * will not fire if the sender's pid == receiver's pid.
+     *
+     * NOTE: The 'size' parameter is in machine-dependent words and
+     *       that the actual size of any binary terms in the message
+     *       are not included.
+     *
+     * NOTE: In cases of messages in external format (i.e. from another
+     *       Erlang node), we probably don't know the message size
+     *       without performing substantial extra computation.  To
+     *       avoid the extra CPU overhead, the message size may be
+     *       reported as -1, which can appear to a D script as 4294967295.
+     *
+     * @param receiver the PID (string form) of the receiver
+     * @param size the size of the message being delivered (words)
+     * @param queue_len length of the queue of the receiving process
+     * @param token_label for the sender's sequential trace token
+     * @param token_previous count for the sender's sequential trace token
+     * @param token_current count for the sender's sequential trace token
+     */
+    probe message__queued(char *receiver, uint32_t size, uint32_t queue_len,
+                        int token_label, int token_previous, int token_current);
+
+    /**
+     * Fired when a message is 'receive'd by a local process and removed
+     * from its mailbox.
+     *
+     * NOTE: The 'size' parameter is in machine-dependent words and
+     *       that the actual size of any binary terms in the message
+     *       are not included.
+     *
+     * NOTE: In cases of messages in external format (i.e. from another
+     *       Erlang node), we probably don't know the message size
+     *       without performing substantial extra computation.  To
+     *       avoid the extra CPU overhead, the message size may be
+     *       reported as -1, which can appear to a D script as 4294967295.
+     *
+     * @param receiver the PID (string form) of the receiver
+     * @param size the size of the message being delivered (words)
+     * @param queue_len length of the queue of the receiving process
+     * @param token_label for the sender's sequential trace token
+     * @param token_previous count for the sender's sequential trace token
+     * @param token_current count for the sender's sequential trace token
+     */
+    probe message__receive(char *receiver, uint32_t size, uint32_t queue_len,
+                        int token_label, int token_previous, int token_current);
+
+    /**
+     * Fired when an Eterm structure is being copied.
+     *
+     * NOTE: Due to the placement of this probe, the process ID of
+     *       owner of the Eterm is not available.
+     *
+     * @param size the size of the structure
+     */
+    probe copy__struct(uint32_t size);
+
+    /**
+     * Fired when an Eterm is being copied onto a process.
+     *
+     * @param proc the PID (string form) of the recipient process
+     * @param size the size of the structure
+     */
+    probe copy__object(char *proc, uint32_t size);
+
+    /* PID, Module, Function, Arity */
+
+    /**
+     * Fired whenever a user function is being called locally.
+     *
+     * @param p the PID (string form) of the process
+     * @param mfa the m:f/a of the function
+     * @param depth the stack depth
+     */
+    probe local__function__entry(char *p, char *mfa, int depth);
+
+    /**
+     * Fired whenever a user function is called externally
+     * (through an export entry).
+     *
+     * @param p the PID (string form) of the process
+     * @param mfa the m:f/a of the function
+     * @param depth the stack depth
+     */
+    probe global__function__entry(char *p, char *mfa, int depth);
+
+    /**
+     * Fired whenever a user function returns.
+     *
+     * @param p the PID (string form) of the process
+     * @param mfa the m:f/a of the function
+     * @param depth the stack depth
+     */
+    probe function__return(char *p, char *mfa, int depth);
+
+    /**
+     * Fired whenever a Built In Function is called.
+     *
+     * @param p the PID (string form) of the process
+     * @param mfa the m:f/a of the function
+     */
+    probe bif__entry(char *p, char *mfa);
+
+    /**
+     * Fired whenever a Built In Function returns.
+     *
+     * @param p the PID (string form) of the process
+     * @param mfa the m:f/a of the function
+     */
+    probe bif__return(char *p, char *mfa);
+
+    /**
+     * Fired whenever a Native Function is called.
+     *
+     * @param p the PID (string form) of the process
+     * @param mfa the m:f/a of the function
+     */
+    probe nif__entry(char *p, char *mfa);
+
+    /**
+     * Fired whenever a Native Function returns.
+     *
+     * @param p the PID (string form) of the process
+     * @param mfa the m:f/a of the function
+     */
+    probe nif__return(char *p, char *mfa);
+
+    /**
+     * Fired when a major GC is starting.
+     *
+     * @param p the PID (string form) of the exiting process
+     * @param need the number of words needed on the heap
+     */
+    probe gc_major__start(char *p, int need);
+
+    /**
+     * Fired when a minor GC is starting.
+     *
+     * @param p the PID (string form) of the exiting process
+     * @param need the number of words needed on the heap
+     */
+    probe gc_minor__start(char *p, int need);
+
+    /**
+     * Fired when a major GC is starting.
+     *
+     * @param p the PID (string form) of the exiting process
+     * @param reclaimed the amount of space reclaimed
+     */
+    probe gc_major__end(char *p, int reclaimed);
+
+    /**
+     * Fired when a minor GC is starting.
+     *
+     * @param p the PID (string form) of the exiting process
+     * @param reclaimed the amount of space reclaimed
+     */
+    probe gc_minor__end(char *p, int reclaimed);
+
+    /**
+     * Fired when a process is spawned.
+     *
+     * @param p the PID (string form) of the new process.
+     * @param mfa the m:f/a of the function
+     */
+    probe process__spawn(char *p, char *mfa);
+
+    /**
+     * Fired when a process is exiting.
+     *
+     * @param p the PID (string form) of the exiting process
+     * @param reason the reason for the exit (may be truncated)
+     */
+    probe process__exit(char *p, char *reason);
+
+    /**
+     * Fired when exit signal is delivered to a local process.
+     *
+     * @param sender the PID (string form) of the exiting process
+     * @param receiver the PID (string form) of the process receiving EXIT signal
+     * @param reason the reason for the exit (may be truncated)
+     */
+    probe process__exit_signal(char *sender, char *receiver, char *reason);
+
+    /**
+     * Fired when exit signal is delivered to a remote process.
+     *
+     * @param sender the PID (string form) of the exiting process
+     * @param node_name the Erlang node name (string form) of the receiver
+     * @param receiver the PID (string form) of the process receiving EXIT signal
+     * @param reason the reason for the exit (may be truncated)
+     * @param token_label for the sender's sequential trace token
+     * @param token_previous count for the sender's sequential trace token
+     * @param token_current count for the sender's sequential trace token
+     */
+    probe process__exit_signal__remote(char *sender, char *node_name,
+                                       char *receiver, char *reason,
+                        int token_label, int token_previous, int token_current);
+
+    /**
+     * Fired when a process is scheduled.
+     *
+     * @param p the PID (string form) of the newly scheduled process
+     * @param mfa the m:f/a of the function it should run next
+     */
+    probe process__scheduled(char *p, char *mfa);
+
+    /**
+     * Fired when a process is unscheduled.
+     *
+     * @param p the PID (string form) of the process that has been
+     * unscheduled.
+     */
+    probe process__unscheduled(char *p);
+
+    /**
+     * Fired when a process goes into hibernation.
+     *
+     * @param p the PID (string form) of the process entering hibernation
+     * @param mfa the m:f/a of the location to resume
+     */
+    probe process__hibernate(char *p, char *mfa);
+
+    /**
+     * Fired when a process is unblocked after a port has been unblocked.
+     *
+     * @param p the PID (string form) of the process that has been
+     * unscheduled.
+     * @param port the port that is no longer busy (i.e., is now unblocked)
+     */
+    probe process__port_unblocked(char *p, char *port);
+
+    /**
+     * Fired when process' heap is growing.
+     *
+     * @param p the PID (string form)
+     * @param old_size the size of the old heap
+     * @param new_size the size of the new heap
+     */
+    probe process__heap_grow(char *p, int old_size, int new_size);
+
+    /**
+     * Fired when process' heap is shrinking.
+     *
+     * @param p the PID (string form)
+     * @param old_size the size of the old heap
+     * @param new_size the size of the new heap
+     */
+    probe process__heap_shrink(char *p, int old_size, int new_size);
+
+    /* network distribution */
+
+    /**
+     * Fired when network distribution event monitor events are triggered.
+     *
+     * @param node the name of the reporting node
+     * @param what the type of event, e.g., nodeup, nodedown
+     * @param monitored_node the name of the monitored node
+     * @param type the type of node, e.g., visible, hidden
+     * @param reason the reason term, e.g., normal, connection_closed, term()
+     */
+    probe dist__monitor(char *node, char *what, char *monitored_node,
+                        char *type, char *reason);
+
+    /**
+     * Fired when network distribution port is busy (i.e. blocked),
+     * usually due to the remote node not consuming distribution
+     * data quickly enough.
+     *
+     * @param node the name of the reporting node
+     * @param port the port ID of the busy port
+     * @param remote_node the name of the remote node.
+     * @param pid the PID (string form) of the local process that has
+     *        become unschedulable until the port becomes unblocked.
+     */
+    probe dist__port_busy(char *node, char *port, char *remote_node,
+                          char *pid);
+
+    /**
+     * Fired when network distribution's driver's "output" callback is called
+     *
+     * @param node the name of the reporting node
+     * @param port the port ID of the busy port
+     * @param remote_node the name of the remote node.
+     * @param bytes the number of bytes written
+     */
+    probe dist__output(char *node, char *port, char *remote_node, int bytes);
+
+    /**
+     * Fired when network distribution's driver's "outputv" callback is called
+     *
+     * @param node the name of the reporting node
+     * @param port the port ID of the busy port
+     * @param remote_node the name of the remote node.
+     * @param bytes the number of bytes written
+     */
+    probe dist__outputv(char *node, char *port, char *remote_node, int bytes);
+
+    /**
+     * Fired when network distribution port is no longer busy (i.e. blocked).
+     *
+     * NOTE: This probe may fire multiple times after the same single
+     *       dist-port_busy probe firing.
+     *
+     * @param node the name of the reporting node
+     * @param port the port ID of the busy port
+     * @param remote_node the name of the remote node.
+     */
+    probe dist__port_not_busy(char *node, char *port, char *remote_node);
+
+    /* ports */
+
+    /**
+     * Fired when new port is opened.
+     *
+     * @param process the PID (string form)
+     * @param port_name the string used when the port was opened
+     * @param port the Port (string form) of the new port
+     */
+    probe port__open(char *process, char *port_name, char *port);
+
+    /**
+     * Fired when port_command is issued.
+     *
+     * @param process the PID (string form)
+     * @param port the Port (string form)
+     * @param port_name the string used when the port was opened
+     * @param command_type type of the issued command, one of: "close", "command" or "connect"
+     */
+    probe port__command(char *process, char *port, char *port_name, char *command_type);
+
+    /**
+     * Fired when port_control is issued.
+     *
+     * @param process the PID (string form)
+     * @param port the Port (string form)
+     * @param port_name the string used when the port was opened
+     * @param command_no command number that has been issued to the port
+     */
+    probe port__control(char *process, char *port, char *port_name, int command_no);
+
+    /**
+     * Fired when port is closed via port_close/1 (reason = 'normal')
+     * or is sent an exit signal.
+     *
+     * @param process the PID (string form)
+     * @param port the Port (string form)
+     * @param port_name the string used when the port was opened
+     * @param reason Erlang term representing the exit signal, e.g. 'normal'
+     */
+    probe port__exit(char *process, char *port, char *port_name,
+                     char *new_process);
+
+    /**
+     * Fired when port_connect is issued.
+     *
+     * @param process the PID (string form) of the current port owner
+     * @param port the Port (string form)
+     * @param port_name the string used when the port was opened
+     * @param new_process the PID (string form) of the new port owner
+     */
+    probe port__connect(char *process, char *port, char *port_name,
+                        char *new_process);
+
+    /**
+     * Fired when a port is busy (i.e. blocked)
+     *
+     * @param port the port ID of the busy port
+     */
+    probe port__busy(char *port);
+
+    /**
+     * Fired when a port is no longer busy (i.e. no longer blocked)
+     *
+     * @param port the port ID of the not busy port
+     */
+    probe port__not_busy(char *port);
+
+    /* drivers */
+
+    /**
+     * Fired when drivers's "init" callback is called.
+     *
+     * @param name the name of the driver
+     * @param major the major version number
+     * @param minor the minor version number
+     * @param flags the flags argument
+     */
+    probe driver__init(char *name, int major, int minor, int flags);
+
+    /**
+     * Fired when drivers's "start" callback is called.
+     *
+     * @param process the PID (string form) of the calling process
+     * @param name the name of the driver
+     * @param port the Port (string form) of the driver's port
+     */
+     probe driver__start(char *process, char *name, char *port);
+
+    /**
+     * Fired when drivers's "stop" callback is called.
+     *
+     * @param process the PID (string form) of the calling process
+     * @param name the name of the driver
+     * @param port the Port (string form) of the driver's port
+     */
+     probe driver__stop(char *process, char *name, char *port);
+
+    /**
+     * Fired when drivers's "finish" callback is called.
+     *
+     * @param name the name of the driver
+     */
+     probe driver__finish(char *name);
+
+    /**
+     * Fired when drivers's "flush" callback is called.
+     *
+     * @param process the PID (string form)
+     * @param port the Port (string form)
+     * @param port_name the string used when the port was opened
+     */
+    probe driver__flush(char *process, char *port, char *port_name);
+
+    /**
+     * Fired when driver's "output" callback is called
+     *
+     * @param process the PID (string form)
+     * @param port the Port (string form)
+     * @param port_name the string used when the port was opened
+     * @param bytes the number of bytes written
+     */
+    probe driver__output(char *node, char *port, char *port_name, int bytes);
+
+    /**
+     * Fired when driver's "outputv" callback is called
+     *
+     * @param process the PID (string form)
+     * @param port the Port (string form)
+     * @param port_name the string used when the port was opened
+     * @param bytes the number of bytes written
+     */
+    probe driver__outputv(char *node, char *port, char *port_name, int bytes);
+
+    /**
+     * Fired when driver's "control" callback is called
+     *
+     * @param process the PID (string form)
+     * @param port the Port (string form)
+     * @param port_name the string used when the port was opened
+     * @param command the command #
+     * @param bytes the number of bytes written
+     */
+    probe driver__control(char *node, char *port, char *port_name,
+                          int command, int bytes);
+
+    /**
+     * Fired when driver's "call" callback is called
+     *
+     * @param process the PID (string form)
+     * @param port the Port (string form)
+     * @param port_name the string used when the port was opened
+     * @param command the command #
+     * @param bytes the number of bytes written
+     */
+    probe driver__call(char *node, char *port, char *port_name,
+                       int command, int bytes);
+
+    /**
+     * Fired when driver's "event" callback is called
+     *
+     * @param process the PID (string form)
+     * @param port the Port (string form)
+     * @param port_name the string used when the port was opened
+     */
+    probe driver__event(char *node, char *port, char *port_name);
+
+    /**
+     * Fired when driver's "ready_input" callback is called
+     *
+     * @param process the PID (string form)
+     * @param port the Port (string form)
+     * @param port_name the string used when the port was opened
+     */
+    probe driver__ready_input(char *node, char *port, char *port_name);
+
+    /**
+     * Fired when driver's "read_output" callback is called
+     *
+     * @param process the PID (string form)
+     * @param port the Port (string form)
+     * @param port_name the string used when the port was opened
+     */
+    probe driver__ready_output(char *node, char *port, char *port_name);
+
+    /**
+     * Fired when driver's "timeout" callback is called
+     *
+     * @param process the PID (string form)
+     * @param port the Port (string form)
+     * @param port_name the string used when the port was opened
+     */
+    probe driver__timeout(char *node, char *port, char *port_name);
+
+    /**
+     * Fired when drivers's "ready_async" callback is called.
+     *
+     * @param process the PID (string form)
+     * @param port the Port (string form)
+     * @param port_name the string used when the port was opened
+     */
+    probe driver__ready_async(char *process, char *port, char *port_name);
+
+    /**
+     * Fired when driver's "process_exit" callback is called
+     *
+     * @param process the PID (string form)
+     * @param port the Port (string form)
+     * @param port_name the string used when the port was opened
+     */
+    probe driver__process_exit(char *node, char *port, char *port_name);
+
+    /**
+     * Fired when driver's "stop_select" callback is called
+     *
+     * @param name the name of the driver
+     */
+    probe driver__stop_select(char *name);
+
+
+    /* Async driver pool */
+
+    /**
+     * Show the post-add length of the async driver thread pool member's queue.
+     *
+     * NOTE: The port name is not available: additional lock(s) must
+     *       be acquired in order to get the port name safely in an SMP
+     *       environment.  The same is true for the aio__pool_get probe.
+     *
+     * @param port the Port (string form)
+     * @param new queue length
+     */
+    probe aio_pool__add(char *, int);
+
+    /**
+     * Show the post-get length of the async driver thread pool member's queue.
+     *
+     * @param port the Port (string form)
+     * @param new queue length
+     */
+    probe aio_pool__get(char *, int);
+
+    /* Probes for efile_drv.c */
+
+    /**
+     * Entry into the efile_drv.c file I/O driver
+     *
+     * For a list of command numbers used by this driver, see the section
+     * "Guide to probe arguments" in ../../../README.md.  That section
+     * also contains explanation of the various integer and string
+     * arguments that may be present when any particular probe fires.
+     *
+     * NOTE: Not all Linux platforms (using SystemTap) can support
+     *       arguments beyond arg9.
+     *
+     *
+     * TODO: Adding the port string, args[10], is a pain.  Making that
+     *       port string available to all the other efile_drv.c probes
+     *       will be more pain.  Is the pain worth it?  If yes, then
+     *       add them everywhere else and grit our teeth.  If no, then
+     *       rip it out.
+     *
+     * @param thread-id number of the scheduler Pthread                   arg0
+     * @param tag number: {thread-id, tag} uniquely names a driver operation
+     * @param user-tag string                                             arg2
+     * @param command number                                              arg3
+     * @param string argument 1                                           arg4
+     * @param string argument 2                                           arg5
+     * @param integer argument 1                                          arg6
+     * @param integer argument 2                                          arg7
+     * @param integer argument 3                                          arg8
+     * @param integer argument 4                                          arg9
+     * @param port the port ID of the busy port                       args[10]
+     */
+    probe efile_drv__entry(int, int, char *, int, char *, char *,
+                           int64_t, int64_t, int64_t, int64_t, char *);
+
+    /**
+     * Entry into the driver's internal work function.  Computation here
+     * is performed by a async worker pool Pthread.
+     *
+     * @param thread-id number
+     * @param tag number
+     * @param command number
+     */
+    probe efile_drv__int_entry(int, int, int);
+
+    /**
+     * Return from the driver's internal work function.
+     *
+     * @param thread-id number
+     * @param tag number
+     * @param command number
+     */
+    probe efile_drv__int_return(int, int, int);
+
+    /**
+     * Return from the efile_drv.c file I/O driver
+     *
+     * @param thread-id number                                            arg0
+     * @param tag number                                                  arg1
+     * @param user-tag string                                             arg2
+     * @param command number                                              arg3
+     * @param Success? 1 is success, 0 is failure                         arg4
+     * @param If failure, the errno of the error.                         arg5
+     */
+    probe efile_drv__return(int, int, char *, int, int, int);
+
+/*
+ * NOTE:
+ * For formatting int64_t arguments within a D script, see:
+ *
+ *   http://mail.opensolaris.org/pipermail/dtrace-discuss/2006-November/002830.html
+ *   Summary:
+ *       "1) you don't need the 'l' printf() modifiers with DTrace ever"
+ */
+
+/*
+ * NOTE: For file_drv_return + SMP + R14B03 (and perhaps other
+ *       releases), the sched-thread-id will be the same as the
+ *       work-thread-id: erl_async.c's async_main() function
+ *       will call the asynchronous invoke function and then
+ *       immediately call the drivers ready_async function while
+ *       inside the same I/O worker pool thread.
+ *       For R14B03's source, see erl_async.c lines 302-317.
+ */
+};
+
+#pragma D attributes Evolving/Evolving/Common provider erlang provider
+#pragma D attributes Private/Private/Common provider erlang module
+#pragma D attributes Private/Private/Common provider erlang function
+#pragma D attributes Evolving/Evolving/Common provider erlang name
+#pragma D attributes Evolving/Evolving/Common provider erlang args
diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h
index 4930def4ed..ddc2c1396d 100644
--- a/erts/emulator/beam/error.h
+++ b/erts/emulator/beam/error.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -187,10 +187,10 @@ extern Eterm exception_tag[NUMBER_EXC_TAGS];
 struct StackTrace {
     Eterm header;	/* bignum header - must be first in struct */
     Eterm freason; /* original exception reason is saved in the struct */
-    Eterm* pc;
-    Eterm* current;
+    BeamInstr* pc;
+    BeamInstr* current;
     int depth;	/* number of saved pointers in trace[] */
-    Eterm *trace[1];  /* varying size - must be last in struct */
+    BeamInstr *trace[1];  /* varying size - must be last in struct */
 };
 
 #endif /* __ERROR_H__ */
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 271b40cf0f..fb0ee99119 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -43,11 +43,9 @@ static erts_smp_rwmtx_t export_table_lock; /* Locks the secondary export table.
 #define export_read_unlock()	erts_smp_rwmtx_runlock(&export_table_lock)
 #define export_write_lock()	erts_smp_rwmtx_rwlock(&export_table_lock)
 #define export_write_unlock()	erts_smp_rwmtx_rwunlock(&export_table_lock)
-#define export_init_lock()	erts_smp_rwmtx_init(&export_table_lock, \
-						    "export_tab")
 
-extern Eterm* em_call_error_handler;
-extern Uint* em_call_traced_function;
+extern BeamInstr* em_call_error_handler;
+extern BeamInstr* em_call_traced_function;
 
 void
 export_info(int to, void *to_arg)
@@ -93,7 +91,7 @@ export_alloc(Export* tmpl)
     obj->code[2] = tmpl->code[2];
     obj->slot.index = -1;
     obj->address = obj->code+3;
-    obj->code[3] = (Eterm) em_call_error_handler;
+    obj->code[3] = (BeamInstr) em_call_error_handler;
     obj->code[4] = 0;
     obj->match_prog_set = NULL;
     return obj;
@@ -111,8 +109,12 @@ void
 init_export_table(void)
 {
     HashFunctions f;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
+
+    erts_smp_rwmtx_init_opt(&export_table_lock, &rwmtx_opt, "export_tab");
 
-    export_init_lock();
     f.hash = (H_FUN) export_hash;
     f.cmp  = (HCMP_FUN) export_cmp;
     f.alloc = (HALLOC_FUN) export_alloc;
@@ -140,7 +142,7 @@ init_export_table(void)
 Export*
 erts_find_export_entry(Eterm m, Eterm f, unsigned int a)
 {
-    HashValue hval = EXPORT_HASH(m, f, a);
+    HashValue hval = EXPORT_HASH((BeamInstr) m, (BeamInstr) f, (BeamInstr) a);
     int ix;
     HashBucket* b;
 
@@ -185,7 +187,7 @@ erts_find_function(Eterm m, Eterm f, unsigned int a)
 
     ep = hash_get(&export_table.htable, (void*) &e);
     if (ep != NULL && ep->address == ep->code+3 &&
-	ep->code[3] != (Uint) em_call_traced_function) {
+	ep->code[3] != (BeamInstr) em_call_traced_function) {
 	ep = NULL;
     }
     return ep;
@@ -206,7 +208,8 @@ erts_export_put(Eterm mod, Eterm func, unsigned int arity)
     Export e;
     int ix;
 
-    ERTS_SMP_LC_ASSERT(erts_initialized == 0 || erts_smp_is_system_blocked(0));
+    ERTS_SMP_LC_ASSERT(erts_initialized == 0
+		       || erts_smp_thr_progress_is_blocking());
     ASSERT(is_atom(mod));
     ASSERT(is_atom(func));
     e.code[0] = mod;
@@ -263,7 +266,8 @@ erts_export_consolidate(void)
     HashInfo hi;
 #endif
 
-    ERTS_SMP_LC_ASSERT(erts_initialized == 0 || erts_smp_is_system_blocked(0));
+    ERTS_SMP_LC_ASSERT(erts_initialized == 0
+		       || erts_smp_thr_progress_is_blocking());
 
     export_write_lock();
     erts_index_merge(&secondary_export_table, &export_table);
diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h
index cd6af6dd85..c604fdf7c3 100644
--- a/erts/emulator/beam/export.h
+++ b/erts/emulator/beam/export.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -37,7 +37,7 @@ typedef struct export
     void* address;		/* Pointer to code for function. */
     struct binary* match_prog_set; /* Match program for tracing. */
 
-    Eterm fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */
+    BeamInstr fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */
     /*
      * code[0]: Tagged atom for module.
      * code[1]: Tagged atom for function.
@@ -52,7 +52,7 @@ typedef struct export
      *		on_load function that has not been run yet.
      *		Otherwise: 0.
      */
-    Eterm code[5];
+    BeamInstr code[5];
 } Export;
 
 
@@ -74,6 +74,6 @@ Export *export_get(Export*);
 #include "beam_load.h" /* For em_* extern declarations */ 
 #define ExportIsBuiltIn(EntryPtr) 			\
 (((EntryPtr)->address == (EntryPtr)->code + 3) && 	\
- ((EntryPtr)->code[3] == (Uint) em_apply_bif))
+ ((EntryPtr)->code[3] == (BeamInstr) em_apply_bif))
 
 #endif
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 099eddd195..52f45b924f 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -46,13 +46,11 @@
 #ifdef HIPE
 #include "hipe_mode_switch.h"
 #endif
-#define in_area(ptr,start,nbytes) ((Uint)((char*)(ptr) - (char*)(start)) < (nbytes))
+#define in_area(ptr,start,nbytes) ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
 
 #define MAX_STRING_LEN 0xffff
-#define dec_set_creation(nodename,creat)				\
-  (((nodename) == erts_this_node->sysname && (creat) == ORIG_CREATION)	\
-   ? erts_this_node->creation						\
-   : (creat))
+
+#define is_valid_creation(Cre) ((unsigned)(Cre) < MAX_CREATION || (Cre) == INTERNAL_CREATION)
 
 #undef ERTS_DEBUG_USE_DIST_SEP
 #ifdef DEBUG
@@ -65,11 +63,9 @@
 #  endif
 #endif
 
-/*
- * For backward compatibility reasons, only encode integers that
- * fit in 28 bits (signed) using INTEGER_EXT.
+/* Does Sint fit in Sint32?
  */
-#define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2)
+#define IS_SSMALL32(x) (((Uint) (((x) >> (32-1)) + 1)) < 2)
 
 /*
  *   Valid creations for nodes are 1, 2, or 3. 0 can also be sent
@@ -85,14 +81,14 @@
  *
  */
 
-static byte* enc_term(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
+static byte* enc_term(ErtsAtomCacheMap *, Eterm, byte*, Uint32, struct erl_off_heap_header** off_heap);
 static Uint is_external_string(Eterm obj, int* p_is_string);
 static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
 static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32);
 static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*);
 static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*);
 static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*);
-static Sint decoded_size(byte *ep, byte* endp, int only_heap_bins);
+static Sint decoded_size(byte *ep, byte* endp, int internal_tags);
 
 
 static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, unsigned);
@@ -463,6 +459,18 @@ Uint erts_encode_ext_size(Eterm term)
 	+ 1 /* VERSION_MAGIC */;
 }
 
+Uint erts_encode_ext_size_2(Eterm term, unsigned dflags)
+{
+    return encode_size_struct2(NULL, term, TERM_TO_BINARY_DFLAGS|dflags)
+        + 1 /* VERSION_MAGIC */;
+}
+
+Uint erts_encode_ext_size_ets(Eterm term)
+{
+    return encode_size_struct2(NULL, term, TERM_TO_BINARY_DFLAGS|DFLAGS_INTERNAL_TAGS);
+}
+
+
 void erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap *acmp)
 {
     byte *ep = *ext;
@@ -470,7 +478,7 @@ void erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap
     if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE))
 #endif
 	*ep++ = VERSION_MAGIC;
-    ep = enc_term(acmp, term, ep, flags);
+    ep = enc_term(acmp, term, ep, flags, NULL);
     if (!ep)
 	erl_exit(ERTS_ABORT_EXIT,
 		 "%s:%d:erts_encode_dist_ext(): Internal data structure error\n",
@@ -482,7 +490,7 @@ void erts_encode_ext(Eterm term, byte **ext)
 {
     byte *ep = *ext;
     *ep++ = VERSION_MAGIC;
-    ep = enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS);
+    ep = enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS, NULL);
     if (!ep)
 	erl_exit(ERTS_ABORT_EXIT,
 		 "%s:%d:erts_encode_ext(): Internal data structure error\n",
@@ -490,6 +498,12 @@ void erts_encode_ext(Eterm term, byte **ext)
     *ext = ep;
 }
 
+byte* erts_encode_ext_ets(Eterm term, byte *ep, struct erl_off_heap_header** off_heap)
+{
+    return enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS|DFLAGS_INTERNAL_TAGS,
+		    off_heap);
+}
+
 ErtsDistExternal *
 erts_make_dist_ext_copy(ErtsDistExternal *edep, Uint xsize)
 {
@@ -504,7 +518,7 @@ erts_make_dist_ext_copy(ErtsDistExternal *edep, Uint xsize)
     ASSERT(edep->ext_endp >= edep->extp);
     ext_sz = edep->ext_endp - edep->extp;
 
-    align_sz = ERTS_WORD_ALIGN_PAD_SZ(dist_ext_sz + ext_sz);
+    align_sz = ERTS_EXTRA_DATA_ALIGN_SZ(dist_ext_sz + ext_sz);
 
     new_edep = erts_alloc(ERTS_ALC_T_EXT_TERM_DATA,
 			  dist_ext_sz + ext_sz + align_sz + xsize);
@@ -796,7 +810,7 @@ bad_dist_ext(ErtsDistExternal *edep)
 }
 
 Sint
-erts_decode_dist_ext_size(ErtsDistExternal *edep, int no_refc_bins)
+erts_decode_dist_ext_size(ErtsDistExternal *edep)
 {
     Sint res;
     byte *ep;
@@ -815,7 +829,7 @@ erts_decode_dist_ext_size(ErtsDistExternal *edep, int no_refc_bins)
 	    goto fail;
 	ep = edep->extp+1;
     }
-    res = decoded_size(ep, edep->ext_endp, no_refc_bins);
+    res = decoded_size(ep, edep->ext_endp, 0);
     if (res >= 0)
 	return res;
  fail:
@@ -823,13 +837,21 @@ erts_decode_dist_ext_size(ErtsDistExternal *edep, int no_refc_bins)
     return -1;
 }
 
-Sint erts_decode_ext_size(byte *ext, Uint size, int no_refc_bins)
+Sint erts_decode_ext_size(byte *ext, Uint size)
 {
     if (size == 0 || *ext != VERSION_MAGIC)
 	return -1;
-    return decoded_size(ext+1, ext+size, no_refc_bins);
+    return decoded_size(ext+1, ext+size, 0);
 }
 
+Sint erts_decode_ext_size_ets(byte *ext, Uint size)
+{
+    Sint sz = decoded_size(ext, ext+size, 1);
+    ASSERT(sz >= 0);
+    return sz;
+}
+
+
 /*
 ** hpp is set to either a &p->htop or
 ** a pointer to a memory pointer (form message buffers)
@@ -889,7 +911,13 @@ Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext)
     return obj;
 }
 
-
+Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext)
+{
+    Eterm obj;
+    ext = dec_term(NULL, hpp, ext, off_heap, &obj);
+    ASSERT(ext);
+    return obj;
+}
 
 /**********************************************************************/
 
@@ -940,7 +968,7 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2)
     ede.extp = binary_bytes(real_bin)+offset;
     ede.ext_endp = ede.extp + size;
 
-    hsz = erts_decode_dist_ext_size(&ede, 0);
+    hsz = erts_decode_dist_ext_size(&ede);
     if (hsz < 0)
 	goto badarg;
 
@@ -960,15 +988,16 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2)
 }
 
 
-Eterm
-term_to_binary_1(Process* p, Eterm Term)
+BIF_RETTYPE term_to_binary_1(BIF_ALIST_1)
 {
-    return erts_term_to_binary(p, Term, 0, TERM_TO_BINARY_DFLAGS);
+    return erts_term_to_binary(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS);
 }
 
-Eterm
-term_to_binary_2(Process* p, Eterm Term, Eterm Flags)
+BIF_RETTYPE term_to_binary_2(BIF_ALIST_2)
 {
+    Process* p = BIF_P;
+    Eterm Term = BIF_ARG_1;
+    Eterm Flags = BIF_ARG_2;
     int level = 0;
     Uint flags = TERM_TO_BINARY_DFLAGS;
 
@@ -1185,7 +1214,8 @@ BIF_RETTYPE binary_to_term_2(BIF_ALIST_2)
         opt = CAR(list_val(opts));
         if (opt == am_safe) {
 	    fakedep.flags |= ERTS_DIST_EXT_BTT_SAFE;
-        } else {
+        }
+	else {
             goto error;
         }
         opts = CDR(list_val(opts));
@@ -1226,8 +1256,11 @@ BIF_RETTYPE binary_to_term_2(BIF_ALIST_2)
 }
 
 Eterm
-external_size_1(Process* p, Eterm Term)
+external_size_1(BIF_ALIST_1)
 {
+    Process* p = BIF_P;
+    Eterm Term = BIF_ARG_1;
+
     Uint size = erts_encode_ext_size(Term);
     if (IS_USMALL(0, size)) {
 	BIF_RET(make_small(size));
@@ -1238,9 +1271,52 @@ external_size_1(Process* p, Eterm Term)
 }
 
 Eterm
+external_size_2(BIF_ALIST_2)
+{
+    Uint size;
+    Uint flags = TERM_TO_BINARY_DFLAGS;
+
+    while (is_list(BIF_ARG_2)) {
+        Eterm arg = CAR(list_val(BIF_ARG_2));
+        Eterm* tp;
+
+        if (is_tuple(arg) && *(tp = tuple_val(arg)) == make_arityval(2)) {
+            if (tp[1] == am_minor_version && is_small(tp[2])) {
+                switch (signed_val(tp[2])) {
+                case 0:
+                    break;
+                case 1:
+                    flags |= DFLAG_NEW_FLOATS;
+                    break;
+                default:
+                    goto error;
+                }
+            } else {
+                goto error;
+            }
+        } else {
+        error:
+            BIF_ERROR(BIF_P, BADARG);
+        }
+        BIF_ARG_2 = CDR(list_val(BIF_ARG_2));
+    }
+    if (is_not_nil(BIF_ARG_2)) {
+        goto error;
+    }
+
+    size = erts_encode_ext_size_2(BIF_ARG_1, flags);
+    if (IS_USMALL(0, size)) {
+        BIF_RET(make_small(size));
+    } else {
+        Eterm* hp = HAlloc(BIF_P, BIG_UINT_HEAP_SIZE);
+        BIF_RET(uint_to_big(size, hp));
+    }
+}
+
+Eterm
 erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags)
 {
-    int size;
+    Uint size;
     Eterm bin;
     size_t real_size;
     byte* endp;
@@ -1257,7 +1333,7 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags)
 	    bytes = erts_alloc(ERTS_ALC_T_TMP, size);
 	}
 
-	if ((endp = enc_term(NULL, Term, bytes, flags))
+	if ((endp = enc_term(NULL, Term, bytes, flags, NULL))
 	    == NULL) {
 	    erl_exit(1, "%s, line %d: bad term: %x\n",
 		     __FILE__, __LINE__, Term);
@@ -1302,7 +1378,7 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags)
 	bin = new_binary(p, (byte *)NULL, size);
 	bytes = binary_bytes(bin);
 	bytes[0] = VERSION_MAGIC;
-	if ((endp = enc_term(NULL, Term, bytes+1, flags))	
+	if ((endp = enc_term(NULL, Term, bytes+1, flags, NULL))
 	    == NULL) {
 	    erl_exit(1, "%s, line %d: bad term: %x\n",
 		     __FILE__, __LINE__, Term);
@@ -1332,6 +1408,21 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags)
 
     ASSERT(is_atom(atom));
 
+    if (dflags & DFLAGS_INTERNAL_TAGS) {
+	Uint aval = atom_val(atom);
+	ASSERT(aval < (1<<24));
+	if (aval >= (1 << 16)) {
+	    *ep++ = ATOM_INTERNAL_REF3;
+	    put_int24(aval, ep);
+	    ep += 3;
+	}
+	else {
+	    *ep++ = ATOM_INTERNAL_REF2;
+	    put_int16(aval, ep);
+	    ep += 2;
+	}
+	return ep;
+    }
     /*
      * term_to_binary/1,2 and the initial distribution message
      * don't use the cache.
@@ -1381,7 +1472,8 @@ enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags)
     ep += 4;
     put_int32(os, ep);
     ep += 4;
-    *ep++ = pid_creation(pid);
+    *ep++ = (is_internal_pid(pid) && (dflags & DFLAGS_INTERNAL_TAGS)) ?
+	INTERNAL_CREATION : pid_creation(pid);
     return ep;
 }
 
@@ -1420,6 +1512,23 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp)
         }
 	ep += len;
 	break;
+    case ATOM_INTERNAL_REF2:
+	n = get_int16(ep);
+	ep += 2;
+	if (n >= atom_table_size()) {
+	    goto error;
+	}
+	*objp = make_atom(n);
+	break;
+    case ATOM_INTERNAL_REF3:
+	n = get_int24(ep);
+	ep += 3;
+	if (n >= atom_table_size()) {
+	    goto error;
+	}
+	*objp = make_atom(n);
+	break;
+
     default:
     error:
 	*objp = NIL;	/* Don't leave a hole in the heap */
@@ -1428,6 +1537,19 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp)
     return ep;
 }
 
+static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation)
+{
+    switch (creation) {
+    case INTERNAL_CREATION:
+	return erts_this_node;
+    case ORIG_CREATION:
+	if (sysname == erts_this_node->sysname) {
+	    creation = erts_this_node->creation;
+	}
+    }
+    return erts_find_or_insert_node(sysname,creation);
+}
+
 static byte*
 dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Eterm* objp)
 {
@@ -1451,18 +1573,20 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete
     ep += 4;
     if (ser > ERTS_MAX_PID_SERIAL)
 	return NULL;
-    if ((cre = get_int8(ep)) >= MAX_CREATION)
-	return NULL;
+    cre = get_int8(ep);
     ep += 1;
 
+    if (!is_valid_creation(cre)) {
+	return NULL;
+    }
+    data = make_pid_data(ser, num);
+
     /*
      * We are careful to create the node entry only after all
      * validity tests are done.
      */
-    cre = dec_set_creation(sysname,cre);
-    node = erts_find_or_insert_node(sysname,cre);
+    node = dec_get_node(sysname, cre);
 
-    data = make_pid_data(ser, num);
     if(node == erts_this_node) {
 	*objp = make_internal_pid(data);
     } else {
@@ -1470,11 +1594,11 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete
 	*hpp += EXTERNAL_THING_HEAD_SIZE + 1;
 
 	etp->header = make_external_pid_header(1);
-	etp->next = off_heap->externals;
+	etp->next = off_heap->first;
 	etp->node = node;
 	etp->data.ui[0] = data;
 
-	off_heap->externals = etp;
+	off_heap->first = (struct erl_off_heap_header*) etp;
 	*objp = make_external_pid(etp);
     }
     return ep;
@@ -1487,22 +1611,31 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete
 #define ENC_LAST_ARRAY_ELEMENT ((Eterm) 3)
 
 static byte*
-enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
+enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags,
+	 struct erl_off_heap_header** off_heap)
 {
-    DECLARE_ESTACK(s);
+    DECLARE_WSTACK(s);
     Uint n;
     Uint i;
     Uint j;
     Uint* ptr;
     Eterm val;
     FloatDef f;
+#if HALFWORD_HEAP
+    UWord wobj;
+#endif
+
 
     goto L_jump_start;
 
  outer_loop:
-    while (!ESTACK_ISEMPTY(s)) {
-	obj = ESTACK_POP(s);
-	switch (val = ESTACK_POP(s)) {
+    while (!WSTACK_ISEMPTY(s)) {
+#if HALFWORD_HEAP
+	obj = (Eterm) (wobj = WSTACK_POP(s));
+#else
+	obj = WSTACK_POP(s);
+#endif
+	switch (val = WSTACK_POP(s)) {
 	case ENC_TERM:
 	    break;
 	case ENC_ONE_CONS:
@@ -1513,29 +1646,40 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 
 		obj = CAR(cons);
 		tl = CDR(cons);
-		ESTACK_PUSH(s, is_list(tl) ? ENC_ONE_CONS : ENC_TERM);
-		ESTACK_PUSH(s, tl);
+		WSTACK_PUSH(s, is_list(tl) ? ENC_ONE_CONS : ENC_TERM);
+		WSTACK_PUSH(s, tl);
 	    }
 	    break;
 	case ENC_PATCH_FUN_SIZE:
 	    {
+#if HALFWORD_HEAP
+		byte* size_p = (byte *) wobj;
+#else
 		byte* size_p = (byte *) obj;
-
+#endif
 		put_int32(ep - size_p, size_p);
 	    }
 	    goto outer_loop;
 	case ENC_LAST_ARRAY_ELEMENT:
 	    {
+#if HALFWORD_HEAP
+		Eterm* ptr = (Eterm *) wobj;
+#else
 		Eterm* ptr = (Eterm *) obj;
+#endif
 		obj = *ptr;
 	    }
 	    break;
 	default:		/* ENC_LAST_ARRAY_ELEMENT+1 and upwards */
 	    {
+#if HALFWORD_HEAP
+		Eterm* ptr = (Eterm *) wobj;
+#else
 		Eterm* ptr = (Eterm *) obj;
+#endif
 		obj = *ptr++;
-		ESTACK_PUSH(s, val-1);
-		ESTACK_PUSH(s, (Eterm) ptr);
+		WSTACK_PUSH(s, val-1);
+		WSTACK_PUSH(s, (UWord) ptr);
 	    }
 	    break;
 	}
@@ -1552,19 +1696,23 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 
 	case SMALL_DEF:
 	    {
+		/* From R14B we no longer restrict INTEGER_EXT to 28 bits,
+		 * as done earlier for backward compatibility reasons. */
 		Sint val = signed_val(obj);
 
 		if ((Uint)val < 256) {
 		    *ep++ = SMALL_INTEGER_EXT;
 		    put_int8(val, ep);
 		    ep++;
-		} else if (sizeof(Sint) == 4 || IS_SSMALL28(val)) {
+		} else if (sizeof(Sint) == 4 || IS_SSMALL32(val)) {
 		    *ep++ = INTEGER_EXT;
 		    put_int32(val, ep);
 		    ep += 4;
 		} else {
-		    Eterm tmp_big[2];
-		    Eterm big = small_to_big(val, tmp_big);
+		    DeclareTmpHeapNoproc(tmp_big,2);
+		    Eterm big;
+		    UseTmpHeapNoproc(2);
+		    big = small_to_big(val, tmp_big);
 		    *ep++ = SMALL_BIG_EXT;
 		    n = big_bytes(big);
 		    ASSERT(n < 256);
@@ -1572,23 +1720,38 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 		    ep += 1;
 		    *ep++ = big_sign(big);
 		    ep = big_to_bytes(big, ep);
+		    UnUseTmpHeapNoproc(2);
 		}
 	    }
 	    break;
 
 	case BIG_DEF:
-	    if ((n = big_bytes(obj)) < 256) {
-		*ep++ = SMALL_BIG_EXT;
-		put_int8(n, ep);
-		ep += 1;
-	    }
-	    else {
-		*ep++ = LARGE_BIG_EXT;
-		put_int32(n, ep);
-		ep += 4;
+	    {
+		int sign = big_sign(obj);
+		n = big_bytes(obj);
+		if (sizeof(Sint)==4 && n<=4) {
+		    Uint dig = big_digit(obj,0);		   
+		    Sint val = sign ? -dig : dig;
+		    if ((val<0) == sign) {
+			*ep++ = INTEGER_EXT;
+			put_int32(val, ep);
+			ep += 4;
+			break;
+		    }
+		}
+		if (n < 256) {
+		    *ep++ = SMALL_BIG_EXT;
+		    put_int8(n, ep);
+		    ep += 1;
+		}
+		else {
+		    *ep++ = LARGE_BIG_EXT;
+		    put_int32(n, ep);
+		    ep += 4;
+		}
+		*ep++ = sign;
+		ep = big_to_bytes(obj, ep);
 	    }
-	    *ep++ = big_sign(obj);
-	    ep = big_to_bytes(obj, ep);
 	    break;
 
 	case PID_DEF:
@@ -1601,12 +1764,14 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 	    Uint32 *ref_num;
 
 	    ASSERT(dflags & DFLAG_EXTENDED_REFERENCES);
+
 	    *ep++ = NEW_REFERENCE_EXT;
 	    i = ref_no_of_numbers(obj);
 	    put_int16(i, ep);
 	    ep += 2;
 	    ep = enc_atom(acmp,ref_node_name(obj),ep,dflags);
-	    *ep++ = ref_creation(obj);
+	    *ep++ = ((dflags & DFLAGS_INTERNAL_TAGS) && is_internal_ref(obj)) ?
+		INTERNAL_CREATION : ref_creation(obj);
 	    ref_num = ref_numbers(obj);
 	    for (j = 0; j < i; j++) {
 		put_int32(ref_num[j], ep);
@@ -1622,7 +1787,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 	    j = port_number(obj);
 	    put_int32(j, ep);
 	    ep += 4;
-	    *ep++ = port_creation(obj);
+	    *ep++ = ((dflags & DFLAGS_INTERNAL_TAGS) && is_internal_port(obj)) ?
+		INTERNAL_CREATION : port_creation(obj);
 	    break;
 
 	case LIST_DEF:
@@ -1662,8 +1828,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 		ep += 4;
 	    }
 	    if (i > 0) {
-		ESTACK_PUSH(s, ENC_LAST_ARRAY_ELEMENT+i-1);
-		ESTACK_PUSH(s, (Eterm) ptr);
+		WSTACK_PUSH(s, ENC_LAST_ARRAY_ELEMENT+i-1);
+		WSTACK_PUSH(s, (UWord) ptr);
 	    }
 	    break;
 
@@ -1702,6 +1868,43 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 		byte* bytes;
 
 		ERTS_GET_BINARY_BYTES(obj, bytes, bitoffs, bitsize);
+		if (dflags & DFLAGS_INTERNAL_TAGS) {
+		    ProcBin* pb = (ProcBin*) binary_val(obj);
+		    Uint bytesize = pb->size;
+		    if (pb->thing_word == HEADER_SUB_BIN) {
+			ErlSubBin* sub = (ErlSubBin*)pb;
+			pb = (ProcBin*) binary_val(sub->orig);
+			ASSERT(bytesize == sub->size);
+			bytesize += (bitoffs + bitsize + 7) / 8;
+		    }
+		    if (pb->thing_word == HEADER_PROC_BIN
+			&& heap_bin_size(bytesize) > PROC_BIN_SIZE) {
+			ProcBin tmp;
+			if (bitoffs || bitsize) {
+			    *ep++ = BIT_BINARY_INTERNAL_REF;
+			    *ep++ = bitoffs;
+			    *ep++ = bitsize;
+			}
+			else {
+			    *ep++ = BINARY_INTERNAL_REF;
+			}
+			if (pb->flags) {
+			    char* before_realloc = pb->val->orig_bytes; 
+			    erts_emasculate_writable_binary(pb);
+			    bytes += (pb->val->orig_bytes - before_realloc);
+			}
+			erts_refc_inc(&pb->val->refc, 2);
+
+			sys_memcpy(&tmp, pb, sizeof(ProcBin));
+			tmp.next = *off_heap;
+			tmp.bytes = bytes;
+			tmp.size = bytesize;
+			sys_memcpy(ep, &tmp, sizeof(ProcBin));
+			*off_heap = (struct erl_off_heap_header*) ep;
+			ep += sizeof(ProcBin);
+			break;
+		    }
+		}
 		if (bitsize == 0) {
 		    /* Plain old byte-sized binary. */
 		    *ep++ = BINARY_EXT;
@@ -1737,16 +1940,16 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 		    *ep++ = SMALL_INTEGER_EXT;
 		    *ep++ = bitsize;
 		}
-		break;
 	    }
+	    break;
 	case EXPORT_DEF:
 	    {
-		Export* exp = (Export *) (export_val(obj))[1];
+		Export* exp = *((Export **) (export_val(obj) + 1));
 		if ((dflags & DFLAG_EXPORT_PTR_TAG) != 0) {
 		    *ep++ = EXPORT_EXT;
 		    ep = enc_atom(acmp, exp->code[0], ep, dflags);
 		    ep = enc_atom(acmp, exp->code[1], ep, dflags);
-		    ep = enc_term(acmp, make_small(exp->code[2]), ep, dflags);
+		    ep = enc_term(acmp, make_small(exp->code[2]), ep, dflags, off_heap);
 		} else {
 		    /* Tag, arity */
 		    *ep++ = SMALL_TUPLE_EXT;
@@ -1770,8 +1973,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 		    int ei;
 
 		    *ep++ = NEW_FUN_EXT;
-		    ESTACK_PUSH(s, ENC_PATCH_FUN_SIZE);
-		    ESTACK_PUSH(s, (Eterm) ep); /* Position for patching in size */
+		    WSTACK_PUSH(s, ENC_PATCH_FUN_SIZE);
+		    WSTACK_PUSH(s, (UWord) ep); /* Position for patching in size */
 		    ep += 4;
 		    *ep = funp->arity;
 		    ep += 1;
@@ -1782,14 +1985,14 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 		    put_int32(funp->num_free, ep);
 		    ep += 4;
 		    ep = enc_atom(acmp, funp->fe->module, ep, dflags);
-		    ep = enc_term(acmp, make_small(funp->fe->old_index), ep, dflags);
-		    ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags);
+		    ep = enc_term(acmp, make_small(funp->fe->old_index), ep, dflags, off_heap);
+		    ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags, off_heap);
 		    ep = enc_pid(acmp, funp->creator, ep, dflags);
 
 		fun_env:
 		    for (ei = funp->num_free-1; ei > 0; ei--) {
-			ESTACK_PUSH(s, ENC_TERM);
-			ESTACK_PUSH(s, funp->env[ei]);
+			WSTACK_PUSH(s, ENC_TERM);
+			WSTACK_PUSH(s, (UWord) funp->env[ei]);
 		    }
 		    if (funp->num_free != 0) {
 			obj = funp->env[0];
@@ -1832,11 +2035,12 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 	    break;
 	}
     }
-    DESTROY_ESTACK(s);
+    DESTROY_WSTACK(s);
     return ep;
 }
 
-static Uint
+static
+Uint
 is_external_string(Eterm list, int* p_is_string)
 {
     Uint len = 0;
@@ -1874,69 +2078,30 @@ is_external_string(Eterm list, int* p_is_string)
     return len;
 }
 
-/* Assumes that the ones to undo are preluding the lists. */ 
+/* Assumes that the ones to undo are preluding the list. */ 
 static void
 undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end)
 {
     const Uint area_sz = (end - start) * sizeof(Eterm);
-    struct proc_bin* mso;
-    struct proc_bin** mso_nextp = NULL;
-#ifndef HYBRID /* FIND ME! */
-    struct erl_fun_thing* funs;
-    struct erl_fun_thing** funs_nextp = NULL;
-#endif
-    struct external_thing_* ext;
-    struct external_thing_** ext_nextp = NULL;
-
-    for (mso = off_heap->mso; ; mso=mso->next) {
-	if (!in_area(mso, start, area_sz)) {
-	    if (mso_nextp != NULL) {
-		*mso_nextp = NULL;
-		erts_cleanup_mso(off_heap->mso);
-		off_heap->mso = mso;
+    struct erl_off_heap_header* hdr;
+    struct erl_off_heap_header** hdr_nextp = NULL;
+
+    for (hdr = off_heap->first; ; hdr=hdr->next) {
+	if (!in_area(hdr, start, area_sz)) {
+	    if (hdr_nextp != NULL) {
+		*hdr_nextp = NULL;
+		erts_cleanup_offheap(off_heap);
+		off_heap->first = hdr;
 	    }
 	    break;
 	}
-	mso_nextp = &mso->next;
+	hdr_nextp = &hdr->next;
     }    
 
-#ifndef HYBRID /* FIND ME! */
-    for (funs = off_heap->funs; ; funs=funs->next) {
-	if (!in_area(funs, start, area_sz)) {
-	    if (funs_nextp != NULL) {
-		*funs_nextp = NULL;
-		erts_cleanup_funs(off_heap->funs);
-		off_heap->funs = funs;
-	    }
-	    break;
-	}
-	funs_nextp = &funs->next;
-    }    
-#endif
-    for (ext = off_heap->externals; ; ext=ext->next) {
-	if (!in_area(ext, start, area_sz)) {
-	    if (ext_nextp != NULL) {
-		*ext_nextp = NULL;
-		erts_cleanup_externals(off_heap->externals);
-		off_heap->externals = ext;
-	    }
-	    break;
-	}
-	ext_nextp = &ext->next;
-    }
-
-    /* Assert that the ones to undo were indeed preluding the lists. */ 
+    /* Assert that the ones to undo were indeed preluding the list. */ 
 #ifdef DEBUG
-    for (mso = off_heap->mso; mso != NULL; mso=mso->next) {
-	ASSERT(!in_area(mso, start, area_sz));
-    }    
-# ifndef HYBRID /* FIND ME! */
-    for (funs = off_heap->funs; funs != NULL; funs=funs->next) {
-	ASSERT(!in_area(funs, start, area_sz));
-    }    
-# endif
-    for (ext = off_heap->externals; ext != NULL; ext=ext->next) {
-	ASSERT(!in_area(ext, start, area_sz));
+    for (hdr = off_heap->first; hdr != NULL; hdr = hdr->next) {
+	ASSERT(!in_area(hdr, start, area_sz));
     }    
 #endif /* DEBUG */
 }
@@ -1952,11 +2117,11 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et
     register Eterm* hp = *hpp;	/* Please don't take the address of hp */
     Eterm* next = objp;
 
-    *next = (Eterm) NULL;
+    *next = (Eterm) (UWord) NULL;
 
     while (next != NULL) {
 	objp = next;
-	next = (Eterm *) (*objp);
+	next = (Eterm *) EXPAND_POINTER(*objp);
 
 	switch (*ep++) {
 	case INTEGER_EXT:
@@ -1964,7 +2129,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Et
 		Sint sn = get_int32(ep);
 
 		ep += 4;
-#if defined(ARCH_64)
+#if defined(ARCH_64) && !HALFWORD_HEAP
 		*objp = make_small(sn);
 #else
 		if (MY_IS_SSMALL(sn)) {
@@ -2061,7 +2226,7 @@ dec_term_atom_common:
 	    hp += n;
 	    objp = hp - 1;
 	    while (n-- > 0) {
-		objp[0] = (Eterm) next;
+		objp[0] = (Eterm) COMPRESS_POINTER(next);
 		next = objp;
 		objp--;
 	    }
@@ -2079,12 +2244,12 @@ dec_term_atom_common:
 	    *objp = make_list(hp);
 	    hp += 2*n;
 	    objp = hp - 2;
-	    objp[0] = (Eterm) (objp+1);
-	    objp[1] = (Eterm) next;
+	    objp[0] = (Eterm) COMPRESS_POINTER((objp+1));
+	    objp[1] = (Eterm) COMPRESS_POINTER(next);
 	    next = objp;
 	    objp -= 2;
 	    while (--n > 0) {
-		objp[0] = (Eterm) next;
+		objp[0] = (Eterm) COMPRESS_POINTER(next);
 		objp[1] = make_list(objp + 2);
 		next = objp;
 		objp -= 2;
@@ -2165,13 +2330,13 @@ dec_term_atom_common:
 		    goto error;
 		}
 		ep += 4;
-		if ((cre = get_int8(ep)) >= MAX_CREATION) {
+		cre = get_int8(ep);
+		ep++;
+		if (!is_valid_creation(cre)) {
 		    goto error;
 		}
-		ep++;
-		cre = dec_set_creation(sysname,cre);
-		node = erts_find_or_insert_node(sysname, cre);
 
+		node = dec_get_node(sysname, cre);
 		if(node == erts_this_node) {
 		    *objp = make_internal_port(num);
 		}
@@ -2180,11 +2345,11 @@ dec_term_atom_common:
 		    hp += EXTERNAL_THING_HEAD_SIZE + 1;
 		    
 		    etp->header = make_external_port_header(1);
-		    etp->next = off_heap->externals;
+		    etp->next = off_heap->first;
 		    etp->node = node;
 		    etp->data.ui[0] = num;
 
-		    off_heap->externals = etp;
+		    off_heap->first = (struct erl_off_heap_header*)etp;
 		    *objp = make_external_port(etp);
 		}
 
@@ -2208,9 +2373,11 @@ dec_term_atom_common:
 		    goto error;
 		ep += 4;
 
-		if ((cre = get_int8(ep)) >= MAX_CREATION)
-		    goto error;
+		cre = get_int8(ep);
 		ep += 1;
+		if (!is_valid_creation(cre)) {
+		    goto error;
+		}
 		goto ref_ext_common;
 
 	    case NEW_REFERENCE_EXT:
@@ -2223,10 +2390,11 @@ dec_term_atom_common:
 		if ((ep = dec_atom(edep, ep, &sysname)) == NULL)
 		    goto error;
 
-		if ((cre = get_int8(ep)) >= MAX_CREATION)
-		    goto error;
+		cre = get_int8(ep);
 		ep += 1;
-
+		if (!is_valid_creation(cre)) {
+		    goto error;
+		}
 		r0 = get_int32(ep);
 		ep += 4;
 		if (r0 >= MAX_REFERENCE)
@@ -2234,36 +2402,42 @@ dec_term_atom_common:
 
 	    ref_ext_common:
 
-		cre = dec_set_creation(sysname, cre);
-		node = erts_find_or_insert_node(sysname, cre);
+		node = dec_get_node(sysname, cre);
 		if(node == erts_this_node) {
 		    RefThing *rtp = (RefThing *) hp;
-		    hp += REF_THING_HEAD_SIZE;
-#ifdef ARCH_64
+		    ref_num = (Uint32 *) (hp + REF_THING_HEAD_SIZE);
+
+#if defined(ARCH_64) && !HALFWORD_HEAP
+		    hp += REF_THING_HEAD_SIZE + ref_words/2 + 1;
 		    rtp->header = make_ref_thing_header(ref_words/2 + 1);
 #else
+		    hp += REF_THING_HEAD_SIZE + ref_words;
 		    rtp->header = make_ref_thing_header(ref_words);
 #endif
 		    *objp = make_internal_ref(rtp);
 		}
 		else {
 		    ExternalThing *etp = (ExternalThing *) hp;
-		    hp += EXTERNAL_THING_HEAD_SIZE;
-		    
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
+		    hp += EXTERNAL_THING_HEAD_SIZE + ref_words/2 + 1;
+#else
+		    hp += EXTERNAL_THING_HEAD_SIZE + ref_words;
+#endif
+
+#if defined(ARCH_64) && !HALFWORD_HEAP
 		    etp->header = make_external_ref_header(ref_words/2 + 1);
 #else
 		    etp->header = make_external_ref_header(ref_words);
 #endif
-		    etp->next = off_heap->externals;
+		    etp->next = off_heap->first;
 		    etp->node = node;
 
-		    off_heap->externals = etp;
+		    off_heap->first = (struct erl_off_heap_header*)etp;
 		    *objp = make_external_ref(etp);
+		    ref_num = &(etp->data.ui32[0]);
 		}
 
-		ref_num = (Uint32 *) hp;
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 		*(ref_num++) = ref_words /* 32-bit arity */;
 #endif
 		ref_num[0] = r0;
@@ -2271,12 +2445,9 @@ dec_term_atom_common:
 		    ref_num[i] = get_int32(ep);
 		    ep += 4;
 		}
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 		if ((1 + ref_words) % 2)
 		    ref_num[ref_words] = 0;
-		hp += ref_words/2 + 1;
-#else
-		hp += ref_words;
 #endif
 		break;
 	    }
@@ -2285,7 +2456,7 @@ dec_term_atom_common:
 		n = get_int32(ep);
 		ep += 4;
 	    
-		if (n <= ERL_ONHEAP_BIN_LIMIT || off_heap == NULL) {
+		if (n <= ERL_ONHEAP_BIN_LIMIT) {
 		    ErlHeapBin* hb = (ErlHeapBin *) hp;
 
 		    hb->thing_word = header_heap_bin(n);
@@ -2304,8 +2475,8 @@ dec_term_atom_common:
 		    hp += PROC_BIN_SIZE;
 		    pb->thing_word = HEADER_PROC_BIN;
 		    pb->size = n;
-		    pb->next = off_heap->mso;
-		    off_heap->mso = pb;
+		    pb->next = off_heap->first;
+		    off_heap->first = (struct erl_off_heap_header*)pb;
 		    pb->val = dbin;
 		    pb->bytes = (byte*) dbin->orig_bytes;
 		    pb->flags = 0;
@@ -2323,7 +2494,7 @@ dec_term_atom_common:
 		n = get_int32(ep);
 		bitsize = ep[4];
 		ep += 5;
-		if (n <= ERL_ONHEAP_BIN_LIMIT || off_heap == NULL) {
+		if (n <= ERL_ONHEAP_BIN_LIMIT) {
 		    ErlHeapBin* hb = (ErlHeapBin *) hp;
 
 		    hb->thing_word = header_heap_bin(n);
@@ -2341,8 +2512,8 @@ dec_term_atom_common:
 		    pb = (ProcBin *) hp;
 		    pb->thing_word = HEADER_PROC_BIN;
 		    pb->size = n;
-		    pb->next = off_heap->mso;
-		    off_heap->mso = pb;
+		    pb->next = off_heap->first;
+		    off_heap->first = (struct erl_off_heap_header*)pb;
 		    pb->val = dbin;
 		    pb->bytes = (byte*) dbin->orig_bytes;
 		    pb->flags = 0;
@@ -2398,7 +2569,12 @@ dec_term_atom_common:
                 }
 		*objp = make_export(hp);
 		*hp++ = HEADER_EXPORT;
+#if HALFWORD_HEAP
+		*((UWord *) (UWord) hp) =  (UWord) erts_export_get_or_make_stub(mod, name, arity);
+		hp += 2;
+#else
 		*hp++ = (Eterm) erts_export_get_or_make_stub(mod, name, arity);
+#endif
 		break;
 	    }
 	    break;
@@ -2452,14 +2628,12 @@ dec_term_atom_common:
 		}
 		old_uniq = unsigned_val(temp);
 
-#ifndef HYBRID /* FIND ME! */
 		/*
 		 * It is safe to link the fun into the fun list only when
 		 * no more validity tests can fail.
 		 */
-		funp->next = off_heap->funs;
-		off_heap->funs = funp;
-#endif
+		funp->next = off_heap->first;
+		off_heap->first = (struct erl_off_heap_header*)funp;
 
 		funp->fe = erts_put_fun_entry2(module, old_uniq, old_index,
 					       uniq, index, arity);
@@ -2474,11 +2648,11 @@ dec_term_atom_common:
 
 		/* Environment */
 		for (i = num_free-1; i >= 0; i--) {
-		    funp->env[i] = (Eterm) next;
+		    funp->env[i] = (Eterm) COMPRESS_POINTER(next);
 		    next = funp->env + i;
 		}
 		/* Creator */
-		funp->creator = (Eterm) next;
+		funp->creator = (Eterm) COMPRESS_POINTER(next);
 		next = &(funp->creator);
 		break;
 	    }
@@ -2530,14 +2704,12 @@ dec_term_atom_common:
 		    goto error;
 		}
 		
-#ifndef HYBRID /* FIND ME! */
 		/*
 		 * It is safe to link the fun into the fun list only when
 		 * no more validity tests can fail.
 		 */
-		funp->next = off_heap->funs;
-		off_heap->funs = funp;
-#endif
+		funp->next = off_heap->first;
+		off_heap->first = (struct erl_off_heap_header*)funp;
 		old_uniq = unsigned_val(temp);
 
 		funp->fe = erts_put_fun_entry(module, old_uniq, old_index);
@@ -2549,11 +2721,71 @@ dec_term_atom_common:
 
 		/* Environment */
 		for (i = num_free-1; i >= 0; i--) {
-		    funp->env[i] = (Eterm) next;
+		    funp->env[i] = (Eterm) COMPRESS_POINTER(next);
 		    next = funp->env + i;
 		}
 		break;
 	    }
+	case ATOM_INTERNAL_REF2:
+	    n = get_int16(ep);
+	    ep += 2;
+	    if (n >= atom_table_size()) {
+		goto error;
+	    }
+	    *objp = make_atom(n);
+	    break;
+	case ATOM_INTERNAL_REF3:
+	    n = get_int24(ep);
+	    ep += 3;
+	    if (n >= atom_table_size()) {
+		goto error;
+	    }
+	    *objp = make_atom(n);
+	    break;
+
+	case BINARY_INTERNAL_REF:
+	    {
+		ProcBin* pb = (ProcBin*) hp;
+		sys_memcpy(pb, ep, sizeof(ProcBin));
+		ep += sizeof(ProcBin);
+
+		erts_refc_inc(&pb->val->refc, 1);
+		hp += PROC_BIN_SIZE;
+		pb->next = off_heap->first;
+		off_heap->first = (struct erl_off_heap_header*)pb;
+		pb->flags = 0;
+		*objp = make_binary(pb);
+		break;
+	    }
+	case BIT_BINARY_INTERNAL_REF:
+	    {
+		Sint bitoffs = *ep++;
+		Sint bitsize = *ep++;
+		ProcBin* pb = (ProcBin*) hp;
+		ErlSubBin* sub;
+		sys_memcpy(pb, ep, sizeof(ProcBin));
+		ep += sizeof(ProcBin);
+
+		erts_refc_inc(&pb->val->refc, 1);
+		hp += PROC_BIN_SIZE;
+		pb->next = off_heap->first;
+		off_heap->first = (struct erl_off_heap_header*)pb;
+		pb->flags = 0;
+
+		sub = (ErlSubBin*)hp;
+		sub->thing_word = HEADER_SUB_BIN;
+		sub->size = pb->size - (bitoffs + bitsize + 7)/8;
+		sub->offs = 0;
+		sub->bitoffs = bitoffs;
+		sub->bitsize = bitsize;
+		sub->is_writable = 0;
+		sub->orig = make_binary(pb);
+
+		hp += ERL_SUB_BIN_SIZE;
+		*objp = make_binary(sub);
+		break;
+	    }
+
 	default:
 	error:
 	    /* UNDO:
@@ -2580,26 +2812,35 @@ dec_term_atom_common:
 static Uint
 encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
 {
-    DECLARE_ESTACK(s);
+    DECLARE_WSTACK(s);
     Uint m, i, arity;
     Uint result = 0;
+#if HALFWORD_HEAP
+    UWord wobj = 0;
+#endif
 
     goto L_jump_start;
 
  outer_loop:
-    while (!ESTACK_ISEMPTY(s)) {
-	obj = ESTACK_POP(s);
-
+    while (!WSTACK_ISEMPTY(s)) {
+#if HALFWORD_HEAP
+	obj = (Eterm) (wobj = WSTACK_POP(s));
+#else
+	obj = WSTACK_POP(s);
+#endif
     handle_popped_obj:
-	if (is_CP(obj)) {
+	if (is_CP(obj)) { /* Does not look for CP, looks for "no tag" */
+#if HALFWORD_HEAP
+	    Eterm* ptr = (Eterm *) wobj;
+#else
 	    Eterm* ptr = (Eterm *) obj;
-
+#endif
 	    /*
 	     * Pointer into a tuple.
 	     */
 	    obj = *ptr--;
 	    if (!is_header(obj)) {
-		ESTACK_PUSH(s, (Eterm)ptr);
+		WSTACK_PUSH(s, (UWord)ptr);
 	    } else {
 		/* Reached tuple header */
 		ASSERT(header_is_arityval(obj));
@@ -2611,7 +2852,7 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
 
 	    tl = CDR(cons);
 	    obj = CAR(cons);
-	    ESTACK_PUSH(s, tl);
+	    WSTACK_PUSH(s, tl);
 	} else if (is_nil(obj)) {
 	    result++;
 	    goto outer_loop;
@@ -2627,37 +2868,51 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
 	case NIL_DEF:
 	    result++;
 	    break;
-	case ATOM_DEF: {
-	    int alen = atom_tab(atom_val(obj))->len;
-	    if ((MAX_ATOM_LENGTH <= 255 || alen <= 255)
-		&& (dflags & DFLAG_SMALL_ATOM_TAGS)) {
-		/* Make sure a SMALL_ATOM_EXT fits: SMALL_ATOM_EXT l t1 t2... */
-		result += 1 + 1 + alen;
+	case ATOM_DEF:
+	    if (dflags & DFLAGS_INTERNAL_TAGS) {
+		if (atom_val(obj) >= (1<<16)) {
+		    result += 1 + 3;
+		}
+		else {
+		    result += 1 + 2;
+		}
 	    }
 	    else {
-		/* Make sure an ATOM_EXT fits: ATOM_EXT l1 l0 t1 t2... */
-		result += 1 + 2 + alen;
+		int alen = atom_tab(atom_val(obj))->len;
+		if ((MAX_ATOM_LENGTH <= 255 || alen <= 255)
+		    && (dflags & DFLAG_SMALL_ATOM_TAGS)) {
+		    /* Make sure a SMALL_ATOM_EXT fits: SMALL_ATOM_EXT l t1 t2... */
+			result += 1 + 1 + alen;
+		}
+		else {
+		    /* Make sure an ATOM_EXT fits: ATOM_EXT l1 l0 t1 t2... */
+			result += 1 + 2 + alen;
+		}
+		insert_acache_map(acmp, obj);
 	    }
-	    insert_acache_map(acmp, obj);
 	    break;
-	}
 	case SMALL_DEF:
 	    {
 		Sint val = signed_val(obj);
 
 		if ((Uint)val < 256)
 		    result += 1 + 1;		/* SMALL_INTEGER_EXT */
-		else if (sizeof(Sint) == 4 || IS_SSMALL28(val))
+		else if (sizeof(Sint) == 4 || IS_SSMALL32(val))
 		    result += 1 + 4;		/* INTEGER_EXT */
 		else {
-		    Eterm tmp_big[2];
+		    DeclareTmpHeapNoproc(tmp_big,2);
+		    UseTmpHeapNoproc(2);
 		    i = big_bytes(small_to_big(val, tmp_big));
 		    result += 1 + 1 + 1 + i;	/* SMALL_BIG_EXT */
+		    UnUseTmpHeapNoproc(2);
 		}
 	    }
 	    break;
 	case BIG_DEF:
-	    if ((i = big_bytes(obj)) < 256)
+	    i = big_bytes(obj);
+	    if (sizeof(Sint)==4 && i <= 4 && (big_digit(obj,0)-big_sign(obj)) < (1<<31))
+		result += 1 + 4;          /* INTEGER_EXT */
+	    else if (i < 256)
 		result += 1 + 1 + 1 + i;  /* tag,size,sign,digits */
 	    else
 		result += 1 + 4 + 1 + i;  /* tag,size,sign,digits */
@@ -2698,7 +2953,11 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
 		    result += 1 + 4;
 		}
 		ptr += arity;
+#if HALFWORD_HEAP
+		obj = (Eterm) (wobj = (UWord) ptr);
+#else
 		obj = (Eterm) ptr;
+#endif
 		goto handle_popped_obj;
 	    }
 	    break;
@@ -2710,8 +2969,25 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
 	    }
 	    break;
 	case BINARY_DEF:
+	    if (dflags & DFLAGS_INTERNAL_TAGS) {
+		ProcBin* pb = (ProcBin*) binary_val(obj);
+		Uint sub_extra = 0;
+		Uint tot_bytes = pb->size;
+		if (pb->thing_word == HEADER_SUB_BIN) {
+		    ErlSubBin* sub = (ErlSubBin*) pb;
+		    pb = (ProcBin*) binary_val(sub->orig);
+		    sub_extra = 2;  /* bitoffs and bitsize */
+		    tot_bytes += (sub->bitoffs + sub->bitsize+ 7) / 8;
+		}
+		if (pb->thing_word == HEADER_PROC_BIN
+		    && heap_bin_size(tot_bytes) > PROC_BIN_SIZE) {
+
+		    result += 1 + sub_extra + sizeof(ProcBin);
+		    break;
+		}
+	    }
 	    result += 1 + 4 + binary_size(obj) +
-		5;			/* For unaligned binary */
+		    5;			/* For unaligned binary */
 	    break;
 	case FUN_DEF:
 	    {
@@ -2740,14 +3016,14 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
 
 		    if (is_not_list(obj)) {
 			/* Push any non-list terms on the stack */
-			ESTACK_PUSH(s, obj);
+			WSTACK_PUSH(s, obj);
 		    } else {
 			/* Lists must be handled specially. */
 			if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) {
 			    result += m + 2 + 1;
 			} else {
 			    result += 5;
-			    ESTACK_PUSH(s, obj);
+			    WSTACK_PUSH(s, obj);
 			}
 		    }
 		}
@@ -2760,8 +3036,12 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
 
 	case EXPORT_DEF:
 	    {
-		Export* ep = (Export *) (export_val(obj))[1];
+		Export* ep = *((Export **) (export_val(obj) + 1));
+#if HALFWORD_HEAP
+		result += 2;
+#else
 		result += 1;
+#endif
 		result += encode_size_struct2(acmp, ep->code[0], dflags);
 		result += encode_size_struct2(acmp, ep->code[1], dflags);
 		result += encode_size_struct2(acmp, make_small(ep->code[2]), dflags);
@@ -2774,12 +3054,12 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
 	}
     }
 
-    DESTROY_ESTACK(s);
+    DESTROY_WSTACK(s);
     return result;
 }
 
 static Sint
-decoded_size(byte *ep, byte* endp, int no_refc_bins)
+decoded_size(byte *ep, byte* endp, int internal_tags)
 {
     int heap_size = 0;
     int terms;
@@ -2836,6 +3116,9 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins)
 	case LARGE_BIG_EXT:
 	    CHKSIZE(4);
 	    n = get_int32(ep);
+	    if (n > BIG_ARITY_MAX*sizeof(ErtsDigit)) {
+		return -1;
+	    }
 	    SKIP2(n,4+1);		/* skip, size,sign,digits */
 	    heap_size += 1+1+(n+sizeof(Eterm)-1)/sizeof(Eterm); /* XXX: 1 too much? */
 	    break;
@@ -2886,7 +3169,7 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins)
 		ep += 2;
 		atom_extra_skip = 1 + 4*id_words;
 		/* In case it is an external ref */
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 		heap_size += EXTERNAL_THING_HEAD_SIZE + id_words/2 + 1;
 #else
 		heap_size += EXTERNAL_THING_HEAD_SIZE + id_words;
@@ -2941,7 +3224,7 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins)
 	    CHKSIZE(4);
 	    n = get_int32(ep);
 	    SKIP2(n, 4);
-	    if (n <= ERL_ONHEAP_BIN_LIMIT || no_refc_bins) {
+	    if (n <= ERL_ONHEAP_BIN_LIMIT) {
 		heap_size += heap_bin_size(n);
 	    } else {
 		heap_size += PROC_BIN_SIZE;
@@ -2952,7 +3235,7 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins)
 		CHKSIZE(5);
 		n = get_int32(ep);
 		SKIP2(n, 5);
-		if (n <= ERL_ONHEAP_BIN_LIMIT || no_refc_bins) {
+		if (n <= ERL_ONHEAP_BIN_LIMIT) {
 		    heap_size += heap_bin_size(n) + ERL_SUB_BIN_SIZE;
 		} else {
 		    heap_size += PROC_BIN_SIZE + ERL_SUB_BIN_SIZE;
@@ -2961,7 +3244,11 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins)
 	    break;
 	case EXPORT_EXT:
 	    terms += 3;
+#if HALFWORD_HEAP
+	    heap_size += 3;
+#else
 	    heap_size += 2;
+#endif
 	    break;
 	case NEW_FUN_EXT:
 	    {
@@ -2985,6 +3272,29 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins)
 		heap_size += ERL_FUN_SIZE + num_free;
 		break;
 	    }
+	case ATOM_INTERNAL_REF2:
+	    SKIP(2+atom_extra_skip);
+	    atom_extra_skip = 0;
+	    break;
+	case ATOM_INTERNAL_REF3:
+	    SKIP(3+atom_extra_skip);
+	    atom_extra_skip = 0;
+	    break;
+
+	case BINARY_INTERNAL_REF:
+	    if (!internal_tags) {
+		return -1;
+	    }
+	    SKIP(sizeof(ProcBin));
+	    heap_size += PROC_BIN_SIZE;
+	    break;
+	case BIT_BINARY_INTERNAL_REF:
+	    if (!internal_tags) {
+		return -1;
+	    }
+	    SKIP(2+sizeof(ProcBin));
+	    heap_size += PROC_BIN_SIZE + ERL_SUB_BIN_SIZE;
+	    break;
 	default:
 	    return -1;
 	}
diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h
index eada6d4f27..eddd4571dd 100644
--- a/erts/emulator/beam/external.h
+++ b/erts/emulator/beam/external.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -54,6 +54,10 @@
 
 #define DIST_HEADER       'D'
 #define ATOM_CACHE_REF    'R'
+#define ATOM_INTERNAL_REF2 'I'
+#define ATOM_INTERNAL_REF3 'K'
+#define BINARY_INTERNAL_REF 'J'
+#define BIT_BINARY_INTERNAL_REF 'L'
 #define COMPRESSED        'P'
 
 #if 0
@@ -156,7 +160,10 @@ Uint erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap *);
 void erts_encode_dist_ext(Eterm, byte **, Uint32, ErtsAtomCacheMap *);
 
 Uint erts_encode_ext_size(Eterm);
+Uint erts_encode_ext_size_2(Eterm, unsigned);
+Uint erts_encode_ext_size_ets(Eterm);
 void erts_encode_ext(Eterm, byte **);
+byte* erts_encode_ext_ets(Eterm, byte *, struct erl_off_heap_header** ext_off_heap);
 
 #ifdef ERTS_WANT_EXTERNAL_TAGS
 ERTS_GLB_INLINE void erts_peek_dist_header(ErtsDistHeaderPeek *, byte *, Uint);
@@ -168,11 +175,13 @@ void *erts_dist_ext_trailer(ErtsDistExternal *);
 void erts_destroy_dist_ext_copy(ErtsDistExternal *);
 int erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint,
 			  DistEntry *, ErtsAtomCache *);
-Sint erts_decode_dist_ext_size(ErtsDistExternal *, int);
+Sint erts_decode_dist_ext_size(ErtsDistExternal *);
 Eterm erts_decode_dist_ext(Eterm **, ErlOffHeap *, ErtsDistExternal *);
 
-Sint erts_decode_ext_size(byte*, Uint, int);
+Sint erts_decode_ext_size(byte*, Uint);
+Sint erts_decode_ext_size_ets(byte*, Uint);
 Eterm erts_decode_ext(Eterm **, ErlOffHeap *, byte**);
+Eterm erts_decode_ext_ets(Eterm **, ErlOffHeap *, byte*);
 
 Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags);
 
@@ -211,8 +220,8 @@ ERTS_GLB_INLINE void *
 erts_dist_ext_trailer(ErtsDistExternal *edep)
 {
     void *res = (void *) (edep->ext_endp
-			  + ERTS_WORD_ALIGN_PAD_SZ(edep->ext_endp));
-    ASSERT((((Uint) res) % sizeof(Uint)) == 0);
+			  + ERTS_EXTRA_DATA_ALIGN_SZ(edep->ext_endp));
+    ASSERT((((UWord) res) % sizeof(Uint)) == 0);
     return res;
 }
 
diff --git a/erts/emulator/beam/fix_alloc.c b/erts/emulator/beam/fix_alloc.c
deleted file mode 100644
index 5637281597..0000000000
--- a/erts/emulator/beam/fix_alloc.c
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-/* General purpose Memory allocator for fixed block size objects         */
-/* This allocater is at least an order of magnitude faster than malloc() */
-
-
-#define NOPERBLOCK 20
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-
-#include "sys.h"
-#include "erl_vm.h"
-#include "global.h"
-#include "erl_db.h"
-
-#ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE
-
-#if ERTS_ALC_MTA_FIXED_SIZE
-#include "erl_threads.h"
-#include "erl_smp.h"
-#  ifdef ERTS_SMP
-#    define FA_LOCK(FA) erts_smp_spin_lock(&(FA)->slck)
-#    define FA_UNLOCK(FA) erts_smp_spin_unlock(&(FA)->slck)
-#  else
-#    define FA_LOCK(FA) erts_mtx_lock(&(FA)->mtx)
-#    define FA_UNLOCK(FA) erts_mtx_unlock(&(FA)->mtx)
-#  endif
-#else
-#  define FA_LOCK(FA)
-#  define FA_UNLOCK(FA)
-#endif
-
-typedef union {double d; long l;} align_t;
-
-typedef struct fix_alloc_block {
-    struct fix_alloc_block *next;
-    align_t mem[1];
-} FixAllocBlock;
-
-typedef struct fix_alloc {
-    Uint item_size;
-    void *freelist;
-    Uint no_free;
-    Uint no_blocks;
-    FixAllocBlock *blocks;
-#if ERTS_ALC_MTA_FIXED_SIZE
-#  ifdef ERTS_SMP
-    erts_smp_spinlock_t slck;
-#  else
-    erts_mtx_t mtx;
-#  endif
-#endif
-} FixAlloc;
-
-static void *(*core_alloc)(Uint); 
-static Uint xblk_sz;
-
-static FixAlloc **fa;
-#define FA_SZ (1 + ERTS_ALC_N_MAX_A_FIXED_SIZE - ERTS_ALC_N_MIN_A_FIXED_SIZE)
-
-#define FIX_IX(N) ((N) - ERTS_ALC_N_MIN_A_FIXED_SIZE)
-
-#define FIX_POOL_SZ(I_SZ) \
-  ((I_SZ)*NOPERBLOCK + sizeof(FixAllocBlock) - sizeof(align_t))
-
-#if defined(DEBUG) && !ERTS_ALC_MTA_FIXED_SIZE
-static int first_time;
-#endif
-
-void erts_init_fix_alloc(Uint extra_block_size,
-			 void *(*alloc)(Uint))
-{
-    int i;
-
-    xblk_sz = extra_block_size;
-    core_alloc = alloc;
-
-    fa = (FixAlloc **) (*core_alloc)(FA_SZ * sizeof(FixAlloc *));
-    if (!fa)
-	erts_alloc_enomem(ERTS_ALC_T_UNDEF, FA_SZ * sizeof(FixAlloc *));
-
-    for (i = 0; i < FA_SZ; i++)
-	fa[i] = NULL;
-#if defined(DEBUG) && !ERTS_ALC_MTA_FIXED_SIZE
-    first_time = 1;
-#endif
-}
-
-Uint
-erts_get_fix_size(ErtsAlcType_t type)
-{
-    Uint i = FIX_IX(ERTS_ALC_T2N(type));
-    return i < FA_SZ && fa[i] ? fa[i]->item_size : 0;
-}
-
-void
-erts_set_fix_size(ErtsAlcType_t type, Uint size)
-{
-    Uint sz;
-    Uint i;
-    FixAlloc *fs;
-    ErtsAlcType_t t_no = ERTS_ALC_T2N(type);
-    sz = xblk_sz + size;
-
-#ifdef DEBUG
-    ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= t_no);
-    ASSERT(t_no <= ERTS_ALC_N_MAX_A_FIXED_SIZE);
-#endif
-
-    while (sz % sizeof(align_t) != 0)     /* Alignment */
-	sz++;
-
-    i = FIX_IX(t_no);
-    fs = (FixAlloc *) (*core_alloc)(sizeof(FixAlloc));
-    if (!fs)
-	erts_alloc_n_enomem(t_no, sizeof(FixAlloc));
-    
-    fs->item_size = sz;
-    fs->no_blocks = 0;
-    fs->no_free = 0;
-    fs->blocks = NULL;
-    fs->freelist = NULL;
-    if (fa[i])
-	erl_exit(-1, "Attempt to overwrite existing fix size (%d)", i);
-    fa[i] = fs;
-
-#if ERTS_ALC_MTA_FIXED_SIZE
-#ifdef ERTS_SMP
-    erts_smp_spinlock_init_x(&fs->slck, "fix_alloc", make_small(i));
-#else
-    erts_mtx_init_x(&fs->mtx, "fix_alloc", make_small(i));
-#endif
-#endif
-
-}
-
-void
-erts_fix_info(ErtsAlcType_t type, ErtsFixInfo *efip)
-{
-    Uint i;
-    FixAlloc *f;
-#ifdef DEBUG
-    FixAllocBlock *b;
-    void *fp;
-#endif
-    Uint real_item_size;
-    ErtsAlcType_t t_no = ERTS_ALC_T2N(type);
-
-    ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= t_no);
-    ASSERT(t_no <= ERTS_ALC_N_MAX_A_FIXED_SIZE);
-
-    i = FIX_IX(t_no);
-    f = fa[i];
-
-    efip->total	= sizeof(FixAlloc *);
-    efip->used	= 0;
-    if (!f)
-	return;
-
-    real_item_size = f->item_size - xblk_sz;
-
-    FA_LOCK(f);
-
-    efip->total += sizeof(FixAlloc);
-    efip->total += f->no_blocks*FIX_POOL_SZ(real_item_size);
-    efip->used = efip->total - f->no_free*real_item_size;
-
-#ifdef DEBUG
-    ASSERT(efip->total >= efip->used);
-    for(i = 0, b = f->blocks; b; i++, b = b->next);
-    ASSERT(f->no_blocks == i);
-    for (i = 0, fp = f->freelist; fp; i++, fp = *((void **) fp));
-    ASSERT(f->no_free == i);
-#endif
-
-    FA_UNLOCK(f);
-
-}
-
-void
-erts_fix_free(ErtsAlcType_t t_no, void *extra, void* ptr)
-{
-    Uint i;
-    FixAlloc *f;
-
-    ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= t_no);
-    ASSERT(t_no <= ERTS_ALC_N_MAX_A_FIXED_SIZE);
-
-    i = FIX_IX(t_no);
-    f = fa[i];
-
-    FA_LOCK(f);
-    *((void **) ptr) = f->freelist;
-    f->freelist = ptr;
-    f->no_free++;
-    FA_UNLOCK(f);
-}
-
-
-void *erts_fix_realloc(ErtsAlcType_t t_no, void *extra, void* ptr, Uint size)
-{
-    erts_alc_fatal_error(ERTS_ALC_E_NOTSUP, ERTS_ALC_O_REALLOC, t_no);
-    return NULL;
-}
-
-void *erts_fix_alloc(ErtsAlcType_t t_no, void *extra, Uint size)
-{
-    void *ret;
-    int i;
-    FixAlloc *f;
-
-#if defined(DEBUG) && !ERTS_ALC_MTA_FIXED_SIZE
-    ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= t_no);
-    ASSERT(t_no <= ERTS_ALC_N_MAX_A_FIXED_SIZE);
-    if (first_time) { /* Check that all sizes have been initialized */
-	int i;
-	for (i = 0; i < FA_SZ; i++)
-	    ASSERT(fa[i]);
-	first_time = 0;
-    }
-#endif
-
-
-    i = FIX_IX(t_no);
-    f = fa[i];
-
-    ASSERT(f);
-    ASSERT(f->item_size >= size);
-
-    FA_LOCK(f);
-    if (f->freelist == NULL) {  /* Gotta alloc some more mem */
-	char *ptr;
-	FixAllocBlock *bl;
-	Uint n;
-
-
-	FA_UNLOCK(f);
-	bl = (*core_alloc)(FIX_POOL_SZ(f->item_size));
-	if (!bl)
-	    return NULL;
-
-	FA_LOCK(f);
-	bl->next = f->blocks;  /* link in first */
-	f->blocks = bl;
-
-	n = NOPERBLOCK;
-	ptr = (char *) &f->blocks->mem[0];
-	while(n--) {
-	    *((void **) ptr) = f->freelist;
-	    f->freelist = (void *) ptr;
-	    ptr += f->item_size;
-	}
-#if !ERTS_ALC_MTA_FIXED_SIZE 
-	ASSERT(f->no_free == 0);
-#endif
-	f->no_free += NOPERBLOCK;
-	f->no_blocks++;
-    }
-
-    ret = f->freelist;
-    f->freelist = *((void **) f->freelist);
-    ASSERT(f->no_free > 0);
-    f->no_free--;
-
-    FA_UNLOCK(f);
-
-    return ret;
-}
-
-#endif /* #ifdef ERTS_ALC_N_MIN_A_FIXED_SIZE */
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index cab249a53f..1b15c4ac3b 100644..100755
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -37,22 +37,17 @@
 #include "erl_process.h"
 #include "erl_sys_driver.h"
 #include "erl_debug.h"
+#include "error.h"
 
 typedef struct port Port;
 #include "erl_port_task.h"
 
-#define ERTS_MAX_NO_OF_ASYNC_THREADS 1024
-extern int erts_async_max_threads;
-#define ERTS_ASYNC_THREAD_MIN_STACK_SIZE 16	/* Kilo words */
-#define ERTS_ASYNC_THREAD_MAX_STACK_SIZE 8192	/* Kilo words */
-extern int erts_async_thread_suggested_stack_size;
-
 typedef struct erts_driver_t_ erts_driver_t;
 
 #define SMALL_IO_QUEUE 5   /* Number of fixed elements */
 
 typedef struct {
-    int size;       /* total size in bytes */
+    ErlDrvSizeT size;       /* total size in bytes */
 
     SysIOVec* v_start;
     SysIOVec* v_end;
@@ -68,20 +63,14 @@ typedef struct {
 } ErlIOQueue;
 
 typedef struct line_buf {  /* Buffer used in line oriented I/O */
-    int bufsiz;              /* Size of character buffer */
-    int ovlen;               /* Length of overflow data */
-    int ovsiz;               /* Actual size of overflow buffer */
+    ErlDrvSizeT bufsiz;      /* Size of character buffer */
+    ErlDrvSizeT ovlen;       /* Length of overflow data */
+    ErlDrvSizeT ovsiz;       /* Actual size of overflow buffer */
     char data[1];            /* Starting point of buffer data,
 			      data[0] is a flag indicating an unprocess CR,
 			      The rest is the overflow buffer. */
 } LineBuf;
 
-/* Temporary object header, auto-deallocated when NIF returns. */
-struct enif_tmp_obj_t {
-    struct enif_tmp_obj_t* next;
-    void (*dtor)(struct enif_tmp_obj_t*);
-    /*char data[];*/
-};
 struct enif_environment_t /* ErlNifEnv */
 {
     struct erl_module_nif* mod_nif;
@@ -183,13 +172,14 @@ struct port {
     DistEntry *dist_entry;       /* Dist entry used in DISTRIBUTION */
     char *name;		         /* String used in the open */
     erts_driver_t* drv_ptr;
-    long drv_data;
+    UWord drv_data;
+    SWord os_pid;                 /* Child process ID */
     ErtsProcList *suspended;	 /* List of suspended processes. */
     LineBuf *linebuf;            /* Buffer to hold data not ready for
 				    process to get (line oriented I/O)*/
     Uint32 status;		 /* Status and type flags */
     int control_flags;		 /* Flags for port_control()  */
-    Uint32 snapshot;             /* Next snapshot that port should be part of */
+    erts_aint32_t snapshot;      /* Next snapshot that port should be part of */
     struct reg_proc *reg;
     ErlDrvPDL port_data_lock;
 
@@ -206,17 +196,17 @@ erts_port_runq(Port *prt)
 {
 #ifdef ERTS_SMP
     ErtsRunQueue *rq1, *rq2;
-    rq1 = (ErtsRunQueue *) erts_smp_atomic_read(&prt->run_queue);
+    rq1 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
     while (1) {
 	erts_smp_runq_lock(rq1);
-	rq2 = (ErtsRunQueue *) erts_smp_atomic_read(&prt->run_queue);
+	rq2 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
 	if (rq1 == rq2)
 	    return rq1;
 	erts_smp_runq_unlock(rq1);
 	rq1 = rq2;
     }
 #else
-    return erts_common_run_queue;
+    return ERTS_RUNQ_IX(0);
 #endif
 }
 
@@ -334,12 +324,15 @@ struct erts_driver_t_ {
     void (*stop)(ErlDrvData drv_data);
     void (*finish)(void);
     void (*flush)(ErlDrvData drv_data);
-    void (*output)(ErlDrvData drv_data, char *buf, int len);
+    void (*output)(ErlDrvData drv_data, char *buf, ErlDrvSizeT len);
     void (*outputv)(ErlDrvData drv_data, ErlIOVec *ev); /* Might be NULL */
-    int (*control)(ErlDrvData drv_data, unsigned int command, char *buf, 
-		   int len, char **rbuf, int rlen); /* Might be NULL */
-    int (*call)(ErlDrvData drv_data, unsigned int command, char *buf, 
-		int len, char **rbuf, int rlen, unsigned int *flags); /* Might be NULL */ 
+    ErlDrvSSizeT (*control)(ErlDrvData drv_data, unsigned int command,
+			    char *buf, ErlDrvSizeT len,
+			    char **rbuf, ErlDrvSizeT rlen); /* Might be NULL */
+    ErlDrvSSizeT (*call)(ErlDrvData drv_data, unsigned int command,
+			 char *buf, ErlDrvSizeT len,
+			 char **rbuf, ErlDrvSizeT rlen, /* Might be NULL */
+			 unsigned int *flags);
     void (*event)(ErlDrvData drv_data, ErlDrvEvent event,
 		  ErlDrvEventData event_data);
     void (*ready_input)(ErlDrvData drv_data, ErlDrvEvent event); 
@@ -403,13 +396,13 @@ extern Eterm erts_ddll_monitor_driver(Process *p,
 
 /* Add fields in ERTS_INTERNAL_BINARY_FIELDS, otherwise the drivers crash */
 #define ERTS_INTERNAL_BINARY_FIELDS				\
-    Uint flags;							\
+    UWord flags;							\
     erts_refc_t refc;						\
     ERTS_BINARY_STRUCT_ALIGNMENT
 
 typedef struct binary {
     ERTS_INTERNAL_BINARY_FIELDS
-    long orig_size;
+    SWord orig_size;
     char orig_bytes[1]; /* to be continued */
 } Binary;
 
@@ -418,7 +411,7 @@ typedef struct binary {
 
 typedef struct {
     ERTS_INTERNAL_BINARY_FIELDS
-    long orig_size;
+    SWord orig_size;
     void (*destructor)(Binary *);
     char magic_bin_data[1];
 } ErtsMagicBinary;
@@ -470,7 +463,10 @@ typedef union {
 typedef struct proc_bin {
     Eterm thing_word;		/* Subtag REFC_BINARY_SUBTAG. */
     Uint size;			/* Binary size in bytes. */
-    struct proc_bin *next;	/* Pointer to next ProcBin. */
+#if HALFWORD_HEAP
+    void* dummy_ptr_padding__;
+#endif
+    struct erl_off_heap_header *next;
     Binary *val;		/* Pointer to Binary structure. */
     byte *bytes;		/* Pointer to the actual data bytes. */
     Uint flags;			/* Flag word. */
@@ -500,8 +496,8 @@ erts_mk_magic_binary_term(Eterm **hpp, ErlOffHeap *ohp, Binary *mbp)
 
     pb->thing_word = HEADER_PROC_BIN;
     pb->size = 0;
-    pb->next = ohp->mso;
-    ohp->mso = pb;
+    pb->next = ohp->first;
+    ohp->first = (struct erl_off_heap_header*) pb;
     pb->val = mbp;
     pb->bytes = (byte *) mbp->orig_bytes;
     pb->flags = 0;
@@ -518,13 +514,22 @@ erts_mk_magic_binary_term(Eterm **hpp, ErlOffHeap *ohp, Binary *mbp)
    && (thing_subtag(*binary_val((T))) == REFC_BINARY_SUBTAG) \
    && (((ProcBin *) binary_val((T)))->val->flags & BIN_FLAG_MAGIC))
 
+
+union erl_off_heap_ptr {
+    struct erl_off_heap_header* hdr;
+    ProcBin *pb;
+    struct erl_fun_thing* fun;
+    struct external_thing_* ext;
+    Eterm* ep;
+    void* voidp;
+};
+
 /* arrays that get malloced at startup */
 extern Port* erts_port;
-extern erts_smp_atomic_t erts_ports_alive;
 
 extern Uint erts_max_ports;
 extern Uint erts_port_tab_index_mask;
-extern erts_smp_atomic_t erts_ports_snapshot;
+extern erts_smp_atomic32_t erts_ports_snapshot;
 extern erts_smp_atomic_t erts_dead_ports_ptr;
 
 ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt);
@@ -534,12 +539,13 @@ ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt);
 ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt)
 {
     ERTS_SMP_LC_ASSERT(erts_smp_lc_spinlock_is_locked(&prt->state_lck));
-    if (prt->snapshot != erts_smp_atomic_read(&erts_ports_snapshot)) {
+    if (prt->snapshot != erts_smp_atomic32_read_acqb(&erts_ports_snapshot)) {
 	/* Dead ports are added from the end of the snapshot buffer */
-	Eterm* tombstone = (Eterm*) erts_smp_atomic_addtest(&erts_dead_ports_ptr,
-							    -(long)sizeof(Eterm));
+	Eterm* tombstone;
+	tombstone = (Eterm*) erts_smp_atomic_add_read_nob(&erts_dead_ports_ptr,
+							  -(erts_aint_t)sizeof(Eterm));
 	ASSERT(tombstone+1 != NULL);
-	ASSERT(prt->snapshot == (Uint32) erts_smp_atomic_read(&erts_ports_snapshot) - 1);
+	ASSERT(prt->snapshot == erts_smp_atomic32_read_nob(&erts_ports_snapshot) - 1);
 	*tombstone = prt->id;
     }
     /*else no ongoing snapshot or port was already included or created after snapshot */
@@ -553,99 +559,12 @@ extern Eterm node_cookie;
 extern erts_smp_atomic_t erts_bytes_out;	/* no bytes written out */
 extern erts_smp_atomic_t erts_bytes_in;		/* no bytes sent into the system */
 extern Uint display_items;	/* no of items to display in traces etc */
-extern Uint display_loads;	/* print info about loaded modules */
 
 extern int erts_backtrace_depth;
-extern erts_smp_atomic_t erts_max_gen_gcs;
+extern erts_smp_atomic32_t erts_max_gen_gcs;
 
 extern int erts_disable_tolerant_timeofday;
 
-#ifdef HYBRID
-
-/* Message Area heap pointers */
-extern Eterm *global_heap;             /* Heap start */
-extern Eterm *global_hend;             /* Heap end */
-extern Eterm *global_htop;             /* Heap top (heap pointer) */
-extern Eterm *global_saved_htop;       /* Saved heap top (heap pointer) */
-extern Uint   global_heap_sz;          /* Heap size, in words */
-extern Eterm *global_old_heap;         /* Old generation */
-extern Eterm *global_old_hend;
-extern ErlOffHeap erts_global_offheap; /* Global MSO (OffHeap) list */
-
-extern Uint16 global_gen_gcs;
-extern Uint16 global_max_gen_gcs;
-extern Uint   global_gc_flags;
-
-#ifdef INCREMENTAL
-#define ACTIVATE(p)
-#define DEACTIVATE(p)
-#define IS_ACTIVE(p) 1
-
-#define INC_ACTIVATE(p) do {                                           \
-    if ((p)->active) {                                                 \
-        if ((p)->active_next != NULL) {                                \
-            (p)->active_next->active_prev = (p)->active_prev;          \
-            if ((p)->active_prev) {                                    \
-                (p)->active_prev->active_next = (p)->active_next;      \
-            } else {                                                   \
-                inc_active_proc = (p)->active_next;                    \
-            }                                                          \
-            inc_active_last->active_next = (p);                        \
-            (p)->active_next = NULL;                                   \
-            (p)->active_prev = inc_active_last;                        \
-            inc_active_last = (p);                                     \
-        }                                                              \
-    } else {                                                           \
-        (p)->active_next = NULL;                                       \
-        (p)->active_prev = inc_active_last;                            \
-        if (inc_active_last) {                                         \
-            inc_active_last->active_next = (p);                        \
-        } else {                                                       \
-            inc_active_proc = (p);                                     \
-        }                                                              \
-        inc_active_last = (p);                                         \
-        (p)->active = 1;                                               \
-    }                                                                  \
-} while(0);
-
-#define INC_DEACTIVATE(p) do {                                         \
-    ASSERT((p)->active == 1);                                          \
-    if ((p)->active_next == NULL) {                                    \
-        inc_active_last = (p)->active_prev;                            \
-    } else {                                                           \
-        (p)->active_next->active_prev = (p)->active_prev;              \
-    }                                                                  \
-    if ((p)->active_prev == NULL) {                                    \
-        inc_active_proc = (p)->active_next;                            \
-    } else {                                                           \
-        (p)->active_prev->active_next = (p)->active_next;              \
-    }                                                                  \
-    (p)->active = 0;                                                   \
-} while(0);
-
-#define INC_IS_ACTIVE(p)  ((p)->active != 0)
-
-#else
-extern Eterm *global_old_htop;
-extern Eterm *global_high_water;
-#define ACTIVATE(p)   (p)->active = 1;
-#define DEACTIVATE(p) (p)->active = 0;
-#define IS_ACTIVE(p)  ((p)->active != 0)
-#define INC_ACTIVATE(p)
-#define INC_IS_ACTIVE(p) 1
-#endif /* INCREMENTAL */
-
-#else
-#  define ACTIVATE(p)
-#  define DEACTIVATE(p)
-#  define IS_ACTIVE(p) 1
-#  define INC_ACTIVATE(p)
-#endif /* HYBRID */
-
-#ifdef HYBRID
-extern Uint global_heap_min_sz;
-#endif
-
 extern int bif_reductions;      /* reductions + fcalls (when doing call_bif) */
 extern int stackdump_on_exit;
 
@@ -723,6 +642,60 @@ do {									\
 #define ESTACK_POP(s) (*(--ESTK_CONCAT(s,_sp)))
 
 
+void erl_grow_wstack(UWord** start, UWord** sp, UWord** end);
+#define WSTK_CONCAT(a,b) a##b
+#define WSTK_SUBSCRIPT(s,i) *((UWord *)((byte *)WSTK_CONCAT(s,_start) + (i)))
+#define DEF_WSTACK_SIZE (16)
+
+#define DECLARE_WSTACK(s)						\
+    UWord WSTK_CONCAT(s,_default_stack)[DEF_WSTACK_SIZE];		\
+    UWord* WSTK_CONCAT(s,_start) = WSTK_CONCAT(s,_default_stack);	\
+    UWord* WSTK_CONCAT(s,_sp) = WSTK_CONCAT(s,_start);			\
+    UWord* WSTK_CONCAT(s,_end) = WSTK_CONCAT(s,_start) + DEF_WSTACK_SIZE
+
+#define DESTROY_WSTACK(s)						\
+do {									\
+    if (WSTK_CONCAT(s,_start) != WSTK_CONCAT(s,_default_stack)) {	\
+	erts_free(ERTS_ALC_T_ESTACK, WSTK_CONCAT(s,_start));		\
+    }									\
+} while(0)
+
+#define WSTACK_PUSH(s, x)						\
+do {									\
+    if (WSTK_CONCAT(s,_sp) == WSTK_CONCAT(s,_end)) {			\
+	erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp),	\
+	               &WSTK_CONCAT(s,_end));				\
+    }									\
+    *WSTK_CONCAT(s,_sp)++ = (x);					\
+} while(0)
+
+#define WSTACK_PUSH2(s, x, y)						\
+do {									\
+    if (WSTK_CONCAT(s,_sp) > WSTK_CONCAT(s,_end) - 2) {			\
+	erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp),	\
+		&WSTK_CONCAT(s,_end));	\
+    }									\
+    *WSTK_CONCAT(s,_sp)++ = (x);					\
+    *WSTK_CONCAT(s,_sp)++ = (y);					\
+} while(0)
+
+#define WSTACK_PUSH3(s, x, y, z)					\
+do {									\
+    if (WSTK_CONCAT(s,_sp) > WSTK_CONCAT(s,_end) - 3) {			\
+	erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp),	\
+		&WSTK_CONCAT(s,_end));					\
+    }									\
+    *WSTK_CONCAT(s,_sp)++ = (x);					\
+    *WSTK_CONCAT(s,_sp)++ = (y);					\
+    *WSTK_CONCAT(s,_sp)++ = (z);					\
+} while(0)
+
+#define WSTACK_COUNT(s) (WSTK_CONCAT(s,_sp) - WSTK_CONCAT(s,_start))
+
+#define WSTACK_ISEMPTY(s) (WSTK_CONCAT(s,_sp) == WSTK_CONCAT(s,_start))
+#define WSTACK_POP(s) (*(--WSTK_CONCAT(s,_sp)))
+
+
 /* port status flags */
 
 #define ERTS_PORT_SFLG_CONNECTED	((Uint32) (1 <<  0))
@@ -748,6 +721,8 @@ do {									\
 /* Port uses port specific locking (opposed to driver specific locking) */
 #define ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK ((Uint32) (1 << 13))
 #define ERTS_PORT_SFLG_INVALID		((Uint32) (1 << 14))
+/* Last port to terminate halts the emulator */
+#define ERTS_PORT_SFLG_HALT		((Uint32) (1 << 15))
 #ifdef DEBUG
 /* Only debug: make sure all flags aren't cleared unintentionally */
 #define ERTS_PORT_SFLG_PORT_DEBUG	((Uint32) (1 << 31))
@@ -773,9 +748,8 @@ do {									\
 void erts_emasculate_writable_binary(ProcBin* pb);
 Eterm erts_new_heap_binary(Process *p, byte *buf, int len, byte** datap);
 Eterm erts_new_mso_binary(Process*, byte*, int);
-Eterm new_binary(Process*, byte*, int);
+Eterm new_binary(Process*, byte*, Uint);
 Eterm erts_realloc_binary(Eterm bin, size_t size);
-void erts_cleanup_mso(ProcBin* pb);
 
 /* erl_bif_info.c */
 
@@ -791,18 +765,41 @@ void erts_queue_monitor_message(Process *,
 				Eterm,
 				Eterm);
 void erts_init_bif(void);
+Eterm erl_send(Process *p, Eterm to, Eterm msg);
+
+/* erl_bif_op.c */
+
+Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2);
 
 /* erl_bif_port.c */
 
 /* erl_bif_trace.c */
+Eterm erl_seq_trace_info(Process *p, Eterm arg1);
 void erts_system_monitor_clear(Process *c_p);
 void erts_system_profile_clear(Process *c_p);
 
 /* beam_load.c */
-int erts_load_module(Process *c_p, ErtsProcLocks c_p_locks,
-		     Eterm group_leader, Eterm* mod, byte* code, int size);
+typedef struct {
+    BeamInstr* current;		/* Pointer to: Mod, Name, Arity */
+    Uint needed;		/* Heap space needed for entire tuple */
+    Uint32 loc;			/* Location in source code */
+    Eterm* fname_ptr;		/* Pointer to fname table */
+} FunctionInfo;
+
+struct LoaderState* erts_alloc_loader_state(void);
+Eterm erts_prepare_loading(struct LoaderState*,  Process *c_p,
+			   Eterm group_leader, Eterm* modp,
+			   byte* code, Uint size);
+Eterm erts_finish_loading(struct LoaderState* stp, Process* c_p,
+			  ErtsProcLocks c_p_locks, Eterm* modp);
+Eterm erts_load_module(Process *c_p, ErtsProcLocks c_p_locks,
+		      Eterm group_leader, Eterm* mod, byte* code, Uint size);
 void init_load(void);
-Eterm* find_function_from_pc(Eterm* pc);
+BeamInstr* find_function_from_pc(BeamInstr* pc);
+Eterm* erts_build_mfa_item(FunctionInfo* fi, Eterm* hp,
+			   Eterm args, Eterm* mfa_p);
+void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info);
+void erts_set_current_function(FunctionInfo* fi, BeamInstr* current);
 Eterm erts_module_info_0(Process* p, Eterm module);
 Eterm erts_module_info_1(Process* p, Eterm module, Eterm what);
 Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info);
@@ -819,130 +816,38 @@ void loaded(int, void *);
 /* config.c */
 
 __decl_noreturn void __noreturn erl_exit(int n, char*, ...);
-__decl_noreturn void __noreturn erl_exit0(char *, int, int n, char*, ...);
+__decl_noreturn void __noreturn erl_exit_flush_async(int n, char*, ...);
 void erl_error(char*, va_list);
 
-#define ERL_EXIT0(n,f)		erl_exit0(__FILE__, __LINE__, n, f)
-#define ERL_EXIT1(n,f,a)	erl_exit0(__FILE__, __LINE__, n, f, a)
-#define ERL_EXIT2(n,f,a,b)	erl_exit0(__FILE__, __LINE__, n, f, a, b)
-#define ERL_EXIT3(n,f,a,b,c)	erl_exit0(__FILE__, __LINE__, n, f, a, b, c)
-
 /* copy.c */
-void init_copy(void);
 Eterm copy_object(Eterm, Process*);
-Uint size_object(Eterm);
-Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*);
-Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*);
-
-#ifdef HYBRID
-#define RRMA_DEFAULT_SIZE 256
-#define RRMA_STORE(p,ptr,src) do {                                      \
-  ASSERT((p)->rrma != NULL);                                            \
-  ASSERT((p)->rrsrc != NULL);                                           \
-  (p)->rrma[(p)->nrr] = (ptr);                                          \
-  (p)->rrsrc[(p)->nrr++] = (src);                                       \
-  if ((p)->nrr == (p)->rrsz)                                            \
-  {                                                                     \
-      (p)->rrsz *= 2;                                                   \
-      (p)->rrma = (Eterm *) erts_realloc(ERTS_ALC_T_ROOTSET,            \
-                                         (void*)(p)->rrma,              \
-                                         sizeof(Eterm) * (p)->rrsz);    \
-      (p)->rrsrc = (Eterm **) erts_realloc(ERTS_ALC_T_ROOTSET,          \
-                                           (void*)(p)->rrsrc,           \
-                                            sizeof(Eterm) * (p)->rrsz); \
-  }                                                                     \
-} while(0)
-
-/* Note that RRMA_REMOVE decreases the given index after deletion. 
- * This is done so that a loop with an increasing index can call
- * remove without having to decrease the index to see the element
- * placed in the hole after the deleted element.
- */
-#define RRMA_REMOVE(p,index) do {                                 \
-        p->rrsrc[index] = p->rrsrc[--p->nrr];                     \
-        p->rrma[index--] = p->rrma[p->nrr];                       \
-    } while(0);
 
+#if HALFWORD_HEAP
+Uint size_object_rel(Eterm, Eterm*);
+#  define size_object(A) size_object_rel(A,NULL)
 
-/* The MessageArea STACKs are used while copying messages to the
- * message area.
- */
-#define MA_STACK_EXTERNAL_DECLARE(type,_s_)     \
-    typedef type ma_##_s_##_type;               \
-    extern ma_##_s_##_type *ma_##_s_##_stack;   \
-    extern Uint ma_##_s_##_top;                 \
-    extern Uint ma_##_s_##_size;
-
-#define MA_STACK_DECLARE(_s_)                                           \
-    ma_##_s_##_type *ma_##_s_##_stack; Uint ma_##_s_##_top; Uint ma_##_s_##_size;
-
-#define MA_STACK_ALLOC(_s_) do {                                        \
-    ma_##_s_##_top = 0;                                                 \
-    ma_##_s_##_size = 512;                                              \
-    ma_##_s_##_stack = (ma_##_s_##_type*)erts_alloc(ERTS_ALC_T_OBJECT_STACK, \
-                       sizeof(ma_##_s_##_type) * ma_##_s_##_size);      \
-} while(0)
-
-
-#define MA_STACK_PUSH(_s_,val) do {                                     \
-    ma_##_s_##_stack[ma_##_s_##_top++] = (val);                         \
-    if (ma_##_s_##_top == ma_##_s_##_size)                              \
-    {                                                                   \
-        ma_##_s_##_size *= 2;                                           \
-        ma_##_s_##_stack =                                              \
-            (ma_##_s_##_type*) erts_realloc(ERTS_ALC_T_OBJECT_STACK,    \
-                                           (void*)ma_##_s_##_stack,     \
-                            sizeof(ma_##_s_##_type) * ma_##_s_##_size); \
-    }                                                                   \
-} while(0)
+Eterm copy_struct_rel(Eterm, Uint, Eterm**, ErlOffHeap*, Eterm* src_base, Eterm* dst_base);
+#  define copy_struct(OBJ,SZ,HPP,OH) copy_struct_rel(OBJ,SZ,HPP,OH, NULL,NULL)
 
-#define MA_STACK_POP(_s_) (ma_##_s_##_top != 0 ? ma_##_s_##_stack[--ma_##_s_##_top] : 0)
-#define MA_STACK_TOP(_s_) (ma_##_s_##_stack[ma_##_s_##_top - 1])
-#define MA_STACK_UPDATE(_s_,offset,value)                               \
-  *(ma_##_s_##_stack[ma_##_s_##_top - 1] + (offset)) = (value)
-#define MA_STACK_SIZE(_s_) (ma_##_s_##_top)
-#define MA_STACK_ELM(_s_,i) ma_##_s_##_stack[i]
+Eterm copy_shallow_rel(Eterm*, Uint, Eterm**, ErlOffHeap*, Eterm* src_base);
+#  define copy_shallow(A,B,C,D) copy_shallow_rel(A,B,C,D,NULL)
 
-MA_STACK_EXTERNAL_DECLARE(Eterm,src);
-MA_STACK_EXTERNAL_DECLARE(Eterm*,dst);
-MA_STACK_EXTERNAL_DECLARE(Uint,offset);
+#else /* !HALFWORD_HEAP */
 
+Uint size_object(Eterm);
+#  define size_object_rel(A,B) size_object(A)
 
-#ifdef INCREMENTAL
-extern Eterm *ma_pending_stack;
-extern Uint ma_pending_top;
-extern Uint ma_pending_size;
-
-#define NO_COPY(obj) (IS_CONST(obj) ||                         \
-                      (((ptr_val(obj) >= global_heap) &&       \
-                        (ptr_val(obj) < global_htop)) ||       \
-                       ((ptr_val(obj) >= inc_fromspc) &&       \
-                        (ptr_val(obj) < inc_fromend)) ||       \
-                       ((ptr_val(obj) >= global_old_heap) &&   \
-                        (ptr_val(obj) < global_old_hend))))
+Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*);
+#  define copy_struct_rel(OBJ,SZ,HPP,OH, SB,DB) copy_struct(OBJ,SZ,HPP,OH)
 
-#else
+Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*);
+#  define copy_shallow_rel(A,B,C,D, BASE) copy_shallow(A,B,C,D)
 
-#define NO_COPY(obj) (IS_CONST(obj) ||                        \
-                      (((ptr_val(obj) >= global_heap) &&      \
-                        (ptr_val(obj) < global_htop)) ||      \
-                       ((ptr_val(obj) >= global_old_heap) &&  \
-                        (ptr_val(obj) < global_old_hend))))
-
-#endif /* INCREMENTAL */
-
-#define LAZY_COPY(from,obj) do {                     \
-  if (!NO_COPY(obj)) {                               \
-      BM_LAZY_COPY_START;                            \
-      BM_COUNT(messages_copied);                     \
-      obj = copy_struct_lazy(from,obj,0);            \
-      BM_LAZY_COPY_STOP;                             \
-  }                                                  \
-} while(0)
+#endif
 
-Eterm copy_struct_lazy(Process*, Eterm, Uint);
 
-#endif /* HYBRID */
+void move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first,
+		      Eterm* refs, unsigned nrefs);
 
 /* Utilities */
 extern void erts_delete_nodes_monitors(Process *, ErtsProcLocks);
@@ -952,7 +857,8 @@ extern int erts_do_net_exits(DistEntry*, Eterm);
 extern int distribution_info(int, void *);
 extern int is_node_name_atom(Eterm a);
 
-extern int erts_net_message(Port *, DistEntry *, byte *, int, byte *, int);
+extern int erts_net_message(Port *, DistEntry *,
+			    byte *, ErlDrvSizeT, byte *, ErlDrvSizeT);
 
 extern void init_dist(void);
 extern int stop_dist(void);
@@ -965,10 +871,11 @@ void print_pass_through(int, byte*, int);
 
 /* beam_emu.c */
 int catchlevel(Process*);
-void init_emulator(_VOID_);
+void init_emulator(void);
 void process_main(void);
 Eterm build_stacktrace(Process* c_p, Eterm exc);
 Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value);
+void erts_save_stacktrace(Process* p, struct StackTrace* s, int depth);
 
 /* erl_init.c */
 
@@ -990,6 +897,7 @@ extern ErtsModifiedTimings erts_modified_timings[];
 #define ERTS_MODIFIED_TIMING_INPUT_REDS \
   (erts_modified_timings[erts_modified_timing_level].input_reds)
 
+extern int erts_no_line_info;
 extern Eterm erts_error_logger_warnings;
 extern int erts_initialized;
 extern int erts_compat_rel;
@@ -1023,17 +931,16 @@ void erts_init_gc(void);
 int erts_garbage_collect(Process*, int, Eterm*, int);
 void erts_garbage_collect_hibernate(Process* p);
 Eterm erts_gc_after_bif_call(Process* p, Eterm result, Eterm* regs, Uint arity);
-void erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size);
+void erts_garbage_collect_literals(Process* p, Eterm* literals,
+				   Uint lit_size,
+				   struct erl_off_heap_header* oh);
 Uint erts_next_heap_size(Uint, Uint);
 Eterm erts_heap_sizes(Process* p);
 
 void erts_offset_off_heap(ErlOffHeap *, Sint, Eterm*, Eterm*);
 void erts_offset_heap_ptr(Eterm*, Uint, Sint, Eterm*, Eterm*);
 void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*);
-
-#ifdef HYBRID
-int erts_global_garbage_collect(Process*, int, Eterm*, int);
-#endif
+void erts_free_heap_frags(Process* p);
 
 /* io.c */
 
@@ -1080,6 +987,10 @@ void erts_fire_port_monitor(Port *prt, Eterm ref);
 void erts_smp_xports_unlock(Port *);
 #endif
 
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT)
+void erts_lcnt_enable_io_lock_count(int enable);
+#endif
+
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
 int erts_lc_is_port_locked(Port *);
 #endif
@@ -1116,11 +1027,11 @@ erts_smp_port_trylock(Port *prt)
 #ifdef ERTS_SMP
     int res;
 
-    ASSERT(erts_smp_atomic_read(&prt->refc) > 0);
-    erts_smp_atomic_inc(&prt->refc);
+    ASSERT(erts_smp_atomic_read_nob(&prt->refc) > 0);
+    erts_smp_atomic_inc_nob(&prt->refc);
     res = erts_smp_mtx_trylock(prt->lock);
     if (res == EBUSY) {
-	erts_smp_atomic_dec(&prt->refc);
+	erts_smp_atomic_dec_nob(&prt->refc);
     }
 
     return res;
@@ -1133,8 +1044,8 @@ ERTS_GLB_INLINE void
 erts_smp_port_lock(Port *prt)
 {
 #ifdef ERTS_SMP
-    ASSERT(erts_smp_atomic_read(&prt->refc) > 0);
-    erts_smp_atomic_inc(&prt->refc);
+    ASSERT(erts_smp_atomic_read_nob(&prt->refc) > 0);
+    erts_smp_atomic_inc_nob(&prt->refc);
     erts_smp_mtx_lock(prt->lock);
 #endif
 }
@@ -1143,13 +1054,12 @@ ERTS_GLB_INLINE void
 erts_smp_port_unlock(Port *prt)
 {
 #ifdef ERTS_SMP
-    long refc;
-    refc = erts_smp_atomic_dectest(&prt->refc);
+    erts_aint_t refc;
+    erts_smp_mtx_unlock(prt->lock);
+    refc = erts_smp_atomic_dec_read_nob(&prt->refc);
     ASSERT(refc >= 0);
     if (refc == 0)
 	erts_port_cleanup(prt);
-    else
-	erts_smp_mtx_unlock(prt->lock);
 #endif
 }
 
@@ -1171,7 +1081,7 @@ erts_smp_port_unlock(Port *prt)
   ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP)
 
 #define ERTS_PORT_SCHED_ID(P, ID) \
-  ((Uint) erts_prtsd_set((P), ERTS_PSD_SCHED_ID, (void *) (ID)))
+  ((Uint) (UWord) erts_prtsd_set((P), ERTS_PSD_SCHED_ID, (void *) (UWord) (ID)))
 
 #ifdef ERTS_SMP
 Port *erts_de2port(DistEntry *, Process *, ErtsProcLocks);
@@ -1215,7 +1125,7 @@ erts_id2port_sflgs(Eterm id, Process *c_p, ErtsProcLocks c_p_locks, Uint32 sflgs
     }
 #ifdef ERTS_SMP
     else {
-	erts_smp_atomic_inc(&prt->refc);
+	erts_smp_atomic_inc_nob(&prt->refc);
 	erts_smp_port_state_unlock(prt);
 
 	if (no_proc_locks)
@@ -1363,84 +1273,6 @@ void erl_drv_thr_init(void);
 
 /* time.c */
 
-ERTS_GLB_INLINE long do_time_read_and_reset(void);
-#ifdef ERTS_TIMER_THREAD
-ERTS_GLB_INLINE int next_time(void);
-ERTS_GLB_INLINE void bump_timer(long);
-#else
-int next_time(void);
-void bump_timer(long);
-extern erts_smp_atomic_t do_time;	/* set at clock interrupt */
-ERTS_GLB_INLINE void do_time_add(long);
-#endif
-
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-
-#ifdef ERTS_TIMER_THREAD
-ERTS_GLB_INLINE long do_time_read_and_reset(void) { return 0; }
-ERTS_GLB_INLINE int next_time(void) { return -1; }
-ERTS_GLB_INLINE void bump_timer(long ignore) { }
-#else
-ERTS_GLB_INLINE long do_time_read_and_reset(void)
-{
-    return erts_smp_atomic_xchg(&do_time, 0L);
-}
-ERTS_GLB_INLINE void do_time_add(long elapsed)
-{
-    erts_smp_atomic_add(&do_time, elapsed);
-}
-#endif
-
-#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
-
-void init_time(void);
-void erl_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint);
-void erl_cancel_timer(ErlTimer*);
-Uint time_left(ErlTimer *);
-
-Uint erts_timer_wheel_memory_size(void);
-
-#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME))
-#  ifndef HAVE_ERTS_NOW_CPU
-#    define HAVE_ERTS_NOW_CPU
-#    ifdef HAVE_GETHRVTIME
-#      define erts_start_now_cpu() sys_start_hrvtime()
-#      define erts_stop_now_cpu()  sys_stop_hrvtime()
-#    endif
-#  endif
-void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec);
-#endif
-
-void erts_get_timeval(SysTimeval *tv);
-long erts_get_time(void);
-
-extern SysTimeval erts_first_emu_time;
-
-void erts_get_emu_time(SysTimeval *);
-
-ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p);
-
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-
-ERTS_GLB_INLINE int
-erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p)
-{
-    if (t1p->tv_sec == t2p->tv_sec) {
-	if (t1p->tv_usec < t2p->tv_usec)
-	    return -1;
-	else if (t1p->tv_usec > t2p->tv_usec)
-	    return 1;
-	return 0;
-    }
-    return t1p->tv_sec < t2p->tv_sec ? -1 : 1;
-}
-
-#endif
-
-#ifdef DEBUG
-void p_slpq(_VOID_);
-#endif
-
 /* utils.c */
 
 /*
@@ -1449,7 +1281,6 @@ void p_slpq(_VOID_);
 void erts_silence_warn_unused_result(long unused);
 
 void erts_cleanup_offheap(ErlOffHeap *offheap);
-void erts_cleanup_externals(ExternalThing *);
 
 Uint erts_fit_in_bits(Uint);
 int list_length(Eterm);
@@ -1463,6 +1294,7 @@ Uint32 make_hash(Eterm);
 
 Eterm erts_bld_atom(Uint **hpp, Uint *szp, char *str);
 Eterm erts_bld_uint(Uint **hpp, Uint *szp, Uint ui);
+Eterm erts_bld_uword(Uint **hpp, Uint *szp, UWord uw);
 Eterm erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64);
 Eterm erts_bld_sint64(Uint **hpp, Uint *szp, Sint64 si64);
 Eterm erts_bld_cons(Uint **hpp, Uint *szp, Eterm car, Eterm cdr);
@@ -1481,7 +1313,7 @@ erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length,
 			      Eterm atoms[], Uint uints1[], Uint uints2[]);
 
 Eterm store_external_or_ref_in_proc_(Process *, Eterm);
-Eterm store_external_or_ref_(Uint **, ExternalThing **, Eterm);
+Eterm store_external_or_ref_(Uint **, ErlOffHeap*, Eterm);
 
 #define NC_HEAP_SIZE(NC) \
  (ASSERT_EXPR(is_node_container((NC))), \
@@ -1499,23 +1331,39 @@ void erts_init_utils_mem(void);
 erts_dsprintf_buf_t *erts_create_tmp_dsbuf(Uint);
 void erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *);
 
+#if HALFWORD_HEAP
+int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base);
+#  define eq(A,B) eq_rel(A,NULL,B,NULL)
+#else
 int eq(Eterm, Eterm);
+#  define eq_rel(A,A_BASE,B,B_BASE) eq(A,B)
+#endif
+
 #define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y))))
 
+#if HALFWORD_HEAP
+Sint cmp_rel(Eterm, Eterm*, Eterm, Eterm*);
+#define CMP(A,B) cmp_rel(A,NULL,B,NULL)
+#else
 Sint cmp(Eterm, Eterm);
-#define cmp_lt(a,b)	(cmp((a),(b)) < 0)
-#define cmp_le(a,b)	(cmp((a),(b)) <= 0)
-#define cmp_eq(a,b)	(cmp((a),(b)) == 0)
-#define cmp_ne(a,b)	(cmp((a),(b)) != 0)
-#define cmp_ge(a,b)	(cmp((a),(b)) >= 0)
-#define cmp_gt(a,b)	(cmp((a),(b)) > 0)
+#define cmp_rel(A,A_BASE,B,B_BASE) cmp(A,B)
+#define CMP(A,B) cmp(A,B)
+#endif
+#define cmp_lt(a,b)	(CMP((a),(b)) < 0)
+#define cmp_le(a,b)	(CMP((a),(b)) <= 0)
+#define cmp_eq(a,b)	(CMP((a),(b)) == 0)
+#define cmp_ne(a,b)	(CMP((a),(b)) != 0)
+#define cmp_ge(a,b)	(CMP((a),(b)) >= 0)
+#define cmp_gt(a,b)	(CMP((a),(b)) > 0)
 
 #define CMP_LT(a,b)	((a) != (b) && cmp_lt((a),(b)))
 #define CMP_GE(a,b)	((a) == (b) || cmp_ge((a),(b)))
 #define CMP_EQ(a,b)	((a) == (b) || cmp_eq((a),(b)))
 #define CMP_NE(a,b)	((a) != (b) && cmp_ne((a),(b)))
 
+/* duplicates from big.h */
 int term_to_Uint(Eterm term, Uint *up);
+int term_to_UWord(Eterm, UWord*);
 
 #ifdef HAVE_ERTS_NOW_CPU
 extern int erts_cpu_timestamp;
@@ -1525,9 +1373,26 @@ void erts_init_bif_chksum(void);
 /* erl_bif_re.c */
 void erts_init_bif_re(void);
 Sint erts_re_set_loop_limit(Sint limit);
+/* erl_bif_binary.c */
+void erts_init_bif_binary(void);
+Sint erts_binary_set_loop_limit(Sint limit);
+
 /* erl_unicode.c */
 void erts_init_unicode(void);
 Sint erts_unicode_set_loop_limit(Sint limit);
+
+void erts_native_filename_put(Eterm ioterm, int encoding, byte *p) ;
+Sint erts_native_filename_need(Eterm ioterm, int encoding);
+void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars);
+int erts_analyze_utf8(byte *source, Uint size, 
+			byte **err_pos, Uint *num_chars, int *left);
+char *erts_convert_filename_to_native(Eterm name, ErtsAlcType_t alloc_type, int allow_empty);
+
+#define ERTS_UTF8_OK 0
+#define ERTS_UTF8_INCOMPLETE 1
+#define ERTS_UTF8_ERROR 2
+#define ERTS_UTF8_ANALYZE_MORE 3
+
 /* erl_trace.c */
 void erts_init_trace(void);
 void erts_trace_check_exiting(Eterm exiting);
@@ -1554,12 +1419,12 @@ void erts_queue_error_logger_message(Eterm, Eterm, ErlHeapFragment *);
 void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *);
 void trace_send(Process*, Eterm, Eterm);
 void trace_receive(Process*, Eterm);
-Uint32 erts_call_trace(Process *p, Eterm mfa[], Binary *match_spec, Eterm* args,
+Uint32 erts_call_trace(Process *p, BeamInstr mfa[], Binary *match_spec, Eterm* args,
 		       int local, Eterm *tracer_pid);
-void erts_trace_return(Process* p, Eterm* fi, Eterm retval, Eterm *tracer_pid);
-void erts_trace_exception(Process* p, Eterm mfa[], Eterm class, Eterm value, 
+void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid);
+void erts_trace_exception(Process* p, BeamInstr mfa[], Eterm class, Eterm value,
 			  Eterm *tracer);
-void erts_trace_return_to(Process *p, Uint *pc);
+void erts_trace_return_to(Process *p, BeamInstr *pc);
 void trace_sched(Process*, Eterm);
 void trace_proc(Process*, Process*, Eterm, Eterm);
 void trace_proc_spawn(Process*, Eterm pid, Eterm mod, Eterm func, Eterm args);
@@ -1588,8 +1453,7 @@ void monitor_generic(Process *p, Eterm type, Eterm spec);
 Uint erts_trace_flag2bit(Eterm flag);
 int erts_trace_flags(Eterm List, 
 		 Uint *pMask, Eterm *pTracer, int *pCpuTimestamp);
-Eterm erts_bif_trace(int bif_index, Process* p, 
-		     Eterm arg1, Eterm arg2, Eterm arg3, Uint *I);
+Eterm erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr *I);
 
 #ifdef ERTS_SMP
 void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp);
@@ -1602,11 +1466,11 @@ do {									\
 #define ERTS_SMP_CHK_PEND_TRACE_MSGS(ESDP)
 #endif
 
-void bin_write(int, void*, byte*, int);
+void bin_write(int, void*, byte*, size_t);
 int intlist_to_buf(Eterm, char*, int); /* most callers pass plain char*'s */
 
 struct Sint_buf {
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
     char s[22];
 #else
     char s[12];
@@ -1614,12 +1478,16 @@ struct Sint_buf {
 };	
 char* Sint_to_buf(Sint, struct Sint_buf*);
 
-Eterm buf_to_intlist(Eterm**, char*, int, Eterm); /* most callers pass plain char*'s */
+#define ERTS_IOLIST_OK 0
+#define ERTS_IOLIST_OVERFLOW 1
+#define ERTS_IOLIST_TYPE 2
+
+Eterm buf_to_intlist(Eterm**, char*, size_t, Eterm); /* most callers pass plain char*'s */
 int io_list_to_buf(Eterm, char*, int);
 int io_list_to_buf2(Eterm, char*, int);
-int io_list_len(Eterm);
+int erts_iolist_size(Eterm, Uint *);
 int is_string(Eterm);
-void erl_at_exit(FUNCTION(void,(*),(void*)), void*);
+void erl_at_exit(void (*) (void*), void*);
 Eterm collect_memory(Process *);
 void dump_memory_to_fd(int);
 int dump_memory_data(const char *);
@@ -1654,17 +1522,14 @@ Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live);
 Eterm erts_gc_float_1(Process* p, Eterm* reg, Uint live);
 Eterm erts_gc_round_1(Process* p, Eterm* reg, Uint live);
 Eterm erts_gc_trunc_1(Process* p, Eterm* reg, Uint live);
+Eterm erts_gc_binary_part_3(Process* p, Eterm* reg, Uint live);
+Eterm erts_gc_binary_part_2(Process* p, Eterm* reg, Uint live);
 
 Uint erts_current_reductions(Process* current, Process *p);
 
 int erts_print_system_version(int to, void *arg, Process *c_p);
 
-/*
- * Interface to erl_init
- */
-void erl_init(void);
-void erts_first_process(Eterm modname, void* code, unsigned size, int argc, char** argv);
-
+int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg);
 #define seq_trace_output(token, msg, type, receiver, process) \
 seq_trace_output_generic((token), (msg), (type), (receiver), (process), NIL)
 #define seq_trace_output_exit(token, msg, type, receiver, exitfrom) \
@@ -1683,8 +1548,10 @@ struct trace_pattern_flags {
     unsigned int local      : 1; /* Local call trace breakpoint */
     unsigned int meta       : 1; /* Metadata trace breakpoint */
     unsigned int call_count : 1; /* Fast call count breakpoint */
+    unsigned int call_time  : 1; /* Fast call time breakpoint */
 };
 extern const struct trace_pattern_flags erts_trace_pattern_flags_off;
+extern int erts_call_time_breakpoint_tracing;
 int erts_set_trace_pattern(Eterm* mfa, int specified, 
 			   Binary* match_prog_set, Binary *meta_match_prog_set,
 			   int on, struct trace_pattern_flags,
@@ -1720,26 +1587,35 @@ do {								\
 extern Binary *erts_match_set_compile(Process *p, Eterm matchexpr);
 Eterm erts_match_set_lint(Process *p, Eterm matchexpr); 
 extern void erts_match_set_release_result(Process* p);
+
+enum erts_pam_run_flags {
+    ERTS_PAM_TMP_RESULT=0,
+    ERTS_PAM_COPY_RESULT=1,
+    ERTS_PAM_CONTIGUOUS_TUPLE=2
+};
 extern Eterm erts_match_set_run(Process *p, Binary *mpsp, 
 				Eterm *args, int num_args,
+				enum erts_pam_run_flags in_flags,
 				Uint32 *return_flags);
 extern Eterm erts_match_set_get_source(Binary *mpsp);
 extern void erts_match_prog_foreach_offheap(Binary *b,
 					    void (*)(ErlOffHeap *, void *),
 					    void *);
 
-#define MATCH_SET_RETURN_TRACE 0x1 /* return trace requested */
-#define MATCH_SET_RETURN_TO_TRACE 0x2 /* Misleading name, it is not actually
-					 set by the match program, but by the
-					 breakpoint functions */
-#define MATCH_SET_EXCEPTION_TRACE 0x4 /* exception trace requested */
+#define MATCH_SET_RETURN_TRACE    (0x1) /* return trace requested */
+#define MATCH_SET_RETURN_TO_TRACE (0x2) /* Misleading name, it is not actually
+					   set by the match program, but by the
+					   breakpoint functions */
+#define MATCH_SET_EXCEPTION_TRACE (0x4) /* exception trace requested */
 #define MATCH_SET_RX_TRACE (MATCH_SET_RETURN_TRACE|MATCH_SET_EXCEPTION_TRACE)
 /*
  * Flag values when tracing bif
+ * Future note: flag field is 8 bits
  */
-#define BIF_TRACE_AS_LOCAL  0x1
-#define BIF_TRACE_AS_GLOBAL 0x2
-#define BIF_TRACE_AS_META   0x4
+#define BIF_TRACE_AS_LOCAL      (0x1)
+#define BIF_TRACE_AS_GLOBAL     (0x2)
+#define BIF_TRACE_AS_META       (0x4)
+#define BIF_TRACE_AS_CALL_TIME  (0x8)
 
 extern erts_driver_t vanilla_driver;
 extern erts_driver_t spawn_driver;
@@ -1781,7 +1657,7 @@ erts_alloc_message_heap(Uint size,
 #endif
 
     if (size > (Uint) INT_MAX)
-	erl_exit(ERTS_ABORT_EXIT, "HUGE size (%bpu)\n", size);
+	erl_exit(ERTS_ABORT_EXIT, "HUGE size (%beu)\n", size);
 
     if (
 #if defined(ERTS_SMP)
@@ -1839,4 +1715,109 @@ erts_alloc_message_heap(Uint size,
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
-#endif
+#if !HEAP_ON_C_STACK
+#  if defined(DEBUG)
+#    define DeclareTmpHeap(VariableName,Size,Process) \
+       Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,Process)
+#    define DeclareTypedTmpHeap(Type,VariableName,Process)		\
+      Type *VariableName = (Type *) erts_debug_allocate_tmp_heap(sizeof(Type)/sizeof(Eterm),Process)
+#    define DeclareTmpHeapNoproc(VariableName,Size) \
+       Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,NULL)
+#    define UseTmpHeap(Size,Proc) \
+       do { \
+         erts_debug_use_tmp_heap((Size),(Proc)); \
+       } while (0)
+#    define UnUseTmpHeap(Size,Proc) \
+       do { \
+         erts_debug_unuse_tmp_heap((Size),(Proc)); \
+       } while (0)
+#    define UseTmpHeapNoproc(Size) \
+       do { \
+         erts_debug_use_tmp_heap(Size,NULL); \
+       } while (0)
+#    define UnUseTmpHeapNoproc(Size) \
+       do { \
+         erts_debug_unuse_tmp_heap(Size,NULL); \
+       } while (0)
+#  else
+#    define DeclareTmpHeap(VariableName,Size,Process) \
+       Eterm *VariableName = (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used)
+#    define DeclareTypedTmpHeap(Type,VariableName,Process)		\
+      Type *VariableName = (Type *) (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used)
+#    define DeclareTmpHeapNoproc(VariableName,Size) \
+       Eterm *VariableName = (erts_get_scheduler_data()->tmp_heap)+(erts_get_scheduler_data()->num_tmp_heap_used)
+#    define UseTmpHeap(Size,Proc) \
+       do { \
+         ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used += (Size); \
+       } while (0)
+#    define UnUseTmpHeap(Size,Proc) \
+       do { \
+         ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used -= (Size); \
+       } while (0)
+#    define UseTmpHeapNoproc(Size) \
+       do { \
+         erts_get_scheduler_data()->num_tmp_heap_used += (Size); \
+       } while (0)
+#    define UnUseTmpHeapNoproc(Size) \
+       do { \
+         erts_get_scheduler_data()->num_tmp_heap_used -= (Size); \
+       } while (0)
+
+
+#  endif
+
+#else
+#  define DeclareTmpHeap(VariableName,Size,Process) \
+     Eterm VariableName[Size]
+#  define DeclareTypedTmpHeap(Type,VariableName,Process)	\
+     Type VariableName[1]
+#  define DeclareTmpHeapNoproc(VariableName,Size) \
+     Eterm VariableName[Size]
+#  define UseTmpHeap(Size,Proc) /* Nothing */
+#  define UnUseTmpHeap(Size,Proc) /* Nothing */
+#  define UseTmpHeapNoproc(Size) /* Nothing */
+#  define UnUseTmpHeapNoproc(Size) /* Nothing */
+#endif /* HEAP_ON_C_STACK */
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+#include "dtrace-wrapper.h"
+
+ERTS_GLB_INLINE void
+dtrace_pid_str(Eterm pid, char *process_buf)
+{
+    erts_snprintf(process_buf, DTRACE_TERM_BUF_SIZE, "<%lu.%lu.%lu>",
+                  pid_channel_no(pid),
+                  pid_number(pid),
+                  pid_serial(pid));
+}
+
+ERTS_GLB_INLINE void
+dtrace_proc_str(Process *process, char *process_buf)
+{
+    dtrace_pid_str(process->id, process_buf);
+}
+
+ERTS_GLB_INLINE void
+dtrace_port_str(Port *port, char *port_buf)
+{
+    erts_snprintf(port_buf, DTRACE_TERM_BUF_SIZE, "#Port<%lu.%lu>",
+                  port_channel_no(port->id),
+                  port_number(port->id));
+}
+
+ERTS_GLB_INLINE void
+dtrace_fun_decode(Process *process,
+                  Eterm module, Eterm function, int arity,
+                  char *process_buf, char *mfa_buf)
+{
+    if (process_buf) {
+        dtrace_proc_str(process, process_buf);
+    }
+
+    erts_snprintf(mfa_buf, DTRACE_TERM_BUF_SIZE, "%T:%T/%d",
+                  module, function, arity);
+}
+#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+#endif /* !__GLOBAL_H__ */
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 3309b77086..204bff299e 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -42,6 +42,8 @@
 #include "erl_bits.h"
 #include "erl_version.h"
 #include "error.h"
+#include "erl_async.h"
+#include "dtrace-wrapper.h"
 
 extern ErlDrvEntry fd_driver_entry;
 extern ErlDrvEntry vanilla_driver_entry;
@@ -56,7 +58,6 @@ static erts_smp_tsd_key_t driver_list_last_error_key;  /* Save last DDLL error o
 							  per thread basis (for BC interfaces) */
 
 Port*      erts_port; /* The port table */
-erts_smp_atomic_t erts_ports_alive;
 erts_smp_atomic_t erts_bytes_out;	/* No bytes sent out of the system */
 erts_smp_atomic_t erts_bytes_in;	/* No bytes gotten into the system */
 
@@ -72,6 +73,18 @@ erts_driver_t fd_driver;
 static int init_driver(erts_driver_t *, ErlDrvEntry *, DE_Handle *);
 static void terminate_port(Port *p);
 static void pdl_init(void);
+#ifdef ERTS_SMP
+static void driver_monitor_lock_pdl(Port *p);
+static void driver_monitor_unlock_pdl(Port *p);
+#define DRV_MONITOR_LOCK_PDL(Port) driver_monitor_lock_pdl(Port)
+#define DRV_MONITOR_UNLOCK_PDL(Port) driver_monitor_unlock_pdl(Port)
+#else
+#define DRV_MONITOR_LOCK_PDL(Port) /* nothing */
+#define DRV_MONITOR_UNLOCK_PDL(Port) /* nothing */
+#endif
+
+#define ERL_SMALL_IO_BIN_LIMIT (4*ERL_ONHEAP_BIN_LIMIT)
+#define SMALL_WRITE_VEC  16
 
 static ERTS_INLINE ErlIOQueue*
 drvport2ioq(ErlDrvPort drvport)
@@ -152,8 +165,8 @@ erts_port_ioq_size(Port *pp)
 typedef struct line_buf_context {
     LineBuf **b;
     char *buf;
-    int left;
-    int retlen;
+    ErlDrvSizeT left;
+    ErlDrvSizeT retlen;
 } LineBufContext;
 
 #define LINEBUF_EMPTY 0
@@ -168,6 +181,20 @@ typedef struct line_buf_context {
 
 #define LINEBUF_INITIAL 100
 
+#ifdef USE_VM_PROBES
+#define DTRACE_FORMAT_COMMON_PID_AND_PORT(PID, PORT)         \
+    DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE);       \
+    DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE);          \
+                                                             \
+    dtrace_pid_str((PID), process_str);                      \
+    dtrace_port_str((PORT), port_str);
+#define DTRACE_FORMAT_COMMON_PROC_AND_PORT(PID, PORT)        \
+    DTRACE_CHARBUF(process_str, DTRACE_TERM_BUF_SIZE);       \
+    DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE);          \
+                                                             \
+    dtrace_proc_str((PID), process_str);                     \
+    dtrace_port_str((PORT), port_str);
+#endif
 
 /* The 'number' field in a port now has two parts: the lowest bits
    contain the index in the port table, and the higher bits are a counter
@@ -181,7 +208,7 @@ typedef struct line_buf_context {
 static erts_smp_spinlock_t get_free_port_lck;
 static Uint last_port_num;
 static Uint port_num_mask;
-erts_smp_atomic_t erts_ports_snapshot; /* Identifies the _next_ snapshot (not the ongoing) */
+erts_smp_atomic32_t erts_ports_snapshot; /* Identifies the _next_ snapshot (not the ongoing) */
 
 
 static ERTS_INLINE void
@@ -233,8 +260,8 @@ get_free_port(void)
     }
     port->status = ERTS_PORT_SFLG_INITIALIZING;
 #ifdef ERTS_SMP
-    ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&port->refc) == 0);
-    erts_smp_atomic_set(&port->refc, 2); /* Port alive + lock */
+    ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 0);
+    erts_smp_atomic_set_nob(&port->refc, 2); /* Port alive + lock */
 #endif	
     erts_smp_port_state_unlock(port);
     return num & port_num_mask;
@@ -271,10 +298,36 @@ erts_test_next_port(int set, Uint next)
     return res;
 }
 
+
+static void port_cleanup(Port *prt);
+
+#ifdef ERTS_SMP
+
+static void
+sched_port_cleanup(void *vprt)
+{
+    Port *prt = (Port *) vprt;
+    erts_smp_mtx_lock(prt->lock);
+    port_cleanup(prt);
+}
+
+#endif
+
 void
 erts_port_cleanup(Port *prt)
 {
 #ifdef ERTS_SMP
+    if (erts_smp_mtx_trylock(prt->lock) == EBUSY)
+	erts_schedule_misc_op(sched_port_cleanup, (void *) prt);
+    else
+#endif
+	port_cleanup(prt);
+}
+
+void
+port_cleanup(Port *prt)
+{
+#ifdef ERTS_SMP
     Uint32 port_specific;
     erts_smp_mtx_t *mtx;
 #endif
@@ -290,7 +343,7 @@ erts_port_cleanup(Port *prt)
 #ifdef ERTS_SMP
 
     ASSERT(prt->status & ERTS_PORT_SFLG_FREE_SCHEDULED);
-    ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&prt->refc) == 0);
+    ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&prt->refc) == 0);
 
     port_specific = (prt->status & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK);
 
@@ -386,14 +439,14 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver,
     new_name = (char*) erts_alloc(ERTS_ALC_T_PORT_NAME, sys_strlen(name)+1);
     sys_strcpy(new_name, name);
     erts_smp_runq_lock(runq);
-    erts_smp_atomic_inc(&erts_ports_alive);
     erts_smp_port_state_lock(prt);    
+    prt->os_pid = -1;
     prt->status = ERTS_PORT_SFLG_CONNECTED | xstatus;
-    prt->snapshot = (Uint32) erts_smp_atomic_read(&erts_ports_snapshot);    
+    prt->snapshot = erts_smp_atomic32_read_nob(&erts_ports_snapshot);    
     old_name = prt->name;
     prt->name = new_name;
 #ifdef ERTS_SMP
-    erts_smp_atomic_set(&prt->run_queue, (long) runq);
+    erts_smp_atomic_set_nob(&prt->run_queue, (erts_aint_t) runq);
 #endif
     ASSERT(!prt->drv_ptr);
     prt->drv_ptr = driver;
@@ -408,7 +461,7 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver,
 
     prt->control_flags = 0;
     prt->connected = pid;
-    prt->drv_data = (long) drv_data;
+    prt->drv_data = (SWord) drv_data;
     prt->bytes_in = 0;
     prt->bytes_out = 0;
     prt->dist_entry = NULL;
@@ -554,8 +607,8 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
 	erts_smp_port_state_lock(port);
 	port->status = ERTS_PORT_SFLG_FREE;
 #ifdef ERTS_SMP
-	ERTS_SMP_LC_ASSERT(erts_smp_atomic_read(&port->refc) == 2);
-	erts_smp_atomic_set(&port->refc, 0); 
+	ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&port->refc) == 2);
+	erts_smp_atomic_set_nob(&port->refc, 0); 
 #endif	
 	erts_smp_port_state_unlock(port);
 	return -3;
@@ -573,7 +626,11 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
 	port->lock = erts_alloc(ERTS_ALC_T_PORT_LOCK,
 					      sizeof(erts_smp_mtx_t));
 	erts_smp_mtx_init_x(port->lock,
+#ifdef ERTS_ENABLE_LOCK_COUNT
+			    (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK) ? "port_lock" : NULL,
+#else
 			    "port_lock",
+#endif
 			    port->id);
 	xstatus |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK;
     }
@@ -602,16 +659,21 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
 	    trace_sched_ports_where(port, am_in, am_start);
 	}
 	port->caller = pid;
+#ifdef USE_VM_PROBES
+        if (DTRACE_ENABLED(driver_start)) {
+            DTRACE_FORMAT_COMMON_PID_AND_PORT(pid, port)
+            DTRACE3(driver_start, process_str, driver->name, port_str);
+        }
+#endif
 	fpe_was_unmasked = erts_block_fpe();
 	drv_data = (*driver->start)((ErlDrvPort)(port_ix),
 				    name, opts);
 	erts_unblock_fpe(fpe_was_unmasked);
 	port->caller = NIL;
-	erts_unblock_fpe(fpe_was_unmasked);
 	if (IS_TRACED_FL(port, F_TRACE_SCHED_PORTS)) {
 	    trace_sched_ports_where(port, am_out, am_start);
 	}
-	if (error_number_ptr && ((long) drv_data) == (long) -2)
+	if (error_number_ptr && ((SWord) drv_data) == (SWord) -2)
 	    *error_number_ptr = errno;
 #ifdef ERTS_SMP
 	if (port->xports)
@@ -620,10 +682,10 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
 #endif
     }
 
-    if (((long)drv_data) == -1 || 
-	((long)drv_data) == -2 || 
-	((long)drv_data) == -3) {
-	int res = (int) ((long) drv_data);
+    if (((SWord)drv_data) == -1 || 
+	((SWord)drv_data) == -2 || 
+	((SWord)drv_data) == -3) {
+	int res = (int) ((SWord) drv_data);
 
 	if (res == -3 && error_number_ptr) {
 	    *error_number_ptr = BADARG;
@@ -635,7 +697,7 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
 #ifdef ERTS_SMP
 	erts_cancel_smp_ptimer(port->ptimer);
 #else
-	erl_cancel_timer(&(port->tm));
+	erts_cancel_timer(&(port->tm));
 #endif
 	stopq(port);
 	kill_port(port);
@@ -652,7 +714,7 @@ erts_open_driver(erts_driver_t* driver,	/* Pointer to driver. */
 	erts_port_release(port);
 	return res;
     }
-    port->drv_data = (long) drv_data;
+    port->drv_data = (SWord) drv_data;
     return port_ix;
 }
 
@@ -707,7 +769,7 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */
 	return (ErlDrvTermData) -1;   /* pid does not exist */
     }
     if ((port_num = get_free_port()) < 0) {
-	errno = ENFILE;
+	errno = SYSTEM_LIMIT;
 	erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
 	erts_smp_mtx_unlock(&erts_driver_list_lock);
 	return (ErlDrvTermData) -1;
@@ -726,7 +788,13 @@ driver_create_port(ErlDrvPort creator_port_ix, /* Creating port */
 	creator_port->xports = xplp;
 	port->lock = erts_alloc(ERTS_ALC_T_PORT_LOCK,
 					      sizeof(erts_smp_mtx_t));
-	erts_smp_mtx_init_locked_x(port->lock, "port_lock", port_id);
+	erts_smp_mtx_init_locked_x(port->lock,
+#ifdef ERTS_ENABLE_LOCK_COUNT
+				   (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK) ? "port_lock" : NULL,
+#else
+				   "port_lock",
+#endif
+				   port_id);
 	xstatus |= ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK;
     }
 
@@ -782,6 +850,11 @@ erts_smp_xports_unlock(Port *prt)
 #define SET_VEC(iov, bv, bin, ptr, len, vlen) do {	\
    (iov)->iov_base = (ptr);				\
    (iov)->iov_len = (len);				\
+   if (sizeof((iov)->iov_len) < sizeof(len)				\
+       /* Check if (len) overflowed (iov)->iov_len */			\
+       && ((len) >> (sizeof((iov)->iov_len)*CHAR_BIT)) != 0) {		\
+       goto L_overflow;							\
+   }									\
    *(bv)++ = (bin);					\
    (iov)++;						\
    (vlen)++;						\
@@ -792,13 +865,13 @@ io_list_to_vec(Eterm obj,	/* io-list */
 	       SysIOVec* iov,	/* io vector */
 	       ErlDrvBinary** binv, /* binary reference vector */
 	       ErlDrvBinary* cbin, /* binary to store characters */
-	       int bin_limit)	/* small binaries limit */
+	       ErlDrvSizeT bin_limit)	/* small binaries limit */
 {
     DECLARE_ESTACK(s);
     Eterm* objp;
     char *buf  = cbin->orig_bytes;
-    int len    = cbin->orig_size;
-    int csize  = 0;
+    ErlDrvSizeT len = cbin->orig_size;
+    ErlDrvSizeT csize  = 0;
     int vlen   = 0;
     char* cptr = buf;
 
@@ -838,7 +911,7 @@ io_list_to_vec(Eterm obj,	/* io-list */
 	    Eterm real_bin;
 	    Uint offset;
 	    Eterm* bptr;
-	    int size;
+	    ErlDrvSizeT size;
 	    int bitoffs;
 	    int bitsize;
 
@@ -913,19 +986,20 @@ io_list_to_vec(Eterm obj,	/* io-list */
 
 #define IO_LIST_VEC_COUNT(obj)						\
 do {									\
-    int _size = binary_size(obj);					\
+    ErlDrvSizeT _size = binary_size(obj);				\
     Eterm _real;							\
-    Uint _offset;							\
+    ERTS_DECLARE_DUMMY(Uint _offset);					\
     int _bitoffs;							\
     int _bitsize;							\
     ERTS_GET_REAL_BIN(obj, _real, _offset, _bitoffs, _bitsize);		\
-    ASSERT(_bitsize == 0);						\
+    if (_bitsize != 0) goto L_type_error;				\
     if (thing_subtag(*binary_val(_real)) == REFC_BINARY_SUBTAG &&	\
 	_bitoffs == 0) {						\
 	b_size += _size;						\
+        if (b_size < _size) goto L_overflow_error;			\
 	in_clist = 0;							\
 	v_size++;							\
-        if (_size >= bin_limit) {					\
+        if (_size >= ERL_SMALL_IO_BIN_LIMIT) {				\
             p_in_clist = 0;						\
             p_v_size++;							\
         } else {							\
@@ -937,6 +1011,7 @@ do {									\
         }								\
     } else {								\
 	c_size += _size;						\
+        if (c_size < _size) goto L_overflow_error;			\
 	if (!in_clist) {						\
 	    in_clist = 1;						\
 	    v_size++;							\
@@ -951,29 +1026,30 @@ do {									\
 
 
 /* 
-** Size of a io list in bytes
-** return -1 if error
-** returns:            - Total size of io list
-**           vsize     - SysIOVec size needed for a writev
-**           csize     - Number of bytes not in binary (in the common binary)
-**           pvsize    - SysIOVec size needed if packing small binaries
-**           pcsize    - Number of bytes in the common binary if packing
-*/
+ * Returns 0 if successful and a non-zero value otherwise.
+ *
+ * Return values through pointers:
+ *    *vsize      - SysIOVec size needed for a writev
+ *    *csize      - Number of bytes not in binary (in the common binary)
+ *    *pvsize     - SysIOVec size needed if packing small binaries
+ *    *pcsize     - Number of bytes in the common binary if packing
+ *    *total_size - Total size of iolist in bytes
+ */
 
 static int 
-io_list_vec_len(Eterm obj, int* vsize, int* csize, 
-		int bin_limit, /* small binaries limit */
-		int * pvsize, int * pcsize)
+io_list_vec_len(Eterm obj, Uint* vsize, Uint* csize,
+		Uint* pvsize, Uint* pcsize, Uint* total_size)
 {
     DECLARE_ESTACK(s);
     Eterm* objp;
-    int v_size = 0;
-    int c_size = 0;
-    int b_size = 0;
-    int in_clist = 0;
-    int p_v_size = 0;
-    int p_c_size = 0;
-    int p_in_clist = 0;
+    Uint v_size = 0;
+    Uint c_size = 0;
+    Uint b_size = 0;
+    Uint in_clist = 0;
+    Uint p_v_size = 0;
+    Uint p_c_size = 0;
+    Uint p_in_clist = 0;
+    Uint total;
 
     goto L_jump_start;  /* avoid a push */
 
@@ -987,6 +1063,9 @@ io_list_vec_len(Eterm obj, int* vsize, int* csize,
 
 	    if (is_byte(obj)) {
 		c_size++;
+		if (c_size == 0) {
+		    goto L_overflow_error;
+		}
 		if (!in_clist) {
 		    in_clist = 1;
 		    v_size++;
@@ -1026,32 +1105,31 @@ io_list_vec_len(Eterm obj, int* vsize, int* csize,
 	}
     }
 
+    total = c_size + b_size;
+    if (total < c_size) {
+	goto L_overflow_error;
+    }
+    *total_size = total;
+
     DESTROY_ESTACK(s);
-    if (vsize != NULL)
-	*vsize = v_size;
-    if (csize != NULL)
-	*csize = c_size;
-    if (pvsize != NULL)
-	*pvsize = p_v_size;
-    if (pcsize != NULL)
-	*pcsize = p_c_size;
-    return c_size + b_size;
+    *vsize = v_size;
+    *csize = c_size;
+    *pvsize = p_v_size;
+    *pcsize = p_c_size;
+    return 0;
 
  L_type_error:
+ L_overflow_error:
     DESTROY_ESTACK(s);
-    return -1;
+    return 1;
 }
 
-#define ERL_SMALL_IO_BIN_LIMIT (4*ERL_ONHEAP_BIN_LIMIT)
-#define SMALL_WRITE_VEC  16
-
-
 /* write data to a port */
 int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
 {
     char *buf;
     erts_driver_t *drv = p->drv_ptr;
-    int size;
+    Uint size;
     int fpe_was_unmasked;
     
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p));
@@ -1059,11 +1137,11 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
 
     p->caller = caller_id;
     if (drv->outputv != NULL) {
-	int vsize;
-	int csize;
-	int pvsize;
-	int pcsize;
-	int blimit;
+	Uint vsize;
+	Uint csize;
+	Uint pvsize;
+	Uint pcsize;
+	ErlDrvSizeT blimit;
 	SysIOVec iv[SMALL_WRITE_VEC];
 	ErlDrvBinary* bv[SMALL_WRITE_VEC];
 	SysIOVec* ivp;
@@ -1071,9 +1149,8 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
 	ErlDrvBinary* cbin;
 	ErlIOVec ev;
 
-	if ((size = io_list_vec_len(list, &vsize, &csize, 
-				    ERL_SMALL_IO_BIN_LIMIT,
-				    &pvsize, &pcsize)) < 0) {
+	if (io_list_vec_len(list, &vsize, &csize,
+			    &pvsize, &pcsize, &size)) {
 	    goto bad_value;
 	}
 	/* To pack or not to pack (small binaries) ...? */
@@ -1106,15 +1183,31 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
 	ivp[0].iov_len = 0;
 	bvp[0] = NULL;
 	ev.vsize = io_list_to_vec(list, ivp+1, bvp+1, cbin, blimit);
+	if (ev.vsize < 0) {
+	    if (ivp != iv) {
+		erts_free(ERTS_ALC_T_TMP, (void *) ivp);
+	    }
+	    if (bvp != bv) {
+		erts_free(ERTS_ALC_T_TMP, (void *) bvp);
+	    }
+	    driver_free_binary(cbin);
+	    goto bad_value;
+	}
 	ev.vsize++;
 #if 0
 	/* This assertion may say something useful, but it can
 	   be falsified during the emulator test suites. */
-	ASSERT((ev.vsize >= 0) && (ev.vsize == vsize));
+	ASSERT(ev.vsize == vsize);
 #endif
 	ev.size = size;  /* total size */
 	ev.iov = ivp;
 	ev.binv = bvp;
+#ifdef USE_VM_PROBES
+        if (DTRACE_ENABLED(driver_outputv)) {
+            DTRACE_FORMAT_COMMON_PID_AND_PORT(caller_id, p)
+            DTRACE4(driver_outputv, process_str, port_str, p->name, size);
+        }
+#endif
 	fpe_was_unmasked = erts_block_fpe();
 	(*drv->outputv)((ErlDrvData)p->drv_data, &ev);
 	erts_unblock_fpe(fpe_was_unmasked);
@@ -1134,8 +1227,21 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
 	buf = erts_alloc(ERTS_ALC_T_TMP, size+1);
 	r = io_list_to_buf(list, buf, size);
 
+#ifdef USE_VM_PROBES
+	if(DTRACE_ENABLED(port_command)) {
+            DTRACE_FORMAT_COMMON_PID_AND_PORT(caller_id, p)
+	    DTRACE4(port_command, process_str, port_str, p->name, "command");
+	}
+#endif
+
 	if (r >= 0) {
 	    size -= r;
+#ifdef USE_VM_PROBES
+            if (DTRACE_ENABLED(driver_output)) {
+                DTRACE_FORMAT_COMMON_PID_AND_PORT(caller_id, p)
+                DTRACE4(driver_output, process_str, port_str, p->name, size);
+            }
+#endif
 	    fpe_was_unmasked = erts_block_fpe();
 	    (*drv->output)((ErlDrvData)p->drv_data, buf, size);
 	    erts_unblock_fpe(fpe_was_unmasked);
@@ -1148,7 +1254,7 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
 	else {
 	    ASSERT(r == -1); /* Overflow */
 	    erts_free(ERTS_ALC_T_TMP, buf);
-	    if ((size = io_list_len(list)) < 0) {
+	    if (erts_iolist_size(list, &size)) {
 		goto bad_value;
 	    }
 
@@ -1159,6 +1265,12 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
 	     */
 	    buf = erts_alloc(ERTS_ALC_T_TMP, size+1); 
 	    r = io_list_to_buf(list, buf, size);
+#ifdef USE_VM_PROBES
+            if (DTRACE_ENABLED(driver_output)) {
+                DTRACE_FORMAT_COMMON_PID_AND_PORT(caller_id, p)
+                DTRACE4(driver_output, process_str, port_str, p->name, size);
+            }
+#endif
 	    fpe_was_unmasked = erts_block_fpe();
 	    (*drv->output)((ErlDrvData)p->drv_data, buf, size);
 	    erts_unblock_fpe(fpe_was_unmasked);
@@ -1166,7 +1278,7 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
 	}
     }
     p->bytes_out += size;
-    erts_smp_atomic_add(&erts_bytes_out, size);
+    erts_smp_atomic_add_nob(&erts_bytes_out, size);
 
 #ifdef ERTS_SMP
     if (p->xports)
@@ -1191,7 +1303,6 @@ void init_io(void)
 {
     int i;
     ErlDrvEntry** dp;
-    ErlDrvEntry* drv;
     char maxports[21]; /* enough for any 64-bit integer */
     size_t maxportssize = sizeof(maxports);
     Uint ports_bits = ERTS_PORTS_BITS;
@@ -1238,17 +1349,22 @@ void init_io(void)
     erts_port = (Port *) erts_alloc(ERTS_ALC_T_PORT_TABLE,
 				    erts_max_ports * sizeof(Port));
 
-    erts_smp_atomic_init(&erts_bytes_out, 0);
-    erts_smp_atomic_init(&erts_bytes_in, 0);
-    erts_smp_atomic_init(&erts_ports_alive, 0);
+    erts_smp_atomic_init_nob(&erts_bytes_out, 0);
+    erts_smp_atomic_init_nob(&erts_bytes_in, 0);
 
     for (i = 0; i < erts_max_ports; i++) {
 	erts_port_task_init_sched(&erts_port[i].sched);
 #ifdef ERTS_SMP
-	erts_smp_atomic_init(&erts_port[i].refc, 0);
+	erts_smp_atomic_init_nob(&erts_port[i].refc, 0);
 	erts_port[i].lock = NULL;
 	erts_port[i].xports = NULL;
-	erts_smp_spinlock_init_x(&erts_port[i].state_lck, "port_state", make_small(i));
+	erts_smp_spinlock_init_x(&erts_port[i].state_lck,
+#ifdef ERTS_ENABLE_LOCK_COUNT
+				 (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK) ? "port_state" : NULL,
+#else
+				 "port_state",
+#endif
+				 make_small(0));
 #endif
 	erts_port[i].tracer_proc = NIL;
 	erts_port[i].trace_flags = 0;
@@ -1262,7 +1378,7 @@ void init_io(void)
 	erts_port[i].port_data_lock = NULL;
     }
 
-    erts_smp_atomic_init(&erts_ports_snapshot, (long) 0);
+    erts_smp_atomic32_init_nob(&erts_ports_snapshot, (erts_aint32_t) 0);
     last_port_num = 0;
     erts_smp_spinlock_init(&get_free_port_lck, "get_free_port");
 
@@ -1274,15 +1390,34 @@ void init_io(void)
     init_driver(&fd_driver, &fd_driver_entry, NULL);
     init_driver(&vanilla_driver, &vanilla_driver_entry, NULL);
     init_driver(&spawn_driver, &spawn_driver_entry, NULL);
-    for (dp = driver_tab; *dp != NULL; dp++) {
-	drv = *dp;
+    for (dp = driver_tab; *dp != NULL; dp++)
 	erts_add_driver_entry(*dp, NULL, 1);
-    }
 
     erts_smp_tsd_set(driver_list_lock_status_key, NULL);
     erts_smp_mtx_unlock(&erts_driver_list_lock);
 }
 
+#if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP)
+void erts_lcnt_enable_io_lock_count(int enable) {
+    int i;
+
+    for (i = 0; i < erts_max_ports; i++) {
+	Port* p = &erts_port[i];
+	if (enable) {
+	    erts_lcnt_init_lock_x(&p->state_lck.lcnt, "port_state", ERTS_LCNT_LT_SPINLOCK, make_small(i));
+	    if (p->lock) {
+		erts_lcnt_init_lock_x(&p->lock->lcnt, "port_lock", ERTS_LCNT_LT_MUTEX, make_small(i));
+	    }
+	} else {
+	    erts_lcnt_destroy_lock(&p->state_lck.lcnt);
+	    if (p->lock) {
+		erts_lcnt_destroy_lock(&p->lock->lcnt);
+	    }
+	}
+    }
+}
+#endif
+
 /*
  * Buffering of data when using line oriented I/O on ports
  */
@@ -1324,7 +1459,8 @@ int bufsiz;
  * buf - A buffer containing the data to be read and split to lines.
  * len - The number of bytes in buf.
  */
-static int init_linebuf_context(LineBufContext *lc, LineBuf **lb, char *buf, int len)
+static int init_linebuf_context(LineBufContext *lc, LineBuf **lb,
+				char *buf, ErlDrvSizeT len)
 {
     if(lc == NULL || lb == NULL)
 	return -1;
@@ -1477,7 +1613,11 @@ deliver_result(Eterm sender, Eterm pid, Eterm res)
 	    hp = erts_alloc_message_heap(sz_res + 3, &bp, &ohp, rp, &rp_locks);
 	    res = copy_struct(res, sz_res, &hp, ohp);
 	    tuple = TUPLE2(hp, sender, res);
-	erts_queue_message(rp, &rp_locks, bp, tuple, NIL);
+	erts_queue_message(rp, &rp_locks, bp, tuple, NIL
+#ifdef USE_VM_PROBES
+			   , NIL
+#endif
+			   );
 	erts_smp_proc_unlock(rp, rp_locks);
 	erts_smp_proc_dec_refc(rp);
     }
@@ -1493,10 +1633,10 @@ deliver_result(Eterm sender, Eterm pid, Eterm res)
  */
 
 static void deliver_read_message(Port* prt, Eterm to,
-				 char *hbuf, int hlen,
-				 char *buf, int len, int eol)
+				 char *hbuf, ErlDrvSizeT hlen,
+				 char *buf, ErlDrvSizeT len, int eol)
 {
-    int need;
+    ErlDrvSizeT need;
     Eterm listp;
     Eterm tuple;
     Process* rp;
@@ -1540,14 +1680,14 @@ static void deliver_read_message(Port* prt, Eterm to,
 	pb = (ProcBin *) hp;
 	pb->thing_word = HEADER_PROC_BIN;
 	pb->size = len;
-	pb->next = ohp->mso;
-	ohp->mso = pb;
+	pb->next = ohp->first;
+	ohp->first = (struct erl_off_heap_header*)pb;
 	pb->val = bptr;
 	pb->bytes = (byte*) bptr->orig_bytes;
 	pb->flags = 0;
 	hp += PROC_BIN_SIZE;
 
-	ohp->overhead += pb->size / sizeof(Eterm);
+	OH_OVERHEAD(ohp, pb->size / sizeof(Eterm));
 	listp = make_binary(pb);
     }
 
@@ -1566,7 +1706,11 @@ static void deliver_read_message(Port* prt, Eterm to,
     tuple = TUPLE2(hp, prt->id, tuple);
     hp += 3;
 
-    erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined);
+    erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined
+#ifdef USE_VM_PROBES
+			   , NIL
+#endif
+		       );
     erts_smp_proc_unlock(rp, rp_locks);
     erts_smp_proc_dec_refc(rp);
 }
@@ -1576,8 +1720,8 @@ static void deliver_read_message(Port* prt, Eterm to,
  * deliver_read_message, and takes the same parameters.
  */
 static void deliver_linebuf_message(Port* prt, Eterm to, 
-				    char* hbuf, int hlen,
-				    char *buf, int len)
+				    char* hbuf, ErlDrvSizeT hlen,
+				    char *buf, ErlDrvSizeT len)
 {
     LineBufContext lc;
     int ret;
@@ -1620,14 +1764,14 @@ static void
 deliver_vec_message(Port* prt,			/* Port */
 		    Eterm to,			/* Receiving pid */
 		    char* hbuf,			/* "Header" buffer... */
-		    int hlen,			/* ... and its length */
+		    ErlDrvSizeT hlen,		/* ... and its length */
 		    ErlDrvBinary** binv,	/* Vector of binaries */
 		    SysIOVec* iov,		/* I/O vector */
 		    int vsize,			/* Size of binv & iov */
-		    int csize)			/* Size of characters in 
+		    ErlDrvSizeT csize)		/* Size of characters in
 						   iov (not hlen) */
 {
-    int need;
+    ErlDrvSizeT need;
     Eterm listp;
     Eterm tuple;
     Process* rp;
@@ -1690,14 +1834,14 @@ deliver_vec_message(Port* prt,			/* Port */
 	    }
 	    pb->thing_word = HEADER_PROC_BIN;
 	    pb->size = iov->iov_len;
-	    pb->next = ohp->mso;
-	    ohp->mso = pb;
+	    pb->next = ohp->first;
+	    ohp->first = (struct erl_off_heap_header*)pb;
 	    pb->val = ErlDrvBinary2Binary(b);
 	    pb->bytes = base;
 	    pb->flags = 0;
 	    hp += PROC_BIN_SIZE;
 	    
-	    ohp->overhead += iov->iov_len / sizeof(Eterm);
+	    OH_OVERHEAD(ohp, iov->iov_len / sizeof(Eterm));
 
 	    if (listp == NIL) {  /* compatible with deliver_bin_message */
 		listp = make_binary(pb);
@@ -1708,7 +1852,7 @@ deliver_vec_message(Port* prt,			/* Port */
 	}
     }
 
-    if (hlen > 0) {		/* Prepend the header */
+    if (hlen != 0) {		/* Prepend the header */
 	Eterm* thp = hp;
 	listp = buf_to_intlist(&thp, hbuf, hlen, listp);
 	hp = thp;
@@ -1719,7 +1863,11 @@ deliver_vec_message(Port* prt,			/* Port */
     tuple = TUPLE2(hp, prt->id, tuple);
     hp += 3;
 
-    erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined);
+    erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined
+#ifdef USE_VM_PROBES
+		       , NIL
+#endif
+		       );
     erts_smp_proc_unlock(rp, rp_locks);
     erts_smp_proc_dec_refc(rp);
 }
@@ -1728,10 +1876,10 @@ deliver_vec_message(Port* prt,			/* Port */
 static void deliver_bin_message(Port*  prt,         /* port */
 				Eterm to,           /* receiving pid */
 				char* hbuf,         /* "header" buffer */
-				int hlen,           /* and it's length */
+				ErlDrvSizeT hlen,   /* and it's length */
 				ErlDrvBinary* bin,  /* binary data */
-				int offs,           /* offset into binary */
-				int len)            /* length of binary */
+				ErlDrvSizeT offs,   /* offset into binary */
+				ErlDrvSizeT len)    /* length of binary */
 {
     SysIOVec vec;
 
@@ -1758,6 +1906,12 @@ static void flush_port(Port *p)
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p));
 
     if (p->drv_ptr->flush != NULL) {
+#ifdef USE_VM_PROBES
+        if (DTRACE_ENABLED(driver_flush)) {
+            DTRACE_FORMAT_COMMON_PID_AND_PORT(p->connected, p)
+            DTRACE3(driver_flush, process_str, port_str, p->name);
+        }
+#endif
         if (IS_TRACED_FL(p, F_TRACE_SCHED_PORTS)) {
 	    trace_sched_ports_where(p, am_in, am_flush);
 	}
@@ -1785,6 +1939,7 @@ terminate_port(Port *prt)
     Eterm send_closed_port_id;
     Eterm connected_id = NIL /* Initialize to silence compiler */;
     erts_driver_t *drv;
+    int halt;
 
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
@@ -1792,6 +1947,8 @@ terminate_port(Port *prt)
     ASSERT(!prt->nlinks);
     ASSERT(!prt->monitors);
 
+    /* prt->status may be altered by kill_port()below */
+    halt = (prt->status & ERTS_PORT_SFLG_HALT) != 0;
     if (prt->status & ERTS_PORT_SFLG_SEND_CLOSED) {
 	erts_port_status_band_set(prt, ~ERTS_PORT_SFLG_SEND_CLOSED);
 	send_closed_port_id = prt->id;
@@ -1804,12 +1961,18 @@ terminate_port(Port *prt)
 #ifdef ERTS_SMP
     erts_cancel_smp_ptimer(prt->ptimer);
 #else
-    erl_cancel_timer(&prt->tm);
+    erts_cancel_timer(&prt->tm);
 #endif
 
     drv = prt->drv_ptr;
     if ((drv != NULL) && (drv->stop != NULL)) {
 	int fpe_was_unmasked = erts_block_fpe();
+#ifdef USE_VM_PROBES
+        if (DTRACE_ENABLED(driver_stop)) {
+            DTRACE_FORMAT_COMMON_PID_AND_PORT(prt->connected, prt)
+            DTRACE3(driver_stop, process_str, drv->name, port_str);
+        }
+#endif
 	(*drv->stop)((ErlDrvData)prt->drv_data);
 	erts_unblock_fpe(fpe_was_unmasked);
 #ifdef ERTS_SMP
@@ -1843,6 +2006,10 @@ terminate_port(Port *prt)
      * We don't want to send the closed message until after the
      * port has been removed from the port table (in kill_port()).
      */
+    if (halt && (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0)) {
+	erts_smp_port_unlock(prt); /* We will exit and never return */
+	erl_exit_flush_async(erts_halt_code, "");
+    }
     if (is_internal_port(send_closed_port_id))
 	deliver_result(send_closed_port_id, connected_id, am_closed);
 
@@ -1967,6 +2134,19 @@ erts_do_exit_port(Port *p, Eterm from, Eterm reason)
 
    rreason = (reason == am_kill) ? am_killed : reason;
 
+#ifdef USE_VM_PROBES
+   if (DTRACE_ENABLED(port_exit)) {
+       DTRACE_CHARBUF(from_str, DTRACE_TERM_BUF_SIZE);
+       DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE);
+       DTRACE_CHARBUF(rreason_str, 64);
+
+       erts_snprintf(from_str, sizeof(from_str), "%T", from);
+       dtrace_port_str(p, port_str);
+       erts_snprintf(rreason_str, sizeof(rreason_str), "%T", rreason);
+       DTRACE4(port_exit, from_str, port_str, p->name, rreason_str);
+   }
+#endif
+
    if ((p->status & (ERTS_PORT_SFLGS_DEAD
 		     | ERTS_PORT_SFLG_EXITING
 		     | ERTS_PORT_SFLG_IMMORTAL))
@@ -1998,12 +2178,13 @@ erts_do_exit_port(Port *p, Eterm from, Eterm reason)
        p->nlinks = NULL;
        erts_sweep_links(lnk, &sweep_one_link, &sc);
    }
+   DRV_MONITOR_LOCK_PDL(p);
    {
        ErtsMonitor *moni = p->monitors;
        p->monitors = NULL;
        erts_sweep_monitors(moni, &sweep_one_monitor, NULL);
    } 
-
+   DRV_MONITOR_UNLOCK_PDL(p);
 
    if ((p->status & ERTS_PORT_SFLG_DISTRIBUTION) && p->dist_entry) {
        erts_do_net_exits(p->dist_entry, rreason);
@@ -2066,6 +2247,13 @@ void erts_port_command(Process *proc,
 	    if (tp[2] == am_close) {
 		erts_port_status_bor_set(port, ERTS_PORT_SFLG_SEND_CLOSED);
 		erts_do_exit_port(port, pid, am_normal);
+
+#ifdef USE_VM_PROBES
+		if(DTRACE_ENABLED(port_command)) {
+                    DTRACE_FORMAT_COMMON_PROC_AND_PORT(proc, port)
+		    DTRACE4(port_command, process_str, port_str, port->name, "close");
+		}
+#endif
 		goto done;
 	    } else if (is_tuple_arity(tp[2], 2)) {
 		tp = tuple_val(tp[2]);
@@ -2073,6 +2261,12 @@ void erts_port_command(Process *proc,
 		    if (erts_write_to_port(caller_id, port, tp[2]) == 0)
 			goto done;
 		} else if ((tp[1] == am_connect) && is_internal_pid(tp[2])) {
+#ifdef USE_VM_PROBES
+		    if(DTRACE_ENABLED(port_command)) {
+                        DTRACE_FORMAT_COMMON_PROC_AND_PORT(proc, port)
+			DTRACE4(port_command, process_str, port_str, port->name, "connect");
+		    }
+#endif
 		    port->connected = tp[2];
 		    deliver_result(port->id, pid, am_connected);
 		    goto done;
@@ -2114,12 +2308,13 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist)
     byte* to_port = NULL;	/* Buffer to write to port. */
 				/* Initialization is for shutting up
 				   warning about use before set. */
-    int to_len = 0;		/* Length of buffer. */
+    Uint to_len = 0;		/* Length of buffer. */
     int must_free = 0;		/* True if the buffer should be freed. */
     char port_result[ERL_ONHEAP_BIN_LIMIT];  /* Default buffer for result from port. */
     char* port_resp;		/* Pointer to result buffer. */
-    int n;
-    int (*control)(ErlDrvData, unsigned, char*, int, char**, int);
+    ErlDrvSSizeT n;
+    ErlDrvSSizeT (*control)
+	(ErlDrvData, unsigned, char*, ErlDrvSizeT, char**, ErlDrvSizeT);
     int fpe_was_unmasked;
 
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
@@ -2133,8 +2328,8 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist)
      * and with its length in to_len.
      */
     if (is_binary(iolist) && binary_bitoffset(iolist) == 0) {
-	Uint bitoffs;
-	Uint bitsize;
+	ERTS_DECLARE_DUMMY(Uint bitoffs);
+	ERTS_DECLARE_DUMMY(Uint bitsize);
 	ERTS_GET_BINARY_BYTES(iolist, to_port, bitoffs, bitsize);
 	to_len = binary_size(iolist);
     } else {
@@ -2159,7 +2354,7 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist)
 	} else {
 	    ASSERT(r == -1);	/* Overflow */
 	    erts_free(ERTS_ALC_T_TMP, (void *) to_port);
-	    if ((to_len = io_list_len(iolist)) < 0) { /* Type error */
+	    if (erts_iolist_size(iolist, &to_len)) { /* Type error */
 		return THE_NON_VALUE;
 	    }
 	    must_free = 1;
@@ -2174,6 +2369,15 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist)
     erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN);
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
+#ifdef USE_VM_PROBES
+    if (DTRACE_ENABLED(port_control) || DTRACE_ENABLED(driver_control)) {
+        DTRACE_FORMAT_COMMON_PROC_AND_PORT(p, prt);
+        DTRACE4(port_control, process_str, port_str, prt->name, command);
+        DTRACE5(driver_control, process_str, port_str, prt->name,
+                command, to_len);
+    }
+#endif
+
     /*
      * Call the port's control routine.
      */
@@ -2223,12 +2427,12 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist)
 		ProcBin* pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE);
 		pb->thing_word = HEADER_PROC_BIN;
 		pb->size = dbin->orig_size;
-		pb->next = MSO(p).mso;
-		MSO(p).mso = pb;
+		pb->next = MSO(p).first;
+		MSO(p).first = (struct erl_off_heap_header*)pb;
 		pb->val = ErlDrvBinary2Binary(dbin);
 		pb->bytes = (byte*) dbin->orig_bytes;
 		pb->flags = 0;
-		MSO(p).overhead += dbin->orig_size / sizeof(Eterm);
+		OH_OVERHEAD(&(MSO(p)), dbin->orig_size / sizeof(Eterm));
 		return make_binary(pb);
 	    }
 	    port_resp = dbin->orig_bytes;
@@ -2314,6 +2518,10 @@ print_port_info(int to, void *arg, int i)
 void
 set_busy_port(ErlDrvPort port_num, int on)
 {
+#ifdef USE_VM_PROBES
+    DTRACE_CHARBUF(port_str, 16);
+#endif
+
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(&erts_port[port_num]));
@@ -2321,12 +2529,26 @@ set_busy_port(ErlDrvPort port_num, int on)
     if (on) {
         erts_port_status_bor_set(&erts_port[port_num],
 				 ERTS_PORT_SFLG_PORT_BUSY);
+#ifdef USE_VM_PROBES
+        if (DTRACE_ENABLED(port_busy)) {
+            erts_snprintf(port_str, sizeof(port_str),
+                          "%T", erts_port[port_num].id);
+            DTRACE1(port_busy, port_str);
+        }
+#endif
     } else {
         ErtsProcList* plp = erts_port[port_num].suspended;
         erts_port_status_band_set(&erts_port[port_num],
 				  ~ERTS_PORT_SFLG_PORT_BUSY);
         erts_port[port_num].suspended = NULL;
 
+#ifdef USE_VM_PROBES
+        if (DTRACE_ENABLED(port_not_busy)) {
+            erts_snprintf(port_str, sizeof(port_str),
+                          "%T", erts_port[port_num].id);
+            DTRACE1(port_not_busy, port_str);
+        }
+#endif
 	if (erts_port[port_num].dist_entry) {
 	    /*
 	     * Processes suspended on distribution ports are
@@ -2344,6 +2566,28 @@ set_busy_port(ErlDrvPort port_num, int on)
 	 */
 
         if (plp) {
+#ifdef USE_VM_PROBES
+            /*
+             * Hrm, for blocked dist ports, plp always seems to be NULL.
+             * That's not so fun.
+             * Well, another way to get the same info is using a D
+             * script to correlate an earlier process-port_blocked+pid
+             * event with a later process-scheduled event.  That's
+             * subject to the multi-CPU races with how events are
+             * handled, but hey, that way works most of the time.
+             */
+            if (DTRACE_ENABLED(process_port_unblocked)) {
+                DTRACE_CHARBUF(pid_str, 16);
+                ErtsProcList* plp2 = plp;
+
+                erts_snprintf(port_str, sizeof(port_str),
+                             "%T", erts_port[port_num]);
+                while (plp2 != NULL) {
+                    erts_snprintf(pid_str, sizeof(pid_str), "%T", plp2->pid);
+                    DTRACE2(process_port_unblocked, pid_str, port_str);
+                }
+            }
+#endif
             /* First proc should be resumed last */
 	    if (plp->next) {
 		erts_resume_processes(plp->next);
@@ -2384,12 +2628,20 @@ void erts_raw_port_command(Port* p, byte* buf, Uint len)
 
     if (len > (Uint) INT_MAX)
 	erl_exit(ERTS_ABORT_EXIT,
-		 "Absurdly large data buffer (%bpu bytes) passed to"
+		 "Absurdly large data buffer (%beu bytes) passed to"
 		 "output callback of %s driver.\n",
 		 len,
 		 p->drv_ptr->name ? p->drv_ptr->name : "unknown");
 
     p->caller = NIL;
+#ifdef USE_VM_PROBES
+    if (DTRACE_ENABLED(driver_output)) {
+        DTRACE_CHARBUF(port_str, DTRACE_TERM_BUF_SIZE);
+
+        dtrace_port_str(p, port_str);
+        DTRACE4(driver_output, "-raw-", port_str, p->name, len);
+    }
+#endif
     fpe_was_unmasked = erts_block_fpe();
     (*p->drv_ptr->output)((ErlDrvData)p->drv_data, (char*) buf, (int) len);
     erts_unblock_fpe(fpe_was_unmasked);
@@ -2405,6 +2657,12 @@ int async_ready(Port *p, void* data)
 	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p));
 	ASSERT(!(p->status & ERTS_PORT_SFLGS_DEAD));
 	if (p->drv_ptr->ready_async != NULL) {
+#ifdef USE_VM_PROBES
+            if (DTRACE_ENABLED(driver_ready_async)) {
+                DTRACE_FORMAT_COMMON_PID_AND_PORT(p->connected, p)
+                DTRACE3(driver_ready_async, process_str, port_str, p->name);
+            }
+#endif
 	    (*p->drv_ptr->ready_async)((ErlDrvData)p->drv_data, data);
 	    need_free = 0;
 #ifdef ERTS_SMP
@@ -2599,7 +2857,11 @@ void driver_report_exit(int ix, int status)
    hp += 3;
    tuple = TUPLE2(hp, prt->id, tuple);
 
-   erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined);
+   erts_queue_message(rp, &rp_locks, bp, tuple, am_undefined
+#ifdef USE_VM_PROBES
+		      , NIL
+#endif
+		      );
 
    erts_smp_proc_unlock(rp, rp_locks);
    erts_smp_proc_dec_refc(rp);
@@ -2766,17 +3028,25 @@ driver_deliver_term(ErlDrvPort port,
 	    break;
 	case ERL_DRV_INT:  /* signed int argument */
 	    ERTS_DDT_CHK_ENOUGH_ARGS(1);
+#if HALFWORD_HEAP
+	    erts_bld_sint64(NULL, &need, (Sint64)ptr[0]);
+#else
 	    /* check for bignum */
 	    if (!IS_SSMALL((Sint)ptr[0]))
 		need += BIG_UINT_HEAP_SIZE;  /* use small_to_big */
+#endif
 	    ptr++;
 	    depth++;
 	    break;
 	case ERL_DRV_UINT:  /* unsigned int argument */
 	    ERTS_DDT_CHK_ENOUGH_ARGS(1);
+#if HALFWORD_HEAP
+	    erts_bld_uint64(NULL, &need, (Uint64)ptr[0]);
+#else
 	    /* check for bignum */
 	    if (!IS_USMALL(0, (Uint)ptr[0]))
 		need += BIG_UINT_HEAP_SIZE;  /* use small_to_big */
+#endif
 	    ptr++;
 	    depth++;
 	    break;
@@ -2943,22 +3213,30 @@ driver_deliver_term(ErlDrvPort port,
 	    break;
 
 	case ERL_DRV_INT:  /* signed int argument */
+#if HALFWORD_HEAP
+	    mess = erts_bld_sint64(&hp, NULL, (Sint64)ptr[0]);
+#else
 	    if (IS_SSMALL((Sint)ptr[0]))
 		mess = make_small((Sint)ptr[0]);
 	    else {
 		mess = small_to_big((Sint)ptr[0], hp);
 		hp += BIG_UINT_HEAP_SIZE;
 	    }
+#endif
 	    ptr++;
 	    break;
 
 	case ERL_DRV_UINT:  /* unsigned int argument */
+#if HALFWORD_HEAP
+	    mess = erts_bld_uint64(&hp, NULL, (Uint64)ptr[0]);
+#else
 	    if (IS_USMALL(0, (Uint)ptr[0]))
 		mess = make_small((Uint)ptr[0]);
 	    else {
 		mess = uint_to_big((Uint)ptr[0], hp);
 		hp += BIG_UINT_HEAP_SIZE;
 	    }
+#endif
 	    ptr++;
 	    break;
 
@@ -2982,6 +3260,8 @@ driver_deliver_term(ErlDrvPort port,
 	    Uint size = ptr[1];
 	    Uint offset = ptr[2];
 
+	    erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) size);
+
 	    if (size <= ERL_ONHEAP_BIN_LIMIT) {
 		ErlHeapBin* hbp = (ErlHeapBin *) hp;
 		hp += heap_bin_size(size);
@@ -2997,14 +3277,14 @@ driver_deliver_term(ErlDrvPort port,
 		driver_binary_inc_refc(b);  /* caller will free binary */
 		pb->thing_word = HEADER_PROC_BIN;
 		pb->size = size;
-		pb->next = ohp->mso;
-		ohp->mso = pb;
+		pb->next = ohp->first;
+		ohp->first = (struct erl_off_heap_header*)pb;
 		pb->val = ErlDrvBinary2Binary(b);
 		pb->bytes = ((byte*) b->orig_bytes) + offset;
 		pb->flags = 0;
 		mess =  make_binary(pb);
 		hp += PROC_BIN_SIZE;
-		ohp->overhead += pb->size / sizeof(Eterm);
+		OH_OVERHEAD(ohp, pb->size / sizeof(Eterm));
 	    }
 	    ptr += 3;
 	    break;
@@ -3013,6 +3293,9 @@ driver_deliver_term(ErlDrvPort port,
 	case ERL_DRV_BUF2BINARY: { /* char*, size */
 	    byte *bufp = (byte *) ptr[0];
 	    Uint size = (Uint) ptr[1];
+
+	    erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) size);
+
 	    if (size <= ERL_ONHEAP_BIN_LIMIT) {
 		ErlHeapBin* hbp = (ErlHeapBin *) hp;
 		hp += heap_bin_size(size);
@@ -3029,19 +3312,19 @@ driver_deliver_term(ErlDrvPort port,
 		Binary* bp = erts_bin_nrml_alloc(size);
 		ASSERT(bufp);
 		bp->flags = 0;
-		bp->orig_size = (long) size;
+		bp->orig_size = (SWord) size;
 		erts_refc_init(&bp->refc, 1);
 		sys_memcpy((void *) bp->orig_bytes, (void *) bufp, size);
 		pbp = (ProcBin *) hp;
 		hp += PROC_BIN_SIZE;
 		pbp->thing_word = HEADER_PROC_BIN;
 		pbp->size = size;
-		pbp->next = ohp->mso;
-		ohp->mso = pbp;
+		pbp->next = ohp->first;
+		ohp->first = (struct erl_off_heap_header*)pbp;
 		pbp->val = bp;
 		pbp->bytes = (byte*) bp->orig_bytes;
 		pbp->flags = 0;
-		ohp->overhead += (pbp->size / sizeof(Eterm));
+		OH_OVERHEAD(ohp, pbp->size / sizeof(Eterm));
 		mess = make_binary(pbp);
 	    }
 	    ptr += 2;
@@ -3049,6 +3332,7 @@ driver_deliver_term(ErlDrvPort port,
 	}
 
 	case ERL_DRV_STRING: /* char*, length */
+	    erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) ptr[1]);
 	    mess = buf_to_intlist(&hp, (char*)ptr[0], ptr[1], NIL);
 	    ptr += 2;
 	    break;
@@ -3133,7 +3417,11 @@ driver_deliver_term(ErlDrvPort port,
 	    HRelease(rp, hp_end, hp);	    
 	}
 	/* send message */
-	erts_queue_message(rp, &rp_locks, bp, mess, am_undefined);
+	erts_queue_message(rp, &rp_locks, bp, mess, am_undefined
+#ifdef USE_VM_PROBES
+			   , NIL
+#endif
+			   );
     }
     else {
 	if (b2t.ix > b2t.used)
@@ -3186,8 +3474,8 @@ driver_send_term(ErlDrvPort ix, ErlDrvTermData to, ErlDrvTermData* data, int len
  * and data is len length of bin starting from offset offs.
  */
 
-int driver_output_binary(ErlDrvPort ix, char* hbuf, int hlen,
-			 ErlDrvBinary* bin, int offs, int len)
+int driver_output_binary(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
+			 ErlDrvBinary* bin, ErlDrvSizeT offs, ErlDrvSizeT len)
 {
     Port* prt = erts_drvport2port(ix);
 
@@ -3200,7 +3488,7 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, int hlen,
 	return 0;
 
     prt->bytes_in += (hlen + len);
-    erts_smp_atomic_add(&erts_bytes_in, (long) (hlen + len));
+    erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len));
     if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) {
 	return erts_net_message(prt,
 				prt->dist_entry,
@@ -3220,7 +3508,8 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, int hlen,
 ** Example: if hlen = 3 then the port owner will receive the data
 ** [H1,H2,H3 | T]
 */
-int driver_output2(ErlDrvPort ix, char* hbuf, int hlen, char* buf, int len)
+int driver_output2(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
+		   char* buf, ErlDrvSizeT len)
 {
     Port* prt = erts_drvport2port(ix);
 
@@ -3235,7 +3524,7 @@ int driver_output2(ErlDrvPort ix, char* hbuf, int hlen, char* buf, int len)
 	return 0;
     
     prt->bytes_in += (hlen + len);
-    erts_smp_atomic_add(&erts_bytes_in, (long) (hlen + len));
+    erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + len));
     if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) {
 	if (len == 0)
 	    return erts_net_message(prt,
@@ -3257,27 +3546,29 @@ int driver_output2(ErlDrvPort ix, char* hbuf, int hlen, char* buf, int len)
 
 /* Interface functions available to driver writers */
 
-int driver_output(ErlDrvPort ix, char* buf, int len)
+int driver_output(ErlDrvPort ix, char* buf, ErlDrvSizeT len)
 {
     ERTS_SMP_CHK_NO_PROC_LOCKS;
     return driver_output2(ix, NULL, 0, buf, len);
 }
 
-int driver_outputv(ErlDrvPort ix, char* hbuf, int hlen, ErlIOVec* vec, int skip)
+int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen,
+		   ErlIOVec* vec, ErlDrvSizeT skip)
 {
     int n;
-    int len;
-    int size;
+    ErlDrvSizeT len;
+    ErlDrvSizeT size;
     SysIOVec* iov;
     ErlDrvBinary** binv;
     Port* prt;
 
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
-    size = vec->size - skip;   /* Size of remaining bytes in vector */
-    ASSERT(size >= 0);
-    if (size <= 0)
+    ASSERT(vec->size >= skip);
+    if (vec->size <= skip)
 	return driver_output2(ix, hbuf, hlen, NULL, 0);
+    size = vec->size - skip;   /* Size of remaining bytes in vector */
+
     ASSERT(hlen >= 0);       /* debug only */
     if (hlen < 0)
 	hlen = 0;
@@ -3312,7 +3603,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, int hlen, ErlIOVec* vec, int skip)
 
     /* XXX handle distribution !!! */
     prt->bytes_in += (hlen + size);
-    erts_smp_atomic_add(&erts_bytes_in, (long) (hlen + size));
+    erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) (hlen + size));
     deliver_vec_message(prt, prt->connected, hbuf, hlen, binv, iov, n, size);
     return 0;
 }
@@ -3321,17 +3612,14 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, int hlen, ErlIOVec* vec, int skip)
 ** input is a vector a buffer and a max length
 ** return bytes copied
 */
-int driver_vec_to_buf(vec, buf, len)
-ErlIOVec* vec; 
-char* buf;
-int len;
+ErlDrvSizeT driver_vec_to_buf(ErlIOVec *vec, char *buf, ErlDrvSizeT len)
 {
     SysIOVec* iov = vec->iov;
     int n = vec->vsize;
-    int orig_len = len;
+    ErlDrvSizeT orig_len = len;
 
     while(n--) {
-	int ilen = iov->iov_len;
+	size_t ilen = iov->iov_len;
 	if (ilen < len) {
 	    sys_memcpy(buf, iov->iov_base, ilen);
 	    len -= ilen;
@@ -3356,25 +3644,25 @@ int len;
  * reference count on driver binaries...
  */
 
-long
+ErlDrvSInt
 driver_binary_get_refc(ErlDrvBinary *dbp)
 {
     Binary* bp = ErlDrvBinary2Binary(dbp);
-    return erts_refc_read(&bp->refc, 1);
+    return (ErlDrvSInt) erts_refc_read(&bp->refc, 1);
 }
 
-long
+ErlDrvSInt
 driver_binary_inc_refc(ErlDrvBinary *dbp)
 {
     Binary* bp = ErlDrvBinary2Binary(dbp);
-    return erts_refc_inctest(&bp->refc, 2);
+    return (ErlDrvSInt) erts_refc_inctest(&bp->refc, 2);
 }
 
-long
+ErlDrvSInt
 driver_binary_dec_refc(ErlDrvBinary *dbp)
 {
     Binary* bp = ErlDrvBinary2Binary(dbp);
-    return erts_refc_dectest(&bp->refc, 1);
+    return (ErlDrvSInt) erts_refc_dectest(&bp->refc, 1);
 }
 
 
@@ -3383,43 +3671,34 @@ driver_binary_dec_refc(ErlDrvBinary *dbp)
 */
 
 ErlDrvBinary*
-driver_alloc_binary(int size)
+driver_alloc_binary(ErlDrvSizeT size)
 {
     Binary* bin;
 
-    if (size < 0)
-	return NULL;
-
     bin = erts_bin_drv_alloc_fnf((Uint) size);
     if (!bin)
 	return NULL; /* The driver write must take action */
     bin->flags = BIN_FLAG_DRV;
     erts_refc_init(&bin->refc, 1);
-    bin->orig_size = (long) size;
+    bin->orig_size = (SWord) size;
     return Binary2ErlDrvBinary(bin);
 }
 
 /* Reallocate space hold by binary */
 
-ErlDrvBinary* driver_realloc_binary(ErlDrvBinary* bin, int size)
+ErlDrvBinary* driver_realloc_binary(ErlDrvBinary* bin, ErlDrvSizeT size)
 {
     Binary* oldbin;
     Binary* newbin;
 
-    if (!bin || size < 0) {
+    if (!bin) {
 	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
 	erts_dsprintf(dsbufp,
-		      "Bad use of driver_realloc_binary(%p, %d): "
+		      "Bad use of driver_realloc_binary(%p, %lu): "
 		      "called with ",
-		      bin, size);
+		      bin, (unsigned long)size);
 	if (!bin) {
 	    erts_dsprintf(dsbufp, "NULL pointer as first argument");
-	    if (size < 0)
-		erts_dsprintf(dsbufp, ", and ");
-	}
-	if (size < 0) {
-	    erts_dsprintf(dsbufp, "negative size as second argument");
-	    size = 0;
 	}
 	erts_send_warning_to_logger_nogl(dsbufp);
 	if (!bin)
@@ -3459,12 +3738,12 @@ ErlDrvBinary* dbin;
  * Allocation/deallocation of memory for drivers 
  */
 
-void *driver_alloc(size_t size)
+void *driver_alloc(ErlDrvSizeT size)
 {
     return erts_alloc_fnf(ERTS_ALC_T_DRV, (Uint) size);
 }
 
-void *driver_realloc(void *ptr, size_t size)
+void *driver_realloc(void *ptr, ErlDrvSizeT size)
 {
     return erts_realloc_fnf(ERTS_ALC_T_DRV, ptr, (Uint) size);
 }
@@ -3486,47 +3765,47 @@ pdl_init(void)
 static ERTS_INLINE void
 pdl_init_refc(ErlDrvPDL pdl)
 {
-    erts_atomic_init(&pdl->refc, 1);
+    erts_atomic_init_nob(&pdl->refc, 1);
 }
 
-static ERTS_INLINE long
+static ERTS_INLINE ErlDrvSInt
 pdl_read_refc(ErlDrvPDL pdl)
 {
-    long refc = erts_atomic_read(&pdl->refc);
+    erts_aint_t refc = erts_atomic_read_nob(&pdl->refc);
     ERTS_LC_ASSERT(refc >= 0);
-    return refc;
+    return (ErlDrvSInt) refc;
 }
 
 static ERTS_INLINE void
 pdl_inc_refc(ErlDrvPDL pdl)
 {
-    erts_atomic_inc(&pdl->refc);
+    erts_atomic_inc_nob(&pdl->refc);
     ERTS_LC_ASSERT(driver_pdl_get_refc(pdl) > 1);
 }
 
-static ERTS_INLINE long
+static ERTS_INLINE ErlDrvSInt
 pdl_inctest_refc(ErlDrvPDL pdl)
 {
-    long refc = erts_atomic_inctest(&pdl->refc);
+    erts_aint_t refc = erts_atomic_inc_read_nob(&pdl->refc);
     ERTS_LC_ASSERT(refc > 1);
-    return refc;
+    return (ErlDrvSInt) refc;
 }
 
 #if 0 /* unused */
 static ERTS_INLINE void
 pdl_dec_refc(ErlDrvPDL pdl)
 {
-    erts_atomic_dec(&pdl->refc);
+    erts_atomic_dec_nob(&pdl->refc);
     ERTS_LC_ASSERT(driver_pdl_get_refc(pdl) > 0);
 }
 #endif
 
-static ERTS_INLINE long
+static ERTS_INLINE ErlDrvSInt
 pdl_dectest_refc(ErlDrvPDL pdl)
 {
-    long refc = erts_atomic_dectest(&pdl->refc);
+    erts_aint_t refc = erts_atomic_dec_read_nob(&pdl->refc);
     ERTS_LC_ASSERT(refc >= 0);
-    return refc;
+    return (ErlDrvSInt) refc;
 }
 
 static ERTS_INLINE void pdl_destroy(ErlDrvPDL pdl)
@@ -3536,6 +3815,32 @@ static ERTS_INLINE void pdl_destroy(ErlDrvPDL pdl)
     erts_free(ERTS_ALC_T_PORT_DATA_LOCK, pdl);
 }
 
+#ifdef ERTS_SMP
+
+static void driver_monitor_lock_pdl(Port *p) {
+    if (p->port_data_lock) {
+	driver_pdl_lock(p->port_data_lock);
+    }
+    /* Now we either have the port lock or the port_data_lock */
+    ERTS_LC_ASSERT(!p->port_data_lock
+		   || erts_lc_mtx_is_locked(&(p->port_data_lock->mtx)));
+    ERTS_SMP_LC_ASSERT(p->port_data_lock
+		       || erts_lc_is_port_locked(p));
+}
+
+static void driver_monitor_unlock_pdl(Port *p) {
+    /* We should either have the port lock or the port_data_lock */
+    ERTS_LC_ASSERT(!p->port_data_lock
+		   || erts_lc_mtx_is_locked(&(p->port_data_lock->mtx)));
+    ERTS_SMP_LC_ASSERT(p->port_data_lock
+		       || erts_lc_is_port_locked(p));
+    if (p->port_data_lock) {
+	driver_pdl_unlock(p->port_data_lock);
+    }
+}
+
+#endif
+
 /*
  * exported driver_pdl_* functions ...
  */
@@ -3571,7 +3876,7 @@ driver_pdl_lock(ErlDrvPDL pdl)
 void
 driver_pdl_unlock(ErlDrvPDL pdl)
 {
-    long refc;
+    ErlDrvSInt refc;
 #ifdef HARDDEBUG
     erts_fprintf(stderr, "driver_pdl_unlock(0x%08X)\r\n",(unsigned) pdl);
 #endif
@@ -3581,28 +3886,30 @@ driver_pdl_unlock(ErlDrvPDL pdl)
 	pdl_destroy(pdl);
 }
 
-long
+ErlDrvSInt
 driver_pdl_get_refc(ErlDrvPDL pdl)
 {
     return pdl_read_refc(pdl);
 }
 
-long
+ErlDrvSInt
 driver_pdl_inc_refc(ErlDrvPDL pdl)
 {
-    long refc = pdl_inctest_refc(pdl);
+    ErlDrvSInt refc = pdl_inctest_refc(pdl);
 #ifdef HARDDEBUG
-    erts_fprintf(stderr, "driver_pdl_inc_refc(0x%08X) -> %ld\r\n",(unsigned) pdl, refc);
+    erts_fprintf(stderr, "driver_pdl_inc_refc(%p) -> %bed\r\n",
+		 pdl, refc);
 #endif
     return refc;
 }
 
-long
+ErlDrvSInt
 driver_pdl_dec_refc(ErlDrvPDL pdl)
 {
-    long refc = pdl_dectest_refc(pdl);
+    ErlDrvSInt refc = pdl_dectest_refc(pdl);
 #ifdef HARDDEBUG
-    erts_fprintf(stderr, "driver_pdl_dec_refc(0x%08X) -> %ld\r\n",(unsigned) pdl, refc);
+    erts_fprintf(stderr, "driver_pdl_dec_refc(%p) -> %bpd\r\n",
+		 pdl, refc);
 #endif
     if (!refc)
 	pdl_destroy(pdl);
@@ -3698,11 +4005,11 @@ static int expandq(ErlIOQueue* q, int n, int tail)
 
 
 /* Put elements from vec at q tail */
-int driver_enqv(ErlDrvPort ix, ErlIOVec* vec, int skip)
+int driver_enqv(ErlDrvPort ix, ErlIOVec* vec, ErlDrvSizeT skip)
 {
     int n;
-    int len;
-    int size;
+    size_t len;
+    ErlDrvSizeT size;
     SysIOVec* iov;
     ErlDrvBinary** binv;
     ErlDrvBinary*  b;
@@ -3711,10 +4018,10 @@ int driver_enqv(ErlDrvPort ix, ErlIOVec* vec, int skip)
     if (q == NULL)
 	return -1;
 
-    size = vec->size - skip;
-    ASSERT(size >= 0);       /* debug only */
-    if (size <= 0)
+    ASSERT(vec->size >= skip);       /* debug only */
+    if (vec->size <= skip)
 	return 0;
+    size = vec->size - skip;
 
     iov = vec->iov;
     binv = vec->binv;
@@ -3764,11 +4071,11 @@ int driver_enqv(ErlDrvPort ix, ErlIOVec* vec, int skip)
 }
 
 /* Put elements from vec at q head */
-int driver_pushqv(ErlDrvPort ix, ErlIOVec* vec, int skip)
+int driver_pushqv(ErlDrvPort ix, ErlIOVec* vec, ErlDrvSizeT skip)
 {
     int n;
-    int len;
-    int size;
+    size_t len;
+    ErlDrvSizeT size;
     SysIOVec* iov;
     ErlDrvBinary** binv;
     ErlDrvBinary* b;
@@ -3777,8 +4084,10 @@ int driver_pushqv(ErlDrvPort ix, ErlIOVec* vec, int skip)
     if (q == NULL)
 	return -1;
 
-    if ((size = vec->size - skip) <= 0)
+    if (vec->size <= skip)
 	return 0;
+    size = vec->size - skip;
+
     iov = vec->iov;
     binv = vec->binv;
     n = vec->vsize;
@@ -3833,15 +4142,14 @@ int driver_pushqv(ErlDrvPort ix, ErlIOVec* vec, int skip)
 ** Remove size bytes from queue head
 ** Return number of bytes that remain in queue
 */
-int driver_deq(ErlDrvPort ix, int size)
+ErlDrvSizeT driver_deq(ErlDrvPort ix, ErlDrvSizeT size)
 {
     ErlIOQueue* q = drvport2ioq(ix);
-    int len;
-    int sz;
+    ErlDrvSizeT len;
 
-    if ((q == NULL) || (sz = (q->size - size)) < 0)
+    if ((q == NULL) || (q->size < size))
 	return -1;
-    q->size = sz;
+    q->size -= size;
     while (size > 0) {
 	ASSERT(q->v_head != q->v_tail);
 
@@ -3864,16 +4172,16 @@ int driver_deq(ErlDrvPort ix, int size)
 	q->v_head = q->v_tail = q->v_start;
 	q->b_head = q->b_tail = q->b_start;
     }
-    return sz;
+    return q->size;
 }
 
 
-int driver_peekqv(ErlDrvPort ix, ErlIOVec *ev) {
+ErlDrvSizeT driver_peekqv(ErlDrvPort ix, ErlIOVec *ev) {
     ErlIOQueue *q = drvport2ioq(ix);
     ASSERT(ev);
 
     if (! q) {
-	return -1;
+	return (ErlDrvSizeT) -1;
     } else {
 	if ((ev->vsize = q->v_tail - q->v_head) == 0) {
 	    ev->size = 0;
@@ -3902,12 +4210,12 @@ SysIOVec* driver_peekq(ErlDrvPort ix, int* vlenp)  /* length of io-vector */
 }
 
 
-int driver_sizeq(ErlDrvPort ix)
+ErlDrvSizeT driver_sizeq(ErlDrvPort ix)
 {
     ErlIOQueue* q = drvport2ioq(ix);
 
     if (q == NULL)
-	return -1;
+	return (size_t) -1;
     return q->size;
 }
 
@@ -3915,7 +4223,8 @@ int driver_sizeq(ErlDrvPort ix)
 /* Utils */
 
 /* Enqueue a binary */
-int driver_enq_bin(ErlDrvPort ix, ErlDrvBinary* bin, int offs, int len)
+int driver_enq_bin(ErlDrvPort ix, ErlDrvBinary* bin,
+		   ErlDrvSizeT offs, ErlDrvSizeT len)
 {
     SysIOVec      iov;
     ErlIOVec      ev;
@@ -3932,7 +4241,7 @@ int driver_enq_bin(ErlDrvPort ix, ErlDrvBinary* bin, int offs, int len)
     return driver_enqv(ix, &ev, 0);
 }
 
-int driver_enq(ErlDrvPort ix, char* buffer, int len)
+int driver_enq(ErlDrvPort ix, char* buffer, ErlDrvSizeT len)
 {
     int code;
     ErlDrvBinary* bin;
@@ -3948,7 +4257,8 @@ int driver_enq(ErlDrvPort ix, char* buffer, int len)
     return code;
 }
 
-int driver_pushq_bin(ErlDrvPort ix, ErlDrvBinary* bin, int offs, int len)
+int driver_pushq_bin(ErlDrvPort ix, ErlDrvBinary* bin,
+		     ErlDrvSizeT offs, ErlDrvSizeT len)
 {
     SysIOVec      iov;
     ErlIOVec      ev;
@@ -3965,7 +4275,7 @@ int driver_pushq_bin(ErlDrvPort ix, ErlDrvBinary* bin, int offs, int len)
     return driver_pushqv(ix, &ev, 0);
 }
 
-int driver_pushq(ErlDrvPort ix, char* buffer, int len)
+int driver_pushq(ErlDrvPort ix, char* buffer, ErlDrvSizeT len)
 {
     int code;
     ErlDrvBinary* bin;
@@ -3988,13 +4298,13 @@ drv_cancel_timer(Port *prt)
 #ifdef ERTS_SMP
     erts_cancel_smp_ptimer(prt->ptimer);
 #else
-    erl_cancel_timer(&prt->tm);
+    erts_cancel_timer(&prt->tm);
 #endif
     if (erts_port_task_is_scheduled(&prt->timeout_task))
 	erts_port_task_abort(prt->id, &prt->timeout_task);
 }
 
-int driver_set_timer(ErlDrvPort ix, Uint t)
+int driver_set_timer(ErlDrvPort ix, unsigned long t)
 {
     Port* prt = erts_drvport2port(ix);
 
@@ -4012,7 +4322,7 @@ int driver_set_timer(ErlDrvPort ix, Uint t)
 			   (ErlTimeoutProc) schedule_port_timeout,
 			   t);
 #else
-    erl_set_timer(&prt->tm,
+    erts_set_timer(&prt->tm,
 		  (ErlTimeoutProc) schedule_port_timeout,
 		  NULL,
 		  prt,
@@ -4043,9 +4353,9 @@ driver_read_timer(ErlDrvPort ix, unsigned long* t)
 	return -1;
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
 #ifdef ERTS_SMP
-    *t = prt->ptimer ? time_left(&prt->ptimer->timer.tm) : 0;
+    *t = prt->ptimer ? erts_time_left(&prt->ptimer->timer.tm) : 0;
 #else
-    *t = time_left(&prt->tm);
+    *t = erts_time_left(&prt->tm);
 #endif
     return 0;
 }
@@ -4053,12 +4363,16 @@ driver_read_timer(ErlDrvPort ix, unsigned long* t)
 int 
 driver_get_now(ErlDrvNowData *now_data)
 {
+    Uint mega,secs,micro;
     ERTS_SMP_CHK_NO_PROC_LOCKS;
 
     if (now_data == NULL) {
 	return -1;
     }
-    get_now(&(now_data->megasecs),&(now_data->secs),&(now_data->microsecs));
+    get_now(&mega,&secs,&micro);
+    now_data->megasecs = (unsigned long) mega;
+    now_data->secs = (unsigned long) secs;
+    now_data->microsecs = (unsigned long) micro;
     return 0;
 }
 
@@ -4072,14 +4386,15 @@ static void ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon)
     memcpy(mon,refp,sizeof(RefThing));
 }
 
-int driver_monitor_process(ErlDrvPort port, 
-			   ErlDrvTermData process, 
-			   ErlDrvMonitor *monitor)
+
+static int do_driver_monitor_process(Port *prt,
+				     Eterm *buf,
+				     ErlDrvTermData process,
+				     ErlDrvMonitor *monitor)
 {
-    Port *prt = erts_drvport2port(port);
     Process *rp;
     Eterm ref;
-    Eterm buf[REF_THING_SIZE];
+
     if (prt->drv_ptr->process_exit == NULL) {
 	return -1;
     }
@@ -4089,22 +4404,76 @@ int driver_monitor_process(ErlDrvPort port,
     if (!rp) {
 	return 1;
     }
+
     ref = erts_make_ref_in_buffer(buf);
     erts_add_monitor(&(prt->monitors), MON_ORIGIN, ref, rp->id, NIL);
     erts_add_monitor(&(rp->monitors), MON_TARGET, ref, prt->id, NIL);
-    
+
     erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_LINK);
     ref_to_driver_monitor(ref,monitor);
     return 0;
 }
 
-int driver_demonitor_process(ErlDrvPort port,
-			     const ErlDrvMonitor *monitor)
+/*
+ * This can be called from a non scheduler thread iff a port_data_lock exists
+ */
+int driver_monitor_process(ErlDrvPort port,
+			   ErlDrvTermData process,
+			   ErlDrvMonitor *monitor)
+{
+    Port *prt;
+    int ret;
+    Uint32 status;
+    ErtsSchedulerData *sched = erts_get_scheduler_data();
+    int ix = (int) port;
+    if (ix < 0 || erts_max_ports <= ix) {
+	return -1;
+    }
+    prt = &erts_port[ix];
+
+    DRV_MONITOR_LOCK_PDL(prt);
+
+    if (sched) {
+	status = erts_port[ix].status;
+    } else {
+	erts_smp_port_state_lock(prt);
+	status = erts_port[ix].status;
+	erts_smp_port_state_unlock(prt);
+    }
+
+    if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) {
+	DRV_MONITOR_UNLOCK_PDL(prt);
+	return -1;
+    }
+
+    /* Now (in SMP) we should have either the port lock (if we have a scheduler) or the port data lock
+       (if we're a driver thread) */
+    ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock));
+
+#if !HEAP_ON_C_STACK
+    if (!sched) {
+	/* Need a separate allocation for the ref :( */
+	Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM,
+				sizeof(Eterm)*REF_THING_SIZE);
+	ret = do_driver_monitor_process(prt,buf,process,monitor);
+	erts_free(ERTS_ALC_T_TEMP_TERM,buf);
+    } else
+#endif
+    {
+	DeclareTmpHeapNoproc(buf,REF_THING_SIZE);
+	UseTmpHeapNoproc(REF_THING_SIZE);
+	ret = do_driver_monitor_process(prt,buf,process,monitor);
+	UnUseTmpHeapNoproc(REF_THING_SIZE);
+    }
+    DRV_MONITOR_UNLOCK_PDL(prt);
+    return ret;
+}
+
+static int do_driver_demonitor_process(Port *prt, Eterm *buf,
+				       const ErlDrvMonitor *monitor)
 {
-    Port *prt = erts_drvport2port(port);
     Process *rp;
     Eterm ref;
-    Eterm buf[REF_THING_SIZE];
     ErtsMonitor *mon;
     Eterm to;
 
@@ -4137,12 +4506,60 @@ int driver_demonitor_process(ErlDrvPort port,
     return 0;
 }
 
-ErlDrvTermData driver_get_monitored_process(ErlDrvPort port,
+int driver_demonitor_process(ErlDrvPort port,
+			     const ErlDrvMonitor *monitor)
+{
+    Port *prt;
+    int ret;
+    Uint32 status;
+    ErtsSchedulerData *sched = erts_get_scheduler_data();
+    int ix = (int) port;
+    if (ix < 0 || erts_max_ports <= ix) {
+	return -1;
+    }
+    prt = &erts_port[ix];
+
+    DRV_MONITOR_LOCK_PDL(prt);
+
+    if (sched) {
+	status = erts_port[ix].status;
+    } else {
+	erts_smp_port_state_lock(prt);
+	status = erts_port[ix].status;
+	erts_smp_port_state_unlock(prt);
+    }
+
+    if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) {
+	DRV_MONITOR_UNLOCK_PDL(prt);
+	return -1;
+    }
+
+    /* Now we should have either the port lock (if we have a scheduler) or the port data lock
+       (if we're a driver thread) */
+    ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock));
+#if !HEAP_ON_C_STACK
+    if (!sched) {
+	/* Need a separate allocation for the ref :( */
+	Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM,
+				sizeof(Eterm)*REF_THING_SIZE);
+	ret = do_driver_demonitor_process(prt,buf,monitor);
+	erts_free(ERTS_ALC_T_TEMP_TERM,buf);
+    } else
+#endif
+    {
+	DeclareTmpHeapNoproc(buf,REF_THING_SIZE);
+	UseTmpHeapNoproc(REF_THING_SIZE);
+	ret = do_driver_demonitor_process(prt,buf,monitor);
+	UnUseTmpHeapNoproc(REF_THING_SIZE);
+    }
+    DRV_MONITOR_UNLOCK_PDL(prt);
+    return ret;
+}
+
+static ErlDrvTermData do_driver_get_monitored_process(Port *prt, Eterm *buf,
 					    const ErlDrvMonitor *monitor)
 {
-    Port *prt = erts_drvport2port(port);
     Eterm ref;
-    Eterm buf[REF_THING_SIZE];
     ErtsMonitor *mon;
     Eterm to;
 
@@ -4158,6 +4575,59 @@ ErlDrvTermData driver_get_monitored_process(ErlDrvPort port,
     return (ErlDrvTermData) to;
 }
 
+
+ErlDrvTermData driver_get_monitored_process(ErlDrvPort port,
+					    const ErlDrvMonitor *monitor)
+{
+    Port *prt;
+    ErlDrvTermData ret;
+    Uint32 status;
+    ErtsSchedulerData *sched = erts_get_scheduler_data();
+    int ix = (int) port;
+    if (ix < 0 || erts_max_ports <= ix) {
+	return driver_term_nil;
+    }
+    prt = &erts_port[ix];
+
+    DRV_MONITOR_LOCK_PDL(prt);
+
+    if (sched) {
+	status = erts_port[ix].status;
+    } else {
+	erts_smp_port_state_lock(prt);
+	status = erts_port[ix].status;
+	erts_smp_port_state_unlock(prt);
+    }
+
+    if (status & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) {
+	DRV_MONITOR_UNLOCK_PDL(prt);
+	return driver_term_nil;
+    }
+
+    /* Now we should have either the port lock (if we have a scheduler) or the port data lock
+       (if we're a driver thread) */
+    ERTS_SMP_LC_ASSERT((sched != NULL || prt->port_data_lock));
+
+#if !HEAP_ON_C_STACK
+    if (!sched) {
+	/* Need a separate allocation for the ref :( */
+	Eterm *buf = erts_alloc(ERTS_ALC_T_TEMP_TERM,
+				sizeof(Eterm)*REF_THING_SIZE);
+	ret = do_driver_get_monitored_process(prt,buf,monitor);
+	erts_free(ERTS_ALC_T_TEMP_TERM,buf);
+    } else
+#endif
+    {
+	DeclareTmpHeapNoproc(buf,REF_THING_SIZE);
+	UseTmpHeapNoproc(REF_THING_SIZE);
+	ret = do_driver_get_monitored_process(prt,buf,monitor);
+	UnUseTmpHeapNoproc(REF_THING_SIZE);
+    }
+    DRV_MONITOR_UNLOCK_PDL(prt);
+    return ret;
+}
+
+
 int driver_compare_monitors(const ErlDrvMonitor *monitor1,
 			    const ErlDrvMonitor *monitor2)
 {
@@ -4173,18 +4643,28 @@ void erts_fire_port_monitor(Port *prt, Eterm ref)
 
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
     ASSERT(prt->drv_ptr != NULL);    
-
+    DRV_MONITOR_LOCK_PDL(prt);
     if (erts_lookup_monitor(prt->monitors,ref) == NULL) {
+	DRV_MONITOR_UNLOCK_PDL(prt);
 	return;
     }
     callback = prt->drv_ptr->process_exit;
     ASSERT(callback != NULL);
     ref_to_driver_monitor(ref,&drv_monitor);
+    DRV_MONITOR_UNLOCK_PDL(prt);
+#ifdef USE_VM_PROBES
+    if (DTRACE_ENABLED(driver_process_exit)) {
+        DTRACE_FORMAT_COMMON_PID_AND_PORT(prt->connected, prt)
+        DTRACE3(driver_process_exit, process_str, port_str, prt->name);
+    }
+#endif
     fpe_was_unmasked = erts_block_fpe();
     (*callback)((ErlDrvData) (prt->drv_data), &drv_monitor);
     erts_unblock_fpe(fpe_was_unmasked);
+    DRV_MONITOR_LOCK_PDL(prt);
     /* remove monitor *after* callback */
     rmon = erts_remove_monitor(&(prt->monitors),ref);
+    DRV_MONITOR_UNLOCK_PDL(prt);
     if (rmon) {
 	erts_destroy_monitor(rmon);
     }
@@ -4334,7 +4814,10 @@ int driver_lock_driver(ErlDrvPort ix)
 
     erts_smp_mtx_lock(&erts_driver_list_lock);
 
-    if (prt == NULL) return -1;
+    if (prt == NULL) {
+	erts_smp_mtx_unlock(&erts_driver_list_lock);
+	return -1;
+    }
 
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
     if ((dh = (DE_Handle*)prt->drv_ptr->handle ) == NULL) {
@@ -4519,7 +5002,7 @@ get_current_port(void)
  */
 
 static void
-no_output_callback(ErlDrvData drv_data, char *buf, int len)
+no_output_callback(ErlDrvData drv_data, char *buf, ErlDrvSizeT len)
 {
 
 }
@@ -4570,16 +5053,11 @@ static int
 init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
 {
     drv->name = de->driver_name;
-    if (de->extended_marker == ERL_DRV_EXTENDED_MARKER) {
-	drv->version.major = de->major_version;
-	drv->version.minor = de->minor_version;
-	drv->flags = de->driver_flags;
-    }
-    else {
-	drv->version.major = 0;
-	drv->version.minor = 0;
-	drv->flags = 0;
-    }
+    ASSERT(de->extended_marker == ERL_DRV_EXTENDED_MARKER);
+    ASSERT(de->major_version >= 2);
+    drv->version.major = de->major_version;
+    drv->version.minor = de->minor_version;
+    drv->flags = de->driver_flags;
     drv->handle = handle;
 #ifdef ERTS_SMP
     if (drv->flags & ERL_DRV_FLAG_USE_PORT_LOCKING)
@@ -4612,11 +5090,8 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
     drv->ready_output = de->ready_output ? de->ready_output : no_ready_output_callback;
     drv->timeout = de->timeout ? de->timeout : no_timeout_callback;
     drv->ready_async = de->ready_async;
-    if (de->extended_marker == ERL_DRV_EXTENDED_MARKER)
-	drv->process_exit = de->process_exit;
-    else
-	drv->process_exit = NULL;
-    if (de->minor_version >= 3/*R13A*/ && de->stop_select)
+    drv->process_exit = de->process_exit;
+    if (de->stop_select)
 	drv->stop_select = de->stop_select;
     else
 	drv->stop_select = no_stop_select_callback;
@@ -4626,6 +5101,8 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
     else {
 	int res;
 	int fpe_was_unmasked = erts_block_fpe();
+        DTRACE4(driver_init, drv->name, drv->version.major, drv->version.minor,
+                drv->flags);
 	res = (*de->init)();
 	erts_unblock_fpe(fpe_was_unmasked);
 	return res;
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index 91e4ccce70..b93b1ad09a 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -108,7 +108,8 @@ erts_put_module(Eterm mod)
     int index;
 
     ASSERT(is_atom(mod));
-    ERTS_SMP_LC_ASSERT(erts_initialized == 0 || erts_smp_is_system_blocked(0));
+    ERTS_SMP_LC_ASSERT(erts_initialized == 0
+		       || erts_smp_thr_progress_is_blocking());
     e.module = atom_val(mod);
     index = index_put(&module_table, (void*) &e);
     return (Module*) erts_index_lookup(&module_table, index);
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index 87d13b3607..694e4ab72f 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -29,8 +29,8 @@ typedef struct erl_module {
     IndexSlot slot;		/* Must be located at top of struct! */
     int module;			/* Atom index for module (not tagged). */
 
-    Eterm* code;
-    Eterm* old_code;
+    BeamInstr* code;
+    BeamInstr* old_code;
     int code_length;		/* Length of loaded code in bytes. */
     int old_code_length;	/* Length of old loaded code in bytes */
     unsigned catches, old_catches;
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index ce1df74f03..9b168889dd 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -1,19 +1,19 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 1997-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 1997-2012. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
 
@@ -25,30 +25,12 @@
 # instruction transformations; thus, they never occur in BEAM files.
 #
 
-# Special instruction used to generate an error message when
-# trying to load a module compiled by the V1 compiler (R5 & R6).
-# (Specially treated in beam_load.c.)
+# The too_old_compiler/0 instruction is specially handled in beam_load.c
+# to produce a user-friendly message informing the user that the module
+# needs to be re-compiled with a modern compiler.
 
 too_old_compiler/0
-too_old_compiler
-
-#
-# Obsolete instruction usage follow. (Nowdays we use f with
-# a zero label instead of p.)
-#
-
-is_list p S => too_old_compiler
-is_nonempty_list p R => too_old_compiler
-is_nil p R => too_old_compiler
-
-is_tuple p S => too_old_compiler
-test_arity p S Arity => too_old_compiler
-
-is_integer p R => too_old_compiler
-is_float p R => too_old_compiler
-is_atom p R => too_old_compiler
-
-is_eq_exact p S1 S2 => too_old_compiler
+too_old_compiler | never() =>
 
 # In R9C and earlier, the loader used to insert special instructions inside
 # the module_info/0,1 functions. (In R10B and later, the compiler inserts
@@ -60,12 +42,17 @@ func_info M=a a==am_module_info A=u==0 | label L | move n r => too_old_compiler
 func_info M=a a==am_module_info A=u==1 | label L | move n r => too_old_compiler
 
 # The undocumented and unsupported guard BIF is_constant/1 was removed
-# in R13. The is_constant/2 operation is marked as obosolete in genop.tab,
+# in R13. The is_constant/2 operation is marked as obsolete in genop.tab,
 # so the loader will automatically generate a too_old_compiler message
 # it is used, but we need to handle the is_constant/1 BIF specially here.
 
 bif1 Fail u$func:erlang:is_constant/1 Src Dst => too_old_compiler
 
+# Since the constant pool was introduced in R12B, empty tuples ({})
+# are literals. Therefore we no longer need to allow put_tuple/2
+# with a tuple size of zero.
+
+put_tuple u==0 d => too_old_compiler
 
 #
 # All the other instructions.
@@ -79,14 +66,46 @@ i_trace_breakpoint
 i_mtrace_breakpoint
 i_debug_breakpoint
 i_count_breakpoint
+i_time_breakpoint
+i_return_time_trace
 i_return_to_trace
 i_yield
-i_global_cons
-i_global_tuple
-i_global_copy
 
 return
 
+#
+# To ensure that a "move Src x(0)" instruction can be combined
+# with the following call instruction, we need to make sure that
+# there is no line/1 instruction between the move and the call.
+#
+
+move S r | line Loc | call_ext Ar Func => \
+     line Loc | move S r | call_ext Ar Func
+move S r | line Loc | call_ext_last Ar Func=u$is_bif D => \
+     line Loc | move S r | call_ext_last Ar Func D
+move S r | line Loc | call_ext_only Ar Func=u$is_bif => \
+     line Loc | move S r | call_ext_only Ar Func
+move S r | line Loc | call Ar Func => \
+     line Loc | move S r | call Ar Func
+
+#
+# A tail-recursive call to an external function (non-BIF) will
+# never be saved on the stack, so there is no reason to keep
+# the line instruction. (The compiler did not remove the line
+# instruction because it cannot tell the difference between
+# BIFs and ordinary Erlang functions.)
+#
+
+line Loc | call_ext_last Ar Func=u$is_not_bif D => \
+     call_ext_last Ar Func D
+line Loc | call_ext_only Ar Func=u$is_not_bif => \
+     call_ext_only Ar Func
+
+line Loc | func_info M F A => func_info M F A | line Loc
+
+line I
+
+
 %macro: allocate Allocate -pack
 %macro: allocate_zero AllocateZero -pack
 %macro: allocate_heap AllocateHeap -pack
@@ -94,16 +113,16 @@ return
 %macro: test_heap TestHeap -pack
 
 allocate t t
-allocate_heap I I I
+allocate_heap t I t
 deallocate I
 init y
 allocate_zero t t
-allocate_heap_zero I I I
+allocate_heap_zero t I t
 
 trim N Remaining => i_trim N
 i_trim I
 
-test_heap I I
+test_heap I t
 
 allocate_heap S u==0 R => allocate S R
 allocate_heap_zero S u==0 R => allocate_zero S R
@@ -115,15 +134,9 @@ init Y1 | init Y2 => init2 Y1 Y2
 %macro: init2 Init2 -pack
 %macro: init3 Init3 -pack
 
-#
-# Warning: The put_string instruction is specially treated in the loader.
-# Don't change the instruction format unless you change the loader too.
-#
-put_string I I d
-
 # Selecting values
 
-select_val S=q Fail=f Size=u Rest=* => const_select_val(S, Fail, Size, Rest)
+select_val S=aiq Fail=f Size=u Rest=* => const_select_val(S, Fail, Size, Rest)
 
 select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \
   gen_jump_tab(S, Fail, Size, Rest)
@@ -131,34 +144,59 @@ select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \
 is_integer Fail=f S | select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \
   gen_jump_tab(S, Fail, Size, Rest)
 
+is_integer TypeFail=f S | select_val S=s Fail=f Size=u Rest=* | \
+	   mixed_types(Size, Rest) => \
+  gen_split_values(S, TypeFail, Fail, Size, Rest)
+
 select_val S=s Fail=f Size=u Rest=* | mixed_types(Size, Rest) => \
-  gen_split_values(S, Fail, Size, Rest)
+  gen_split_values(S, Fail, Fail, Size, Rest)
 
-is_integer Fail=f S | select_val S=s Fail=f Size=u Rest=* | \
+is_integer Fail=f S | select_val S=d Fail=f Size=u Rest=* | \
   fixed_size_values(Size, Rest) => gen_select_val(S, Fail, Size, Rest)
 
-is_atom Fail=f S | select_val S=s Fail=f Size=u Rest=* | \
+is_atom Fail=f S | select_val S=d Fail=f Size=u Rest=* | \
   fixed_size_values(Size, Rest) => gen_select_val(S, Fail, Size, Rest)
 
-select_val S=s Fail=f Size=u Rest=* | fixed_size_values(Size, Rest) => \
-  gen_select_val(S, Fail, Size, Rest)
+select_val S=s Fail=f Size=u Rest=* | floats_or_bignums(Size, Rest) => \
+  gen_select_literals(S, Fail, Size, Rest)
 
-select_val S=s Fail=f Size=u Rest=* | all_values_are_big(Size, Rest) => \
-  gen_select_big(S, Fail, Size, Rest)
+select_val S=d Fail=f Size=u Rest=* | fixed_size_values(Size, Rest) => \
+  gen_select_val(S, Fail, Size, Rest)
 
-is_tuple Fail=f S | select_tuple_arity S=s Fail=f Size=u Rest=* => \
+is_tuple Fail=f S | select_tuple_arity S=d Fail=f Size=u Rest=* => \
   gen_select_tuple_arity(S, Fail, Size, Rest)
 
-select_tuple_arity S=s Fail=f Size=u Rest=* => \
+select_tuple_arity S=d Fail=f Size=u Rest=* => \
   gen_select_tuple_arity(S, Fail, Size, Rest)
 
-i_select_val s f I
-i_select_tuple_arity s f I
-i_select_big s f
-i_select_float s f I
+i_select_val r f I
+i_select_val x f I
+i_select_val y f I
+
+i_select_val2 r f c f c f
+i_select_val2 x f c f c f
+i_select_val2 y f c f c f
+
+i_select_tuple_arity2 r f A f A f
+i_select_tuple_arity2 x f A f A f
+i_select_tuple_arity2 y f A f A f
+
+i_select_tuple_arity r f I
+i_select_tuple_arity x f I
+i_select_tuple_arity y f I
 
-i_jump_on_val_zero s f I
-i_jump_on_val s f I I
+i_jump_on_val_zero r f I
+i_jump_on_val_zero x f I
+i_jump_on_val_zero y f I
+
+i_jump_on_val r f I I
+i_jump_on_val x f I I
+i_jump_on_val y f I I
+
+jump Target | label Lbl | same_label(Target, Lbl) => label Lbl
+
+is_ne_exact L1 S1 S2 | jump Fail | label L2 | same_label(L1, L2) => \
+  is_eq_exact Fail S1 S2 | label L2
 
 %macro: get_list GetList -pack
 get_list x x x
@@ -233,11 +271,17 @@ is_number Fail Literal=q => move Literal x | is_number Fail x
 
 jump f
 
-case_end Literal=q => move Literal x | case_end x
-badmatch Literal=q => move Literal x | badmatch x
+case_end Literal=cq => move Literal x | case_end x
+badmatch Literal=cq => move Literal x | badmatch x
+
+case_end r
+case_end x
+case_end y
+
+badmatch r
+badmatch x
+badmatch y
 
-case_end s
-badmatch s
 if_end
 raise s s
 
@@ -245,14 +289,33 @@ raise s s
 badarg j
 system_limit j
 
-move R R =>
+move C=cxy r | jump Lbl => move_jump Lbl C
+
+%macro: move_jump MoveJump -nonext
+move_jump f n
+move_jump f c
+move_jump f x
+move_jump f y
 
 move X1=x Y1=y | move X2=x Y2=y => move2 X1 Y1 X2 Y2
 move Y1=y X1=x | move Y2=y X2=x => move2 Y1 X1 Y2 X2
+move X1=x X2=x | move X3=x X4=x => move2 X1 X2 X3 X4
+
+move C=aiq X=x==1 => move_x1 C
+move C=aiq X=x==2 => move_x2 C
+
+move_x1 c
+move_x2 c
 
 %macro: move2 Move2 -pack
 move2 x y x y
 move2 y x y x
+move2 x x x x
+
+# The compiler almost never generates a "move Literal y(Y)" instruction,
+# so let's cheat if we encounter one.
+move S=n D=y => init D
+move S=c D=y => move S x | move x D
 
 %macro:move Move -pack -gen_dest
 move x x
@@ -264,15 +327,10 @@ move r x
 move r y
 move c r
 move c x
-move c y
 move n x
 move n r
 move y y
 
-%cold
-move s d
-%hot
-
 # Receive operations.
 
 loop_rec Fail Src | smp_mark_target_label(Fail) => i_loop_rec Fail Src
@@ -305,58 +363,78 @@ i_wait_error_locked
 send
 
 #
-# Comparisions.
+# Optimized comparisons with one immediate/literal operand.
+#
+
+is_eq_exact Lbl R=rxy C=ian => i_is_eq_exact_immed Lbl R C
+is_eq_exact Lbl R=rxy C=q => i_is_eq_exact_literal R Lbl C
+
+is_ne_exact Lbl R=rxy C=ian => i_is_ne_exact_immed Lbl R C
+is_ne_exact Lbl R=rxy C=q => i_is_ne_exact_literal R Lbl C
+
+%macro: i_is_eq_exact_immed EqualImmed -fail_action
+i_is_eq_exact_immed f r c
+i_is_eq_exact_immed f x c
+i_is_eq_exact_immed f y c
+
+i_is_eq_exact_literal r f c
+i_is_eq_exact_literal x f c
+i_is_eq_exact_literal y f c
+
+%macro: i_is_ne_exact_immed NotEqualImmed -fail_action
+i_is_ne_exact_immed f r c
+i_is_ne_exact_immed f x c
+i_is_ne_exact_immed f y c
+
+i_is_ne_exact_literal r f c
+i_is_ne_exact_literal x f c
+i_is_ne_exact_literal y f c
+
+#
+# All other comparisons.
 #
 
-is_eq_exact Lbl=f R=rxy C=ian => i_is_eq_immed Lbl R C
-is_eq Lbl=f R=rxy C=an => i_is_eq_immed Lbl R C
+is_eq_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl
+is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl
 
 is_ge Lbl S1 S2 => i_fetch S1 S2 | i_is_ge Lbl
 is_lt Lbl S1 S2 => i_fetch S1 S2 | i_is_lt Lbl
 is_eq Lbl S1 S2 => i_fetch S1 S2 | i_is_eq Lbl
 is_ne Lbl S1 S2 => i_fetch S1 S2 | i_is_ne Lbl
 
-is_eq_exact Lbl=f S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl
-is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl
-
+i_is_eq_exact f
+i_is_ne_exact f
 i_is_lt f
 i_is_ge f
 i_is_eq f
 i_is_ne f
-i_is_eq_exact f
-i_is_ne_exact f
-
-%macro: i_is_eq_immed EqualImmed -fail_action
-i_is_eq_immed f r c
-i_is_eq_immed f x c
-i_is_eq_immed f y c
 
 #
 # Putting things.
 #
 
-put_tuple u==0 Dst => i_put_tuple_only u Dst
-put_tuple Arity Dst | put V => i_put_tuple Arity V Dst
+put_tuple Arity Dst => i_put_tuple Dst u
 
-i_put_tuple_only A d
+i_put_tuple Dst Arity Puts=* | put S1 | put S2 | \
+  put S3 | put S4 | put S5 => \
+	    tuple_append_put5(Arity, Dst, Puts, S1, S2, S3, S4, S5)
 
-%macro: i_put_tuple PutTuple -pack
-i_put_tuple A x x
-i_put_tuple A y x
-i_put_tuple A r x
-i_put_tuple A n x
-i_put_tuple A c x
-i_put_tuple A x y
-i_put_tuple A x r
-i_put_tuple A y r
-i_put_tuple A n r
-i_put_tuple A c r
+i_put_tuple Dst Arity Puts=* | put S => \
+	    tuple_append_put(Arity, Dst, Puts, S)
 
-%cold
-i_put_tuple A r y
-i_put_tuple A y y
-i_put_tuple A c y
-%hot
+i_put_tuple/2
+
+%macro:i_put_tuple PutTuple -pack -goto:do_put_tuple
+i_put_tuple r I
+i_put_tuple x I
+i_put_tuple y I
+
+#
+# The instruction "put_list Const [] Dst" will not be generated by
+# the current BEAM compiler. But until R15A, play it safe by handling
+# that instruction with the following transformation.
+#
+put_list Const=c n Dst => move Const x | put_list x n Dst
 
 %macro:put_list PutList -pack -gen_dest
 
@@ -364,10 +442,8 @@ put_list x n x
 put_list y n x
 put_list x x x
 put_list y x x
-put_list c n x
 put_list x x r
 put_list y r r
-put_list c n r
 
 put_list y y x
 put_list x y x
@@ -378,6 +454,13 @@ put_list y y r
 put_list y r x
 put_list r n x
 
+put_list x r x
+put_list x y r
+put_list y x r
+put_list y x x
+
+put_list x r r
+
 # put_list SrcReg Constant Dst
 put_list r c r
 put_list r c x
@@ -405,17 +488,9 @@ put_list c y x
 put_list c y y
 
 %cold
-put_list x r r
 put_list s s d
 %hot
 
-%macro: put Put
-put x
-put r
-put y
-put c
-put n
-
 %macro: i_fetch FetchArgs -pack
 i_fetch c c
 i_fetch c r
@@ -466,19 +541,20 @@ move_return n r
 
 move S r | deallocate D | return => move_deallocate_return S r D
 
-%macro: move_deallocate_return MoveDeallocateReturn -nonext
-move_deallocate_return x r P
-move_deallocate_return y r P
-move_deallocate_return c r P
-move_deallocate_return n r P
+%macro: move_deallocate_return MoveDeallocateReturn -pack -nonext
+move_deallocate_return x r Q
+move_deallocate_return y r Q
+move_deallocate_return c r Q
+move_deallocate_return n r Q
 
 deallocate D | return => deallocate_return D
 
 %macro: deallocate_return DeallocateReturn -nonext
-deallocate_return P
+deallocate_return Q
 
 test_heap Need u==1 | put_list Y=y r r => test_heap_1_put_list Need Y
 
+%macro: test_heap_1_put_list TestHeapPutList -pack
 test_heap_1_put_list I y
 
 # Test tuple & arity (head)
@@ -519,8 +595,6 @@ get_tuple_element Reg P Dst => i_get_tuple_element Reg P Dst | original_reg Reg
 
 original_reg Reg Pos =>
 
-get_tuple_element Reg P Dst => i_get_tuple_element Reg P Dst
-
 original_reg/2
 
 extract_next_element D1=xy | original_reg Reg P1 | get_tuple_element Reg P2 D2=xy | \
@@ -578,14 +652,14 @@ is_list f y
 
 is_nonempty_list Fail=f S=rx | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs
 
-%macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action
-is_nonempty_list_allocate f x I I
-is_nonempty_list_allocate f r I I
+%macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action -pack
+is_nonempty_list_allocate f x I t
+is_nonempty_list_allocate f r I t
 
 is_nonempty_list F=f r | test_heap I1 I2 => is_non_empty_list_test_heap F r I1 I2
 
-%macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action
-is_non_empty_list_test_heap f r I I
+%macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action -pack
+is_non_empty_list_test_heap f r I t
 
 %macro: is_nonempty_list IsNonemptyList -fail_action
 is_nonempty_list f x
@@ -771,11 +845,11 @@ call_ext_only u==3 u$func:erlang:apply/3 => i_apply_only
 # thus there is no need to generate any return instruction.
 #
 
-call_ext_last u==1 Bif=u$bif:erlang:exit/1 D => call_bif1 Bif
-call_ext_last u==1 Bif=u$bif:erlang:throw/1 D => call_bif1 Bif
+call_ext_last u==1 Bif=u$bif:erlang:exit/1 D => call_bif Bif
+call_ext_last u==1 Bif=u$bif:erlang:throw/1 D => call_bif Bif
 
-call_ext_only u==1 Bif=u$bif:erlang:exit/1 => call_bif1 Bif
-call_ext_only u==1 Bif=u$bif:erlang:throw/1 => call_bif1 Bif
+call_ext_only u==1 Bif=u$bif:erlang:exit/1 => call_bif Bif
+call_ext_only u==1 Bif=u$bif:erlang:throw/1 => call_bif Bif
 
 #
 # The error/1 and error/2 BIFs never execute the instruction following them;
@@ -785,13 +859,13 @@ call_ext_only u==1 Bif=u$bif:erlang:throw/1 => call_bif1 Bif
 # the continuation pointer on the stack.
 #
 
-call_ext_last u==1 Bif=u$bif:erlang:error/1 D => call_bif1 Bif
-call_ext_last u==2 Bif=u$bif:erlang:error/2 D => call_bif2 Bif
+call_ext_last u==1 Bif=u$bif:erlang:error/1 D => call_bif Bif
+call_ext_last u==2 Bif=u$bif:erlang:error/2 D => call_bif Bif
 
 call_ext_only Ar=u==1 Bif=u$bif:erlang:error/1 => \
-  allocate u Ar | call_bif1 Bif
+  allocate u Ar | call_bif Bif
 call_ext_only Ar=u==2 Bif=u$bif:erlang:error/2 => \
-  allocate u Ar | call_bif2 Bif
+  allocate u Ar | call_bif Bif
 
 #
 # The yield/0 BIF is an instruction
@@ -809,21 +883,93 @@ call_ext_last u==3 u$func:erlang:hibernate/3 D => i_hibernate
 call_ext_only u==3 u$func:erlang:hibernate/3 => i_hibernate
 
 #
-# Hybrid memory architecture need special cons and tuple instructions
-# that allocate on the message area. These looks like BIFs in the BEAM code.
-#
-
-call_ext u==2 u$func:hybrid:cons/2 => i_global_cons
-call_ext_last u==2 u$func:hybrid:cons/2 D => i_global_cons | deallocate_return D
-call_ext_only Ar=u==2 u$func:hybrid:cons/2 => i_global_cons | return
-
-call_ext u==1 u$func:hybrid:tuple/1 => i_global_tuple
-call_ext_last u==1 u$func:hybrid:tuple/1 D => i_global_tuple | deallocate_return D
-call_ext_only Ar=u==1 u$func:hybrid:tuple/1 => i_global_tuple | return
-
-call_ext u==1 u$func:hybrid:copy/1 => i_global_copy
-call_ext_last u==1 u$func:hybrid:copy/1 D => i_global_copy | deallocate_return D
-call_ext_only u==1 Ar=u$func:hybrid:copy/1 => i_global_copy | return
+# If VM probes are not enabled, we want to short-circult calls to
+# the dt tag BIFs to make them as cheap as possible.
+#
+
+%unless USE_VM_PROBES
+
+call_ext Arity u$func:erlang:dt_get_tag/0 => \
+    move a=am_undefined r
+call_ext_last Arity u$func:erlang:dt_get_tag/0 D => \
+    move a=am_undefined r | deallocate D | return
+call_ext_only Arity u$func:erlang:dt_get_tag/0 => \
+    move a=am_undefined r | return
+
+move Any r | call_ext Arity u$func:erlang:dt_put_tag/1 => \
+    move a=am_undefined r
+move Any r | call_ext_last Arity u$func:erlang:dt_put_tag/1 D => \
+    move a=am_undefined r | deallocate D | return
+move Any r | call_ext_only Arity u$func:erlang:dt_put_tag/1 => \
+    move a=am_undefined r | return
+call_ext Arity u$func:erlang:dt_put_tag/1 => \
+    move a=am_undefined r
+call_ext_last Arity u$func:erlang:dt_put_tag/1 D => \
+    move a=am_undefined r | deallocate D | return
+call_ext_only Arity u$func:erlang:dt_put_tag/1 => \
+    move a=am_undefined r | return
+
+call_ext Arity u$func:erlang:dt_get_tag_data/0 => \
+    move a=am_undefined r
+call_ext_last Arity u$func:erlang:dt_get_tag_data/0 D => \
+    move a=am_undefined r | deallocate D | return
+call_ext_only Arity u$func:erlang:dt_get_tag_data/0 => \
+    move a=am_undefined r | return
+
+move Any r | call_ext Arity u$func:erlang:dt_spread_tag/1 => \
+    move a=am_true r
+move Any r | call_ext_last Arity u$func:erlang:dt_spread_tag/1 D => \
+    move a=am_true r | deallocate D | return
+move Any r | call_ext_only Arity u$func:erlang:dt_spread_tag/1 => \
+    move a=am_true r | return
+call_ext Arity u$func:erlang:dt_spread_tag/1 => \
+    move a=am_true r
+call_ext_last Arity u$func:erlang:dt_spread_tag/1 D => \
+    move a=am_true r | deallocate D | return
+call_ext_only Arity u$func:erlang:dt_spread_tag/1 => \
+    move a=am_true r | return
+
+move Any r | call_ext Arity u$func:erlang:dt_restore_tag/1 => \
+    move a=am_true r
+move Any r | call_ext_last Arity u$func:erlang:dt_restore_tag/1 D => \
+    move a=am_true r | deallocate D | return
+move Any r | call_ext_only Arity u$func:erlang:dt_restore_tag/1 => \
+    move a=am_true r | return
+call_ext Arity u$func:erlang:dt_restore_tag/1 => \
+    move a=am_true r
+call_ext_last Arity u$func:erlang:dt_restore_tag/1 D => \
+    move a=am_true r | deallocate D | return
+call_ext_only Arity u$func:erlang:dt_restore_tag/1 => \
+    move a=am_true r | return
+
+move Any r | call_ext Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \
+    move Any r
+move Any r | call_ext_last Arity u$func:erlang:dt_prepend_vm_tag_data/1 D => \
+    move Any r | deallocate D | return
+move Any r | call_ext_only Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \
+    move Any r | return
+call_ext Arity u$func:erlang:dt_prepend_vm_tag_data/1 =>
+call_ext_last Arity u$func:erlang:dt_prepend_vm_tag_data/1 D => \
+    deallocate D | return
+call_ext_only Arity u$func:erlang:dt_prepend_vm_tag_data/1 => \
+    return
+
+move Any r | call_ext Arity u$func:erlang:dt_append_vm_tag_data/1 => \
+    move Any r
+move Any r | call_ext_last Arity u$func:erlang:dt_append_vm_tag_data/1 D => \
+    move Any r | deallocate D | return
+move Any r | call_ext_only Arity u$func:erlang:dt_append_vm_tag_data/1 => \
+    move Any r | return
+call_ext Arity u$func:erlang:dt_append_vm_tag_data/1 =>
+call_ext_last Arity u$func:erlang:dt_append_vm_tag_data/1 D => \
+    deallocate D | return
+call_ext_only Arity u$func:erlang:dt_append_vm_tag_data/1 => \
+    return
+
+# Can happen after one of the transformations above.
+move Discarded r | move Something r => move Something r
+
+%endif
 
 #
 # The general case for BIFs that have no special instructions.
@@ -832,24 +978,12 @@ call_ext_only u==1 Ar=u$func:hybrid:copy/1 => i_global_copy | return
 # To make trapping and stack backtraces work correctly, we make sure that
 # the continuation pointer is always stored on the stack.
 
-call_ext u==0 Bif=u$is_bif => call_bif0 Bif
-call_ext u==1 Bif=u$is_bif => call_bif1 Bif
-call_ext u==2 Bif=u$is_bif => call_bif2 Bif
-call_ext u==3 Bif=$is_bif => call_bif3 Bif
+call_ext u Bif=u$is_bif => call_bif Bif
 
-call_ext_last u==0 Bif=u$is_bif D => call_bif0 Bif | deallocate_return D
-call_ext_last u==1 Bif=u$is_bif D => call_bif1 Bif | deallocate_return D
-call_ext_last u==2 Bif=u$is_bif D => call_bif2 Bif | deallocate_return D
-call_ext_last u==3 Bif=u$is_bif D => call_bif3 Bif | deallocate_return D
+call_ext_last u Bif=u$is_bif D => call_bif Bif | deallocate_return D
 
-call_ext_only Ar=u==0 Bif=u$is_bif => \
-  allocate u Ar | call_bif0 Bif | deallocate_return u
-call_ext_only Ar=u==1 Bif=u$is_bif => \
-  allocate u Ar | call_bif1 Bif | deallocate_return u
-call_ext_only Ar=u==2 Bif=u$is_bif => \
-  allocate u Ar | call_bif2 Bif | deallocate_return u
-call_ext_only Ar=u==3 Bif=u$is_bif => \
-  allocate u Ar | call_bif3 Bif | deallocate_return u
+call_ext_only Ar=u Bif=u$is_bif => \
+  allocate u Ar | call_bif Bif | deallocate_return u
 
 #
 # Any remaining calls are calls to Erlang functions, not BIFs.
@@ -862,9 +996,9 @@ move S=c r | call_ext Ar=u Func=u$is_not_bif => i_move_call_ext S r Func
 move S=c r | call_ext_last Ar=u Func=u$is_not_bif D => i_move_call_ext_last Func D S r
 move S=c r | call_ext_only Ar=u Func=u$is_not_bif => i_move_call_ext_only Func S r 
 
-call_ext Ar=u Func        => i_call_ext Func
-call_ext_last Ar=u Func D => i_call_ext_last Func D
-call_ext_only Ar=u Func   => i_call_ext_only Func
+call_ext Ar Func        => i_call_ext Func
+call_ext_last Ar Func D => i_call_ext_last Func D
+call_ext_only Ar Func   => i_call_ext_only Func
 
 i_apply
 i_apply_last P
@@ -876,10 +1010,7 @@ i_apply_fun_only
 
 i_hibernate
 
-call_bif0 e
-call_bif1 e
-call_bif2 e
-call_bif3 e
+call_bif e
 
 #
 # Calls to non-building and guard BIFs.
@@ -898,7 +1029,7 @@ bif1 p Bif S1 Dst => bif1_body Bif S1 Dst
 bif1_body Bif Literal=q Dst => move Literal x | bif1_body Bif x Dst
 
 bif2 p Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2_body Bif Dst
-bif2 Fail=f Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2 Fail Bif Dst
+bif2 Fail Bif S1 S2 Dst => i_fetch S1 S2 | i_bif2 Fail Bif Dst
 
 i_get s d
 
@@ -914,8 +1045,13 @@ node x
 node y
 %hot
 
-i_fast_element j I s d
-i_element j s s d
+i_fast_element r j I d
+i_fast_element x j I d
+i_fast_element y j I d
+
+i_element r j s d
+i_element x j s d
+i_element y j s d
 
 bif1 f b s d
 bif1_body b s d
@@ -942,11 +1078,11 @@ move S r | call_last Ar P=f D => move_call_last S r P D
 
 i_move_call_last f P c r
 
-%macro:move_call_last MoveCallLast -arg_f -nonext
+%macro:move_call_last MoveCallLast -arg_f -nonext -pack
 
 move_call_last/4
-move_call_last x r f P
-move_call_last y r f P
+move_call_last x r f Q
+move_call_last y r f Q
 
 move S=c r | call_only Ar P=f => i_move_call_only P S r
 move S=x r | call_only Ar P=f => move_call_only S r P
@@ -976,8 +1112,8 @@ i_move_call_ext_only e c r
 
 # Fun calls.
 
-call_fun Arity=u | deallocate D | return => i_call_fun_last Arity D
-call_fun Arity=u => i_call_fun Arity
+call_fun Arity | deallocate D | return => i_call_fun_last Arity D
+call_fun Arity => i_call_fun Arity
 
 i_call_fun I
 i_call_fun_last I P
@@ -995,7 +1131,7 @@ is_function f y
 is_function f r
 is_function Fail=f c => jump Fail
 
-func_info M=a F=a A=u | label L => gen_func_info(M, F, A, L)
+func_info M F A => i_func_info u M F A
 
 # ================================================================
 # New bit syntax matching (R11B).
@@ -1165,7 +1301,7 @@ i_bs_init_heap I I I d
 i_bs_init_heap_bin_heap I I I d
 
 
-bs_init_bits Fail Sz Words Regs Flags Dst | binary_too_big_bits(Sz) => system_limit Fail
+bs_init_bits Fail Sz=o Words Regs Flags Dst => system_limit Fail
 
 bs_init_bits Fail Sz=u Words=u==0 Regs Flags Dst => i_bs_init_bits Sz Regs Dst
 bs_init_bits Fail Sz=u Words Regs Flags Dst =>  i_bs_init_bits_heap Sz Words Regs Dst
@@ -1184,12 +1320,6 @@ i_bs_init_bits_fail_heap I j I d
 i_bs_init_bits I I d
 i_bs_init_bits_heap I I I d
 
-bs_bits_to_bytes Fail Src Dst => i_bs_bits_to_bytes Src Fail Dst
-
-i_bs_bits_to_bytes r j d
-i_bs_bits_to_bytes x j d
-i_bs_bits_to_bytes y j d
-
 bs_add Fail S1=i==0 S2 Unit=u==1 D => move S2 D
 bs_add Fail S1 S2 Unit D => i_fetch S1 S2 | i_bs_add Fail Unit D
 
@@ -1239,13 +1369,13 @@ i_bs_utf16_size s d
 
 bs_put_utf8 Fail=j Flags=u Literal=q => \
 	move Literal x | bs_put_utf8 Fail Flags x
-bs_put_utf8 Fail=j u Src=s => i_bs_put_utf8 Fail Src
+bs_put_utf8 Fail u Src=s => i_bs_put_utf8 Fail Src
 
 i_bs_put_utf8 j s
 
 bs_put_utf16 Fail=j Flags=u Literal=q => \
 	move Literal x | bs_put_utf16 Fail Flags x
-bs_put_utf16 Fail=j Flags=u Src=s => i_bs_put_utf16 Fail Flags Src
+bs_put_utf16 Fail Flags=u Src=s => i_bs_put_utf16 Fail Flags Src
 
 i_bs_put_utf16 j I s
 
@@ -1311,10 +1441,12 @@ fmul p FR1 FR2 FR3 => i_fmul FR1 FR2 FR3
 fdiv p FR1 FR2 FR3 => i_fdiv FR1 FR2 FR3
 fnegate p FR1 FR2 => i_fnegate FR1 FR2
 
-fconv Int=iq Dst=l => move Int x | fconv x Dst
+fconv Arg=iqan Dst=l => move Arg x | fconv x Dst
 
 fmove q l
 fmove d l
+fmove l d
+
 fconv d l
 
 i_fadd l l l
@@ -1330,12 +1462,6 @@ fcheckerror p => i_fcheckerror
 i_fcheckerror
 fclearerror
 
-fmove FR=l Dst=d | new_float_allocation() => fmove_new FR Dst
- 
-# The new instruction for moving a float out of a floating point register.
-# (No allocation.)
-fmove_new l d
-
 #
 # New apply instructions in R10B.
 #
@@ -1344,7 +1470,21 @@ apply I
 apply_last I P
 
 #
-# New GCing arithmetic instructions.
+# Optimize addition and subtraction of small literals using
+# the i_increment/4 instruction (in bodies, not in guards).
+#
+
+gc_bif2 p Live u$bif:erlang:splus/2 Int=i Reg=d Dst => \
+	gen_increment(Reg, Int, Live, Dst)
+gc_bif2 p Live u$bif:erlang:splus/2 Reg=d Int=i Dst => \
+	gen_increment(Reg, Int, Live, Dst)
+
+gc_bif2 p Live u$bif:erlang:sminus/2 Reg=d Int=i Dst | \
+	negation_is_small(Int) => \
+	gen_increment_from_minus(Reg, Int, Live, Dst)
+
+#
+# GCing arithmetic instructions.
 #
 
 gc_bif2 Fail I u$bif:erlang:splus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_plus Fail I Dst
@@ -1367,6 +1507,10 @@ gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst
 gc_bif1 Fail I u$bif:erlang:sminus/1 Src Dst=d => i_fetch i Src | i_minus Fail I Dst
 gc_bif1 Fail I u$bif:erlang:splus/1 Src Dst=d => i_fetch i Src | i_plus Fail I Dst
 
+i_increment r I I d
+i_increment x I I d
+i_increment y I I d
+
 i_plus j I d
 i_minus j I d
 i_times j I d
@@ -1396,35 +1540,49 @@ bif1 Fail u$bif:erlang:trunc/1 s d => too_old_compiler
 #
 # Guard BIFs.
 #
-gc_bif1 Fail I Bif=u$bif:erlang:length/1 Src Dst=d => \
-	gen_guard_bif(Fail, I, Bif, Src, Dst)
+gc_bif1 Fail I Bif Src Dst => \
+	gen_guard_bif1(Fail, I, Bif, Src, Dst)
 
-gc_bif1 Fail I Bif=u$bif:erlang:size/1 Src Dst=d => \
-	gen_guard_bif(Fail, I, Bif, Src, Dst)
+gc_bif2 Fail I Bif S1 S2 Dst => \
+	gen_guard_bif2(Fail, I, Bif, S1, S2, Dst)
 
-gc_bif1 Fail I Bif=u$bif:erlang:bit_size/1 Src Dst=d => \
-	gen_guard_bif(Fail, I, Bif, Src, Dst)
+gc_bif3 Fail I Bif S1 S2 S3 Dst => \
+	gen_guard_bif3(Fail, I, Bif, S1, S2, S3, Dst)
 
-gc_bif1 Fail I Bif=u$bif:erlang:byte_size/1 Src Dst=d => \
-	gen_guard_bif(Fail, I, Bif, Src, Dst)
+i_gc_bif1 Fail Bif V=q Live D => move V x | i_gc_bif1 Fail Bif x Live D
 
-gc_bif1 Fail I Bif=u$bif:erlang:abs/1 Src Dst=d => \
-	gen_guard_bif(Fail, I, Bif, Src, Dst)
+i_gc_bif1 j I s I d
 
-gc_bif1 Fail I Bif=u$bif:erlang:float/1 Src Dst=d => \
-	gen_guard_bif(Fail, I, Bif, Src, Dst)
+ii_gc_bif2/6
 
-gc_bif1 Fail I Bif=u$bif:erlang:round/1 Src Dst=d => \
-	gen_guard_bif(Fail, I, Bif, Src, Dst)
+ii_gc_bif2 Fail Bif S1 S2 Live D => i_fetch S1 S2 | i_gc_bif2 Fail Bif Live D
 
-gc_bif1 Fail I Bif=u$bif:erlang:trunc/1 Src Dst=d => \
-	gen_guard_bif(Fail, I, Bif, Src, Dst)
+i_gc_bif2 j I I d
 
-i_gc_bif1 Fail Bif V=q Live D => move V x | i_gc_bif1 Fail Bif x Live D
+ii_gc_bif3/7
 
-i_gc_bif1 j I s I d
+ii_gc_bif3 Fail Bif S1 S2 S3 Live D => move S1 x | i_fetch S2 S3 | i_gc_bif3 Fail Bif x Live D
+
+i_gc_bif3 j I s I d
+
+#
+# The following instruction is specially handled in beam_load.c
+# to produce a user-friendly message if an unsupported guard BIF is
+# encountered.
+#
+unsupported_guard_bif/3
+unsupported_guard_bif A B C | never() =>
 
 #
 # R13B03
 #
 on_load
+
+#
+# R14A.
+#
+recv_mark f
+
+recv_set Fail | label Lbl | loop_rec Lf Reg => \
+   i_recv_set | label Lbl | loop_rec Lf Reg
+i_recv_set
diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c
index 8c8029d450..f1cfa8df39 100644
--- a/erts/emulator/beam/packet_parser.c
+++ b/erts/emulator/beam/packet_parser.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -47,11 +47,6 @@
                              (((unsigned char*) (s))[1] << 8) | \
                              (((unsigned char*) (s))[0]))
 
-#define put_int24(s, x) ((((unsigned char*)(s))[0] = ((x) >> 16) & 0xff), \
-                         (((unsigned char*)(s))[1] = ((x) >> 8) & 0xff), \
-                         (((unsigned char*)(s))[2] = (x) & 0xff))
-
-
 #if !defined(__WIN32__) && !defined(HAVE_STRNCASECMP)
 #define STRNCASECMP my_strncasecmp
 
@@ -306,7 +301,11 @@ int packet_get_length(enum PacketParseType htype,
         /* TCP_PB_LINE_LF:  [Data ... \n]  */
         const char* ptr2;
         if ((ptr2 = memchr(ptr, '\n', n)) == NULL) {
-            if (n >= trunc_len && trunc_len!=0) { /* buffer full */
+            if (n > max_plen && max_plen != 0) { /* packet full */
+                DEBUGF((" => packet full (no NL)=%d\r\n", n));
+                goto error;
+            }
+            else if (n >= trunc_len && trunc_len!=0) { /* buffer full */
                 DEBUGF((" => line buffer full (no NL)=%d\r\n", n));
                 return trunc_len;
             }
@@ -314,6 +313,10 @@ int packet_get_length(enum PacketParseType htype,
         }
         else {
             int len = (ptr2 - ptr) + 1; /* including newline */
+            if (len > max_plen && max_plen!=0) {
+                DEBUGF((" => packet_size %d exceeded\r\n", max_plen));
+                goto error;
+            }
             if (len > trunc_len && trunc_len!=0) {
                 DEBUGF((" => truncated line=%d\r\n", trunc_len));
                 return trunc_len;
@@ -402,33 +405,50 @@ int packet_get_length(enum PacketParseType htype,
             const char* ptr1 = ptr;
             int   len = plen;
             
+	    if (!max_plen) {
+		/* This is for backward compatibility with old user of decode_packet
+		 * that might use option 'line_length' to limit accepted length of
+		 * http lines.
+		 */
+		max_plen = trunc_len;
+	    }
+
             while (1) {
                 const char* ptr2 = memchr(ptr1, '\n', len);
                 
                 if (ptr2 == NULL) {
-                    if (n >= trunc_len && trunc_len!=0) { /* buffer full */
-                        plen = trunc_len;
-                        goto done;
+                    if (max_plen != 0) {
+                        if (n >= max_plen) /* packet full */
+                            goto error;
                     }
                     goto more;
                 }
                 else {
                     plen = (ptr2 - ptr) + 1;
-                    
-                    if (*statep == 0)
+
+                    if (*statep == 0) {
+                        if (max_plen != 0 && plen > max_plen)
+                            goto error;
                         goto done;
-                    
+                    }
+
                     if (plen < n) {
                         if (SP(ptr2+1) && plen>2) {
                             /* header field value continue on next line */
                             ptr1 = ptr2+1;
                             len = n - plen;
                         }
-                        else
+                        else {
+                            if (max_plen != 0 && plen > max_plen)
+                                goto error;
                             goto done;
+                        }
                     }
-                    else
+                    else {
+                        if (max_plen != 0 && plen > max_plen)
+                            goto error;
                         goto more;
+                    }
                 }
             }
         }
@@ -679,7 +699,7 @@ int packet_parse_http(const char* buf, int len, int* statep,
             while (n && SP(ptr)) {
                 ptr++; n--;
             }
-            if (ptr==p0) return -1;
+            if (ptr==p0 && n>0) return -1;
             
             /* NOTE: the syntax allows empty reason phrases */
             (*statep) = !0;
@@ -833,7 +853,7 @@ int packet_parse_ssl(const char* buf, int len,
         char prefix[4];
         /* <<1:8,Length:24,Data/binary>> */
         prefix[0] = 1;
-        put_int24(&prefix[1],len-3);
+        put_int24(len-3,&prefix[1]);
         return pcb->ssl_tls(arg, 22, major, minor, buf+3, len-3, prefix, sizeof(prefix));
     } 
     else {
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index 964c10a380..26d64887d0 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -39,8 +39,6 @@ static Hash process_reg;
 
 static erts_smp_rwmtx_t regtab_rwmtx;
 
-#define reg_lock_init()			erts_smp_rwmtx_init(&regtab_rwmtx, \
-							    "reg_tab")
 #define reg_try_read_lock()		erts_smp_rwmtx_tryrlock(&regtab_rwmtx)
 #define reg_try_write_lock()		erts_smp_rwmtx_tryrwlock(&regtab_rwmtx)
 #define reg_read_lock()			erts_smp_rwmtx_rlock(&regtab_rwmtx)
@@ -147,8 +145,11 @@ static void reg_free(RegProc *obj)
 void init_register_table(void)
 {
     HashFunctions f;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
+    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
+    rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
-    reg_lock_init();
+    erts_smp_rwmtx_init_opt(&regtab_rwmtx, &rwmtx_opt, "reg_tab");
 
     f.hash = (H_FUN) reg_hash;
     f.cmp  = (HCMP_FUN) reg_cmp;
@@ -476,8 +477,9 @@ int erts_unregister_name(Process *c_p,
      *           on c_prt.
      */
 
-    if (!c_p)
+    if (!c_p) {
 	c_p_locks = 0;
+    }
     current_c_p_locks = c_p_locks;
 
  restart:
@@ -489,9 +491,15 @@ int erts_unregister_name(Process *c_p,
     if (is_non_value(name)) {
 	/* Unregister current process name */
 	ASSERT(c_p);
-	if (c_p->reg)
+#ifdef ERTS_SMP
+	if (current_c_p_locks != c_p_locks) {
+	    erts_smp_proc_lock(c_p, c_p_locks);
+	    current_c_p_locks = c_p_locks;
+	}
+#endif
+	if (c_p->reg) {
 	    r.name = c_p->reg->name;
-	else {
+	} else {
 	    /* Name got unregistered while main lock was released */
 	    res = 0;
 	    goto done;
@@ -533,24 +541,25 @@ int erts_unregister_name(Process *c_p,
 	    }
 
 	} else if (rp->p) {
-	    Process* p = rp->p;
+
 #ifdef ERTS_SMP
 	    erts_proc_safelock(c_p,
 			       current_c_p_locks,
 			       c_p_locks,
 			       rp->p,
-			       0,
+			       (c_p == rp->p) ?  current_c_p_locks : 0,
 			       ERTS_PROC_LOCK_MAIN);
 	    current_c_p_locks = c_p_locks;
 #endif
-	    p->reg = NULL;
+	    rp->p->reg = NULL;
+	    if (IS_TRACED_FL(rp->p, F_TRACE_PROCS)) {
+		trace_proc(c_p, rp->p, am_unregister, r.name);
+	    }
 #ifdef ERTS_SMP
-	    if (rp->p != c_p)
+	    if (rp->p != c_p) {
 		erts_smp_proc_unlock(rp->p, ERTS_PROC_LOCK_MAIN);
-#endif
-	    if (IS_TRACED_FL(p, F_TRACE_PROCS)) {
-		trace_proc(c_p, p, am_unregister, r.name);
 	    }
+#endif
 	}
 	hash_erase(&process_reg, (void*) &r);
 	res = 1;
@@ -560,14 +569,17 @@ int erts_unregister_name(Process *c_p,
 
     reg_write_unlock();
     if (c_prt != port) {
-	if (port)
+	if (port) {
 	    erts_smp_port_unlock(port);
-	if (c_prt)
+	}
+	if (c_prt) {
 	    erts_smp_port_lock(c_prt);
+	}
     }
 #ifdef ERTS_SMP
-    if (c_p && !current_c_p_locks)
+    if (c_p && !current_c_p_locks) {
 	erts_smp_proc_lock(c_p, c_p_locks);
+    }
 #endif
     return res;
 }
diff --git a/erts/emulator/beam/register.h b/erts/emulator/beam/register.h
index 97bab3ab71..38e8cfbf28 100644
--- a/erts/emulator/beam/register.h
+++ b/erts/emulator/beam/register.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -41,7 +41,7 @@ struct port;
 typedef struct reg_proc
 {
     HashBucket bucket;  /* MUST BE LOCATED AT TOP OF STRUCT!!! */
-    Process *p;         /* The process registerd (only one of this and
+    Process *p;         /* The process registered (only one of this and
 			   'pt' is non-NULL */
     struct port *pt;    /* The port registered */
     Eterm name;         /* Atom name */
diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c
index 21d6ce9304..3326e5cc2a 100644
--- a/erts/emulator/beam/safe_hash.c
+++ b/erts/emulator/beam/safe_hash.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -61,7 +61,7 @@ static ERTS_INLINE int align_up_pow2(int val)
 */
 static void rehash(SafeHash* h, int grow_limit)
 {
-    if (erts_smp_atomic_xchg(&h->is_rehashing, 1) != 0) {        
+    if (erts_smp_atomic_xchg_acqb(&h->is_rehashing, 1) != 0) {        
 	return; /* already in progress */
     }
     if (h->grow_limit == grow_limit) {
@@ -99,7 +99,7 @@ static void rehash(SafeHash* h, int grow_limit)
 	erts_free(h->type, (void *) old_tab);
     }
     /*else already done */
-    erts_smp_atomic_set(&h->is_rehashing, 0);
+    erts_smp_atomic_set_relb(&h->is_rehashing, 0);
 }
 
 
@@ -166,8 +166,8 @@ SafeHash* safe_hash_init(ErtsAlcType_t type, SafeHash* h, char* name, int size,
     h->name = name;
     h->fun = fun;
     set_size(h,size);
-    erts_smp_atomic_init(&h->is_rehashing, 0);
-    erts_smp_atomic_init(&h->nitems, 0);
+    erts_smp_atomic_init_nob(&h->is_rehashing, 0);
+    erts_smp_atomic_init_nob(&h->nitems, 0);
     for (i=0; i<SAFE_HASH_LOCK_CNT; i++) {
 	erts_smp_mtx_init(&h->lock_vec[i].mtx,"safe_hash");
     }
@@ -222,7 +222,7 @@ void* safe_hash_put(SafeHash* h, void* tmpl)
     *head = b;
     grow_limit = h->grow_limit;
     erts_smp_mtx_unlock(lock);
-    if (erts_smp_atomic_inctest(&h->nitems) > grow_limit) {
+    if (erts_smp_atomic_inc_read_nob(&h->nitems) > grow_limit) {
 	rehash(h, grow_limit);
     }
     return (void*) b;
@@ -245,7 +245,7 @@ void* safe_hash_erase(SafeHash* h, void* tmpl)
 	if ((b->hvalue == hval) && (h->fun.cmp(tmpl, (void*)b) == 0)) {
 	    *prevp = b->next;
 	    erts_smp_mtx_unlock(lock);
-	    erts_smp_atomic_dec(&h->nitems);
+	    erts_smp_atomic_dec_nob(&h->nitems);
 	    h->fun.free((void*)b);
 	    return tmpl;
 	}
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 4b949523fa..7b2bb81f62 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -1,37 +1,30 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
 #ifndef __SYS_H__
 #define __SYS_H__
 
+
 #if defined(VALGRIND) && !defined(NO_FPE_SIGNALS)
 #  define NO_FPE_SIGNALS
 #endif
 
-/* Never use elib-malloc when purify-memory-tracing */
-#if defined(PURIFY)
-#undef ENABLE_ELIB_MALLOC
-#undef ELIB_HEAP_SBRK
-#undef ELIB_ALLOC_IS_CLIB
-#endif
-
-
 /* xxxP __VXWORKS__ */
 #ifdef VXWORKS
 #include <vxWorks.h>
@@ -46,19 +39,10 @@
 #define ENABLE_CHILD_WAITER_THREAD 1
 #endif
 
-/* The ERTS_TIMER_TREAD #define must be visible to the
-   erl_${OS}_sys.h #include files: it controls whether
-   certain optional facilities should be defined or not. */
-#if defined(ERTS_SMP) && 0
-#define ERTS_TIMER_THREAD
-#endif
-
 #if defined (__WIN32__)
 #  include "erl_win_sys.h"
 #elif defined (VXWORKS) 
 #  include "erl_vxworks_sys.h"
-#elif defined (_OSE_) 
-#  include "erl_ose_sys.h"
 #else 
 #  include "erl_unix_sys.h"
 #ifndef UNIX
@@ -119,6 +103,15 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
 #  define ERTS_LIKELY(BOOL)   (BOOL)
 #  define ERTS_UNLIKELY(BOOL) (BOOL)
 #endif
+#ifdef __GNUC__
+#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 5)
+#    define ERTS_DECLARE_DUMMY(X) X __attribute__ ((unused))
+#  else
+#    define ERTS_DECLARE_DUMMY(X) X
+#  endif
+#else
+#  define ERTS_DECLARE_DUMMY(X) X
+#endif
 
 #if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
 #  undef ERTS_CAN_INLINE
@@ -172,23 +165,6 @@ void erl_assert_error(char* expr, char* file, int line);
 
 #include <stdarg.h>
 
-#if defined(__STDC__) || defined(_MSC_VER)
-#  define EXTERN_FUNCTION(t, f, x)  extern t f x
-#  define FUNCTION(t, f, x) t f x
-#  define _DOTS_ ...
-#  define _VOID_      void
-#elif defined(__cplusplus)
-#  define EXTERN_FUNCTION(f, x) extern "C" { f x }
-#  define FUNCTION(t, f, x) t f x
-#  define _DOTS_ ...
-#  define _VOID_    void
-#else
-#  define EXTERN_FUNCTION(t, f, x) extern t f (/*x*/)
-#  define FUNCTION(t, f, x) t f (/*x*/)
-#  define _DOTS_
-#  define _VOID_
-#endif
-
 /* This isn't sys-dependent, but putting it here benefits sys.c and drivers
    - allow use of 'const' regardless of compiler */
 
@@ -198,7 +174,7 @@ void erl_assert_error(char* expr, char* file, int line);
 
 #ifdef VXWORKS
 /* Replace VxWorks' printf with a real one that does fprintf(stdout, ...) */
-EXTERN_FUNCTION(int, real_printf, (const char *fmt, ...));
+int real_printf(const char *fmt, ...);
 #  define printf real_printf
 #endif
 
@@ -230,9 +206,14 @@ EXTERN_FUNCTION(int, real_printf, (const char *fmt, ...));
 ** Data types:
 **
 ** Eterm: A tagged erlang term (possibly 64 bits)
+** BeamInstr: A beam code instruction unit, possibly larger than Eterm, not smaller.
 ** UInt:  An unsigned integer exactly as large as an Eterm.
 ** SInt:  A signed integer exactly as large as an eterm and therefor large
 **        enough to hold the return value of the signed_val() macro.
+** UWord: An unsigned integer at least as large as a void * and also as large
+**          or larger than an Eterm
+** SWord: A signed integer at least as large as a void * and also as large
+**          or larger than an Eterm
 ** Uint32: An unsigned integer of 32 bits exactly
 ** Sint32: A signed integer of 32 bits exactly
 ** Uint16: An unsigned integer of 16 bits exactly
@@ -240,7 +221,8 @@ EXTERN_FUNCTION(int, real_printf, (const char *fmt, ...));
 */
 
 #if !((SIZEOF_VOID_P >= 4) && (SIZEOF_VOID_P == SIZEOF_SIZE_T) \
-      && ((SIZEOF_VOID_P == SIZEOF_INT) || (SIZEOF_VOID_P == SIZEOF_LONG)))
+      && ((SIZEOF_VOID_P == SIZEOF_INT) || (SIZEOF_VOID_P == SIZEOF_LONG) || \
+          (SIZEOF_VOID_P == SIZEOF_LONG_LONG)))
 #error Cannot handle this combination of int/long/void*/size_t sizes
 #endif
 
@@ -253,25 +235,92 @@ EXTERN_FUNCTION(int, real_printf, (const char *fmt, ...));
 #else
 #error Neither 32 nor 64 bit architecture
 #endif
+#if defined(ARCH_64) && defined(HALFWORD_HEAP_EMULATOR)
+#    define HALFWORD_HEAP 1
+#    define HALFWORD_ASSERT 0
+#    define ASSERT_HALFWORD(COND) ASSERT(COND)
+#else
+#    define HALFWORD_HEAP 0
+#    define HALFWORD_ASSERT 0
+#    define ASSERT_HALFWORD(COND)
+#endif
 
 #if SIZEOF_VOID_P != SIZEOF_SIZE_T
 #error sizeof(void*) != sizeof(size_t)
 #endif
 
+#if HALFWORD_HEAP
+
+#if SIZEOF_INT == 4
+typedef unsigned int Eterm;
+typedef unsigned int Uint;
+typedef int          Sint;
+#define ERTS_SIZEOF_ETERM SIZEOF_INT
+#define ErtsStrToSint strtol
+#else
+#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
+#endif
+
+#if SIZEOF_VOID_P == SIZEOF_LONG
+typedef unsigned long UWord;
+typedef long          SWord;
+#define SWORD_CONSTANT(Const) Const##L
+#define UWORD_CONSTANT(Const) Const##UL
+#elif SIZEOF_VOID_P == SIZEOF_INT
+typedef unsigned int UWord;
+typedef int          SWord;
+#define SWORD_CONSTANT(Const) Const
+#define UWORD_CONSTANT(Const) Const##U
+#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
+typedef unsigned long long UWord;
+typedef long long          SWord;
+#define SWORD_CONSTANT(Const) Const##LL
+#define UWORD_CONSTANT(Const) Const##ULL
+#else
+#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
+#endif
+
+#else /* !HALFWORD_HEAP */
+
 #if SIZEOF_VOID_P == SIZEOF_LONG
 typedef unsigned long Eterm;
 typedef unsigned long Uint;
 typedef long          Sint;
+#define SWORD_CONSTANT(Const) Const##L
+#define UWORD_CONSTANT(Const) Const##UL
 #define ERTS_SIZEOF_ETERM SIZEOF_LONG
+#define ErtsStrToSint strtol
 #elif SIZEOF_VOID_P == SIZEOF_INT
 typedef unsigned int Eterm;
 typedef unsigned int Uint;
 typedef int          Sint;
+#define SWORD_CONSTANT(Const) Const
+#define UWORD_CONSTANT(Const) Const##U
 #define ERTS_SIZEOF_ETERM SIZEOF_INT
+#define ErtsStrToSint strtol
+#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
+typedef unsigned long long Eterm;
+typedef unsigned long long Uint;
+typedef long long          Sint;
+#define SWORD_CONSTANT(Const) Const##LL
+#define UWORD_CONSTANT(Const) Const##ULL
+#define ERTS_SIZEOF_ETERM SIZEOF_LONG_LONG
+#if defined(__WIN32__)
+#define ErtsStrToSint _strtoi64
+#else
+#define ErtsStrToSint strtoll
+#endif
 #else
 #error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
 #endif
 
+typedef Uint UWord;
+typedef Sint SWord;
+
+#endif /* HALFWORD_HEAP */
+
+typedef UWord BeamInstr;
+
 #ifndef HAVE_INT64
 #if SIZEOF_LONG == 8
 #define HAVE_INT64 1
@@ -316,23 +365,21 @@ typedef unsigned char byte;
 #error 64-bit architecture, but no appropriate type to use for Uint64 and Sint64 found 
 #endif
 
-#if defined(ARCH_64)
-#  define ERTS_WORD_ALIGN_PAD_SZ(X) \
+#  define ERTS_EXTRA_DATA_ALIGN_SZ(X) \
     (((size_t) 8) - (((size_t) (X)) & ((size_t) 7)))
-#elif defined(ARCH_32)
-#  define ERTS_WORD_ALIGN_PAD_SZ(X) \
-    (((size_t) 4) - (((size_t) (X)) & ((size_t) 3)))
-#else
-#error "Not supported..."
-#endif
 
 #include "erl_lock_check.h"
+
+/* needed by erl_smp.h */
+int erts_send_warning_to_logger_str_nogl(char *);
+
 #include "erl_smp.h"
 
 #ifdef ERTS_WANT_BREAK_HANDLING
 #  ifdef ERTS_SMP
-extern erts_smp_atomic_t erts_break_requested;
-#    define ERTS_BREAK_REQUESTED ((int) erts_smp_atomic_read(&erts_break_requested))
+extern erts_smp_atomic32_t erts_break_requested;
+#    define ERTS_BREAK_REQUESTED \
+  ((int) erts_smp_atomic32_read_nob(&erts_break_requested))
 #  else
 extern volatile int erts_break_requested;
 #    define ERTS_BREAK_REQUESTED erts_break_requested
@@ -345,8 +392,8 @@ void erts_do_break_handling(void);
 #    define ERTS_GOT_SIGUSR1 0
 #  else
 #    ifdef ERTS_SMP
-extern erts_smp_atomic_t erts_got_sigusr1;
-#      define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic_read(&erts_got_sigusr1))
+extern erts_smp_atomic32_t erts_got_sigusr1;
+#      define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic32_read_mb(&erts_got_sigusr1))
 #    else
 extern volatile int erts_got_sigusr1;
 #      define ERTS_GOT_SIGUSR1 erts_got_sigusr1
@@ -355,11 +402,15 @@ extern volatile int erts_got_sigusr1;
 #endif
 
 #ifdef ERTS_SMP
-extern erts_smp_atomic_t erts_writing_erl_crash_dump;
+extern erts_smp_atomic32_t erts_writing_erl_crash_dump;
+extern erts_tsd_key_t erts_is_crash_dumping_key;
+#define ERTS_SOMEONE_IS_CRASH_DUMPING \
+  ((int) erts_smp_atomic32_read_mb(&erts_writing_erl_crash_dump))
 #define ERTS_IS_CRASH_DUMPING \
-  ((int) erts_smp_atomic_read(&erts_writing_erl_crash_dump))
+  ((int) (SWord) erts_tsd_get(erts_is_crash_dumping_key))
 #else
 extern volatile int erts_writing_erl_crash_dump;
+#define ERTS_SOMEONE_IS_CRASH_DUMPING erts_writing_erl_crash_dump
 #define ERTS_IS_CRASH_DUMPING erts_writing_erl_crash_dump
 #endif
 
@@ -413,13 +464,6 @@ extern volatile int erts_writing_erl_crash_dump;
    in non-blocking mode - and ioctl FIONBIO on AIX *doesn't* work for
    pipes or ttys (O_NONBLOCK does)!!! For now, we'll use FIONBIO for AIX. */
 
-# ifdef _OSE_
-static const int zero_value = 0, one_value = 1;
-#      define SET_BLOCKING(fd)	        ioctl((fd), FIONBIO, (char*)&zero_value)
-#      define SET_NONBLOCKING(fd)	ioctl((fd), FIONBIO, (char*)&one_value)
-#      define ERRNO_BLOCK EWOULDBLOCK
-# else
-
 #  ifdef __WIN32__
 
 static unsigned long zero_value = 0, one_value = 1;
@@ -460,26 +504,14 @@ static const int zero_value = 0, one_value = 1;
 #      endif /* !NB_FIONBIO */
 #    endif /* _WXWORKS_ */
 #  endif /* !__WIN32__ */
-# endif /* _OSE_ */
 #endif /* WANT_NONBLOCKING */
 
-extern erts_cpu_info_t *erts_cpuinfo; /* erl_init.c */
-
 __decl_noreturn void __noreturn erl_exit(int n, char*, ...);
 
 /* Some special erl_exit() codes: */
 #define ERTS_INTR_EXIT	INT_MIN		/* called from signal handler */
 #define ERTS_ABORT_EXIT	(INT_MIN + 1)	/* no crash dump; only abort() */
-#define ERTS_DUMP_EXIT	(127)		/* crash dump; then exit() */
-
-
-#ifndef ERTS_SMP
-int check_async_ready(void);
-#ifdef USE_THREADS
-void sys_async_ready(int hndl);
-int erts_register_async_ready_callback(void (*funcp)(void));
-#endif
-#endif
+#define ERTS_DUMP_EXIT	(INT_MIN + 2)	/* crash dump; then exit() */
 
 Eterm erts_check_io_info(void *p);
 
@@ -523,7 +555,8 @@ int erts_send_info_to_logger_nogl(erts_dsprintf_buf_t *);
 int erts_send_warning_to_logger_nogl(erts_dsprintf_buf_t *);
 int erts_send_error_to_logger_nogl(erts_dsprintf_buf_t *);
 int erts_send_info_to_logger_str_nogl(char *);
-int erts_send_warning_to_logger_str_nogl(char *);
+/* needed by erl_smp.h (declared above)
+   int erts_send_warning_to_logger_str_nogl(char *); */
 int erts_send_error_to_logger_str_nogl(char *);
 
 typedef struct preload {
@@ -538,11 +571,9 @@ typedef struct preload {
  * None of the drivers use all of the fields.
  */
 
-/* OSE: Want process_type and priority in here as well! Needs updates in erl_bif_ports.c! */
-
 typedef struct _SysDriverOpts {
-    int ifd;			/* Input file descriptor (fd driver). */
-    int ofd;			/* Outputfile descriptor (fd driver). */
+    Uint ifd;			/* Input file descriptor (fd driver). */
+    Uint ofd;			/* Outputfile descriptor (fd driver). */
     int packet_bytes;		/* Number of bytes in packet header. */
     int read_write;		/* Read and write bits. */
     int use_stdio;		/* Use standard I/O: TRUE or FALSE. */
@@ -556,12 +587,6 @@ typedef struct _SysDriverOpts {
     char *wd;			/* Working directory. */
     unsigned spawn_type;        /* Bitfield of ERTS_SPAWN_DRIVER | 
 				   ERTS_SPAWN_EXTERNAL | both*/ 
-
-#ifdef _OSE_
-    enum PROCESS_TYPE process_type;
-    OSPRIORITY priority;
-#endif /* _OSE_ */
-
 } SysDriverOpts;
 
 extern char *erts_default_arg0;
@@ -569,11 +594,7 @@ extern char *erts_default_arg0;
 extern char os_type[];
 
 extern int sys_init_time(void);
-#if defined(ERTS_TIMER_THREAD)
-#define erts_deliver_time()
-#else
 extern void erts_deliver_time(void);
-#endif
 extern void erts_time_remaining(SysTimeval *);
 extern int erts_init_time_sup(void);
 extern void erts_sys_init_float(void);
@@ -621,16 +642,14 @@ extern char *erts_sys_ddll_error(int code);
 
 
 /*
- * System interfaces for startup/sae code (functions found in respective sys.c)
+ * System interfaces for startup.
  */
+#include "erl_time.h"
 
-
-#ifdef ERTS_SMP
 void erts_sys_schedule_interrupt(int set);
-void erts_sys_schedule_interrupt_timed(int set, long msec);
+#ifdef ERTS_SMP
+void erts_sys_schedule_interrupt_timed(int set, erts_short_time_t msec);
 void erts_sys_main_thread(void);
-#else
-#define erts_sys_schedule_interrupt(Set)
 #endif
 
 extern void erts_sys_prepare_crash_dump(void);
@@ -638,35 +657,38 @@ extern void erts_sys_pre_init(void);
 extern void erl_sys_init(void);
 extern void erl_sys_args(int *argc, char **argv);
 extern void erl_sys_schedule(int);
-#ifdef _OSE_
-extern void erl_sys_init_final(void);
-#else
-void sys_tty_reset(void);
-#endif
+void sys_tty_reset(int);
 
-EXTERN_FUNCTION(int, sys_max_files, (_VOID_));
+int sys_max_files(void);
 void sys_init_io(void);
 Preload* sys_preloaded(void);
-EXTERN_FUNCTION(unsigned char*, sys_preload_begin, (Preload*));
-EXTERN_FUNCTION(void, sys_preload_end, (Preload*));
-EXTERN_FUNCTION(int, sys_get_key, (int));
-void elapsed_time_both(unsigned long *ms_user, unsigned long *ms_sys, 
-		       unsigned long *ms_user_diff, unsigned long *ms_sys_diff);
-void wall_clock_elapsed_time_both(unsigned long *ms_total, 
-				  unsigned long *ms_diff);
+unsigned char* sys_preload_begin(Preload*);
+void sys_preload_end(Preload*);
+int sys_get_key(int);
+void elapsed_time_both(UWord *ms_user, UWord *ms_sys, 
+		       UWord *ms_user_diff, UWord *ms_sys_diff);
+void wall_clock_elapsed_time_both(UWord *ms_total, 
+				  UWord *ms_diff);
 void get_time(int *hour, int *minute, int *second);
 void get_date(int *year, int *month, int *day);
 void get_localtime(int *year, int *month, int *day, 
 		   int *hour, int *minute, int *second);
 void get_universaltime(int *year, int *month, int *day, 
 		       int *hour, int *minute, int *second);
-int univ_to_local(Sint *year, Sint *month, Sint *day, 
+int seconds_to_univ(Sint64 seconds, 
+		    Sint *year, Sint *month, Sint *day, 
+		    Sint *hour, Sint *minute, Sint *second);
+int univ_to_seconds(Sint year, Sint month, Sint day, 
+		    Sint hour, Sint minute, Sint second,
+		    Sint64* seconds);
+int univ_to_local(
+    Sint *year, Sint *month, Sint *day, 
 		  Sint *hour, Sint *minute, Sint *second);
 int local_to_univ(Sint *year, Sint *month, Sint *day, 
 		  Sint *hour, Sint *minute, Sint *second, int isdst);
 void get_now(Uint*, Uint*, Uint*);
 void get_sys_now(Uint*, Uint*, Uint*);
-EXTERN_FUNCTION(void, set_break_quit, (void (*)(void), void (*)(void)));
+void set_break_quit(void (*)(void), void (*)(void));
 
 void os_flavor(char*, unsigned);
 void os_version(int*, int*, int*);
@@ -686,6 +708,8 @@ int erts_sys_putenv(char *key_value, int sep_ix);
    *size), a value > 0 if value buffer is too small (*size is set to needed
    size), and a value < 0 on failure. */
 int erts_sys_getenv(char *key, char *value, size_t *size);
+/* erts_sys_getenv__() is only allowed to be used in early init phase */
+int erts_sys_getenv__(char *key, char *value, size_t *size);
 
 /* Easier to use, but not as efficient, environment functions */
 char *erts_read_env(char *key);
@@ -706,293 +730,16 @@ int erts_write_env(char *key, char *value);
 #define ERTS_DEFAULT_MMAP_THRESHOLD  (128 * 1024)
 #define ERTS_DEFAULT_MMAP_MAX        64
 
-EXTERN_FUNCTION(int, sys_alloc_opt, (int, int));
+int sys_alloc_opt(int, int);
 
 typedef struct {
-  Sint trim_threshold;
-  Sint top_pad;
-  Sint mmap_threshold;
-  Sint mmap_max;
+  int trim_threshold;
+  int top_pad;
+  int mmap_threshold;
+  int mmap_max;
 } SysAllocStat;
 
-EXTERN_FUNCTION(void, sys_alloc_stat, (SysAllocStat *));
-
-/* Block the whole system... */
-
-#define ERTS_BS_FLG_ALLOW_GC				(((Uint32) 1) << 0)
-#define ERTS_BS_FLG_ALLOW_IO				(((Uint32) 1) << 1)
-
-/* Activities... */
-typedef enum {
-    ERTS_ACTIVITY_UNDEFINED,	/* Undefined activity */
-    ERTS_ACTIVITY_WAIT,		/* Waiting */
-    ERTS_ACTIVITY_GC,		/* Garbage collecting */
-    ERTS_ACTIVITY_IO		/* I/O including message passing to erl procs */
-} erts_activity_t;
-
-#ifdef ERTS_SMP
-
-typedef enum {
-    ERTS_ACT_ERR_LEAVE_WAIT_UNLOCKED,
-    ERTS_ACT_ERR_LEAVE_UNKNOWN_ACTIVITY,
-    ERTS_ACT_ERR_ENTER_UNKNOWN_ACTIVITY
-} erts_activity_error_t;
-
-typedef struct {
-    erts_smp_atomic_t do_block;
-    struct {
-	erts_smp_atomic_t wait;
-	erts_smp_atomic_t gc;
-	erts_smp_atomic_t io;
-    } in_activity;
-} erts_system_block_state_t;
-
-extern erts_system_block_state_t erts_system_block_state;
-
-int erts_is_system_blocked(erts_activity_t allowed_activities);
-void erts_block_me(void (*prepare)(void *), void (*resume)(void *), void *arg);
-void erts_register_blockable_thread(void);
-void erts_unregister_blockable_thread(void);
-void erts_note_activity_begin(erts_activity_t activity);
-void
-erts_check_block(erts_activity_t old_activity,
-		 erts_activity_t new_activity,
-		 int locked,
-		 void (*prepare)(void *),
-		 void (*resume)(void *),
-		 void *arg);
-void erts_block_system(Uint32 allowed_activities);
-int erts_emergency_block_system(long timeout, Uint32 allowed_activities);
-void erts_release_system(void);
-void erts_system_block_init(void);
-void erts_set_activity_error(erts_activity_error_t, char *, int);
-#ifdef ERTS_ENABLE_LOCK_CHECK
-void erts_lc_activity_change_begin(void);
-void erts_lc_activity_change_end(void);
-int erts_lc_is_blocking(void);
-#define ERTS_LC_IS_BLOCKING \
-  (erts_smp_pending_system_block() && erts_lc_is_blocking())
-#endif
-#endif
-
-#define erts_smp_activity_begin(NACT, PRP, RSM, ARG)		\
-  erts_smp_set_activity(ERTS_ACTIVITY_UNDEFINED,		\
-			(NACT),					\
-			0,					\
-			(PRP),					\
-			(RSM),					\
-			(ARG),					\
-			__FILE__,				\
-			__LINE__)
-#define erts_smp_activity_change(OACT, NACT, PRP, RSM, ARG)	\
-  erts_smp_set_activity((OACT),					\
-			(NACT),					\
-			0,					\
-			(PRP),					\
-			(RSM),					\
-			(ARG),					\
-			__FILE__,				\
-			__LINE__)
-#define erts_smp_activity_end(OACT, PRP, RSM, ARG)		\
-  erts_smp_set_activity((OACT),					\
-			ERTS_ACTIVITY_UNDEFINED,		\
-			0,					\
-			(PRP),					\
-			(RSM),					\
-			(ARG),					\
-			__FILE__,				\
-			__LINE__)
-
-#define erts_smp_locked_activity_begin(NACT)			\
-  erts_smp_set_activity(ERTS_ACTIVITY_UNDEFINED,		\
-			(NACT),					\
-			1,					\
-			NULL,					\
-			NULL,					\
-			NULL,					\
-			__FILE__,				\
-			__LINE__)
-#define erts_smp_locked_activity_change(OACT, NACT)		\
-  erts_smp_set_activity((OACT),					\
-			(NACT),					\
-			1,					\
-			NULL,					\
-			NULL,					\
-			NULL,					\
-			__FILE__,				\
-			__LINE__)
-#define erts_smp_locked_activity_end(OACT)			\
-  erts_smp_set_activity((OACT),					\
-			ERTS_ACTIVITY_UNDEFINED,		\
-			1,					\
-			NULL,					\
-			NULL,					\
-			NULL,					\
-			__FILE__,				\
-			__LINE__)
-
-
-ERTS_GLB_INLINE int erts_smp_is_system_blocked(erts_activity_t allowed_activities);
-ERTS_GLB_INLINE void erts_smp_block_system(Uint32 allowed_activities);
-ERTS_GLB_INLINE int erts_smp_emergency_block_system(long timeout,
-						    Uint32 allowed_activities);
-ERTS_GLB_INLINE void erts_smp_release_system(void);
-ERTS_GLB_INLINE int erts_smp_pending_system_block(void);
-ERTS_GLB_INLINE void erts_smp_chk_system_block(void (*prepare)(void *),
-					       void (*resume)(void *),
-					       void *arg);
-ERTS_GLB_INLINE void
-erts_smp_set_activity(erts_activity_t old_activity,
-		      erts_activity_t new_activity,
-		      int locked,
-		      void (*prepare)(void *),
-		      void (*resume)(void *),
-		      void *arg,
-		      char *file,
-		      int line);
-
-#if ERTS_GLB_INLINE_INCL_FUNC_DEF
-
-
-ERTS_GLB_INLINE int
-erts_smp_is_system_blocked(erts_activity_t allowed_activities)
-{
-#ifdef ERTS_SMP
-    return erts_is_system_blocked(allowed_activities);
-#else
-    return 1;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_block_system(Uint32 allowed_activities)
-{
-#ifdef ERTS_SMP
-    erts_block_system(allowed_activities);
-#endif
-}
-
-ERTS_GLB_INLINE int
-erts_smp_emergency_block_system(long timeout, Uint32 allowed_activities)
-{
-#ifdef ERTS_SMP
-    return erts_emergency_block_system(timeout, allowed_activities);
-#else
-    return 0;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_release_system(void)
-{
-#ifdef ERTS_SMP
-    erts_release_system();
-#endif
-}
-
-ERTS_GLB_INLINE int
-erts_smp_pending_system_block(void)
-{
-#ifdef ERTS_SMP
-    return erts_smp_atomic_read(&erts_system_block_state.do_block);
-#else
-    return 0;
-#endif
-}
-
-
-ERTS_GLB_INLINE void
-erts_smp_chk_system_block(void (*prepare)(void *),
-			  void (*resume)(void *),
-			  void *arg)
-{
-#ifdef ERTS_SMP
-    if (erts_smp_pending_system_block())
-	erts_block_me(prepare, resume, arg);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_set_activity(erts_activity_t old_activity,
-		      erts_activity_t new_activity,
-		      int locked,
-		      void (*prepare)(void *),
-		      void (*resume)(void *),
-		      void *arg,
-		      char *file,
-		      int line)
-{
-#ifdef ERTS_SMP
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_activity_change_begin();
-#endif
-    switch (old_activity) {
-    case ERTS_ACTIVITY_UNDEFINED:
-	break;
-    case ERTS_ACTIVITY_WAIT:
-	erts_smp_atomic_dec(&erts_system_block_state.in_activity.wait);
-	if (locked) {
-	    /* You are not allowed to leave activity waiting
-	     * without supplying the possibility to block
-	     * unlocked.
-	     */
-	    erts_set_activity_error(ERTS_ACT_ERR_LEAVE_WAIT_UNLOCKED,
-				    file, line);
-	}
-	break;
-    case ERTS_ACTIVITY_GC:
-	erts_smp_atomic_dec(&erts_system_block_state.in_activity.gc);
-	break;
-    case ERTS_ACTIVITY_IO:
-	erts_smp_atomic_dec(&erts_system_block_state.in_activity.io);
-	break;
-    default:
-	erts_set_activity_error(ERTS_ACT_ERR_LEAVE_UNKNOWN_ACTIVITY,
-				file, line);
-	break;
-    }
-
-    /* We are not allowed to block when going to activity waiting... */
-    if (new_activity != ERTS_ACTIVITY_WAIT && erts_smp_pending_system_block())
-	erts_check_block(old_activity,new_activity,locked,prepare,resume,arg);
-
-    switch (new_activity) {
-    case ERTS_ACTIVITY_UNDEFINED:
-	break;
-    case ERTS_ACTIVITY_WAIT:
-	erts_smp_atomic_inc(&erts_system_block_state.in_activity.wait);
-	break;
-    case ERTS_ACTIVITY_GC:
-	erts_smp_atomic_inc(&erts_system_block_state.in_activity.gc);
-	break;
-    case ERTS_ACTIVITY_IO:
-	erts_smp_atomic_inc(&erts_system_block_state.in_activity.io);
-	break;
-    default:
-	erts_set_activity_error(ERTS_ACT_ERR_ENTER_UNKNOWN_ACTIVITY,
-				file, line);
-	break;
-    }
-
-    switch (new_activity) {
-    case ERTS_ACTIVITY_WAIT:
-    case ERTS_ACTIVITY_GC:
-    case ERTS_ACTIVITY_IO:
-	if (erts_smp_pending_system_block())
-	    erts_note_activity_begin(new_activity);
-	break;
-    default:
-	break;
-    }
-
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_activity_change_end();
-#endif
-
-#endif
-}
-
-#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
+void sys_alloc_stat(SysAllocStat *);
 
 #if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
 #undef ERTS_REFC_DEBUG
@@ -1001,40 +748,44 @@ erts_smp_set_activity(erts_activity_t old_activity,
 
 typedef erts_smp_atomic_t erts_refc_t;
 
-ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, long val);
-ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, long min_val);
-ERTS_GLB_INLINE long erts_refc_inctest(erts_refc_t *refcp, long min_val);
-ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, long min_val);
-ERTS_GLB_INLINE long erts_refc_dectest(erts_refc_t *refcp, long min_val);
-ERTS_GLB_INLINE void erts_refc_add(erts_refc_t *refcp, long diff, long min_val);
-ERTS_GLB_INLINE long erts_refc_read(erts_refc_t *refcp, long min_val);
+ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, erts_aint_t val);
+ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val);
+ERTS_GLB_INLINE erts_aint_t erts_refc_inctest(erts_refc_t *refcp,
+					      erts_aint_t min_val);
+ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val);
+ERTS_GLB_INLINE erts_aint_t erts_refc_dectest(erts_refc_t *refcp,
+					      erts_aint_t min_val);
+ERTS_GLB_INLINE void erts_refc_add(erts_refc_t *refcp, erts_aint_t diff,
+				   erts_aint_t min_val);
+ERTS_GLB_INLINE erts_aint_t erts_refc_read(erts_refc_t *refcp,
+					   erts_aint_t min_val);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
 ERTS_GLB_INLINE void
-erts_refc_init(erts_refc_t *refcp, long val)
+erts_refc_init(erts_refc_t *refcp, erts_aint_t val)
 {
-    erts_smp_atomic_init((erts_smp_atomic_t *) refcp, val);
+    erts_smp_atomic_init_nob((erts_smp_atomic_t *) refcp, val);
 }
 
 ERTS_GLB_INLINE void
-erts_refc_inc(erts_refc_t *refcp, long min_val)
+erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val)
 {
 #ifdef ERTS_REFC_DEBUG
-    long val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp);
+    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
     if (val < min_val)
 	erl_exit(ERTS_ABORT_EXIT,
 		 "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n",
 		 val, min_val);
 #else
-    erts_smp_atomic_inc((erts_smp_atomic_t *) refcp);
+    erts_smp_atomic_inc_nob((erts_smp_atomic_t *) refcp);
 #endif
 }
 
-ERTS_GLB_INLINE long
-erts_refc_inctest(erts_refc_t *refcp, long min_val)
+ERTS_GLB_INLINE erts_aint_t
+erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val)
 {
-    long val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp);
+    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
 #ifdef ERTS_REFC_DEBUG
     if (val < min_val)
 	erl_exit(ERTS_ABORT_EXIT,
@@ -1045,23 +796,23 @@ erts_refc_inctest(erts_refc_t *refcp, long min_val)
 }
 
 ERTS_GLB_INLINE void
-erts_refc_dec(erts_refc_t *refcp, long min_val)
+erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val)
 {
 #ifdef ERTS_REFC_DEBUG
-    long val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp);
+    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
     if (val < min_val)
 	erl_exit(ERTS_ABORT_EXIT,
 		 "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n",
 		 val, min_val);
 #else
-    erts_smp_atomic_dec((erts_smp_atomic_t *) refcp);
+    erts_smp_atomic_dec_nob((erts_smp_atomic_t *) refcp);
 #endif
 }
 
-ERTS_GLB_INLINE long
-erts_refc_dectest(erts_refc_t *refcp, long min_val)
+ERTS_GLB_INLINE erts_aint_t
+erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val)
 {
-    long val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp);
+    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
 #ifdef ERTS_REFC_DEBUG
     if (val < min_val)
 	erl_exit(ERTS_ABORT_EXIT,
@@ -1072,23 +823,23 @@ erts_refc_dectest(erts_refc_t *refcp, long min_val)
 }
 
 ERTS_GLB_INLINE void
-erts_refc_add(erts_refc_t *refcp, long diff, long min_val)
+erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val)
 {
 #ifdef ERTS_REFC_DEBUG
-    long val = erts_smp_atomic_addtest((erts_smp_atomic_t *) refcp, diff);
+    erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff);
     if (val < min_val)
 	erl_exit(ERTS_ABORT_EXIT,
 		 "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n",
 		 diff, val, min_val);
 #else
-    erts_smp_atomic_add((erts_smp_atomic_t *) refcp, diff);
+    erts_smp_atomic_add_nob((erts_smp_atomic_t *) refcp, diff);
 #endif
 }
 
-ERTS_GLB_INLINE long
-erts_refc_read(erts_refc_t *refcp, long min_val)
+ERTS_GLB_INLINE erts_aint_t
+erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val)
 {
-    long val = erts_smp_atomic_read((erts_smp_atomic_t *) refcp);
+    erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp);
 #ifdef ERTS_REFC_DEBUG
     if (val < min_val)
 	erl_exit(ERTS_ABORT_EXIT,
@@ -1104,13 +855,10 @@ erts_refc_read(erts_refc_t *refcp, long min_val)
 extern int erts_use_kernel_poll;
 #endif
 
-void elib_ensure_initialized(void);
-
-
-#if (defined(VXWORKS) || defined(_OSE_))
+#if defined(VXWORKS)
 /* NOTE! sys_calloc2 does not exist on other 
    platforms than VxWorks and OSE */
-EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint));
+void* sys_calloc2(Uint, Uint);
 #endif /* VXWORKS || OSE */
 
 
@@ -1150,14 +898,14 @@ EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint));
 
 /* Standard set of integer macros  .. */
 
-#define get_int64(s) ((((unsigned char*) (s))[0] << 56) | \
-                      (((unsigned char*) (s))[1] << 48) | \
-                      (((unsigned char*) (s))[2] << 40) | \
-                      (((unsigned char*) (s))[3] << 32) | \
-                      (((unsigned char*) (s))[4] << 24) | \
-                      (((unsigned char*) (s))[5] << 16) | \
-                      (((unsigned char*) (s))[6] << 8)  | \
-                      (((unsigned char*) (s))[7]))
+#define get_int64(s) (((Uint64)(((unsigned char*) (s))[0]) << 56) | \
+                      (((Uint64)((unsigned char*) (s))[1]) << 48) | \
+                      (((Uint64)((unsigned char*) (s))[2]) << 40) | \
+                      (((Uint64)((unsigned char*) (s))[3]) << 32) | \
+                      (((Uint64)((unsigned char*) (s))[4]) << 24) | \
+                      (((Uint64)((unsigned char*) (s))[5]) << 16) | \
+                      (((Uint64)((unsigned char*) (s))[6]) << 8)  | \
+                      (((Uint64)((unsigned char*) (s))[7])))
 
 #define put_int64(i, s) do {((char*)(s))[0] = (char)((Sint64)(i) >> 56) & 0xff;\
                             ((char*)(s))[1] = (char)((Sint64)(i) >> 48) & 0xff;\
@@ -1180,6 +928,15 @@ EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint));
                             ((char*)(s))[3] = (char)(i)         & 0xff;} \
                         while (0)
 
+#define get_int24(s) ((((unsigned char*) (s))[0] << 16) | \
+                      (((unsigned char*) (s))[1] << 8)  | \
+                      (((unsigned char*) (s))[2]))
+
+#define put_int24(i, s) do {((char*)(s))[0] = (char)((i) >> 16) & 0xff;  \
+                            ((char*)(s))[1] = (char)((i) >> 8)  & 0xff;  \
+                            ((char*)(s))[2] = (char)(i)         & 0xff;} \
+                        while (0)
+
 #define get_int16(s) ((((unsigned char*)  (s))[0] << 8) | \
                       (((unsigned char*)  (s))[1]))
 
@@ -1193,6 +950,7 @@ EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint));
 
 #define put_int8(i, s) do {((unsigned char*)(s))[0] = (i) & 0xff;} while (0)
 
+
 /*
  * Use DEBUGF as you would use printf, but use double parentheses:
  *
@@ -1202,8 +960,8 @@ EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint));
  */
 
 #ifdef DEBUG
-EXTERN_FUNCTION(void, erl_debug, (char* format, ...));
-EXTERN_FUNCTION(void, erl_bin_write, (unsigned char *, int, int));
+void erl_debug(char* format, ...);
+void erl_bin_write(unsigned char *, int, int);
 
 #  define DEBUGF(x) erl_debug x
 #else
@@ -1247,6 +1005,19 @@ EXTERN_FUNCTION(void, erl_bin_write, (unsigned char *, int, int));
 
 #endif
 
+#ifdef __WIN32__
+#ifdef ARCH_64
+#define ERTS_ALLOC_ALIGN_BYTES 16
+#define ERTS_SMALL_ABS(Small) _abs64(Small) 
+#else
+#define ERTS_ALLOC_ALIGN_BYTES 8
+#define ERTS_SMALL_ABS(Small) labs(Small) 
+#endif
+#else
+#define ERTS_ALLOC_ALIGN_BYTES 8
+#define ERTS_SMALL_ABS(Small) labs(Small) 
+#endif
+
 
 #ifdef __WIN32__
 
@@ -1257,6 +1028,22 @@ char* win32_errorstr(int);
 
 #endif
 
+/************************************************************************
+ * Find out the native filename encoding of the process (look at locale of 
+ * Unix processes and just do UTF16 on windows 
+ ************************************************************************/
+#define ERL_FILENAME_UNKNOWN 0
+#define ERL_FILENAME_LATIN1 1
+#define ERL_FILENAME_UTF8 2
+#define ERL_FILENAME_UTF8_MAC 3
+#define ERL_FILENAME_WIN_WCHAR 4
+
+int erts_get_native_filename_encoding(void);
+/* The set function is only to be used by erl_init! */
+void erts_set_user_requested_filename_encoding(int encoding); 
+int erts_get_user_requested_filename_encoding(void);
+
+void erts_init_sys_common_misc(void);
 
 #endif
 
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index a07d6a5327..932d157cd8 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -83,24 +83,8 @@
 #define ASSERT_NO_LOCKED_LOCKS
 #endif
 
+static erts_smp_mtx_t tiw_lock;
 
-#if defined(ERTS_TIMER_THREAD) || 1
-/* I don't yet know why, but using a mutex instead of a spinlock
-   or spin-based rwlock avoids excessive delays at startup. */
-static erts_smp_rwmtx_t tiw_lock;
-#define tiw_read_lock()		erts_smp_rwmtx_rlock(&tiw_lock)
-#define tiw_read_unlock()	erts_smp_rwmtx_runlock(&tiw_lock)
-#define tiw_write_lock()	erts_smp_rwmtx_rwlock(&tiw_lock)
-#define tiw_write_unlock()	erts_smp_rwmtx_rwunlock(&tiw_lock)
-#define tiw_init_lock()		erts_smp_rwmtx_init(&tiw_lock, "timer_wheel")
-#else
-static erts_smp_rwlock_t tiw_lock;
-#define tiw_read_lock()		erts_smp_read_lock(&tiw_lock)
-#define tiw_read_unlock()	erts_smp_read_unlock(&tiw_lock)
-#define tiw_write_lock()	erts_smp_write_lock(&tiw_lock)
-#define tiw_write_unlock()	erts_smp_write_unlock(&tiw_lock)
-#define tiw_init_lock()		erts_smp_rwlock_init(&tiw_lock, "timer_wheel")
-#endif
 
 /* BEGIN tiw_lock protected variables 
 **
@@ -115,84 +99,52 @@ static erts_smp_rwlock_t tiw_lock;
 static ErlTimer** tiw;		/* the timing wheel, allocated in init_time() */
 static Uint tiw_pos;		/* current position in wheel */
 static Uint tiw_nto;		/* number of timeouts in wheel */
+static Uint tiw_min;
+static ErlTimer *tiw_min_ptr;
 
 /* END tiw_lock protected variables */
 
 /* Actual interval time chosen by sys_init_time() */
 static int itime; /* Constant after init */
 
-#if defined(ERTS_TIMER_THREAD)
-static SysTimeval time_start;	/* start of current time interval */
-static long ticks_end;		/* time_start+ticks_end == time_wakeup */
-static long ticks_latest;	/* delta from time_start at latest time update*/
-
-static ERTS_INLINE long time_gettimeofday(SysTimeval *now)
-{
-    long elapsed;
-
-    erts_get_timeval(now);
-    now->tv_usec = 1000 * (now->tv_usec / 1000); /* ms resolution */
-    elapsed = (1000 * (now->tv_sec - time_start.tv_sec) +
-	       (now->tv_usec - time_start.tv_usec) / 1000);
-    // elapsed /= CLOCK_RESOLUTION;
-    return elapsed;
-}
-
-static long do_time_update(void)
-{
-    SysTimeval now;
-    long elapsed;
-
-    elapsed = time_gettimeofday(&now);
-    ticks_latest = elapsed;
-    return elapsed;
-}
-
-static ERTS_INLINE long do_time_read(void)
+erts_smp_atomic32_t do_time;	/* set at clock interrupt */
+static ERTS_INLINE erts_short_time_t do_time_read(void)
 {
-    return ticks_latest;
+    return erts_smp_atomic32_read_acqb(&do_time);
 }
 
-static long do_time_reset(void)
+static ERTS_INLINE erts_short_time_t do_time_update(void)
 {
-    SysTimeval now;
-    long elapsed;
-
-    elapsed = time_gettimeofday(&now);
-    time_start = now;
-    ticks_end = LONG_MAX;
-    ticks_latest = 0;
-    return elapsed;
+    return do_time_read();
 }
 
 static ERTS_INLINE void do_time_init(void)
 {
-    (void)do_time_reset();
+    erts_smp_atomic32_init_nob(&do_time, 0);
 }
 
-#else
-erts_smp_atomic_t do_time;	/* set at clock interrupt */
-static ERTS_INLINE long do_time_read(void) { return erts_smp_atomic_read(&do_time); }
-static ERTS_INLINE long do_time_update(void) { return do_time_read(); }
-static ERTS_INLINE void do_time_init(void) { erts_smp_atomic_init(&do_time, 0L); }
-#endif
-
 /* get the time (in units of itime) to the next timeout,
    or -1 if there are no timeouts                     */
 
-static int next_time_internal(void) /* PRE: tiw_lock taken by caller */
+static erts_short_time_t next_time_internal(void) /* PRE: tiw_lock taken by caller */
 {
     int i, tm, nto;
-    unsigned int min;
+    Uint32 min;
     ErlTimer* p;
-    long dt;
+    erts_short_time_t dt;
   
     if (tiw_nto == 0)
 	return -1;	/* no timeouts in wheel */
+
+    if (tiw_min_ptr) {
+	min = tiw_min;
+	dt  = do_time_read();
+	return ((min >= dt) ? (min - dt) : 0);
+    }
   
     /* start going through wheel to find next timeout */
     tm = nto = 0;
-    min = (unsigned int) -1;	/* max unsigned int */
+    min = (Uint32) -1;	/* max Uint32 */
     i = tiw_pos;
     do {
 	p = tiw[i];
@@ -201,11 +153,17 @@ static int next_time_internal(void) /* PRE: tiw_lock taken by caller */
 	    if (p->count == 0) {
 		/* found next timeout */
 		dt = do_time_read();
+		/* p->count is zero */
+		tiw_min_ptr = p;
+		tiw_min     = tm;
 		return ((tm >= dt) ? (tm - dt) : 0);
 	    } else {
 		/* keep shortest time in 'min' */
-		if (tm + p->count*TIW_SIZE < min)
+		if (tm + p->count*TIW_SIZE < min) {
 		    min = tm + p->count*TIW_SIZE;
+		    tiw_min_ptr = p;
+		    tiw_min     = min;
+		}
 	    }
 	    p = p->next;
 	}
@@ -215,33 +173,60 @@ static int next_time_internal(void) /* PRE: tiw_lock taken by caller */
 	i = (i + 1) % TIW_SIZE;
     } while (i != tiw_pos);
     dt = do_time_read();
-    return ((min >= dt) ? (min - dt) : 0);
+    if (min <= (Uint32) dt)
+	return 0;
+    if ((min - (Uint32) dt) > (Uint32) ERTS_SHORT_TIME_T_MAX)
+	return ERTS_SHORT_TIME_T_MAX;
+    return (erts_short_time_t) (min - (Uint32) dt);
+}
+
+static void remove_timer(ErlTimer *p) {
+    /* first */
+    if (!p->prev) {
+	tiw[p->slot] = p->next;
+	if(p->next)
+	    p->next->prev = NULL;
+    } else {
+	p->prev->next = p->next;
+    }
+
+    /* last */
+    if (!p->next) {
+	if (p->prev)
+	    p->prev->next = NULL;
+    } else {
+	p->next->prev = p->prev;
+    }
+
+    p->next = NULL;
+    p->prev = NULL;
+    /* Make sure cancel callback isn't called */
+    p->active = 0;
+    tiw_nto--;
 }
 
-#if !defined(ERTS_TIMER_THREAD)
 /* Private export to erl_time_sup.c */
-int next_time(void)
+erts_short_time_t erts_next_time(void)
 {
-    int ret;
+    erts_short_time_t ret;
 
-    tiw_write_lock();
+    erts_smp_mtx_lock(&tiw_lock);
     (void)do_time_update();
     ret = next_time_internal();
-    tiw_write_unlock();
+    erts_smp_mtx_unlock(&tiw_lock);
     return ret;
 }
-#endif
 
-static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-locked */
+static ERTS_INLINE void bump_timer_internal(erts_short_time_t dt) /* PRE: tiw_lock is write-locked */
 {
     Uint keep_pos;
     Uint count;
     ErlTimer *p, **prev, *timeout_head, **timeout_tail;
-    Uint dtime = (unsigned long)dt;  
+    Uint dtime = (Uint) dt;  
 
     /* no need to bump the position if there aren't any timeouts */
     if (tiw_nto == 0) {
-	tiw_write_unlock();
+	erts_smp_mtx_unlock(&tiw_lock);
 	return;
     }
 
@@ -258,12 +243,16 @@ static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-l
 	if (tiw_pos == keep_pos) count--;
 	prev = &tiw[tiw_pos];
 	while ((p = *prev) != NULL) {
+	    ASSERT( p != p->next);
 	    if (p->count < count) {     /* we have a timeout */
-		*prev = p->next;	/* Remove from list */
-		tiw_nto--;
-		p->next = NULL;
-		p->active = 0;		/* Make sure cancel callback
-					   isn't called */
+		/* remove min time */
+		if (tiw_min_ptr == p) {
+		    tiw_min_ptr = NULL;
+		    tiw_min = 0;
+		}
+
+		/* Remove from list */
+		remove_timer(p);
 		*timeout_tail = p;	/* Insert in timeout queue */
 		timeout_tail = &p->next;
 	    }
@@ -277,8 +266,10 @@ static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-l
 	dtime--;
     }
     tiw_pos = keep_pos;
+    if (tiw_min_ptr)
+	tiw_min -= dt;
     
-    tiw_write_unlock();
+    erts_smp_mtx_unlock(&tiw_lock);
     
     /* Call timedout timers callbacks */
     while (timeout_head) {
@@ -291,24 +282,17 @@ static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-l
 	 * callback is called.
 	 */
 	p->next = NULL;
+	p->prev = NULL;
 	p->slot = 0;
 	(*p->timeout)(p->arg);
     }
 }
 
-#if defined(ERTS_TIMER_THREAD)
-static void timer_thread_bump_timer(void)
+void erts_bump_timer(erts_short_time_t dt) /* dt is value from do_time */
 {
-    tiw_write_lock();
-    bump_timer_internal(do_time_reset());
-}
-#else
-void bump_timer(long dt) /* dt is value from do_time */
-{
-    tiw_write_lock();
+    erts_smp_mtx_lock(&tiw_lock);
     bump_timer_internal(dt);
 }
-#endif
 
 Uint
 erts_timer_wheel_memory_size(void)
@@ -316,82 +300,10 @@ erts_timer_wheel_memory_size(void)
     return (Uint) TIW_SIZE * sizeof(ErlTimer*);
 }
 
-#if defined(ERTS_TIMER_THREAD)
-static struct erts_iwait *timer_thread_iwait;
-
-static int timer_thread_setup_delay(SysTimeval *rem_time)
-{
-    long elapsed;
-    int ticks;
-
-    tiw_write_lock();
-    elapsed = do_time_update();
-    ticks = next_time_internal();
-    if (ticks == -1)	/* timer queue empty */
-	ticks = 100*1000*1000;
-    if (elapsed > ticks)
-	elapsed = ticks;
-    ticks -= elapsed;
-    //ticks *= CLOCK_RESOLUTION;
-    rem_time->tv_sec = ticks / 1000;
-    rem_time->tv_usec = 1000 * (ticks % 1000);
-    ticks_end = ticks;
-    tiw_write_unlock();
-    return ticks;
-}
-
-static void *timer_thread_start(void *ignore)
-{
-    SysTimeval delay;
-
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_set_thread_name("timer");
-#endif
-    erts_register_blockable_thread();
-
-    for(;;) {
-	if (timer_thread_setup_delay(&delay)) {
-	    erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
-	    ASSERT_NO_LOCKED_LOCKS;
-	    erts_iwait_wait(timer_thread_iwait, &delay);
-	    ASSERT_NO_LOCKED_LOCKS;
-	    erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
-	}
-	else
-	    erts_smp_chk_system_block(NULL, NULL, NULL);
-	timer_thread_bump_timer();
-	ASSERT_NO_LOCKED_LOCKS;
-    }
-    /*NOTREACHED*/
-    return NULL;
-}
-
-static ERTS_INLINE void timer_thread_post_insert(Uint ticks)
-{
-    if ((Sint)ticks < ticks_end)
-	erts_iwait_interrupt(timer_thread_iwait);
-}
-
-static void timer_thread_init(void)
-{
-    erts_thr_opts_t opts = ERTS_THR_OPTS_DEFAULT_INITER;
-    erts_tid_t tid;
-
-    opts->detached = 1;
-
-    timer_thread_iwait = erts_iwait_init();
-    erts_thr_create(&tid, timer_thread_start, NULL, &opts);
-}
-
-#else
-static ERTS_INLINE void timer_thread_post_insert(Uint ticks) { }
-static ERTS_INLINE void timer_thread_init(void) { }
-#endif
-
 /* this routine links the time cells into a free list at the start
    and sets the time queue as empty */
 void
-init_time(void)
+erts_init_time(void)
 {
     int i;
 
@@ -399,7 +311,7 @@ init_time(void)
        if timer thread is enabled */
     itime = erts_init_time_sup();
 
-    tiw_init_lock();
+    erts_smp_mtx_init(&tiw_lock, "timer_wheel");
 
     tiw = (ErlTimer**) erts_alloc(ERTS_ALC_T_TIMER_WHEEL,
 				  TIW_SIZE * sizeof(ErlTimer*));
@@ -407,10 +319,13 @@ init_time(void)
 	tiw[i] = NULL;
     do_time_init();
     tiw_pos = tiw_nto = 0;
-
-    timer_thread_init();
+    tiw_min_ptr = NULL;
+    tiw_min = 0;
 }
 
+
+
+
 /*
 ** Insert a process into the time queue, with a timeout 't'
 */
@@ -440,20 +355,35 @@ insert_timer(ErlTimer* p, Uint t)
   
     /* insert at head of list at slot */
     p->next = tiw[tm];
+    p->prev = NULL;
+    if (p->next != NULL)
+	p->next->prev = p;
     tiw[tm] = p;
-    tiw_nto++;
 
-    timer_thread_post_insert(ticks);
+
+    /* insert min time */
+    if ((tiw_nto == 0) || ((tiw_min_ptr != NULL) && (ticks < tiw_min))) {
+	tiw_min = ticks;
+	tiw_min_ptr = p;
+    }
+    if ((tiw_min_ptr == p) && (ticks > tiw_min)) {
+	/* some other timer might be 'min' now */
+	tiw_min = 0;
+	tiw_min_ptr = NULL;
+    }
+
+    tiw_nto++;
 }
 
 void
-erl_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel,
+erts_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel,
 	      void* arg, Uint t)
 {
+
     erts_deliver_time();
-    tiw_write_lock();
+    erts_smp_mtx_lock(&tiw_lock);
     if (p->active) { /* XXX assert ? */
-	tiw_write_unlock();
+	erts_smp_mtx_unlock(&tiw_lock);
 	return;
     }
     p->timeout = timeout;
@@ -461,45 +391,37 @@ erl_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel,
     p->arg = arg;
     p->active = 1;
     insert_timer(p, t);
-    tiw_write_unlock();
-#if defined(ERTS_SMP) && !defined(ERTS_TIMER_THREAD)
-    if (t <= (Uint) LONG_MAX)
-	erts_sys_schedule_interrupt_timed(1, (long) t);
+    erts_smp_mtx_unlock(&tiw_lock);
+#if defined(ERTS_SMP)
+    if (t <= (Uint) ERTS_SHORT_TIME_T_MAX)
+	erts_sys_schedule_interrupt_timed(1, (erts_short_time_t) t);
 #endif
 }
 
 void
-erl_cancel_timer(ErlTimer* p)
+erts_cancel_timer(ErlTimer* p)
 {
-    ErlTimer *tp;
-    ErlTimer **prev;
-
-    tiw_write_lock();
+    erts_smp_mtx_lock(&tiw_lock);
     if (!p->active) { /* allow repeated cancel (drivers) */
-	tiw_write_unlock();
+	erts_smp_mtx_unlock(&tiw_lock);
 	return;
     }
-    /* find p in linked list at slot p->slot and remove it */
-    prev = &tiw[p->slot];
-    while ((tp = *prev) != NULL) {
-	if (tp == p) {
-	    *prev = p->next;	/* Remove from list */
-	    tiw_nto--;
-	    p->next = NULL;
-	    p->slot = p->count = 0;
-	    p->active = 0;
-	    if (p->cancel != NULL) {
-		tiw_write_unlock();
-		(*p->cancel)(p->arg);
-	    } else {
-		tiw_write_unlock();
-	    }
-	    return;
-	} else {
-	    prev = &tp->next;
-	}
+
+    /* is it the 'min' timer, remove min */
+    if (p == tiw_min_ptr) {
+	tiw_min_ptr = NULL;
+	tiw_min     = 0;
     }
-    tiw_write_unlock();
+
+    remove_timer(p);
+    p->slot = p->count = 0;
+
+    if (p->cancel != NULL) {
+	erts_smp_mtx_unlock(&tiw_lock);
+	(*p->cancel)(p->arg);
+	return;
+    }
+    erts_smp_mtx_unlock(&tiw_lock);
 }
 
 /*
@@ -509,15 +431,15 @@ erl_cancel_timer(ErlTimer* p)
   immediately if it hadn't been cancelled).
 */
 Uint
-time_left(ErlTimer *p)
+erts_time_left(ErlTimer *p)
 {
     Uint left;
-    long dt;
+    erts_short_time_t dt;
 
-    tiw_read_lock();
+    erts_smp_mtx_lock(&tiw_lock);
 
     if (!p->active) {
-	tiw_read_unlock();
+	erts_smp_mtx_unlock(&tiw_lock);
 	return 0;
     }
 
@@ -531,19 +453,18 @@ time_left(ErlTimer *p)
     else
 	left -= dt;
 
-    tiw_read_unlock();
+    erts_smp_mtx_unlock(&tiw_lock);
 
-    return left * itime;
+    return (Uint) left * itime;
 }
 
 #ifdef DEBUG
-
-void p_slpq()
+void erts_p_slpq(void)
 {
     int i;
     ErlTimer* p;
   
-    tiw_read_lock();
+    erts_smp_mtx_lock(&tiw_lock);
 
     /* print the whole wheel, starting at the current position */
     erts_printf("\ntiw_pos = %d tiw_nto %d\n", tiw_pos, tiw_nto);
@@ -565,7 +486,6 @@ void p_slpq()
 	}
     }
 
-    tiw_read_unlock();
+    erts_smp_mtx_unlock(&tiw_lock);
 }
-
 #endif /* DEBUG */
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 31efddc0f2..a36d15204e 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -32,6 +32,7 @@
 #include "erl_binary.h"
 #include "erl_bits.h"
 #include "packet_parser.h"
+#include "erl_gc.h"
 #define ERTS_WANT_DB_INTERNAL__
 #include "erl_db.h"
 #include "erl_threads.h"
@@ -41,30 +42,27 @@
 #include "erl_threads.h"
 #include "erl_smp.h"
 #include "erl_time.h"
+#include "erl_thr_progress.h"
+#include "erl_thr_queue.h"
+#include "erl_sched_spec_pre_alloc.h"
+#include "beam_bp.h"
 
 #undef M_TRIM_THRESHOLD
 #undef M_TOP_PAD
 #undef M_MMAP_THRESHOLD
 #undef M_MMAP_MAX
 
-#if !defined(ELIB_ALLOC_IS_CLIB) && defined(__GLIBC__) && defined(HAVE_MALLOC_H)
+#if defined(__GLIBC__) && defined(HAVE_MALLOC_H)
 #include <malloc.h>
 #endif
 
-#if defined(ELIB_ALLOC_IS_CLIB) || !defined(HAVE_MALLOPT)
+#if !defined(HAVE_MALLOPT)
 #undef  HAVE_MALLOPT
 #define HAVE_MALLOPT 0
 #endif
 
 /* profile_scheduler mini message queue */
 
-#ifdef ERTS_TIMER_THREAD
-/* A timer thread is not welcomed with this lock violation work around.
- * - Bj�rn-Egil
- */
-#error Timer thread may not be enabled due to lock violation.
-#endif
-
 typedef struct {
     Uint scheduler_id;
     Uint no_schedulers;
@@ -81,6 +79,7 @@ typedef struct {
 
 #ifdef ERTS_SMP
 
+#if 0 /* Unused */
 static void 
 dispatch_profile_msg_q(profile_sched_msg_q *psmq)
 {
@@ -92,12 +91,13 @@ dispatch_profile_msg_q(profile_sched_msg_q *psmq)
 	profile_scheduler_q(make_small(msg->scheduler_id), msg->state, am_undefined, msg->Ms, msg->s, msg->us);
     }
 }
+#endif
 
 #endif
 
 
 Eterm*
-erts_heap_alloc(Process* p, Uint need)
+erts_heap_alloc(Process* p, Uint need, Uint xtra)
 {
     ErlHeapFragment* bp;
     Eterm* htop;
@@ -123,9 +123,9 @@ erts_heap_alloc(Process* p, Uint need)
     p->space_verified_from = NULL;
 #endif /* FORCE_HEAP_FRAGS */
 
-    n = need;
+    n = need + xtra;
     bp = MBUF(p);
-    if (bp != NULL && need <= (bp->size - bp->used_size)) {
+    if (bp != NULL && need <= (bp->alloc_size - bp->used_size)) {
 	Eterm* ret = bp->mem + bp->used_size;
 	bp->used_size += need;
 	return ret;
@@ -158,16 +158,11 @@ erts_heap_alloc(Process* p, Uint need)
 
     bp->next = MBUF(p);
     MBUF(p) = bp;
-    bp->size = n;
-    bp->used_size = n;
+    bp->alloc_size = n;
+    bp->used_size = need;
     MBUF_SIZE(p) += n;
-    bp->off_heap.mso = NULL;
-#ifndef HYBRID /* FIND ME! */
-    bp->off_heap.funs = NULL;
-#endif
-    bp->off_heap.externals = NULL;
+    bp->off_heap.first = NULL;
     bp->off_heap.overhead = 0;
-
     return bp->mem;
 }
 
@@ -204,6 +199,25 @@ erl_grow_stack(Eterm** start, Eterm** sp, Eterm** end)
     *end = *start + new_size;
     *sp = *start + sp_offs;
 }
+/*
+ * Helper function for the ESTACK macros defined in global.h.
+ */
+void
+erl_grow_wstack(UWord** start, UWord** sp, UWord** end)
+{
+    Uint old_size = (*end - *start);
+    Uint new_size = old_size * 2;
+    Uint sp_offs = *sp - *start;
+    if (new_size > 2 * DEF_ESTACK_SIZE) {
+	*start = erts_realloc(ERTS_ALC_T_ESTACK, (void *) *start, new_size*sizeof(UWord));
+    } else {
+	UWord* new_ptr = erts_alloc(ERTS_ALC_T_ESTACK, new_size*sizeof(UWord));
+	sys_memcpy(new_ptr, *start, old_size*sizeof(UWord));
+	*start = new_ptr;
+    }
+    *end = *start + new_size;
+    *sp = *start + sp_offs;
+}
 
 /* CTYPE macros */
 
@@ -354,6 +368,31 @@ erts_bld_uint(Uint **hpp, Uint *szp, Uint ui)
     return res;
 }
 
+/*
+ * Erts_bld_uword is more or less similar to erts_bld_uint, but a pointer
+ * can safely be passed.
+ */
+
+Eterm
+erts_bld_uword(Uint **hpp, Uint *szp, UWord uw)
+{
+    Eterm res = THE_NON_VALUE;
+    if (IS_USMALL(0, uw)) {
+	if (hpp)
+	    res = make_small((Uint) uw);
+    }
+    else {
+	if (szp)
+	    *szp += BIG_UWORD_HEAP_SIZE(uw);
+	if (hpp) {
+	    res = uword_to_big(uw, *hpp);
+	    *hpp += BIG_UWORD_HEAP_SIZE(uw);
+	}
+    }
+    return res;
+}
+
+
 Eterm
 erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64)
 {
@@ -364,7 +403,7 @@ erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64)
     }
     else {
 	if (szp)
-	    *szp = ERTS_UINT64_HEAP_SIZE(ui64);
+	    *szp += ERTS_UINT64_HEAP_SIZE(ui64);
 	if (hpp)
 	    res = erts_uint64_to_big(ui64, hpp);
     }
@@ -381,7 +420,7 @@ erts_bld_sint64(Uint **hpp, Uint *szp, Sint64 si64)
     }
     else {
 	if (szp)
-	    *szp = ERTS_SINT64_HEAP_SIZE(si64);
+	    *szp += ERTS_SINT64_HEAP_SIZE(si64);
 	if (hpp)
 	    res = erts_sint64_to_big(si64, hpp);
     }
@@ -465,7 +504,7 @@ erts_bld_string_n(Uint **hpp, Uint *szp, const char *str, Sint len)
     if (hpp) {
 	res = NIL;
 	while (--i >= 0) {
-	    res = CONS(*hpp, make_small(str[i]), res);
+	    res = CONS(*hpp, make_small((byte) str[i]), res);
 	    *hpp += 2;
 	}
     }
@@ -711,7 +750,7 @@ hash_binary_bytes(Eterm bin, Uint sz, Uint32 hash)
 
 Uint32 make_hash(Eterm term_arg)
 {
-    DECLARE_ESTACK(stack);
+    DECLARE_WSTACK(stack);
     Eterm term = term_arg;
     Eterm hash = 0;
     unsigned op;
@@ -770,7 +809,7 @@ tail_recur:
 	    Uint y2 = y1 < 0 ? -(Uint)y1 : y1;
 
 	    UINT32_HASH_STEP(y2, FUNNY_NUMBER2);
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 	    if (y2 >> 32)
 		UINT32_HASH_STEP(y2 >> 32, FUNNY_NUMBER2);
 #endif
@@ -787,7 +826,7 @@ tail_recur:
 	}
     case EXPORT_DEF:
 	{
-	    Export* ep = (Export *) (export_val(term))[1];
+	    Export* ep = *((Export **) (export_val(term) + 1));
 
 	    hash = hash * FUNNY_NUMBER11 + ep->code[2];
 	    hash = hash*FUNNY_NUMBER1 + 
@@ -809,7 +848,7 @@ tail_recur:
 	    hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq;
 	    if (num_free > 0) {
 		if (num_free > 1) {
-		    ESTACK_PUSH3(stack, (Eterm) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP);
+		    WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP);
 		}
 		term = funp->env[0];
 		goto tail_recur;
@@ -837,9 +876,9 @@ tail_recur:
 	}
 
     case MAKE_HASH_CDR_PRE_OP:
-	term = ESTACK_POP(stack);
+	term = (Eterm) WSTACK_POP(stack);
 	if (is_not_list(term)) {
-	    ESTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP);
+	    WSTACK_PUSH(stack, (UWord) MAKE_HASH_CDR_POST_OP);
 	    goto tail_recur;
 	}
 	/* fall through */
@@ -854,13 +893,13 @@ tail_recur:
 		hash = hash*FUNNY_NUMBER2 + unsigned_val(*list);
 		
 		if (is_not_list(CDR(list))) {
-		    ESTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP);
+		    WSTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP);
 		    term = CDR(list);
 		    goto tail_recur;
 		}		
 		list = list_val(CDR(list));
 	    }
-	    ESTACK_PUSH2(stack, CDR(list), MAKE_HASH_CDR_PRE_OP);
+	    WSTACK_PUSH2(stack, CDR(list), MAKE_HASH_CDR_PRE_OP);
 	    term = CAR(list);
 	    goto tail_recur;
 	}
@@ -888,7 +927,7 @@ tail_recur:
 	    }
 	    d = BIG_DIGIT(ptr, k);
 	    k = sizeof(ErtsDigit);
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
 	    if (!(d >> 32))
 		k /= 2;
 #endif
@@ -904,21 +943,21 @@ tail_recur:
 	    Eterm* ptr = tuple_val(term);
 	    Uint arity = arityval(*ptr);
 
-	    ESTACK_PUSH3(stack, arity, (Eterm)(ptr+1), arity);
+	    WSTACK_PUSH3(stack, (UWord) arity, (UWord)(ptr+1), (UWord) arity);
 	    op = MAKE_HASH_TUPLE_OP;	    
 	}/*fall through*/
     case MAKE_HASH_TUPLE_OP:
     case MAKE_HASH_FUN_OP:
 	{
-	    Uint i = ESTACK_POP(stack);
-	    Eterm* ptr = (Eterm*) ESTACK_POP(stack);
+	    Uint i = (Uint) WSTACK_POP(stack);
+	    Eterm* ptr = (Eterm*) WSTACK_POP(stack);
 	    if (i != 0) {
 		term = *ptr;
-		ESTACK_PUSH3(stack, (Eterm)(ptr+1), i-1, op);
+		WSTACK_PUSH3(stack, (UWord)(ptr+1), (UWord) i-1, (UWord) op);
 		goto tail_recur;
 	    }
 	    if (op == MAKE_HASH_TUPLE_OP) {
-		Uint32 arity = ESTACK_POP(stack);
+		Uint32 arity = (Uint32) WSTACK_POP(stack);
 		hash = hash*FUNNY_NUMBER9 + arity;
 	    }
 	    break;
@@ -928,10 +967,10 @@ tail_recur:
 	erl_exit(1, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op);
 	return 0;
       }
-      if (ESTACK_ISEMPTY(stack)) break;
-      op = ESTACK_POP(stack);
+      if (WSTACK_ISEMPTY(stack)) break;
+      op = WSTACK_POP(stack);
     }
-    DESTROY_ESTACK(stack);
+    DESTROY_WSTACK(stack);
     return hash;
 
 #undef UINT32_HASH_STEP
@@ -1002,7 +1041,7 @@ Uint32
 make_hash2(Eterm term)
 {
     Uint32 hash;
-    Eterm tmp_big[2];
+    DeclareTmpHeapNoproc(tmp_big,2);
 
 /* (HCONST * {2, ..., 14}) mod 2^32 */
 #define HCONST_2 0x3c6ef372UL
@@ -1041,7 +1080,6 @@ make_hash2(Eterm term)
 	} while(0)
 
 #define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2)
-
     /* Optimization. Simple cases before declaration of estack. */
     if (primary_tag(term) == TAG_PRIMARY_IMMED1) {
 	switch (term & _TAG_IMMED1_MASK) {
@@ -1070,6 +1108,7 @@ make_hash2(Eterm term)
     Eterm tmp;
     DECLARE_ESTACK(s);
 
+    UseTmpHeapNoproc(2);
     hash = 0;
     for (;;) {
 	switch (primary_tag(term)) {
@@ -1123,7 +1162,7 @@ make_hash2(Eterm term)
 	    break;
 	    case EXPORT_SUBTAG:
 	    {
-		Export* ep = (Export *) (export_val(term))[1];
+		Export* ep = *((Export **) (export_val(term) + 1));
 
 		UINT32_HASH_2
 		    (ep->code[2], 
@@ -1314,6 +1353,7 @@ make_hash2(Eterm term)
 	hash2_common:
 	    if (ESTACK_ISEMPTY(s)) {
 		DESTROY_ESTACK(s);
+		UnUseTmpHeapNoproc(2);
 		return hash;
 	    }
 	    term = ESTACK_POP(s);
@@ -1332,7 +1372,7 @@ make_hash2(Eterm term)
 Uint32 make_broken_hash(Eterm term)
 {
     Uint32 hash = 0;
-    DECLARE_ESTACK(stack);
+    DECLARE_WSTACK(stack);
     unsigned op;
 tail_recur:
     op = tag_val_def(term); 
@@ -1346,7 +1386,7 @@ tail_recur:
 	    (atom_tab(atom_val(term))->slot.bucket.hvalue);
 	break;
     case SMALL_DEF:
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
     {
 	Sint y1 = signed_val(term);
 	Uint y2 = y1 < 0 ? -(Uint)y1 : y1;
@@ -1399,7 +1439,7 @@ tail_recur:
 
     case EXPORT_DEF:
 	{
-	    Export* ep = (Export *) (export_val(term))[1];
+	    Export* ep = *((Export **) (export_val(term) + 1));
 
 	    hash = hash * FUNNY_NUMBER11 + ep->code[2];
 	    hash = hash*FUNNY_NUMBER1 + 
@@ -1421,7 +1461,7 @@ tail_recur:
 	    hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq;
 	    if (num_free > 0) {
 		if (num_free > 1) {
-		    ESTACK_PUSH3(stack, (Eterm) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP);
+		    WSTACK_PUSH3(stack, (UWord) &funp->env[1], (num_free-1), MAKE_HASH_FUN_OP);
 		}
 		term = funp->env[0];
 		goto tail_recur;
@@ -1456,16 +1496,17 @@ tail_recur:
 	break;
 
     case MAKE_HASH_CDR_PRE_OP:
-	term = ESTACK_POP(stack);
+	term = (Eterm) WSTACK_POP(stack);
 	if (is_not_list(term)) {
-	    ESTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP);
+	    WSTACK_PUSH(stack, (UWord) MAKE_HASH_CDR_POST_OP);
 	    goto tail_recur;
 	}
 	/*fall through*/
     case LIST_DEF:
 	{
 	    Eterm* list = list_val(term);
-	    ESTACK_PUSH2(stack, CDR(list), MAKE_HASH_CDR_PRE_OP);
+	    WSTACK_PUSH2(stack, (UWord) CDR(list),
+			 (UWord) MAKE_HASH_CDR_PRE_OP);
 	    term = CAR(list);
 	    goto tail_recur;
 	}
@@ -1538,21 +1579,21 @@ tail_recur:
 	    Eterm* ptr = tuple_val(term);
 	    Uint arity = arityval(*ptr);
 
-	    ESTACK_PUSH3(stack, arity, (Eterm)(ptr+1), arity);
+	    WSTACK_PUSH3(stack, (UWord) arity, (UWord) (ptr+1), (UWord) arity);
 	    op = MAKE_HASH_TUPLE_OP;
 	}/*fall through*/ 
     case MAKE_HASH_TUPLE_OP:
     case MAKE_HASH_FUN_OP:
 	{
-	    Uint i = ESTACK_POP(stack);
-	    Eterm* ptr = (Eterm*) ESTACK_POP(stack);
+	    Uint i = (Uint) WSTACK_POP(stack);
+	    Eterm* ptr = (Eterm*) WSTACK_POP(stack);
 	    if (i != 0) {
 		term = *ptr;
-		ESTACK_PUSH3(stack, (Eterm)(ptr+1), i-1, op);
+		WSTACK_PUSH3(stack, (UWord)(ptr+1), (UWord) i-1, (UWord) op);
 		goto tail_recur;
 	    }
 	    if (op == MAKE_HASH_TUPLE_OP) {
-		Uint32 arity = ESTACK_POP(stack);
+		Uint32 arity = (UWord) WSTACK_POP(stack);
 		hash = hash*FUNNY_NUMBER9 + arity;
 	    }
 	    break;
@@ -1562,11 +1603,11 @@ tail_recur:
 	erl_exit(1, "Invalid tag in make_broken_hash\n");
 	return 0;
       }
-      if (ESTACK_ISEMPTY(stack)) break;
-      op = ESTACK_POP(stack);
+      if (WSTACK_ISEMPTY(stack)) break;
+      op = (Uint) WSTACK_POP(stack);
     }
 
-    DESTROY_ESTACK(stack);
+    DESTROY_WSTACK(stack);
     return hash;
     
 #undef MAKE_HASH_TUPLE_OP
@@ -1656,7 +1697,11 @@ static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len)
 	erts_queue_error_logger_message(from, tuple3, bp);
     }
 #else
-    erts_queue_message(p, NULL /* only used for smp build */, bp, tuple3, NIL);
+    erts_queue_message(p, NULL /* only used for smp build */, bp, tuple3, NIL
+#ifdef USE_VM_PROBES
+		       , NIL
+#endif
+		       );
 #endif
     return 0;
 }
@@ -1859,42 +1904,44 @@ erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *dsbufp)
     erts_free(ERTS_ALC_T_TMP_DSBUF, (void *) dsbufp);
 }
 
-
 /* eq and cmp are written as separate functions a eq is a little faster */
 
 /*
  * Test for equality of two terms.
  * Returns 0 if not equal, or a non-zero value otherwise.
  */
-
+#if HALFWORD_HEAP
+int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base)
+#else
 int eq(Eterm a, Eterm b)
+#endif
 {
-    DECLARE_ESTACK(stack);
+    DECLARE_WSTACK(stack);
     Sint sz;
     Eterm* aa;
-    Eterm* bb;	
+    Eterm* bb;
 
 tailrecur:
-    if (a == b) goto pop_next; 
+    if (is_same(a, a_base, b, b_base)) goto pop_next;
 tailrecur_ne:
 
     switch (primary_tag(a)) {
     case TAG_PRIMARY_LIST:
 	if (is_list(b)) {
-	    Eterm* aval = list_val(a);
-	    Eterm* bval = list_val(b);
+	    Eterm* aval = list_val_rel(a, a_base);
+	    Eterm* bval = list_val_rel(b, b_base);
 	    while (1) {
 		Eterm atmp = CAR(aval);
 		Eterm btmp = CAR(bval);
-		if (atmp != btmp) {
-		    ESTACK_PUSH2(stack,CDR(bval),CDR(aval));
+		if (!is_same(atmp,a_base,btmp,b_base)) {
+		    WSTACK_PUSH2(stack,(UWord) CDR(bval),(UWord) CDR(aval));
 		    a = atmp;
 		    b = btmp;
 		    goto tailrecur_ne;
 		}
 		atmp = CDR(aval);
 		btmp = CDR(bval);
-		if (atmp == btmp) {
+		if (is_same(atmp,a_base,btmp,b_base)) {
 		    goto pop_next;
 		}
 		if (is_not_list(atmp) || is_not_list(btmp)) {
@@ -1902,22 +1949,22 @@ tailrecur_ne:
 		    b = btmp;
 		    goto tailrecur_ne;
 		}
-		aval = list_val(atmp);
-		bval = list_val(btmp);
+		aval = list_val_rel(atmp, a_base);
+		bval = list_val_rel(btmp, b_base);
 	    }
 	}
 	break; /* not equal */
 
     case TAG_PRIMARY_BOXED:
 	{	
-	    Eterm hdr = *boxed_val(a);
+	    Eterm hdr = *boxed_val_rel(a,a_base);
 	    switch (hdr & _TAG_HEADER_MASK) {
 	    case ARITYVAL_SUBTAG:
 		{
-		    aa = tuple_val(a);
-		    if (!is_boxed(b) || *boxed_val(b) != *aa)
+		    aa = tuple_val_rel(a, a_base);
+		    if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa)
 			goto not_equal;
-		    bb = tuple_val(b);
+		    bb = tuple_val_rel(b,b_base);
 		    if ((sz = arityval(*aa)) == 0) goto pop_next;
 		    ++aa;
 		    ++bb;
@@ -1936,16 +1983,16 @@ tailrecur_ne:
 		    Uint a_bitoffs;
 		    Uint b_bitoffs;
 		    
-		    if (is_not_binary(b)) {
+		    if (!is_binary_rel(b,b_base)) {
 			goto not_equal;
 		    }
-		    a_size = binary_size(a);
-		    b_size = binary_size(b); 
+		    a_size = binary_size_rel(a,a_base);
+		    b_size = binary_size_rel(b,b_base);
 		    if (a_size != b_size) {
 			goto not_equal;
 		    }
-		    ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize);
-		    ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize);
+		    ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base);
+		    ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base);
 		    if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) {
 			if (sys_memcmp(a_ptr, b_ptr, a_size) == 0) goto pop_next;
 		    } else if (a_bitsize == b_bitsize) {
@@ -1956,9 +2003,9 @@ tailrecur_ne:
 		}
 	    case EXPORT_SUBTAG:
 		{
-		    if (is_export(b)) {
-			Export* a_exp = (Export *) (export_val(a))[1];
-			Export* b_exp = (Export *) (export_val(b))[1];
+		    if (is_export_rel(b,b_base)) {
+			Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1));
+			Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1));
 			if (a_exp == b_exp) goto pop_next;
 		    }
 		    break; /* not equal */
@@ -1968,10 +2015,10 @@ tailrecur_ne:
 		    ErlFunThing* f1;
 		    ErlFunThing* f2;
   
-		    if (is_not_fun(b))
+		    if (!is_fun_rel(b,b_base))
 			goto not_equal;
-		    f1 = (ErlFunThing *) fun_val(a);
-		    f2 = (ErlFunThing *) fun_val(b);
+		    f1 = (ErlFunThing *) fun_val_rel(a,a_base);
+		    f2 = (ErlFunThing *) fun_val_rel(b,b_base);
 		    if (f1->fe->module != f2->fe->module ||
 			f1->fe->old_index != f2->fe->old_index ||
 			f1->fe->old_uniq != f2->fe->old_uniq ||
@@ -1989,15 +2036,15 @@ tailrecur_ne:
 		ExternalThing *ap;
 		ExternalThing *bp;
 
-		if(is_not_external(b))
+		if(!is_external_rel(b,b_base))
 		    goto not_equal;
 
-		ap = external_thing_ptr(a);
-		bp = external_thing_ptr(b);
+		ap = external_thing_ptr_rel(a,a_base);
+		bp = external_thing_ptr_rel(b,b_base);
 
 		if(ap->header == bp->header && ap->node == bp->node) {
-		    ASSERT(1 == external_data_words(a));
-		    ASSERT(1 == external_data_words(b));
+		    ASSERT(1 == external_data_words_rel(a,a_base));
+		    ASSERT(1 == external_data_words_rel(b,b_base));
 		    
 		    if (ap->data.ui[0] == bp->data.ui[0]) goto pop_next;
 		}
@@ -2015,27 +2062,36 @@ tailrecur_ne:
 		Uint alen;
 		Uint blen;
 		Uint i;
+		ExternalThing* athing;
+		ExternalThing* bthing;
 
-		if(is_not_external_ref(b))
+		if(!is_external_ref_rel(b,b_base))
 		    goto not_equal;
 
-		if(external_node(a) != external_node(b))
+		athing = external_thing_ptr_rel(a,a_base);
+		bthing = external_thing_ptr_rel(b,b_base);
+
+		if(athing->node != bthing->node)
 		    goto not_equal;
 
-		anum = external_ref_numbers(a);
-		bnum = external_ref_numbers(b);
-		alen = external_ref_no_of_numbers(a);
-		blen = external_ref_no_of_numbers(b);
+		anum = external_thing_ref_numbers(athing);
+		bnum = external_thing_ref_numbers(bthing);
+		alen = external_thing_ref_no_of_numbers(athing);
+		blen = external_thing_ref_no_of_numbers(bthing);
 
 		goto ref_common;
 	    case REF_SUBTAG:
-  
-		    if (is_not_internal_ref(b))
+		    if (!is_internal_ref_rel(b,b_base))
 			goto not_equal;
-		    alen = internal_ref_no_of_numbers(a);
-		    blen = internal_ref_no_of_numbers(b);
-		    anum = internal_ref_numbers(a);
-		    bnum = internal_ref_numbers(b);
+
+		    {
+			RefThing* athing = ref_thing_ptr_rel(a,a_base);
+			RefThing* bthing = ref_thing_ptr_rel(b,b_base);
+			alen = internal_thing_ref_no_of_numbers(athing);
+			blen = internal_thing_ref_no_of_numbers(bthing);
+			anum = internal_thing_ref_numbers(athing);
+			bnum = internal_thing_ref_numbers(bthing);
+		    }
 
 	    ref_common:
 		    ASSERT(alen > 0 && blen > 0);
@@ -2080,10 +2136,10 @@ tailrecur_ne:
 		{
 		    int i;
   
-		    if (is_not_big(b))
+		    if (!is_big_rel(b,b_base))
 			goto not_equal;
-		    aa = big_val(a); /* get pointer to thing */
-		    bb = big_val(b);
+		    aa = big_val_rel(a,a_base);
+		    bb = big_val_rel(b,b_base);
 		    if (*aa != *bb)
 			goto not_equal;
 		    i = BIG_ARITY(aa);
@@ -2098,9 +2154,9 @@ tailrecur_ne:
 		    FloatDef af;
 		    FloatDef bf;
   
-		    if (is_float(b)) {
-			GET_DOUBLE(a, af);
-			GET_DOUBLE(b, bf);
+		    if (is_float_rel(b,b_base)) {
+			GET_DOUBLE_REL(a, af, a_base);
+			GET_DOUBLE_REL(b, bf, b_base);
 			if (af.fd == bf.fd) goto pop_next;
 		    }
 		    break; /* not equal */
@@ -2119,7 +2175,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'sz' */
 	Eterm* bp = bb;
 	Sint i = sz;
 	for (;;) {
-	    if (*ap != *bp) break;
+	    if (!is_same(*ap,a_base,*bp,b_base)) break;
 	    if (--i == 0) goto pop_next;
 	    ++ap;
 	    ++bp;
@@ -2130,32 +2186,32 @@ term_array: /* arrays in 'aa' and 'bb', length in 'sz' */
 	    goto not_equal;
 	}
 	if (i > 1) { /* push the rest */
-	    ESTACK_PUSH3(stack, i-1, (Eterm)(bp+1),
-			 ((Eterm)(ap+1)) | TAG_PRIMARY_HEADER);
+	    WSTACK_PUSH3(stack, i-1, (UWord)(bp+1),
+			 ((UWord)(ap+1)) | TAG_PRIMARY_HEADER);
 	    /* We (ab)use TAG_PRIMARY_HEADER to recognize a term_array */
 	}
 	goto tailrecur_ne;
     }
    
 pop_next:
-    if (!ESTACK_ISEMPTY(stack)) {
-	Eterm something  = ESTACK_POP(stack);	
-	if (primary_tag(something) == TAG_PRIMARY_HEADER) { /* a term_array */
+    if (!WSTACK_ISEMPTY(stack)) {
+	UWord something  = WSTACK_POP(stack);
+	if (primary_tag((Eterm) something) == TAG_PRIMARY_HEADER) { /* a term_array */
 	    aa = (Eterm*) something;
-	    bb = (Eterm*) ESTACK_POP(stack);
-	    sz = ESTACK_POP(stack);
+	    bb = (Eterm*) WSTACK_POP(stack);
+	    sz = WSTACK_POP(stack);
 	    goto term_array;
 	}
 	a = something;
-	b = ESTACK_POP(stack);
+	b = WSTACK_POP(stack);
 	goto tailrecur;
     }
 
-    DESTROY_ESTACK(stack);
+    DESTROY_WSTACK(stack);
     return 1;
 
 not_equal:
-    DESTROY_ESTACK(stack);
+    DESTROY_WSTACK(stack);
     return 0;
 }
 
@@ -2208,9 +2264,13 @@ static int cmp_atoms(Eterm a, Eterm b)
 		    bb->name+3, bb->len-3);
 }
 
+#if HALFWORD_HEAP
+Sint cmp_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base)
+#else
 Sint cmp(Eterm a, Eterm b)
+#endif
 {
-    DECLARE_ESTACK(stack);
+    DECLARE_WSTACK(stack);
     Eterm* aa;
     Eterm* bb;
     int i;
@@ -2242,7 +2302,7 @@ Sint cmp(Eterm a, Eterm b)
 
 
 tailrecur:
-    if (a == b) {		/* Equal values or pointers. */
+    if (is_same(a,a_base,b,b_base)) {	/* Equal values or pointers. */
 	goto pop_next;
     }
 tailrecur_ne:
@@ -2268,9 +2328,9 @@ tailrecur_ne:
 	    if (is_internal_port(b)) {
 		bnode = erts_this_node;
 		bdata = internal_port_data(b);
-	    } else if (is_external_port(b)) {
-		bnode = external_port_node(b);
-		bdata = external_port_data(b);
+	    } else if (is_external_port_rel(b,b_base)) {
+		bnode = external_port_node_rel(b,b_base);
+		bdata = external_port_data_rel(b,b_base);
 	    } else {
 		a_tag = PORT_DEF;
 		goto mixed_types;
@@ -2286,9 +2346,9 @@ tailrecur_ne:
 	    if (is_internal_pid(b)) {
 		bnode = erts_this_node;
 		bdata = internal_pid_data(b);
-	    } else if (is_external_pid(b)) {
-		bnode = external_pid_node(b);
-		bdata = external_pid_data(b);
+	    } else if (is_external_pid_rel(b,b_base)) {
+		bnode = external_pid_node_rel(b,b_base);
+		bdata = external_pid_data_rel(b,b_base);
 	    } else {
 		a_tag = PID_DEF;
 		goto mixed_types;
@@ -2321,20 +2381,20 @@ tailrecur_ne:
 	    a_tag = LIST_DEF;
 	    goto mixed_types;
 	}
-	aa = list_val(a);
-	bb = list_val(b);
+	aa = list_val_rel(a,a_base);
+	bb = list_val_rel(b,b_base);
 	while (1) {
 	    Eterm atmp = CAR(aa);
 	    Eterm btmp = CAR(bb);
-	    if (atmp != btmp) {
-		ESTACK_PUSH2(stack,CDR(bb),CDR(aa));
+	    if (!is_same(atmp,a_base,btmp,b_base)) {
+		WSTACK_PUSH2(stack,(UWord) CDR(bb),(UWord) CDR(aa));
 		a = atmp;
 		b = btmp;
 		goto tailrecur_ne;
 	    }
 	    atmp = CDR(aa);
 	    btmp = CDR(bb);
-	    if (atmp == btmp) {
+	    if (is_same(atmp,a_base,btmp,b_base)) {
 		goto pop_next;
 	    }
 	    if (is_not_list(atmp) || is_not_list(btmp)) {
@@ -2342,20 +2402,20 @@ tailrecur_ne:
 		b = btmp;
 		goto tailrecur_ne;
 	    }
-	    aa = list_val(atmp);
-	    bb = list_val(btmp);
+	    aa = list_val_rel(atmp,a_base);
+	    bb = list_val_rel(btmp,b_base);
 	}
     case TAG_PRIMARY_BOXED:
 	{
-	    Eterm ahdr = *boxed_val(a);
+	    Eterm ahdr = *boxed_val_rel(a,a_base);
 	    switch ((ahdr & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE) {
 	    case (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE):
-		if (is_not_tuple(b)) {
+		if (!is_tuple_rel(b,b_base)) {
 		    a_tag = TUPLE_DEF;
 		    goto mixed_types;
 		}
-		aa = tuple_val(a);
-		bb = tuple_val(b);
+		aa = tuple_val_rel(a,a_base);
+		bb = tuple_val_rel(b,b_base);
 		/* compare the arities */
 		i = arityval(ahdr);	/* get the arity*/
 		if (i != arityval(*bb)) {
@@ -2369,31 +2429,31 @@ tailrecur_ne:
 		goto term_array;
 
 	    case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE):
-		if (is_not_float(b)) {
+		if (!is_float_rel(b,b_base)) {
 		    a_tag = FLOAT_DEF;
 		    goto mixed_types;
 		} else {
 		    FloatDef af;
 		    FloatDef bf; 
 
-		    GET_DOUBLE(a, af);
-		    GET_DOUBLE(b, bf);
+		    GET_DOUBLE_REL(a, af, a_base);
+		    GET_DOUBLE_REL(b, bf, b_base);
 		    ON_CMP_GOTO(float_comp(af.fd, bf.fd));
 		}
 	    case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE):
 	    case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE):
-		if (is_not_big(b)) {
+		if (!is_big_rel(b,b_base)) {
 		    a_tag = BIG_DEF;
 		    goto mixed_types;
 		}
-		ON_CMP_GOTO(big_comp(a, b));
+		ON_CMP_GOTO(big_comp(rterm2wterm(a,a_base), rterm2wterm(b,b_base)));
 	    case (_TAG_HEADER_EXPORT >> _TAG_PRIMARY_SIZE):
-		if (is_not_export(b)) {
+		if (!is_export_rel(b,b_base)) {
 		    a_tag = EXPORT_DEF;
 		    goto mixed_types;
 		} else {
-		    Export* a_exp = (Export *) (export_val(a))[1];
-		    Export* b_exp = (Export *) (export_val(b))[1];
+		    Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1));
+		    Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1));
 
 		    if ((j = cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) {
 			RETURN_NEQ(j);
@@ -2405,12 +2465,12 @@ tailrecur_ne:
 		}
 		break;
 	    case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE):
-		if (is_not_fun(b)) {
+		if (!is_fun_rel(b,b_base)) {
 		    a_tag = FUN_DEF;
 		    goto mixed_types;
 		} else {
-		    ErlFunThing* f1 = (ErlFunThing *) fun_val(a);
-		    ErlFunThing* f2 = (ErlFunThing *) fun_val(b);
+		    ErlFunThing* f1 = (ErlFunThing *) fun_val_rel(a,a_base);
+		    ErlFunThing* f2 = (ErlFunThing *) fun_val_rel(b,b_base);
 		    Sint diff;
 
 		    diff = cmpbytes(atom_tab(atom_val(f1->fe->module))->name,
@@ -2442,51 +2502,57 @@ tailrecur_ne:
 		if (is_internal_pid(b)) {
 		    bnode = erts_this_node;
 		    bdata = internal_pid_data(b);
-		} else if (is_external_pid(b)) {
-		    bnode = external_pid_node(b);
-		    bdata = external_pid_data(b);
+		} else if (is_external_pid_rel(b,b_base)) {
+		    bnode = external_pid_node_rel(b,b_base);
+		    bdata = external_pid_data_rel(b,b_base);
 		} else {
 		    a_tag = EXTERNAL_PID_DEF;
 		    goto mixed_types;
 		}
-		anode = external_pid_node(a);
-		adata = external_pid_data(a);
+		anode = external_pid_node_rel(a,a_base);
+		adata = external_pid_data_rel(a,a_base);
 		goto pid_common;
 	    case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE):
 		if (is_internal_port(b)) {
 		    bnode = erts_this_node;
 		    bdata = internal_port_data(b);
-		} else if (is_external_port(b)) {
-		    bnode = external_port_node(b);
-		    bdata = external_port_data(b);
+		} else if (is_external_port_rel(b,b_base)) {
+		    bnode = external_port_node_rel(b,b_base);
+		    bdata = external_port_data_rel(b,b_base);
 		} else {
 		    a_tag = EXTERNAL_PORT_DEF;
 		    goto mixed_types;
 		}
-		anode = external_port_node(a);
-		adata = external_port_data(a);
+		anode = external_port_node_rel(a,a_base);
+		adata = external_port_data_rel(a,a_base);
 		goto port_common;
 	    case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE):
 		/*
 		 * Note! When comparing refs we need to compare ref numbers
 		 * (32-bit words), *not* ref data words.
 		 */
+
 		
-		if (is_internal_ref(b)) {
+		if (is_internal_ref_rel(b,b_base)) {
+		    RefThing* bthing = ref_thing_ptr_rel(b,b_base);
 		    bnode = erts_this_node;
-		    bnum = internal_ref_numbers(b);
-		    blen = internal_ref_no_of_numbers(b);
-		} else if(is_external_ref(b)) {
-		    bnode = external_ref_node(b);
-		    bnum = external_ref_numbers(b);
-		    blen = external_ref_no_of_numbers(b);
+		    bnum = internal_thing_ref_numbers(bthing);
+		    blen = internal_thing_ref_no_of_numbers(bthing);
+		} else if(is_external_ref_rel(b,b_base)) {
+		    ExternalThing* bthing = external_thing_ptr_rel(b,b_base);
+		    bnode = bthing->node;
+		    bnum = external_thing_ref_numbers(bthing);
+		    blen = external_thing_ref_no_of_numbers(bthing);
 		} else {
 		    a_tag = REF_DEF;
 		    goto mixed_types;
 		}
-		anode = erts_this_node;
-		anum = internal_ref_numbers(a);
-		alen = internal_ref_no_of_numbers(a);
+		{
+		    RefThing* athing = ref_thing_ptr_rel(a,a_base);
+		    anode = erts_this_node;
+		    anum = internal_thing_ref_numbers(athing);
+		    alen = internal_thing_ref_no_of_numbers(athing);
+		}
 		
 	    ref_common:
 		CMP_NODES(anode, bnode);
@@ -2515,31 +2581,36 @@ tailrecur_ne:
 			RETURN_NEQ((Sint32) (anum[i] - bnum[i]));
 		goto pop_next;
 	    case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE):
-		if (is_internal_ref(b)) {
+		if (is_internal_ref_rel(b,b_base)) {
+		    RefThing* bthing = ref_thing_ptr_rel(b,b_base);
 		    bnode = erts_this_node;
-		    bnum = internal_ref_numbers(b);
-		    blen = internal_ref_no_of_numbers(b);
-		} else if (is_external_ref(b)) {
-		    bnode = external_ref_node(b);
-		    bnum = external_ref_numbers(b);
-		    blen = external_ref_no_of_numbers(b);
+		    bnum = internal_thing_ref_numbers(bthing);
+		    blen = internal_thing_ref_no_of_numbers(bthing);
+		} else if (is_external_ref_rel(b,b_base)) {
+		    ExternalThing* bthing = external_thing_ptr_rel(b,b_base);
+		    bnode = bthing->node;
+		    bnum = external_thing_ref_numbers(bthing);
+		    blen = external_thing_ref_no_of_numbers(bthing);
 		} else {
 		    a_tag = EXTERNAL_REF_DEF;
 		    goto mixed_types;
 		}
-		anode = external_ref_node(a);
-		anum = external_ref_numbers(a);
-		alen = external_ref_no_of_numbers(a);
+		{
+		    ExternalThing* athing = external_thing_ptr_rel(a,a_base);
+		    anode = athing->node;
+		    anum = external_thing_ref_numbers(athing);
+		    alen = external_thing_ref_no_of_numbers(athing);
+		}
 		goto ref_common;
 	    default:
 		/* Must be a binary */
-		ASSERT(is_binary(a));
-		if (is_not_binary(b)) {
+		ASSERT(is_binary_rel(a,a_base));
+		if (!is_binary_rel(b,b_base)) {
 		    a_tag = BINARY_DEF;
 		    goto mixed_types;
 		} else {
-		    Uint a_size = binary_size(a);
-		    Uint b_size = binary_size(b);
+		    Uint a_size = binary_size_rel(a,a_base);
+		    Uint b_size = binary_size_rel(b,b_base);
 		    Uint a_bitsize;
 		    Uint b_bitsize;
 		    Uint a_bitoffs;
@@ -2548,8 +2619,8 @@ tailrecur_ne:
 		    int cmp;
 		    byte* a_ptr;
 		    byte* b_ptr;
-		    ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize);
-		    ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize);
+		    ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base);
+		    ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base);
 		    if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) {
 			min_size = (a_size < b_size) ? a_size : b_size;
 			if ((cmp = sys_memcmp(a_ptr, b_ptr, min_size)) != 0) {
@@ -2576,47 +2647,114 @@ tailrecur_ne:
      */
 
  mixed_types:
-    b_tag = tag_val_def(b);
 
     {
 	FloatDef f1, f2;
 	Eterm big;
-	Eterm big_buf[2];
+#if HEAP_ON_C_STACK
+	Eterm big_buf[CMP_TMP_HEAP_SIZE]; /* If HEAP_ON_C_STACK */
+#else
+	Eterm *big_buf = erts_get_scheduler_data()->cmp_tmp_heap;
+#endif
+#if HALFWORD_HEAP
+	Wterm aw = is_immed(a) ? a : rterm2wterm(a,a_base);
+	Wterm bw = is_immed(b) ? b : rterm2wterm(b,b_base);
+#else
+	Eterm aw = a;
+	Eterm bw = b;
+#endif
+#define MAX_LOSSLESS_FLOAT ((double)((1LL << 53) - 2))
+#define MIN_LOSSLESS_FLOAT ((double)(((1LL << 53) - 2)*-1))
+#define BIG_ARITY_FLOAT_MAX (1024 / D_EXP) /* arity of max float as a bignum */
+	b_tag = tag_val_def(bw);
 
 	switch(_NUMBER_CODE(a_tag, b_tag)) {
 	case SMALL_BIG:
-	    big = small_to_big(signed_val(a), big_buf);
-	    j = big_comp(big, b);
-	    break;
-	case SMALL_FLOAT:
-	    f1.fd = signed_val(a);
-	    GET_DOUBLE(b, f2);
-	    j = float_comp(f1.fd, f2.fd);
+	    j = big_sign(bw) ? 1 : -1;
 	    break;
 	case BIG_SMALL:
-	    big = small_to_big(signed_val(b), big_buf);
-	    j = big_comp(a, big);
+	    j = big_sign(aw) ? -1 : 1;
 	    break;
+	case SMALL_FLOAT:
+	    GET_DOUBLE(bw, f2);
+	    if (f2.fd < MAX_LOSSLESS_FLOAT && f2.fd > MIN_LOSSLESS_FLOAT) {
+		// Float is within the no loss limit
+		f1.fd = signed_val(aw);
+		j = float_comp(f1.fd, f2.fd);
+#if ERTS_SIZEOF_ETERM == 8
+	    } else if (f2.fd > (double) (MAX_SMALL + 1)) {
+		// Float is a positive bignum, i.e. bigger
+		j = -1;
+	    } else if (f2.fd < (double) (MIN_SMALL - 1)) {
+		// Float is a negative bignum, i.e. smaller
+		j = 1;
+	    } else { // Float is a Sint but less precise
+		j = signed_val(aw) - (Sint) f2.fd;
+	    }
+#else
+	    } else {
+		// If float is positive it is bigger than small
+		j = (f2.fd > 0.0) ? -1 : 1;
+	    }
+#endif // ERTS_SIZEOF_ETERM == 8
+	    break;
+        case FLOAT_BIG:
+	{
+	    Wterm tmp = aw;
+	    aw = bw;
+	    bw = tmp;
+	}/* fall through */
 	case BIG_FLOAT:
-	    if (big_to_double(a, &f1.fd) < 0) {
-		j = big_sign(a) ? -1 : 1;
+	    GET_DOUBLE(bw, f2);
+	    if ((f2.fd < (double) (MAX_SMALL + 1))
+		    && (f2.fd > (double) (MIN_SMALL - 1))) {
+		// Float is a Sint
+		j = big_sign(aw) ? -1 : 1;
+	    } else if (big_arity(aw) > BIG_ARITY_FLOAT_MAX
+		       || pow(2.0,(big_arity(aw)-1)*D_EXP) > fabs(f2.fd)) {
+		// If bignum size shows that it is bigger than the abs float
+		j = big_sign(aw) ? -1 : 1;
+	    } else if (big_arity(aw) < BIG_ARITY_FLOAT_MAX
+		       && (pow(2.0,(big_arity(aw))*D_EXP)-1.0) < fabs(f2.fd)) {
+		// If bignum size shows that it is smaller than the abs float
+		j = f2.fd < 0 ? 1 : -1;
+	    } else if (f2.fd < MAX_LOSSLESS_FLOAT && f2.fd > MIN_LOSSLESS_FLOAT) {
+		// Float is within the no loss limit
+		if (big_to_double(aw, &f1.fd) < 0) {
+		    j = big_sign(aw) ? -1 : 1;
+		} else {
+		    j = float_comp(f1.fd, f2.fd);
+		}
 	    } else {
-		GET_DOUBLE(b, f2);
-		j = float_comp(f1.fd, f2.fd);
+		big = double_to_big(f2.fd, big_buf);
+		j = big_comp(aw, big);
+	    }
+	    if (_NUMBER_CODE(a_tag, b_tag) == FLOAT_BIG) {
+		j = -j;
 	    }
 	    break;
 	case FLOAT_SMALL:
-	    GET_DOUBLE(a, f1);
-	    f2.fd = signed_val(b);
-	    j = float_comp(f1.fd, f2.fd);
-	    break;
-	case FLOAT_BIG:
-	    if (big_to_double(b, &f2.fd) < 0) {
-		j = big_sign(b) ? 1 : -1;
-	    } else {
-		GET_DOUBLE(a, f1);
+	    GET_DOUBLE(aw, f1);
+	    if (f1.fd < MAX_LOSSLESS_FLOAT && f1.fd > MIN_LOSSLESS_FLOAT) {
+		// Float is within the no loss limit
+		f2.fd = signed_val(bw);
 		j = float_comp(f1.fd, f2.fd);
+#if ERTS_SIZEOF_ETERM == 8
+	    } else if (f1.fd > (double) (MAX_SMALL + 1)) {
+		// Float is a positive bignum, i.e. bigger
+		j = 1;
+	    } else if (f1.fd < (double) (MIN_SMALL - 1)) {
+		// Float is a negative bignum, i.e. smaller
+		j = -1;
+	    } else { // Float is a Sint but less precise it
+		j = (Sint) f1.fd - signed_val(bw);
 	    }
+#else
+	    } else {
+		// If float is positive it is bigger than small
+		j = (f1.fd > 0.0) ? 1 : -1;
+	    }
+#endif // ERTS_SIZEOF_ETERM == 8
 	    break;
 	default:
 	    j = b_tag - a_tag;
@@ -2633,7 +2771,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */
     while (--i) {
 	a = *aa++;
 	b = *bb++;
-	if (a != b) {
+	if (!is_same(a,a_base, b,b_base)) {
 	    if (is_atom(a) && is_atom(b)) {
 		if ((j = cmp_atoms(a, b)) != 0) {
 		    goto not_equal;
@@ -2644,7 +2782,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */
 		}
 	    } else {
 		/* (ab)Use TAG_PRIMARY_HEADER to recognize a term_array */
-		ESTACK_PUSH3(stack, i, (Eterm)bb, (Eterm)aa | TAG_PRIMARY_HEADER);
+		WSTACK_PUSH3(stack, i, (UWord)bb, (UWord)aa | TAG_PRIMARY_HEADER);
 		goto tailrecur_ne;
 	    }
 	}
@@ -2654,20 +2792,20 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */
     goto tailrecur;    
    
 pop_next:
-    if (!ESTACK_ISEMPTY(stack)) {
-	Eterm something = ESTACK_POP(stack);
-	if (primary_tag(something) == TAG_PRIMARY_HEADER) { /* a term_array */
+    if (!WSTACK_ISEMPTY(stack)) {
+	UWord something = WSTACK_POP(stack);
+	if (primary_tag((Eterm) something) == TAG_PRIMARY_HEADER) { /* a term_array */
 	    aa = (Eterm*) something;
-	    bb = (Eterm*) ESTACK_POP(stack);
-	    i = ESTACK_POP(stack);
+	    bb = (Eterm*) WSTACK_POP(stack);
+	    i = WSTACK_POP(stack);
 	    goto term_array;
 	}
-	a = something;
-	b = ESTACK_POP(stack);
+	a = (Eterm) something;
+	b = (Eterm) WSTACK_POP(stack);
 	goto tailrecur;
     }
 
-    DESTROY_ESTACK(stack);
+    DESTROY_WSTACK(stack);
     return 0;
 
 not_equal:
@@ -2678,21 +2816,8 @@ not_equal:
 }
 
 
-void
-erts_cleanup_externals(ExternalThing *etp)
-{
-    ExternalThing *tetp;
-
-    tetp = etp;
-
-    while(tetp) {
-	erts_deref_node_entry(tetp->node);
-	tetp = tetp->next;
-    }
-}
-
 Eterm
-store_external_or_ref_(Uint **hpp, ExternalThing **etpp, Eterm ns)
+store_external_or_ref_(Uint **hpp, ErlOffHeap* oh, Eterm ns)
 {
     Uint i;
     Uint size;
@@ -2711,8 +2836,8 @@ store_external_or_ref_(Uint **hpp, ExternalThing **etpp, Eterm ns)
 
 	erts_refc_inc(&((ExternalThing *) to_hp)->node->refc, 2);
 
-	((ExternalThing *) to_hp)->next = *etpp;
-	*etpp = (ExternalThing *) to_hp;
+	((struct erl_off_heap_header*) to_hp)->next = oh->first;
+	oh->first = (struct erl_off_heap_header*) to_hp;
 
 	return make_external(to_hp);
     }
@@ -2741,12 +2866,12 @@ store_external_or_ref_in_proc_(Process *proc, Eterm ns)
     sz = NC_HEAP_SIZE(ns);
     ASSERT(sz > 0);
     hp = HAlloc(proc, sz);
-    return store_external_or_ref_(&hp, &MSO(proc).externals, ns);
+    return store_external_or_ref_(&hp, &MSO(proc), ns);
 }
 
-void bin_write(int to, void *to_arg, byte* buf, int sz)
+void bin_write(int to, void *to_arg, byte* buf, size_t sz)
 {
-    int i;
+    size_t i;
 
     for (i=0;i<sz;i++) {
 	if (IS_DIGIT(buf[i]))
@@ -2821,17 +2946,17 @@ char* Sint_to_buf(Sint n, struct Sint_buf *buf)
 */
 
 Eterm
-buf_to_intlist(Eterm** hpp, char *buf, int len, Eterm tail)
+buf_to_intlist(Eterm** hpp, char *buf, size_t len, Eterm tail)
 {
     Eterm* hp = *hpp;
+    size_t i = len;
 
-    buf += (len-1);
-    while(len > 0) {
-	tail = CONS(hp, make_small((byte)*buf), tail);
+    while(i != 0) {
+	--i;
+	tail = CONS(hp, make_small((Uint)(byte)buf[i]), tail);
 	hp += 2;
-	buf--;
-	len--;
     }
+
     *hpp = hp;
     return tail;
 }
@@ -2962,13 +3087,25 @@ int io_list_to_buf(Eterm obj, char* buf, int len)
     return -1;
 }
 
-int io_list_len(Eterm obj)
+/*
+ * Return 0 if successful, and non-zero if unsuccessful.
+ */
+int erts_iolist_size(Eterm obj, Uint* sizep)
 {
     Eterm* objp;
-    Sint len = 0;
+    Uint size = 0;
     DECLARE_ESTACK(s);
     goto L_again;
 
+#define SAFE_ADD(Var, Val)			\
+    do {					\
+        Uint valvar = (Val);			\
+	Var += valvar;				\
+	if (Var < valvar) {			\
+	    goto L_overflow_error;		\
+	}					\
+    } while (0)
+
     while (!ESTACK_ISEMPTY(s)) {
 	obj = ESTACK_POP(s);
     L_again:
@@ -2978,9 +3115,12 @@ int io_list_len(Eterm obj)
 	    /* Head */
 	    obj = CAR(objp);
 	    if (is_byte(obj)) {
-		len++;
+		size++;
+		if (size == 0) {
+		    goto L_overflow_error;
+		}
 	    } else if (is_binary(obj) && binary_bitsize(obj) == 0) {
-		len += binary_size(obj);
+		SAFE_ADD(size, binary_size(obj));
 	    } else if (is_list(obj)) {
 		ESTACK_PUSH(s, CDR(objp));
 		goto L_iter_list; /* on head */
@@ -2992,23 +3132,29 @@ int io_list_len(Eterm obj)
 	    if (is_list(obj))
 		goto L_iter_list; /* on tail */
 	    else if (is_binary(obj) && binary_bitsize(obj) == 0) {
-		len += binary_size(obj);
+		SAFE_ADD(size, binary_size(obj));
 	    } else if (is_not_nil(obj)) {
 		goto L_type_error;
 	    }
 	} else if (is_binary(obj) && binary_bitsize(obj) == 0) { /* Tail was binary */
-	    len += binary_size(obj);
+	    SAFE_ADD(size, binary_size(obj));
 	} else if (is_not_nil(obj)) {
 	    goto L_type_error;
 	}
     }
+#undef SAFE_ADD
 
     DESTROY_ESTACK(s);
-    return len;
+    *sizep = size;
+    return ERTS_IOLIST_OK;
+
+ L_overflow_error:
+    DESTROY_ESTACK(s);
+    return ERTS_IOLIST_OVERFLOW;
 
  L_type_error:
     DESTROY_ESTACK(s);
-    return -1;
+    return ERTS_IOLIST_TYPE;
 }
 
 /* return 0 if item is not a non-empty flat list of bytes */
@@ -3150,7 +3296,7 @@ erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref,
 
     *timer_ref = res;
 
-    erl_set_timer(&res->timer.tm,
+    erts_set_timer(&res->timer.tm,
 		  (ErlTimeoutProc) ptimer_timeout,
 		  (ErlCancelProc) ptimer_cancelled,
 		  (void*) res,
@@ -3164,16 +3310,16 @@ erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer)
 	ASSERT(*ptimer->timer.timer_ref == ptimer);
 	*ptimer->timer.timer_ref = NULL;
 	ptimer->timer.flags |= ERTS_PTMR_FLG_CANCELLED;
-	erl_cancel_timer(&ptimer->timer.tm);
+	erts_cancel_timer(&ptimer->timer.tm);
     }
 }
 
 #endif
 
-static Sint trim_threshold;
-static Sint top_pad;
-static Sint mmap_threshold;
-static Sint mmap_max;
+static int trim_threshold;
+static int top_pad;
+static int mmap_threshold;
+static int mmap_max;
 
 Uint tot_bin_allocated;
 
@@ -3196,8 +3342,8 @@ int
 sys_alloc_opt(int opt, int value)
 {
 #if HAVE_MALLOPT
-  Sint m_opt;
-  Sint *curr_val;
+  int m_opt;
+  int *curr_val;
 
   switch(opt) {
   case SYS_ALLOC_OPT_TRIM_THRESHOLD:
@@ -3237,7 +3383,7 @@ sys_alloc_opt(int opt, int value)
   }
 
   if(mallopt(m_opt, value)) {
-    *curr_val = (Sint) value;
+    *curr_val = value;
     return 1;
   }
 
@@ -3256,686 +3402,6 @@ sys_alloc_stat(SysAllocStat *sasp)
 
 }
 
-#ifdef ERTS_SMP
-
-/* Local system block state */
-
-struct {
-    int emergency;
-    long emergency_timeout;
-    erts_smp_cnd_t watchdog_cnd;
-    erts_smp_tid_t watchdog_tid;
-    int threads_to_block;
-    int have_blocker;
-    erts_smp_tid_t blocker_tid;
-    int recursive_block;
-    Uint32 allowed_activities;
-    erts_smp_tsd_key_t blockable_key;
-    erts_smp_mtx_t mtx;
-    erts_smp_cnd_t cnd;
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    int activity_changing;
-    int checking;
-#endif
-} system_block_state;
-
-/* Global system block state */
-erts_system_block_state_t erts_system_block_state;
-
-
-static ERTS_INLINE int
-is_blockable_thread(void)
-{
-    return erts_smp_tsd_get(system_block_state.blockable_key) != NULL;
-}
-
-static ERTS_INLINE int
-is_blocker(void)
-{
-    return (system_block_state.have_blocker
-	    && erts_smp_equal_tids(system_block_state.blocker_tid,
-				   erts_smp_thr_self()));
-}
-
-#ifdef ERTS_ENABLE_LOCK_CHECK
-int
-erts_lc_is_blocking(void)
-{
-    int res;
-    erts_smp_mtx_lock(&system_block_state.mtx);
-    res = erts_smp_pending_system_block() && is_blocker();
-    erts_smp_mtx_unlock(&system_block_state.mtx);
-    return res;
-}
-#endif
-
-static ERTS_INLINE void
-block_me(void (*prepare)(void *),
-	 void (*resume)(void *),
-	 void *arg,
-	 int mtx_locked,
-	 int want_to_block,
-	 int update_act_changing,
-	 profile_sched_msg_q *psmq)
-{
-    if (prepare)
-	(*prepare)(arg);
-
-    /* Locks might be held... */
-
-    if (!mtx_locked)
-	erts_smp_mtx_lock(&system_block_state.mtx);
-
-    if (erts_smp_pending_system_block() && !is_blocker()) {
-	int is_blockable = is_blockable_thread();
-	ASSERT(is_blockable);
-
-	if (is_blockable)
-	    system_block_state.threads_to_block--;
-
-	if (erts_system_profile_flags.scheduler && psmq) {
-	    ErtsSchedulerData *esdp = erts_get_scheduler_data();
-	    if (esdp) {
-	    	profile_sched_msg *msg = NULL;
-	        
-		ASSERT(psmq->n < 2);
-		msg = &((psmq->msg)[psmq->n]);
-		msg->scheduler_id = esdp->no;
-		get_now(&(msg->Ms), &(msg->s), &(msg->us));
-		msg->no_schedulers = 0;
-		msg->state = am_inactive;
-	    	psmq->n++;
-	    }
-	}
-
-#ifdef ERTS_ENABLE_LOCK_CHECK
-	if (update_act_changing)
-	    system_block_state.activity_changing--;
-#endif
-
-	erts_smp_cnd_broadcast(&system_block_state.cnd);
-
-	do {
-	    erts_smp_cnd_wait(&system_block_state.cnd, &system_block_state.mtx);
-	} while (erts_smp_pending_system_block()
-		 && !(want_to_block && !system_block_state.have_blocker));
-
-#ifdef ERTS_ENABLE_LOCK_CHECK
-	if (update_act_changing)
-	    system_block_state.activity_changing++;
-#endif
-	if (erts_system_profile_flags.scheduler && psmq) {
-	    ErtsSchedulerData *esdp = erts_get_scheduler_data();
-	    if (esdp) {
-	    	profile_sched_msg *msg = NULL;
-	        
-		ASSERT(psmq->n < 2);
-		msg = &((psmq->msg)[psmq->n]);
-		msg->scheduler_id = esdp->no;
-		get_now(&(msg->Ms), &(msg->s), &(msg->us));
-		msg->no_schedulers = 0;
-		msg->state = am_active;
-	    	psmq->n++;
-	    }
-	}
-
-	if (is_blockable)
-	    system_block_state.threads_to_block++;
-    }
-
-    if (!mtx_locked)
-	erts_smp_mtx_unlock(&system_block_state.mtx);
-
-    if (resume)
-	(*resume)(arg);
-}
-
-void
-erts_block_me(void (*prepare)(void *),
-	      void (*resume)(void *),
-	      void *arg)
-{
-    profile_sched_msg_q psmq;
-    psmq.n = 0;
-    if (prepare)
-	(*prepare)(arg);
-
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_check_exact(NULL, 0); /* No locks should be locked */
-#endif
-
-    block_me(NULL, NULL, NULL, 0, 0, 0, &psmq);
-
-    if (erts_system_profile_flags.scheduler && psmq.n > 0) 
-    	dispatch_profile_msg_q(&psmq);
-
-    if (resume)
-	(*resume)(arg);
-}
-
-void
-erts_register_blockable_thread(void)
-{
-    profile_sched_msg_q psmq;
-    psmq.n = 0;
-    if (!is_blockable_thread()) {
-	erts_smp_mtx_lock(&system_block_state.mtx);
-	system_block_state.threads_to_block++;
-	erts_smp_tsd_set(system_block_state.blockable_key,
-			 (void *) &erts_system_block_state);
-
-	/* Someone might be waiting for us to block... */
-	if (erts_smp_pending_system_block())
-	    block_me(NULL, NULL, NULL, 1, 0, 0, &psmq);
-	erts_smp_mtx_unlock(&system_block_state.mtx);
-
-	if (erts_system_profile_flags.scheduler && psmq.n > 0)
-	    dispatch_profile_msg_q(&psmq);
-    }
-}
-
-void
-erts_unregister_blockable_thread(void)
-{
-    if (is_blockable_thread()) {
-	erts_smp_mtx_lock(&system_block_state.mtx);
-	system_block_state.threads_to_block--;
-	ASSERT(system_block_state.threads_to_block >= 0);
-	erts_smp_tsd_set(system_block_state.blockable_key, NULL);
-
-	/* Someone might be waiting for us to block... */
-	if (erts_smp_pending_system_block())
-	    erts_smp_cnd_broadcast(&system_block_state.cnd);
-	erts_smp_mtx_unlock(&system_block_state.mtx);
-    }
-}
-
-void
-erts_note_activity_begin(erts_activity_t activity)
-{
-    erts_smp_mtx_lock(&system_block_state.mtx);
-    if (erts_smp_pending_system_block()) {
-	Uint32 broadcast = 0;
-	switch (activity) {
-	case ERTS_ACTIVITY_GC:
-	    broadcast = (system_block_state.allowed_activities
-			 & ERTS_BS_FLG_ALLOW_GC);
-	    break;
-	case ERTS_ACTIVITY_IO:
-	    broadcast = (system_block_state.allowed_activities
-			 & ERTS_BS_FLG_ALLOW_IO);
-	    break;
-	case ERTS_ACTIVITY_WAIT:
-	    broadcast = 1;
-	    break;
-	default:
-	    abort();
-	    break;
-	}
-	if (broadcast)
-	    erts_smp_cnd_broadcast(&system_block_state.cnd);
-    }
-    erts_smp_mtx_unlock(&system_block_state.mtx);
-}
-
-void
-erts_check_block(erts_activity_t old_activity,
-		 erts_activity_t new_activity,
-		 int locked,
-		 void (*prepare)(void *),
-		 void (*resume)(void *),
-		 void *arg)
-{
-    int do_block;
-    profile_sched_msg_q psmq;
-
-    psmq.n = 0;
-    if (!locked && prepare)
-	(*prepare)(arg);
-
-    erts_smp_mtx_lock(&system_block_state.mtx);
-
-    /* First check if it is ok to block... */
-    if (!locked)
-	do_block = 1;
-    else {
-	switch (old_activity) {
-	case ERTS_ACTIVITY_UNDEFINED:
-	    do_block = 0;
-	    break;
-	case ERTS_ACTIVITY_GC:
-	    do_block = (system_block_state.allowed_activities
-			& ERTS_BS_FLG_ALLOW_GC);
-	    break;
-	case ERTS_ACTIVITY_IO:
-	    do_block = (system_block_state.allowed_activities
-			& ERTS_BS_FLG_ALLOW_IO);
-	    break;
-	case ERTS_ACTIVITY_WAIT:
-	    /* You are not allowed to leave activity waiting
-	     * without supplying the possibility to block
-	     * unlocked.
-	     */
-	    erts_set_activity_error(ERTS_ACT_ERR_LEAVE_WAIT_UNLOCKED,
-				    __FILE__, __LINE__);
-	    do_block = 0;
-	    break;
-	default:
-	    erts_set_activity_error(ERTS_ACT_ERR_LEAVE_UNKNOWN_ACTIVITY,
-				    __FILE__, __LINE__);
-	    do_block = 0;
-	    break;
-	}
-    }
-
-    if (do_block) {
-	/* ... then check if it is necessary to block... */
-
-	switch (new_activity) {
-	case ERTS_ACTIVITY_UNDEFINED:
-	    do_block = 1;
-	    break;
-	case ERTS_ACTIVITY_GC:
-	    do_block = !(system_block_state.allowed_activities
-			 & ERTS_BS_FLG_ALLOW_GC);
-	break;
-	case ERTS_ACTIVITY_IO:
-	    do_block = !(system_block_state.allowed_activities
-			 & ERTS_BS_FLG_ALLOW_IO);
-	    break;
-	case ERTS_ACTIVITY_WAIT:
-	    /* No need to block if we are going to wait */
-	    do_block = 0;
-	    break;
-	default:
-	    erts_set_activity_error(ERTS_ACT_ERR_ENTER_UNKNOWN_ACTIVITY,
-				    __FILE__, __LINE__);
-	    break;
-	}
-    }
-
-    if (do_block) {
-
-#ifdef ERTS_ENABLE_LOCK_CHECK
-	if (!locked) {
-	    /* Only system_block_state.mtx should be held */
-	    erts_lc_check_exact(&system_block_state.mtx.lc, 1);
-	}
-#endif
-
-	block_me(NULL, NULL, NULL, 1, 0, 1, &psmq);
-
-    }
-
-    erts_smp_mtx_unlock(&system_block_state.mtx);
-
-    if (erts_system_profile_flags.scheduler && psmq.n > 0) 
-	dispatch_profile_msg_q(&psmq);	
-
-    if (!locked && resume)
-	(*resume)(arg);
-}
-
-
-
-void
-erts_set_activity_error(erts_activity_error_t error, char *file, int line)
-{
-    switch (error) {
-    case ERTS_ACT_ERR_LEAVE_WAIT_UNLOCKED:
-	erl_exit(1, "%s:%d: Fatal error: Leaving activity waiting without "
-		 "supplying the possibility to block unlocked.",
-		 file, line);
-	break;
-    case ERTS_ACT_ERR_LEAVE_UNKNOWN_ACTIVITY:
-	erl_exit(1, "%s:%d: Fatal error: Leaving unknown activity.",
-		 file, line);
-	break;
-    case ERTS_ACT_ERR_ENTER_UNKNOWN_ACTIVITY:
-	erl_exit(1, "%s:%d: Fatal error: Leaving unknown activity.",
-		 file, line);
-	break;
-    default:
-	erl_exit(1, "%s:%d: Internal error in erts_smp_set_activity()",
-		 file, line);
-	break;
-    }
-
-}
-
-
-static ERTS_INLINE int
-threads_not_under_control(void)
-{
-    int res = system_block_state.threads_to_block;
-
-    /* Waiting is always an allowed activity... */
-    res -= erts_smp_atomic_read(&erts_system_block_state.in_activity.wait);
-
-    if (system_block_state.allowed_activities & ERTS_BS_FLG_ALLOW_GC)
-	res -= erts_smp_atomic_read(&erts_system_block_state.in_activity.gc);
-
-    if (system_block_state.allowed_activities & ERTS_BS_FLG_ALLOW_IO)
-	res -= erts_smp_atomic_read(&erts_system_block_state.in_activity.io);
-
-    if (res < 0) {
-	ASSERT(0);
-	return 0;
-    }
-    return res;
-}
-
-/*
- * erts_block_system() blocks all threads registered as blockable.
- * It doesn't return until either all threads have blocked (0 is returned)
- * or it has timed out (ETIMEDOUT) is returned.
- *
- * If allowed activities == 0, blocked threads will release all locks
- * before blocking.
- *
- * If allowed_activities is != 0, erts_block_system() will allow blockable
- * threads to continue executing as long as they are doing an allowed
- * activity. When they are done with the allowed activity they will block,
- * *but* they will block holding locks. Therefore, the thread calling
- * erts_block_system() must *not* try to aquire any locks that might be
- * held by blocked threads holding locks from allowed activities.
- *
- * Currently allowed_activities are:
- *	* ERTS_BS_FLG_ALLOW_GC		Thread continues with garbage
- *					collection and blocks with
- *					main process lock on current
- *					process locked.
- *	* ERTS_BS_FLG_ALLOW_IO		Thread continues with I/O
- */
-
-void
-erts_block_system(Uint32 allowed_activities)
-{
-    int do_block;
-    profile_sched_msg_q psmq;
-    
-    psmq.n = 0;
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_check_exact(NULL, 0); /* No locks should be locked */
-#endif
-
-    erts_smp_mtx_lock(&system_block_state.mtx);
-
-    do_block = erts_smp_pending_system_block();
-    if (do_block
-	&& system_block_state.have_blocker
-	&& erts_smp_equal_tids(system_block_state.blocker_tid,
-			       erts_smp_thr_self())) {
-	ASSERT(system_block_state.recursive_block >= 0);
-	system_block_state.recursive_block++;
-
-	/* You are not allowed to restrict allowed activites
-	   in a recursive block! */
-	ERTS_SMP_LC_ASSERT((system_block_state.allowed_activities
-			    & ~allowed_activities) == 0);
-    }
-    else {
-
-	erts_smp_atomic_inc(&erts_system_block_state.do_block);
-
-	/* Someone else might be waiting for us to block... */
-	if (do_block) {
-	do_block_me:
-	    block_me(NULL, NULL, NULL, 1, 1, 0, &psmq);
-	}
-
-	ASSERT(!system_block_state.have_blocker);
-	system_block_state.have_blocker = 1;
-	system_block_state.blocker_tid = erts_smp_thr_self();
-	system_block_state.allowed_activities = allowed_activities;
-
-	if (is_blockable_thread())
-	    system_block_state.threads_to_block--;
-
-	while (threads_not_under_control() && !system_block_state.emergency)
-	    erts_smp_cnd_wait(&system_block_state.cnd, &system_block_state.mtx);
-
-	if (system_block_state.emergency) {
-	    system_block_state.have_blocker = 0;
-	    goto do_block_me;
-	}
-    }
-
-    erts_smp_mtx_unlock(&system_block_state.mtx);
-
-    if (erts_system_profile_flags.scheduler && psmq.n > 0 )
-    	dispatch_profile_msg_q(&psmq);
-}
-
-/*
- * erts_emergency_block_system() should only be called when we are
- * about to write a crash dump...
- */
-
-int
-erts_emergency_block_system(long timeout, Uint32 allowed_activities)
-{
-    int res = 0;
-    long another_blocker;
-
-    erts_smp_mtx_lock(&system_block_state.mtx);
-
-    if (system_block_state.emergency) {
-	 /* Argh... */
-	res = EINVAL;
-	goto done;
-    }
-
-    another_blocker = erts_smp_pending_system_block();
-    system_block_state.emergency = 1;
-    erts_smp_atomic_inc(&erts_system_block_state.do_block);
-
-    if (another_blocker) {
-	if (is_blocker()) {
-	    erts_smp_atomic_dec(&erts_system_block_state.do_block);
-	    res = 0;
-	    goto done;
-	}
-	/* kick the other blocker */
-	erts_smp_cnd_broadcast(&system_block_state.cnd);
-	while (system_block_state.have_blocker)
-	    erts_smp_cnd_wait(&system_block_state.cnd, &system_block_state.mtx);
-    }
-
-    ASSERT(!system_block_state.have_blocker);
-    system_block_state.have_blocker = 1;
-    system_block_state.blocker_tid = erts_smp_thr_self();
-    system_block_state.allowed_activities = allowed_activities;
-
-    if (is_blockable_thread())
-	system_block_state.threads_to_block--;
-
-    if (timeout < 0) {
-	while (threads_not_under_control())
-	    erts_smp_cnd_wait(&system_block_state.cnd, &system_block_state.mtx);
-    }
-    else {	
-	system_block_state.emergency_timeout = timeout;
-	erts_smp_cnd_signal(&system_block_state.watchdog_cnd);
-
-	while (system_block_state.emergency_timeout >= 0
-	       && threads_not_under_control()) {
-	    erts_smp_cnd_wait(&system_block_state.cnd,
-			      &system_block_state.mtx);
-	}
-    }
- done:
-    erts_smp_mtx_unlock(&system_block_state.mtx);
-    return res;
-}
-
-void
-erts_release_system(void)
-{
-    long do_block;
-    profile_sched_msg_q psmq;
-    
-    psmq.n = 0;
-
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_lc_check_exact(NULL, 0); /* No locks should be locked */
-#endif
-
-    erts_smp_mtx_lock(&system_block_state.mtx);
-    ASSERT(is_blocker());
-
-    ASSERT(system_block_state.recursive_block >= 0);
-
-    if (system_block_state.recursive_block)
-	system_block_state.recursive_block--;
-    else {
-	do_block = erts_smp_atomic_dectest(&erts_system_block_state.do_block);
-	system_block_state.have_blocker = 0;
-	if (is_blockable_thread())
-	    system_block_state.threads_to_block++;
-	else
-	    do_block = 0;
-
-	/* Someone else might be waiting for us to block... */
-	if (do_block)
-	    block_me(NULL, NULL, NULL, 1, 0, 0, &psmq);
-	else
-	    erts_smp_cnd_broadcast(&system_block_state.cnd);
-    }
-
-    erts_smp_mtx_unlock(&system_block_state.mtx);
-
-    if (erts_system_profile_flags.scheduler && psmq.n > 0) 
-    	dispatch_profile_msg_q(&psmq);
-}
-
-#ifdef ERTS_ENABLE_LOCK_CHECK
-
-void
-erts_lc_activity_change_begin(void)
-{
-    erts_smp_mtx_lock(&system_block_state.mtx);
-    system_block_state.activity_changing++;
-    erts_smp_mtx_unlock(&system_block_state.mtx);
-}
-
-void
-erts_lc_activity_change_end(void)
-{
-    erts_smp_mtx_lock(&system_block_state.mtx);
-    system_block_state.activity_changing--;
-    if (system_block_state.checking && !system_block_state.activity_changing)
-	erts_smp_cnd_broadcast(&system_block_state.cnd);
-    erts_smp_mtx_unlock(&system_block_state.mtx);
-}
-
-#endif
-
-int
-erts_is_system_blocked(erts_activity_t allowed_activities)
-{
-    int blkd;
-
-    erts_smp_mtx_lock(&system_block_state.mtx);
-    blkd = (erts_smp_pending_system_block()
-	    && system_block_state.have_blocker
-	    && erts_smp_equal_tids(system_block_state.blocker_tid,
-				   erts_smp_thr_self())
-	    && !(system_block_state.allowed_activities & ~allowed_activities));
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    if (blkd) {
-	system_block_state.checking = 1;
-	while (system_block_state.activity_changing)
-	    erts_smp_cnd_wait(&system_block_state.cnd, &system_block_state.mtx);
-	system_block_state.checking = 0;
-	blkd = !threads_not_under_control();
-    }
-#endif
-    erts_smp_mtx_unlock(&system_block_state.mtx);
-    return blkd;
-}
-
-static void *
-emergency_watchdog(void *unused)
-{
-    erts_smp_mtx_lock(&system_block_state.mtx);
-    while (1) {
-	long timeout;
-	while (system_block_state.emergency_timeout < 0)
-	    erts_smp_cnd_wait(&system_block_state.watchdog_cnd, &system_block_state.mtx);
-	timeout = system_block_state.emergency_timeout;
-	erts_smp_mtx_unlock(&system_block_state.mtx);
-
-	if (erts_disable_tolerant_timeofday)
-	    erts_milli_sleep(timeout);
-	else {
-	    SysTimeval to;
-	    erts_get_timeval(&to);
-	    to.tv_sec += timeout / 1000;
-	    to.tv_usec += timeout % 1000;
-
-	    while (1) {
-		SysTimeval curr;
-		erts_milli_sleep(timeout);
-		erts_get_timeval(&curr);
-		if (curr.tv_sec > to.tv_sec
-		    || (curr.tv_sec == to.tv_sec && curr.tv_usec >= to.tv_usec)) {
-		    break;
-		}
-		timeout = (to.tv_sec - curr.tv_sec)*1000;
-		timeout += (to.tv_usec - curr.tv_usec)/1000;
-	    }
-	}
-
-	erts_smp_mtx_lock(&system_block_state.mtx);
-	system_block_state.emergency_timeout = -1;
-	erts_smp_cnd_broadcast(&system_block_state.cnd);
-    }
-    erts_smp_mtx_unlock(&system_block_state.mtx);
-    return NULL;
-}
-
-void
-erts_system_block_init(void)
-{
-    erts_smp_thr_opts_t thr_opts = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
-    /* Local state... */
-    system_block_state.emergency = 0;
-    system_block_state.emergency_timeout = -1;
-    erts_smp_cnd_init(&system_block_state.watchdog_cnd);
-    system_block_state.threads_to_block = 0;
-    system_block_state.have_blocker = 0;
-    /* system_block_state.block_tid */
-    system_block_state.recursive_block = 0;
-    system_block_state.allowed_activities = 0;
-    erts_smp_tsd_key_create(&system_block_state.blockable_key);
-    erts_smp_mtx_init(&system_block_state.mtx, "system_block");
-    erts_smp_cnd_init(&system_block_state.cnd);
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    system_block_state.activity_changing = 0;
-    system_block_state.checking = 0;
-#endif
-
-    thr_opts.suggested_stack_size = 8;
-    erts_smp_thr_create(&system_block_state.watchdog_tid,
-			emergency_watchdog,
-			NULL,
-			&thr_opts);
-
-    /* Global state... */
-
-    erts_smp_atomic_init(&erts_system_block_state.do_block, 0L);
-    erts_smp_atomic_init(&erts_system_block_state.in_activity.wait, 0L);
-    erts_smp_atomic_init(&erts_system_block_state.in_activity.gc, 0L);
-    erts_smp_atomic_init(&erts_system_block_state.in_activity.io, 0L);
-
-    /* Make sure blockable threads unregister when exiting... */
-    erts_smp_install_exit_handler(erts_unregister_blockable_thread);
-}
-
-
-#endif /* #ifdef ERTS_SMP */
-
 char *
 erts_read_env(char *key)
 {
@@ -3997,11 +3463,9 @@ void erts_silence_warn_unused_result(long unused)
  * Handy functions when using a debugger - don't use in the code!
  */
 
-void upp(buf,sz)
-byte* buf;
-int sz;
+void upp(byte *buf, size_t sz)
 {
-    bin_write(ERTS_PRINT_STDERR,NULL,buf,sz);
+    bin_write(ERTS_PRINT_STDERR, NULL, buf, sz);
 }
 
 void pat(Eterm atom)
diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c
index 95510a16b2..347247ee7b 100644
--- a/erts/emulator/drivers/common/efile_drv.c
+++ b/erts/emulator/drivers/common/efile_drv.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /*
@@ -53,6 +53,9 @@
 #define FILE_IPREAD             27
 #define FILE_ALTNAME            28
 #define FILE_READ_LINE          29
+#define FILE_FDATASYNC          30
+#define FILE_FADVISE            31
+#define FILE_SENDFILE           32
 
 /* Return codes */
 
@@ -65,6 +68,9 @@
 #define FILE_RESP_LDATA      6
 #define FILE_RESP_N2DATA     7
 #define FILE_RESP_EOF        8
+#define FILE_RESP_FNAME      9
+#define FILE_RESP_ALL_DATA  10
+#define FILE_RESP_LFNAME    11
 
 /* Options */
 
@@ -93,25 +99,66 @@
 #  include "config.h"
 #endif
 #include <stdlib.h>
+
+/* Need (NON)BLOCKING macros for sendfile */
+#ifndef WANT_NONBLOCKING
+#define WANT_NONBLOCKING
+#endif
+
 #include "sys.h"
+
 #include "erl_driver.h"
 #include "erl_efile.h"
 #include "erl_threads.h"
 #include "zlib.h"
 #include "gzio.h"
+#include "dtrace-wrapper.h" 
 #include <ctype.h>
 #include <sys/types.h>
 
-extern void erl_exit(int n, char *fmt, _DOTS_);
+void erl_exit(int n, char *fmt, ...);
 
 static ErlDrvSysInfo sys_info;
 
+/* For explanation of this var, see comment for same var in erl_async.c */
+static unsigned gcc_optimizer_hack = 0;
+
+#ifdef  USE_VM_PROBES
+
+#define DTRACE_EFILE_BUFSIZ 128
+
+#define DTRACE_INVOKE_SETUP(op) \
+    do { DTRACE3(efile_drv_int_entry, d->sched_i1, d->sched_i2, op); } while (0)
+#define DTRACE_INVOKE_SETUP_BY_NAME(op) \
+    struct t_data *d = (struct t_data *) data ; \
+    DTRACE_INVOKE_SETUP(op)
+#define DTRACE_INVOKE_RETURN(op) \
+    do { DTRACE3(efile_drv_int_return, d->sched_i1, d->sched_i2, \
+                 op); } while (0) ; gcc_optimizer_hack++ ;
 
-/*#define TRACE 1*/
+/* Assign human-friendlier id numbers to scheduler & I/O worker threads */
+int             dt_driver_idnum = 0;
+int             dt_driver_io_worker_base = 5000;
+erts_mtx_t      dt_driver_mutex;
+pthread_key_t   dt_driver_key;
+
+typedef struct {
+    int         thread_num;
+    Uint64      tag;
+} dt_private;
+
+dt_private *get_dt_private(int);
+#else  /* USE_VM_PROBES */
+#define DTRACE_INVOKE_SETUP(op)            do {} while (0)
+#define DTRACE_INVOKE_SETUP_BY_NAME(op)    do {} while (0)
+#define DTRACE_INVOKE_RETURN(op)           do {} while (0)
+#endif  /* USE_VM_PROBES */
+
+/* #define TRACE 1 */
 #ifdef TRACE
-#    define TRACE_C(c) (putchar(c))
-#    define TRACE_S(s) (fputs((s), stdout))
-#    define TRACE_F(args) (printf args)
+#    define TRACE_C(c) do { putchar(c); fflush(stdout); } while (0)
+#    define TRACE_S(s) do { fputs((s), stdout); fflush(stdout); } while (0)
+#    define TRACE_F(args) do { printf args ;fflush(stdout); } while (0)
 #else
 #    define TRACE_C(c) ((void)(0))
 #    define TRACE_S(s) ((void)(0))
@@ -136,23 +183,73 @@ static ErlDrvSysInfo sys_info;
 #endif
 
 
+/**
+ * On DARWIN sendfile can deadlock with close if called in
+ * different threads. So until Apple fixes so that sendfile
+ * is not buggy we disable usage of the async pool for
+ * DARWIN. The testcase t_sendfile_crashduring reproduces
+ * this error when using +A 10.
+ */
+#if defined(__APPLE__) && defined(__MACH__)
+#define USE_THRDS_FOR_SENDFILE 0
+#else
+#define USE_THRDS_FOR_SENDFILE (sys_info.async_threads > 0)
+#endif /* defined(__APPLE__) && defined(__MACH__) */
+
+
 
 #if 0
 /* Experimental, for forcing all file operations to use the same thread. */
-static unsigned file_fixed_key = 1;
-#define KEY(desc) (&file_fixed_key)
+   static unsigned file_fixed_key = 1;
+#  define KEY(desc) (&file_fixed_key)
 #else
-#define KEY(desc) (&(desc)->key)
+#  define KEY(desc) (&(desc)->key)
 #endif
 
 
+#ifdef FILENAMES_16BIT
+#ifdef USE_VM_PROBES
+#error 16bit characters in filenames and dtrace in combination is not supported.
+#endif
+#  define FILENAME_BYTELEN(Str) filename_len_16bit(Str)
+#  define FILENAME_COPY(To,From) filename_cpy_16bit((To),(From)) 
+#  define FILENAME_CHARSIZE 2
+
+   static int filename_len_16bit(char *str) 
+   {
+       char *p = str;
+       while(*p != '\0' || p[1] != '\0') {
+	   p += 2;
+       }
+       return (p - str);
+   }
+
+   static void filename_cpy_16bit(char *to, char *from) 
+   {
+       while(*from != '\0' || from[1] != '\0') {
+	   *to++ = *from++;
+	   *to++ = *from++;
+       }
+       *to++ = *from++;
+       *to++ = *from++;
+   }
+
+#else
+#  define FILENAME_BYTELEN(Str) strlen(Str)
+#  define FILENAME_COPY(To,From) strcpy(To,From) 
+#  define FILENAME_CHARSIZE 1
+#endif
 
-#if     MAXPATHLEN >= BUFSIZ
-#define    RESBUFSIZE  MAXPATHLEN+1
+#if     (MAXPATHLEN+1)*FILENAME_CHARSIZE+1 > BUFSIZ
+#  define    RESBUFSIZE  ((MAXPATHLEN+1)*FILENAME_CHARSIZE+1)
 #else
-#define    RESBUFSIZE  BUFSIZ
+#  define    RESBUFSIZE  BUFSIZ
 #endif
 
+#define READDIR_CHUNKS (5)
+
+
+
 #define GET_TIME(i, b) \
     (i).year  = get_int32((b) + 0 * 4); \
     (i).month = get_int32((b) + 1 * 4); \
@@ -181,24 +278,33 @@ typedef unsigned char uchar;
 static ErlDrvData file_start(ErlDrvPort port, char* command);
 static int file_init(void);
 static void file_stop(ErlDrvData);
-static void file_output(ErlDrvData, char* buf, int len);
-static int file_control(ErlDrvData, unsigned int command, 
-			char* buf, int len, char **rbuf, int rlen);
+static void file_output(ErlDrvData, char* buf, ErlDrvSizeT len);
+static ErlDrvSSizeT file_control(ErlDrvData, unsigned int command,
+				 char* buf, ErlDrvSizeT len,
+				 char **rbuf, ErlDrvSizeT rlen);
 static void file_timeout(ErlDrvData);
 static void file_outputv(ErlDrvData, ErlIOVec*);
 static void file_async_ready(ErlDrvData, ErlDrvThreadData);
 static void file_flush(ErlDrvData);
 
+#ifdef HAVE_SENDFILE
+static void file_ready_output(ErlDrvData data, ErlDrvEvent event);
+static void file_stop_select(ErlDrvEvent event, void* _);
+#endif /* HAVE_SENDFILE */
 
 
 enum e_timer {timer_idle, timer_again, timer_write};
+#ifdef HAVE_SENDFILE
+enum e_sendfile {sending, not_sending};
+static void free_sendfile(void *data);
+#endif /* HAVE_SENDFILE */
 
 struct t_data;
 
 typedef struct {
-    Sint            fd;
+    SWord           fd;
     ErlDrvPort      port;
-    unsigned        key;      /* Async queue key */
+    unsigned int    key;      /* Async queue key */
     unsigned        flags;    /* Original flags from FILE_OPEN. */
     void          (*invoke)(void *);
     struct t_data  *d;
@@ -206,6 +312,9 @@ typedef struct {
     struct t_data  *cq_head;  /* Queue of incoming commands */
     struct t_data  *cq_tail;  /* -""- */
     enum e_timer    timer_state;
+#ifdef HAVE_SENDFILE
+    enum e_sendfile sendfile_state;
+#endif /* HAVE_SENDFILE */
     size_t          read_bufsize;
     ErlDrvBinary   *read_binp;
     size_t          read_offset;
@@ -217,6 +326,10 @@ typedef struct {
     ErlDrvPDL       q_mtx;    /* Mutex for the driver queue, known by the emulator. Also used for
 				 mutual exclusion when accessing field(s) below. */
     size_t          write_buffered;
+#ifdef USE_VM_PROBES
+    int             idnum;      /* Unique ID # for this driver thread/desc */
+    char            port_str[DTRACE_TERM_BUF_SIZE];
+#endif
 } file_descriptor;
 
 
@@ -228,7 +341,11 @@ struct erl_drv_entry efile_driver_entry = {
     file_stop,
     file_output,
     NULL,
+#ifdef HAVE_SENDFILE
+    file_ready_output,
+#else
     NULL,
+#endif /* HAVE_SENDFILE */
     "efile",
     NULL,
     NULL,
@@ -243,7 +360,13 @@ struct erl_drv_entry efile_driver_entry = {
     ERL_DRV_EXTENDED_MAJOR_VERSION,
     ERL_DRV_EXTENDED_MINOR_VERSION,
     ERL_DRV_FLAG_USE_PORT_LOCKING,
+    NULL,
+    NULL,
+#ifdef HAVE_SENDFILE
+    file_stop_select
+#else
     NULL
+#endif /* HAVE_SENDFILE */
 };
 
 
@@ -283,15 +406,16 @@ struct t_preadv {
     Sint64   offsets[1];
 };
 
-#define READDIR_BUFSIZE (8*1024)
-#if READDIR_BUFSIZE < (2*MAXPATHLEN)
-#undef READDIR_BUFSIZE
-#define READDIR_BUFSIZE (2*MAXPATHLEN)
+#define READDIR_BUFSIZE (8*1024)*READDIR_CHUNKS
+#if READDIR_BUFSIZE < (1 + (2 + MAXPATHLEN)*FILENAME_CHARSIZE*READDIR_CHUNKS)
+#  undef READDIR_BUFSIZE
+#  define READDIR_BUFSIZE (1 + (2 + MAXPATHLEN)*FILENAME_CHARSIZE*READDIR_CHUNKS)
 #endif
 
 struct t_readdir_buf {
-    struct t_readdir_buf *next;
-    char buf[READDIR_BUFSIZE];
+     struct t_readdir_buf *next;
+     size_t n;
+     char buf[READDIR_BUFSIZE];
 };
 
 struct t_data
@@ -303,10 +427,15 @@ struct t_data
     void         (*free)(void *);
     int            again;
     int            reply;
+#ifdef  USE_VM_PROBES
+    int               sched_i1;
+    Uint64            sched_i2;
+    char              sched_utag[DTRACE_EFILE_BUFSIZ+1];
+#endif
     int            result_ok;
     Efile_error    errInfo;
     int            flags;
-    Sint           fd;
+    SWord          fd;
     /**/
     Efile_info        info;
     EFILE_DIR_HANDLE  dir_handle; /* Handle to open directory. */
@@ -351,17 +480,30 @@ struct t_data
 	    ErlDrvBinary *binp;
 	    int           size;
 	    int           offset;
-	    char          name[1];
 	} read_file;
 	struct {
 	    struct t_readdir_buf *first_buf;
 	    struct t_readdir_buf *last_buf;
 	} read_dir;
+	struct {
+	    Sint64 offset;
+	    Sint64 length;
+	    int advise;
+	} fadvise;
+#ifdef HAVE_SENDFILE
+	struct {
+	    ErlDrvPort port;
+	    ErlDrvPDL q_mtx;
+	    int out_fd;
+	    off_t offset;
+	    Uint64 nbytes;
+	    Uint64 written;
+	} sendfile;
+#endif /* HAVE_SENDFILE */
     } c;
     char b[1];
 };
 
-
 #define EF_ALLOC(S)		driver_alloc((S))
 #define EF_REALLOC(P, S)	driver_realloc((P), (S))
 #define EF_SAFE_ALLOC(S)	ef_safe_alloc((S))
@@ -371,7 +513,7 @@ struct t_data
 static void *ef_safe_alloc(Uint s)
 {
     void *p = EF_ALLOC(s);
-    if (!p) erl_exit(1, "efile drv: Can't allocate %d bytes of memory\n", s);
+    if (!p) erl_exit(1, "efile drv: Can't allocate %lu bytes of memory\n", (unsigned long)s);
     return p;
 }
 
@@ -380,7 +522,7 @@ static void *ef_safe_alloc(Uint s)
 static void *ef_safe_realloc(void *op, Uint s)
 {
     void *p = EF_REALLOC(op, s);
-    if (!p) erl_exit(1, "efile drv: Can't reallocate %d bytes of memory\n", s);
+    if (!p) erl_exit(1, "efile drv: Can't reallocate %lu bytes of memory\n", (unsigned long)s);
     return p;
 }
 
@@ -390,7 +532,7 @@ static void *ef_safe_realloc(void *op, Uint s)
  * ErlIOVec manipulation functions.
  */
 
-/* char EV_CHAR(ErlIOVec *ev, int p, int q) */
+/* char EV_CHAR_P(ErlIOVec *ev, int p, int q) */
 #define EV_CHAR_P(ev, p, q)                   \
     (((char *)(ev)->iov[(q)].iov_base) + (p))
 
@@ -443,7 +585,6 @@ static void *ef_safe_realloc(void *op, Uint s)
      : 0)
 
 
-
 #if 0
 
 static void ev_clear(ErlIOVec *ev) {
@@ -571,7 +712,6 @@ static struct t_data *cq_deq(file_descriptor *desc) {
 }
 
 
-
 /*********************************************************************
  * Driver entry point -> init
  */
@@ -586,6 +726,12 @@ file_init(void)
 			    ? atoi(buf)
 			    : 0);
     driver_system_info(&sys_info, sizeof(ErlDrvSysInfo));
+
+#ifdef  USE_VM_PROBES
+    erts_mtx_init(&dt_driver_mutex, "efile_drv dtrace mutex");
+    pthread_key_create(&dt_driver_key, NULL);
+#endif  /* USE_VM_PROBES */
+
     return 0;
 }
 
@@ -605,7 +751,7 @@ file_start(ErlDrvPort port, char* command)
     }
     desc->fd = FILE_FD_INVALID;
     desc->port = port;
-    desc->key = (unsigned) (Uint) port;
+    desc->key = (unsigned int) (UWord) port;
     desc->flags = 0;
     desc->invoke = NULL;
     desc->d = NULL;
@@ -613,6 +759,9 @@ file_start(ErlDrvPort port, char* command)
     desc->cq_head = NULL;
     desc->cq_tail = NULL;
     desc->timer_state = timer_idle;
+#ifdef HAVE_SENDFILE
+    desc->sendfile_state = not_sending;
+#endif
     desc->read_bufsize = 0;
     desc->read_binp = NULL;
     desc->read_offset = 0;
@@ -622,6 +771,10 @@ file_start(ErlDrvPort port, char* command)
     desc->write_error = 0;
     MUTEX_INIT(desc->q_mtx, port); /* Refc is one, referenced by emulator now */
     desc->write_buffered = 0;
+#ifdef  USE_VM_PROBES
+    dtrace_drvport_str(port, desc->port_str);
+    get_dt_private(0);           /* throw away return value */
+#endif  /* USE_VM_PROBES */
     return (ErlDrvData) desc;
 }
 
@@ -630,7 +783,7 @@ static void free_data(void *data)
     EF_FREE(data);
 }
 
-static void do_close(int flags, Sint fd) {
+static void do_close(int flags, SWord fd) {
     if (flags & EFILE_COMPRESSED) {
 	erts_gzclose((gzFile)(fd));
     } else {
@@ -641,8 +794,10 @@ static void do_close(int flags, Sint fd) {
 static void invoke_close(void *data)
 {
     struct t_data *d = (struct t_data *) data;
+    DTRACE_INVOKE_SETUP(FILE_CLOSE);
     d->again = 0;
     do_close(d->flags, d->fd);
+    DTRACE_INVOKE_RETURN(FILE_CLOSE);
 }
 
 /*********************************************************************
@@ -709,7 +864,7 @@ static void reply_Uint_posix_error(file_descriptor *desc, Uint num,
     TRACE_C('N');
 
     response[0] = FILE_RESP_NUMERR;
-#if SIZEOF_VOID_P == 4
+#if SIZEOF_VOID_P == 4 || HALFWORD_HEAP
     put_int32(0, response+1);
 #else
     put_int32(num>>32, response+1);
@@ -720,7 +875,16 @@ static void reply_Uint_posix_error(file_descriptor *desc, Uint num,
     driver_output2(desc->port, response, t-response, NULL, 0);
 }
 
+static void reply_string_error(file_descriptor *desc, char* str) {
+    char response[256];		/* Response buffer. */
+    char* s;
+    char* t;
 
+    response[0] = FILE_RESP_ERROR;
+    for (s = str, t = response+1; *s; s++, t++)
+	*t = tolower(*s);
+    driver_output2(desc->port, response, t-response, NULL, 0);
+}
 
 static int reply_error(file_descriptor *desc, 
 		       Efile_error *errInfo) /* The error codes. */
@@ -767,7 +931,7 @@ static int reply_Uint(file_descriptor *desc, Uint result) {
     TRACE_C('R');
 
     tmp[0] = FILE_RESP_NUMBER;
-#if SIZEOF_VOID_P == 4
+#if SIZEOF_VOID_P == 4 || HALFWORD_HEAP
     put_int32(0, tmp+1);
 #else
     put_int32(result>>32, tmp+1);
@@ -851,8 +1015,6 @@ static int reply_eof(file_descriptor *desc) {
     driver_output2(desc->port, &c, 1, NULL, 0);
     return 0;
 }
-
-
  
 static void invoke_name(void *data, int (*f)(Efile_error *, char *))
 {
@@ -865,40 +1027,63 @@ static void invoke_name(void *data, int (*f)(Efile_error *, char *))
 
 static void invoke_mkdir(void *data)
 {
+    DTRACE_INVOKE_SETUP_BY_NAME(FILE_MKDIR);
     invoke_name(data, efile_mkdir);
+    DTRACE_INVOKE_RETURN(FILE_MKDIR);
 }
 
 static void invoke_rmdir(void *data)
 {
+    DTRACE_INVOKE_SETUP_BY_NAME(FILE_RMDIR);
     invoke_name(data, efile_rmdir);
+    DTRACE_INVOKE_RETURN(FILE_RMDIR);
 }
 
 static void invoke_delete_file(void *data)
 {
+    DTRACE_INVOKE_SETUP_BY_NAME(FILE_DELETE);
     invoke_name(data, efile_delete_file);
+    DTRACE_INVOKE_RETURN(FILE_DELETE);
 }
 
 static void invoke_chdir(void *data)
 {
+    DTRACE_INVOKE_SETUP_BY_NAME(FILE_CHDIR);
     invoke_name(data, efile_chdir);
+    DTRACE_INVOKE_RETURN(FILE_CHDIR);
+}
+
+static void invoke_fdatasync(void *data)
+{
+    struct t_data *d = (struct t_data *) data;
+    int fd = (int) d->fd;
+    DTRACE_INVOKE_SETUP(FILE_FDATASYNC);
+
+    d->again = 0;
+    d->result_ok = efile_fdatasync(&d->errInfo, fd);
+    DTRACE_INVOKE_RETURN(FILE_FDATASYNC);
 }
 
 static void invoke_fsync(void *data)
 {
     struct t_data *d = (struct t_data *) data;
     int fd = (int) d->fd;
+    DTRACE_INVOKE_SETUP(FILE_FSYNC);
 
     d->again = 0;
     d->result_ok = efile_fsync(&d->errInfo, fd);
+    DTRACE_INVOKE_RETURN(FILE_FSYNC);
 }
 
 static void invoke_truncate(void *data)
 {
     struct t_data *d = (struct t_data *) data;
     int fd = (int) d->fd;
+    DTRACE_INVOKE_SETUP(FILE_TRUNCATE);
 
     d->again = 0;
     d->result_ok = efile_truncate_file(&d->errInfo, &fd, d->flags);
+    DTRACE_INVOKE_RETURN(FILE_TRUNCATE);
 }
 
 static void invoke_read(void *data)
@@ -906,6 +1091,7 @@ static void invoke_read(void *data)
     struct t_data *d = (struct t_data *) data;
     int status, segment;
     size_t size, read_size;
+    DTRACE_INVOKE_SETUP(FILE_READ);
 
     segment = d->again && d->c.read.bin_size >= 2*FILE_SEGMENT_READ;
     if (segment) {
@@ -940,6 +1126,7 @@ static void invoke_read(void *data)
     } else {
 	d->again = 0;
     }
+    DTRACE_INVOKE_RETURN(FILE_READ);
 }
 
 static void free_read(void *data)
@@ -956,13 +1143,14 @@ static void invoke_read_line(void *data)
     int status;
     size_t read_size;
     int local_loop = (d->again == 0);
+    DTRACE_INVOKE_SETUP(FILE_READ_LINE);
 
     do {
 	size_t size = (d->c.read_line.binp)->orig_size - 
 	    d->c.read_line.read_offset - d->c.read_line.read_size;
 	if (size == 0) {
 	    /* Need more place */
-	    size_t need = (d->c.read_line.read_size >= DEFAULT_LINEBUF_SIZE) ? 
+	    ErlDrvSizeT need = (d->c.read_line.read_size >= DEFAULT_LINEBUF_SIZE) ?
 		d->c.read_line.read_size + DEFAULT_LINEBUF_SIZE : DEFAULT_LINEBUF_SIZE;
 	    ErlDrvBinary   *newbin = driver_alloc_binary(need);
 	    if (newbin == NULL) {
@@ -1047,6 +1235,7 @@ static void invoke_read_line(void *data)
 	    break;
 	}
     } while (local_loop);
+    DTRACE_INVOKE_RETURN(FILE_READ_LINE);
 }
 
 static void free_read_line(void *data)
@@ -1062,13 +1251,14 @@ static void invoke_read_file(void *data)
     struct t_data *d = (struct t_data *) data;
     size_t read_size;
     int chop;
+    DTRACE_INVOKE_SETUP(FILE_READ_FILE);
     
     if (! d->c.read_file.binp) { /* First invocation only */
 	int fd;
 	Sint64 size;
 	
 	if (! (d->result_ok = 
-	       efile_openfile(&d->errInfo, d->c.read_file.name, 
+	       efile_openfile(&d->errInfo, d->b, 
 			      EFILE_MODE_READ, &fd, &size))) {
 	    goto done;
 	}
@@ -1098,12 +1288,14 @@ static void invoke_read_file(void *data)
 		   &read_size);
     if (d->result_ok) {
 	d->c.read_file.offset += read_size;
-	if (chop) return; /* again */
+	if (chop) goto chop_done; /* again */
     }
  close:
     efile_closefile((int) d->fd);
  done:
     d->again = 0;
+ chop_done:
+    DTRACE_INVOKE_RETURN(FILE_READ_FILE);
 }
 
 static void free_read_file(void *data)
@@ -1123,6 +1315,7 @@ static void invoke_preadv(void *data)
     ErlIOVec        *ev = &c->eiov;
     size_t           bytes_read_so_far = 0;
     unsigned char   *p = (unsigned char *)ev->iov[0].iov_base + 4+4+8*c->cnt;
+    DTRACE_INVOKE_SETUP(FILE_PREADV);
 
     while (c->cnt < c->n) {
 	size_t read_size = ev->iov[1 + c->cnt].iov_len - c->size;
@@ -1144,7 +1337,7 @@ static void invoke_preadv(void *data)
 	    bytes_read_so_far += bytes_read;
 	    if (chop && bytes_read == read_size) {
 		c->size += bytes_read;
-		return;
+		goto done;
 	    }
 	    ASSERT(bytes_read <= read_size);
 	    ev->iov[1 + c->cnt].iov_len = bytes_read + c->size;
@@ -1155,7 +1348,7 @@ static void invoke_preadv(void *data)
 	    if (d->again 
 		&& bytes_read_so_far >= FILE_SEGMENT_READ
 		&& c->cnt < c->n) {
-		return;
+		goto done;
 	    }
 	} else {
 	    /* In case of a read error, ev->size will not be correct,
@@ -1166,6 +1359,8 @@ static void invoke_preadv(void *data)
 	}
     }					
     d->again = 0;
+ done:
+    DTRACE_INVOKE_RETURN(FILE_PREADV);
 }
 
 static void free_preadv(void *data) {
@@ -1187,6 +1382,7 @@ static void invoke_ipread(void *data)
     size_t bytes_read = 0;
     char buf[2*sizeof(Uint32)];
     Uint32 offset, size;
+    DTRACE_INVOKE_SETUP(FILE_IPREAD);
     
     /* Read indirection header */
     if (! efile_pread(&d->errInfo, (int) d->fd, c->offsets[0], 
@@ -1225,14 +1421,17 @@ static void invoke_ipread(void *data)
     /* Read data block */
     d->invoke = invoke_preadv;
     invoke_preadv(data);
+    DTRACE_INVOKE_RETURN(FILE_IPREAD);
     return;
  error:
     d->result_ok = 0;
     d->again = 0;
+    DTRACE_INVOKE_RETURN(FILE_IPREAD);
     return;
  done:
     d->result_ok = !0;
     d->again = 0;
+    DTRACE_INVOKE_RETURN(FILE_IPREAD);
 }
 
 /* invoke_writev and invoke_pwritev are the only thread functions that
@@ -1255,6 +1454,7 @@ static void invoke_writev(void *data) {
     size_t         size;
     size_t         p;
     int            segment;
+    DTRACE_INVOKE_SETUP(FILE_WRITE);
 
     segment = d->again && d->c.writev.size >= 2*FILE_SEGMENT_WRITE;
     if (segment) {
@@ -1263,7 +1463,11 @@ static void invoke_writev(void *data) {
 	size = d->c.writev.size;
     }
 
-    /* Copy the io vector to avoid locking the port que while writing */
+    /* Copy the io vector to avoid locking the port que while writing,
+     * also, both we and efile_writev might/will change the SysIOVec
+     * when segmenting or due to partial write and we do not want to
+     * tamper with the actual queue that we get from driver_peekq
+     */
     MUTEX_LOCK(d->c.writev.q_mtx); /* Lock before accessing the port queue */
     iov0 = driver_peekq(d->c.writev.port, &iovlen);
 
@@ -1272,7 +1476,7 @@ static void invoke_writev(void *data) {
 	 p < size && iovcnt < iovlen;
 	 p += iov0[iovcnt++].iov_len)
 	;
-    iov = EF_ALLOC(sizeof(SysIOVec)*iovcnt);
+    iov = EF_SAFE_ALLOC(sizeof(SysIOVec)*iovcnt);
     memcpy(iov,iov0,iovcnt*sizeof(SysIOVec));
     MUTEX_UNLOCK(d->c.writev.q_mtx);
     /* Let go of lock until we deque from original vector */
@@ -1302,7 +1506,7 @@ static void invoke_writev(void *data) {
 	} else {
 	    d->result_ok = efile_writev(&d->errInfo, 
 					d->flags, (int) d->fd,
-					iov, iovcnt, size);
+					iov, iovcnt);
 	}
     } else if (iovlen == 0) {
 	d->result_ok = 1;
@@ -1324,6 +1528,7 @@ static void invoke_writev(void *data) {
 	TRACE_F(("w%lu", (unsigned long)size));
 
     }
+    DTRACE_INVOKE_RETURN(FILE_WRITE);
 }
 
 static void free_writev(void *data) {
@@ -1337,34 +1542,40 @@ static void free_writev(void *data) {
 static void invoke_pwd(void *data)
 {
     struct t_data *d = (struct t_data *) data;
+    DTRACE_INVOKE_SETUP(FILE_PWD);
 
     d->again = 0;
     d->result_ok = efile_getdcwd(&d->errInfo,d->drive, d->b+1,
 				 RESBUFSIZE-1);
+    DTRACE_INVOKE_RETURN(FILE_PWD);
 }
 
 static void invoke_readlink(void *data)
 {
     struct t_data *d = (struct t_data *) data;
     char resbuf[RESBUFSIZE];	/* Result buffer. */
+    DTRACE_INVOKE_SETUP(FILE_READLINK);
 
     d->again = 0;
     d->result_ok = efile_readlink(&d->errInfo, d->b, resbuf+1,
 				  RESBUFSIZE-1);
     if (d->result_ok != 0)
-	strcpy((char *) d->b + 1, resbuf+1);
+	FILENAME_COPY((char *) d->b + 1, resbuf+1);
+    DTRACE_INVOKE_RETURN(FILE_READLINK);
 }
 
 static void invoke_altname(void *data)
 {
     struct t_data *d = (struct t_data *) data;
     char resbuf[RESBUFSIZE];	/* Result buffer. */
+    DTRACE_INVOKE_SETUP(FILE_ALTNAME);
 
     d->again = 0;
     d->result_ok = efile_altname(&d->errInfo, d->b, resbuf+1,
 				  RESBUFSIZE-1);
     if (d->result_ok != 0)
-	strcpy((char *) d->b + 1, resbuf+1);
+	FILENAME_COPY((char *) d->b + 1, resbuf+1);
+    DTRACE_INVOKE_RETURN(FILE_ALTNAME);
 }
 
 static void invoke_pwritev(void *data) {
@@ -1377,6 +1588,7 @@ static void invoke_pwritev(void *data) {
     size_t            p;
     int               segment;
     size_t            size, write_size;
+    DTRACE_INVOKE_SETUP(FILE_PWRITEV);
 
     segment = d->again && c->size >= 2*FILE_SEGMENT_WRITE;
     if (segment) {
@@ -1389,7 +1601,7 @@ static void invoke_pwritev(void *data) {
     /* Lock the queue just for a while, we don't want it locked during write */
     MUTEX_LOCK(c->q_mtx);
     iov0 = driver_peekq(c->port, &iovlen);
-    iov = EF_ALLOC(sizeof(SysIOVec)*iovlen);
+    iov = EF_SAFE_ALLOC(sizeof(SysIOVec)*iovlen);
     memcpy(iov,iov0,sizeof(SysIOVec)*iovlen);
     MUTEX_UNLOCK(c->q_mtx);
 
@@ -1456,6 +1668,7 @@ static void invoke_pwritev(void *data) {
     }
  done:
     EF_FREE(iov); /* Free our copy of the vector, nothing to restore */
+    DTRACE_INVOKE_RETURN(FILE_PWRITEV);
 }
 
 static void free_pwritev(void *data) {
@@ -1471,9 +1684,14 @@ static void invoke_flstat(void *data)
 {
     struct t_data *d = (struct t_data *) data;
 
+    DTRACE3(efile_drv_int_entry, d->sched_i1, d->sched_i2,
+            d->command == FILE_LSTAT ? FILE_LSTAT : FILE_FSTAT);
     d->again = 0;
     d->result_ok = efile_fileinfo(&d->errInfo, &d->info,
 				  d->b, d->command == FILE_LSTAT);
+    DTRACE3(efile_drv_int_entry, d->sched_i1, d->sched_i2,
+            d->command == FILE_LSTAT ? FILE_LSTAT : FILE_FSTAT);
+    gcc_optimizer_hack++;
 }
 
 static void invoke_link(void *data)
@@ -1481,10 +1699,12 @@ static void invoke_link(void *data)
     struct t_data *d = (struct t_data *) data;
     char *name = d->b;
     char *new_name;
+    DTRACE_INVOKE_SETUP(FILE_LINK);
 
     d->again = 0;
-    new_name = name+strlen(name)+1;
+    new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE;
     d->result_ok = efile_link(&d->errInfo, name, new_name);
+    DTRACE_INVOKE_RETURN(FILE_LINK);
 }
 
 static void invoke_symlink(void *data)
@@ -1492,10 +1712,12 @@ static void invoke_symlink(void *data)
     struct t_data *d = (struct t_data *) data;
     char *name = d->b;
     char *new_name;
+    DTRACE_INVOKE_SETUP(FILE_SYMLINK);
 
     d->again = 0;
-    new_name = name+strlen(name)+1;
+    new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE;
     d->result_ok = efile_symlink(&d->errInfo, name, new_name);
+    DTRACE_INVOKE_RETURN(FILE_SYMLINK);
 }
 
 static void invoke_rename(void *data)
@@ -1503,24 +1725,29 @@ static void invoke_rename(void *data)
     struct t_data *d = (struct t_data *) data;
     char *name = d->b;
     char *new_name;
+    DTRACE_INVOKE_SETUP(FILE_RENAME);
 
     d->again = 0;
-    new_name = name+strlen(name)+1;
+    new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE;
     d->result_ok = efile_rename(&d->errInfo, name, new_name);
+    DTRACE_INVOKE_RETURN(FILE_RENAME);
 }
 
 static void invoke_write_info(void *data)
 {
     struct t_data *d = (struct t_data *) data;
+    DTRACE_INVOKE_SETUP(FILE_WRITE_INFO);
 
     d->again = 0;
     d->result_ok = efile_write_info(&d->errInfo, &d->info, d->b);
+    DTRACE_INVOKE_RETURN(FILE_WRITE_INFO);
 }
 
 static void invoke_lseek(void *data)
 {
     struct t_data *d = (struct t_data *) data;
     int status;
+    DTRACE_INVOKE_SETUP(FILE_LSEEK);
 
     d->again = 0;
     if (d->flags & EFILE_COMPRESSED) {
@@ -1545,64 +1772,61 @@ static void invoke_lseek(void *data)
 			    &d->c.lseek.location);
     }
     d->result_ok = status;
+    DTRACE_INVOKE_RETURN(FILE_LSEEK);
 }
 
 static void invoke_readdir(void *data)
 {
     struct t_data *d = (struct t_data *) data;
-    int s;
     char *p = NULL;
-    int buf_sz = 0;
+    size_t file_bs;
+    size_t n = 0, total = 0;
+    struct t_readdir_buf *b = NULL;
+    int res = 0;
+    DTRACE_INVOKE_SETUP(FILE_READDIR);
 
     d->again = 0;
     d->errInfo.posix_errno = 0;
 
-    while (1) {
-	char *str;
-	if (buf_sz < (4 /* sz */ + 1 /* cmd */ + MAXPATHLEN + 1 /* '\0' */)) {
-	    struct t_readdir_buf *b;
-	    if (p) {
-		put_int32(0, p); /* EOB */
-	    }
-	    b = EF_SAFE_ALLOC(sizeof(struct t_readdir_buf));
-	    b->next = NULL;
-	    if (d->c.read_dir.last_buf)
-		d->c.read_dir.last_buf->next = b;
-	    else
-		d->c.read_dir.first_buf = b;
-	    d->c.read_dir.last_buf = b;
-	    p = &b->buf[0];
-	    buf_sz = READDIR_BUFSIZE - 4/* EOB */;
-	}
-
-	p[4] = FILE_RESP_OK;
-	buf_sz -= 4 + 1;
-	str = p + 4 + 1;
-	ASSERT(buf_sz >= MAXPATHLEN + 1);
-	s = efile_readdir(&d->errInfo, d->b, &d->dir_handle, str, buf_sz);
-
-	if (s) {
-	    int str_sz = strlen(str);
-	    int sz = str_sz + 1;
-	    put_int32(sz, p);
-	    p += 4 + sz;
-	    buf_sz -= str_sz;
-	}
-	else {
-	    put_int32(1, p);
-	    p += 4 + 1;
-	    put_int32(0, p); /* EOB */
-	    d->result_ok = (d->errInfo.posix_errno == 0);
-	    break;
+    do {
+	total   = READDIR_BUFSIZE;
+	n       = 1;
+	b       = EF_SAFE_ALLOC(sizeof(struct t_readdir_buf));
+	b->next = NULL;
+	
+	if (d->c.read_dir.last_buf) {
+	    d->c.read_dir.last_buf->next = b;
+	} else {
+	    d->c.read_dir.first_buf = b;
 	}
-    }
+	d->c.read_dir.last_buf = b;
+
+	p       = &b->buf[0];
+	p[0]    = FILE_RESP_LFNAME;
+	file_bs = READDIR_BUFSIZE - n;
+
+	do {
+	    res = efile_readdir(&d->errInfo, d->b, &d->dir_handle, p + n + 2, &file_bs);
+
+	    if (res) {
+		put_int16((Uint16)file_bs, p + n);
+		n += 2 + file_bs;
+		file_bs = READDIR_BUFSIZE - n;
+	    }
+	} while( res && ((total - n - 2) >= MAXPATHLEN*FILENAME_CHARSIZE));
+
+	b->n = n;
+    } while(res);
+
+    d->result_ok = (d->errInfo.posix_errno == 0);
+    DTRACE_INVOKE_RETURN(FILE_READDIR);
 }
 
 static void invoke_open(void *data)
 {
     struct t_data *d = (struct t_data *) data;
-    
     int status = 1;		/* Status of open call. */
+    DTRACE_INVOKE_SETUP(FILE_OPEN);
 
     d->again = 0;
     if ((d->flags & EFILE_COMPRESSED) == 0) {
@@ -1620,7 +1844,7 @@ static void invoke_open(void *data)
 	    status = efile_may_openfile(&d->errInfo, d->b);
 	    if (status || (d->errInfo.posix_errno != EISDIR)) {
 		mode = (d->flags & EFILE_MODE_READ) ? "rb" : "wb";
-		d->fd = (Sint) erts_gzopen(d->b, mode);
+		d->fd = (SWord) erts_gzopen(d->b, mode);
 		if ((gzFile)d->fd) {
 		    status = 1;
 		} else {
@@ -1635,12 +1859,102 @@ static void invoke_open(void *data)
     }
 
     d->result_ok = status;
+    DTRACE_INVOKE_RETURN(FILE_OPEN);
+}
+
+static void invoke_fadvise(void *data)
+{
+    struct t_data *d = (struct t_data *) data;
+    int fd = (int) d->fd;
+    off_t offset = (off_t) d->c.fadvise.offset;
+    off_t length = (off_t) d->c.fadvise.length;
+    int advise = (int) d->c.fadvise.advise;
+    DTRACE_INVOKE_SETUP(FILE_FADVISE);
+
+    d->again = 0;
+    d->result_ok = efile_fadvise(&d->errInfo, fd, offset, length, advise);
+    DTRACE_INVOKE_RETURN(FILE_FADVISE);
+}
+
+#ifdef HAVE_SENDFILE
+static void invoke_sendfile(void *data)
+{
+    struct t_data *d = (struct t_data *)data;
+    int fd = d->fd;
+    int out_fd = (int)d->c.sendfile.out_fd;
+    Uint64 nbytes = d->c.sendfile.nbytes;
+    int result = 0;
+    d->again = 0;
+
+    result = efile_sendfile(&d->errInfo, fd, out_fd, &d->c.sendfile.offset, &nbytes, NULL);
+
+    d->c.sendfile.written += nbytes;
+
+ if (result == 1 || (result == 0 && USE_THRDS_FOR_SENDFILE)) {
+      d->result_ok = 0;
+    } else if (result == 0 && (d->errInfo.posix_errno == EAGAIN
+				 || d->errInfo.posix_errno == EINTR)) {
+      if ((d->c.sendfile.nbytes - nbytes) != 0) {
+	d->result_ok = 1;
+	if (d->c.sendfile.nbytes != 0)
+	  d->c.sendfile.nbytes -= nbytes;
+      } else
+	d->result_ok = 0;
+    } else {
+	d->result_ok = -1;
+    }
+}
+
+static void free_sendfile(void *data) {
+    struct t_data *d = (struct t_data *)data;
+    if (USE_THRDS_FOR_SENDFILE) {
+	SET_NONBLOCKING(d->c.sendfile.out_fd);
+    } else {
+	MUTEX_LOCK(d->c.sendfile.q_mtx);
+	driver_deq(d->c.sendfile.port,1);
+	MUTEX_UNLOCK(d->c.sendfile.q_mtx);
+	driver_select(d->c.sendfile.port, (ErlDrvEvent)(long)d->c.sendfile.out_fd, ERL_DRV_USE_NO_CALLBACK|ERL_DRV_WRITE, 0);
+    }
+    EF_FREE(data);
+}
+
+static void file_ready_output(ErlDrvData data, ErlDrvEvent event)
+{
+    file_descriptor* fd = (file_descriptor*) data;
+
+    switch (fd->d->command) {
+    case FILE_SENDFILE:
+	driver_select(fd->d->c.sendfile.port, event,
+		      (int)ERL_DRV_WRITE,(int) 0);
+	invoke_sendfile((void *)fd->d);
+	file_async_ready(data, (ErlDrvThreadData)fd->d);
+	break;
+    default:
+	break;
+    }
+}
+
+static void file_stop_select(ErlDrvEvent event, void* _)
+{
+
 }
 
+static int flush_sendfile(file_descriptor *desc,void *_) {
+    if (desc->sendfile_state == sending) {
+	desc->d->result_ok = -1;
+	desc->d->errInfo.posix_errno = ECONNABORTED;
+	file_async_ready((ErlDrvData)desc,(ErlDrvThreadData)desc->d);
+    }
+    return 1;
+}
+#endif /* HAVE_SENDFILE */
+
+
 static void free_readdir(void *data)
 {
     struct t_data *d = (struct t_data *) data;
     struct t_readdir_buf *b1 = d->c.read_dir.first_buf;
+
     while (b1) {
 	struct t_readdir_buf *b2 = b1;
 	b1 = b1->next;
@@ -1698,6 +2012,10 @@ static void cq_execute(file_descriptor *desc) {
     register void *void_ptr; /* Soft cast variable */
     if (desc->timer_state == timer_again)
 	return;
+#ifdef HAVE_SENDFILE
+    if (desc->sendfile_state == sending)
+	return;
+#endif
     if (! (d = cq_deq(desc)))
 	return;
     TRACE_F(("x%i", (int) d->command));
@@ -1705,12 +2023,16 @@ static void cq_execute(file_descriptor *desc) {
     DRIVER_ASYNC(d->level, desc, d->invoke, void_ptr=d, d->free);
 }
 
-static int async_write(file_descriptor *desc, int *errp,
-		       int reply, Uint32 reply_size) {
+static struct t_data *async_write(file_descriptor *desc, int *errp,
+		       int reply, Uint32 reply_size
+#ifdef USE_VM_PROBES
+		       ,Sint64 *dt_i1, Sint64 *dt_i2, Sint64 *dt_i3
+#endif
+) {
     struct t_data *d;
     if (! (d = EF_ALLOC(sizeof(struct t_data) - 1))) {
 	if (errp) *errp = ENOMEM;
-	return -1;
+	return NULL;
     }
     TRACE_F(("w%lu", (unsigned long)desc->write_buffered));
     d->command = FILE_WRITE;
@@ -1719,6 +2041,13 @@ static int async_write(file_descriptor *desc, int *errp,
     d->c.writev.port = desc->port;
     d->c.writev.q_mtx = desc->q_mtx;
     d->c.writev.size = desc->write_buffered;
+#ifdef USE_VM_PROBES
+    if (dt_i1 != NULL) {
+        *dt_i1 = d->fd;
+        *dt_i2 = d->flags;
+        *dt_i3 = d->c.writev.size;
+    }
+#endif
     d->reply = reply;
     d->c.writev.free_size = 0;
     d->c.writev.reply_size = reply_size;
@@ -1727,18 +2056,49 @@ static int async_write(file_descriptor *desc, int *errp,
     d->level = 1;
     cq_enq(desc, d);
     desc->write_buffered = 0;
-    return 0;
+    return d;
 }
 
-static int flush_write(file_descriptor *desc, int *errp) {
-    int    result;
+static int flush_write(file_descriptor *desc, int *errp
+#ifdef USE_VM_PROBES
+                       , dt_private *dt_priv, char *dt_utag
+#endif
+) {
+    int    result = 0;
+#ifdef USE_VM_PROBES
+    Sint64 dt_i1 = 0, dt_i2 = 0, dt_i3 = 0;
+#endif
+    struct t_data *d = NULL;
+
     MUTEX_LOCK(desc->q_mtx);
     if (desc->write_buffered > 0) {
-	result = async_write(desc, errp, 0, 0);
-    } else {
-	result = 0;
+	if ((d = async_write(desc, errp, 0, 0
+#ifdef USE_VM_PROBES
+			     ,&dt_i1, &dt_i2, &dt_i3
+#endif
+			     )) == NULL) {
+            result = -1;
+        }
     }
     MUTEX_UNLOCK(desc->q_mtx);
+#ifdef USE_VM_PROBES
+    if (d != NULL) {
+        d->sched_i1 = dt_priv->thread_num;
+        d->sched_i2 = dt_priv->tag;
+        d->sched_utag[0] = '\0';
+        if (dt_utag != NULL) {
+            if (dt_utag[0] == '\0') {
+                dt_utag = NULL;
+            } else {
+                strncpy(d->sched_utag, dt_utag, sizeof(d->sched_utag) - 1);
+                d->sched_utag[sizeof(d->sched_utag) - 1] = '\0';
+            }
+        }
+        DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag++,
+                 dt_utag, FILE_WRITE,
+                 NULL, NULL, dt_i1, dt_i2, dt_i3, 0, desc->port_str);
+    }
+#endif /* USE_VM_PROBES */
     return result;
 }
 
@@ -1751,9 +2111,17 @@ static int check_write_error(file_descriptor *desc, int *errp) {
     return 0;
 }
 
-static int flush_write_check_error(file_descriptor *desc, int *errp) {
+static int flush_write_check_error(file_descriptor *desc, int *errp
+#ifdef USE_VM_PROBES
+                                   , dt_private *dt_priv, char *dt_utag
+#endif
+				   ) {
     int r;
-    if ( (r = flush_write(desc, errp)) != 0) {
+    if ( (r = flush_write(desc, errp
+#ifdef USE_VM_PROBES
+			  , dt_priv, dt_utag
+#endif
+			  )) != 0) {
 	check_write_error(desc, NULL);
 	return r;
     } else {
@@ -1761,12 +2129,16 @@ static int flush_write_check_error(file_descriptor *desc, int *errp) {
     }
 }
 
-static int async_lseek(file_descriptor *desc, int *errp, int reply, 
-		       Sint64 offset, int origin) {
+static struct t_data *async_lseek(file_descriptor *desc, int *errp, int reply,
+				  Sint64 offset, int origin
+#ifdef USE_VM_PROBES
+				  , Sint64 *dt_i1, Sint64 *dt_i2, Sint64 *dt_i3
+#endif
+				  ) {
     struct t_data *d;
     if (! (d = EF_ALLOC(sizeof(struct t_data)))) {
 	*errp = ENOMEM;
-	return -1;
+	return NULL;
     }
     d->flags = desc->flags;
     d->fd = desc->fd;
@@ -1774,11 +2146,18 @@ static int async_lseek(file_descriptor *desc, int *errp, int reply,
     d->reply = reply;
     d->c.lseek.offset = offset;
     d->c.lseek.origin = origin;
+#ifdef USE_VM_PROBES
+    if (dt_i1 != NULL) {
+        *dt_i1 = d->fd;
+        *dt_i2 = d->c.lseek.offset;
+        *dt_i3 = d->c.lseek.origin;
+    }
+#endif
     d->invoke = invoke_lseek;
     d->free = free_data;
     d->level = 1;
     cq_enq(desc, d);
-    return 0;
+    return d;
 }
 
 static void flush_read(file_descriptor *desc) {
@@ -1790,18 +2169,45 @@ static void flush_read(file_descriptor *desc) {
     }
 }
 
-static int lseek_flush_read(file_descriptor *desc, int *errp) {
+static int lseek_flush_read(file_descriptor *desc, int *errp
+#ifdef USE_VM_PROBES
+			    ,dt_private *dt_priv, char *dt_utag
+#endif
+			    ) {
     int r = 0;
     size_t read_size = desc->read_size;
+#ifdef USE_VM_PROBES
+    Sint64 dt_i1 = 0, dt_i2 = 0, dt_i3 = 0;
+#endif
+    struct t_data *d;
+
+    flush_read(desc);
     if (read_size != 0) {
-	flush_read(desc);
-	if ((r = async_lseek(desc, errp, 0, 
-			     -((ssize_t)read_size), EFILE_SEEK_CUR)) 
-	    < 0) {
-	    return r;
-	}
-    } else {
-	flush_read(desc);
+	if ((d = async_lseek(desc, errp, 0,
+                             -((ssize_t)read_size), EFILE_SEEK_CUR
+#ifdef USE_VM_PROBES
+			     , &dt_i1, &dt_i2, &dt_i3
+#endif
+			     )) == NULL) {
+            r = -1;
+        } else {
+#ifdef USE_VM_PROBES
+            d->sched_i1 = dt_priv->thread_num;
+            d->sched_i2 = dt_priv->tag;
+            d->sched_utag[0] = '\0';
+            if (dt_utag != NULL) {
+                if (dt_utag[0] == '\0') {
+                    dt_utag = NULL;
+                } else {
+                    strncpy(d->sched_utag, dt_utag, sizeof(d->sched_utag) - 1);
+                    d->sched_utag[sizeof(d->sched_utag) - 1] = '\0';
+                }
+            }
+            DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag++,
+                     dt_utag, FILE_LSEEK,
+                     NULL, NULL, dt_i1, dt_i2, dt_i3, 0, desc->port_str);
+#endif /* USE_VM_PROBES */
+        }
     }
     return r;
 }
@@ -1818,11 +2224,23 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data)
     struct t_data *d = (struct t_data *) data;
     char header[5];		/* result code + count */
     char resbuf[RESBUFSIZE];	/* Result buffer. */
-    
+#ifdef  USE_VM_PROBES
+    int sched_i1 = d->sched_i1, sched_i2 = d->sched_i2, command = d->command,
+        result_ok = d->result_ok,
+        posix_errno = d->result_ok ? 0 : d->errInfo.posix_errno;
+    DTRACE_CHARBUF(sched_utag, DTRACE_EFILE_BUFSIZ+1);
+
+    sched_utag[0] = '\0';
+    if (DTRACE_ENABLED(efile_drv_return)) {
+        strncpy(sched_utag, d->sched_utag, DTRACE_EFILE_BUFSIZ);
+        sched_utag[DTRACE_EFILE_BUFSIZ] = '\0';
+    }
+#endif  /* USE_VM_PROBES */
 
     TRACE_C('r');
 
     if (try_again(desc, d)) {
+        /* DTRACE TODO: what kind of probe makes sense here? */
 	return;
     }
 
@@ -1883,7 +2301,7 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data)
 	if (!d->result_ok)
 	    reply_error(desc, &d->errInfo);
 	else {
-	    header[0] = FILE_RESP_OK;
+	    header[0] = FILE_RESP_ALL_DATA;
 	    TRACE_C('R');
 	    driver_output_binary(desc->port, header, 1,
 				 d->c.read_file.binp,
@@ -1919,12 +2337,14 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data)
       case FILE_RMDIR:
       case FILE_CHDIR:
       case FILE_DELETE:
+      case FILE_FDATASYNC:
       case FILE_FSYNC:
       case FILE_TRUNCATE:
       case FILE_LINK:
       case FILE_SYMLINK:
       case FILE_RENAME:
       case FILE_WRITE_INFO:
+      case FILE_FADVISE:
 	reply(desc, d->result_ok, &d->errInfo);
 	free_data(data);
 	break;
@@ -1938,10 +2358,10 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data)
 	    if (!d->result_ok)
 		reply_error(desc, &d->errInfo);
 	    else {
-		resbuf[0] = FILE_RESP_OK;
-		length = 1+strlen((char*) resbuf+1);
+		resbuf[0] = FILE_RESP_FNAME;
+		length = 1+FILENAME_BYTELEN((char*) resbuf+1);
 		TRACE_C('R');
-		driver_output2(desc->port, resbuf, length, NULL, 0);
+		driver_output2(desc->port, resbuf, 1, resbuf+1, length-1);
 	    }
 	    free_data(data);
 	    break;
@@ -1962,24 +2382,25 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data)
 	    if (d->result_ok) {
 		resbuf[0] = FILE_RESP_INFO;
 
-		put_int32(d->info.size_high,         &resbuf[1 + (0 * 4)]);
-		put_int32(d->info.size_low,          &resbuf[1 + (1 * 4)]);
-		put_int32(d->info.type,              &resbuf[1 + (2 * 4)]);
-
-		PUT_TIME(d->info.accessTime, resbuf + 1 + 3*4);
-		PUT_TIME(d->info.modifyTime, resbuf + 1 + 9*4);
-		PUT_TIME(d->info.cTime, resbuf + 1 + 15*4);
-
-		put_int32(d->info.mode,              &resbuf[1 + (21 * 4)]);
-		put_int32(d->info.links,             &resbuf[1 + (22 * 4)]);
-		put_int32(d->info.major_device,      &resbuf[1 + (23 * 4)]);
-		put_int32(d->info.minor_device,      &resbuf[1 + (24 * 4)]);
-		put_int32(d->info.inode,             &resbuf[1 + (25 * 4)]);
-		put_int32(d->info.uid,               &resbuf[1 + (26 * 4)]);
-		put_int32(d->info.gid,               &resbuf[1 + (27 * 4)]);
-		put_int32(d->info.access,            &resbuf[1 + (28 * 4)]);
-
-#define RESULT_SIZE (1 + (29 * 4))
+		put_int32(d->info.size_high,         &resbuf[1 + ( 0 * 4)]);
+		put_int32(d->info.size_low,          &resbuf[1 + ( 1 * 4)]);
+		put_int32(d->info.type,              &resbuf[1 + ( 2 * 4)]);
+
+		/* Note 64 bit indexing in resbuf here */
+		put_int64(d->info.accessTime,        &resbuf[1 + ( 3 * 4)]);
+		put_int64(d->info.modifyTime,        &resbuf[1 + ( 5 * 4)]);
+		put_int64(d->info.cTime,             &resbuf[1 + ( 7 * 4)]);
+
+		put_int32(d->info.mode,              &resbuf[1 + ( 9 * 4)]);
+		put_int32(d->info.links,             &resbuf[1 + (10 * 4)]);
+		put_int32(d->info.major_device,      &resbuf[1 + (11 * 4)]);
+		put_int32(d->info.minor_device,      &resbuf[1 + (12 * 4)]);
+		put_int32(d->info.inode,             &resbuf[1 + (13 * 4)]);
+		put_int32(d->info.uid,               &resbuf[1 + (14 * 4)]);
+		put_int32(d->info.gid,               &resbuf[1 + (15 * 4)]);
+		put_int32(d->info.access,            &resbuf[1 + (16 * 4)]);
+
+#define RESULT_SIZE (1 + (17 * 4))
 		TRACE_C('R');
 		driver_output2(desc->port, resbuf, RESULT_SIZE, NULL, 0);
 #undef RESULT_SIZE
@@ -1989,25 +2410,24 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data)
 	free_data(data);
 	break;
       case FILE_READDIR:
-	if (!d->result_ok)
+	if (!d->result_ok) {
 	    reply_error(desc, &d->errInfo);
-	else {
+	} else {
 	    struct t_readdir_buf *b1 = d->c.read_dir.first_buf;
+	    char   op = FILE_RESP_LFNAME;
+
 	    TRACE_C('R');
 	    ASSERT(b1);
+
 	    while (b1) {
 		struct t_readdir_buf *b2 = b1;
 		char *p = &b1->buf[0];
-		int sz = get_int32(p);
-		while (sz) { /* 0 == EOB */
-		    p += 4;
-		    driver_output2(desc->port, p, sz, NULL, 0);
-		    p += sz;
-		    sz = get_int32(p);
-		}
+		driver_output2(desc->port, p, 1, p + 1, b1->n - 1);
 		b1 = b1->next;
 		EF_FREE(b2);
 	    }
+	    driver_output2(desc->port, &op, 1, NULL, 0);
+    
 	    d->c.read_dir.first_buf = NULL;
 	    d->c.read_dir.last_buf = NULL;
 	}
@@ -2018,6 +2438,9 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data)
 	  if (d->reply) {
 	      TRACE_C('K');
 	      reply_ok(desc);
+#ifdef USE_VM_PROBES
+              result_ok = 1;
+#endif
 	  }
 	  free_data(data);
 	  break;
@@ -2047,9 +2470,34 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data)
 	  }
 	  free_preadv(data);
 	  break;
+#ifdef HAVE_SENDFILE
+      case FILE_SENDFILE:
+	  if (d->result_ok == -1) {
+	      if (d->errInfo.posix_errno == ECONNRESET ||
+		  d->errInfo.posix_errno == ENOTCONN ||
+		  d->errInfo.posix_errno == EPIPE)
+		  reply_string_error(desc,"closed");
+	      else
+		  reply_error(desc, &d->errInfo);
+	      desc->sendfile_state = not_sending;
+	      free_sendfile(data);
+	  } else if (d->result_ok == 0) {
+	      reply_Sint64(desc, d->c.sendfile.written);
+	      desc->sendfile_state = not_sending;
+	      free_sendfile(data);
+	  } else if (d->result_ok == 1) { // If we are using select to send the rest of the data
+	      desc->sendfile_state = sending;
+	      desc->d = d;
+	      driver_select(desc->port, (ErlDrvEvent)(long)d->c.sendfile.out_fd,
+			    ERL_DRV_USE_NO_CALLBACK|ERL_DRV_WRITE, 1);
+	  }
+	  break;
+#endif
       default:
 	abort();
     }
+    DTRACE6(efile_drv_return, sched_i1, sched_i2, sched_utag,
+            command, result_ok, posix_errno);
     if (desc->write_buffered != 0 && desc->timer_state == timer_idle) {
 	desc->timer_state = timer_write;
 	driver_set_timer(desc->port, desc->write_delay);
@@ -2057,11 +2505,12 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data)
     cq_execute(desc);
 }
 
+
 /*********************************************************************
  * Driver entry point -> output
  */
 static void 
-file_output(ErlDrvData e, char* buf, int count)
+file_output(ErlDrvData e, char* buf, ErlDrvSizeT count)
 {
     file_descriptor* desc = (file_descriptor*)e;
     Efile_error errInfo;	/* The error codes for the last operation. */
@@ -2071,7 +2520,15 @@ file_output(ErlDrvData e, char* buf, int count)
     char* name;			/* Points to the filename in buf. */
     int command;
     struct t_data *d = NULL;
-
+#ifdef  USE_VM_PROBES
+    char *dt_utag = NULL;
+    char *dt_s1 = NULL, *dt_s2 = NULL;
+    Sint64 dt_i1 = 0;
+    Sint64 dt_i2 = 0;
+    Sint64 dt_i3 = 0;
+    Sint64 dt_i4 = 0;
+    dt_private *dt_priv = get_dt_private(0);
+#endif  /* USE_VM_PROBES */
 
     TRACE_C('o');
 
@@ -2083,9 +2540,13 @@ file_output(ErlDrvData e, char* buf, int count)
 
     case FILE_MKDIR:
     {
-	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1);
+	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE);
 	
-	strcpy(d->b, name);
+	FILENAME_COPY(d->b, name);
+#ifdef USE_VM_PROBES
+	dt_s1 = d->b;
+	dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE;
+#endif
 	d->command = command;
 	d->invoke = invoke_mkdir;
 	d->free = free_data;
@@ -2094,9 +2555,13 @@ file_output(ErlDrvData e, char* buf, int count)
     }
     case FILE_RMDIR:
     {
-	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1);
+	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE);
 	
-	strcpy(d->b, name);
+	FILENAME_COPY(d->b, name);
+#ifdef USE_VM_PROBES
+	dt_s1 = d->b;
+	dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE;
+#endif
 	d->command = command;
 	d->invoke = invoke_rmdir;
 	d->free = free_data;
@@ -2105,9 +2570,13 @@ file_output(ErlDrvData e, char* buf, int count)
     }
     case FILE_DELETE:
     {
-	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1);
+	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE);
 	
-	strcpy(d->b, name);
+	FILENAME_COPY(d->b, name);
+#ifdef USE_VM_PROBES
+	dt_s1 = d->b;
+	dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE;
+#endif
 	d->command = command;
 	d->invoke = invoke_delete_file;
 	d->free = free_data;
@@ -2117,14 +2586,19 @@ file_output(ErlDrvData e, char* buf, int count)
     case FILE_RENAME:
 	{
 	    char* new_name;
-
-	    new_name = name+strlen(name)+1;
+	    int namelen = FILENAME_BYTELEN(name)+FILENAME_CHARSIZE;
+	    new_name = name+namelen;
 	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1
-			      + strlen(name) + 1
-			      + strlen(new_name) + 1);
+			      + namelen
+			      + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE);
 	
-	    strcpy(d->b, name);
-	    strcpy(d->b + strlen(name) + 1, new_name);
+	    FILENAME_COPY(d->b, name);
+	    FILENAME_COPY(d->b + namelen, new_name);
+#ifdef USE_VM_PROBES
+	    dt_s1 = d->b;
+	    dt_s2 = d->b + namelen;
+	    dt_utag = buf + namelen + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE;
+#endif
 	    d->flags = desc->flags;
 	    d->fd = fd;
 	    d->command = command;
@@ -2135,9 +2609,13 @@ file_output(ErlDrvData e, char* buf, int count)
 	}
     case FILE_CHDIR:
     {
-	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1);
+	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE);
 	
-	strcpy(d->b, name);
+	FILENAME_COPY(d->b, name);
+#ifdef USE_VM_PROBES
+	dt_s1 = d->b;
+	dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE;
+#endif
 	d->command = command;
 	d->invoke = invoke_chdir;
 	d->free = free_data;
@@ -2149,6 +2627,9 @@ file_output(ErlDrvData e, char* buf, int count)
 	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + RESBUFSIZE + 1);
 	
 	    d->drive = *(uchar*)buf;
+#ifdef USE_VM_PROBES
+	    dt_utag = buf + 1;
+#endif
 	    d->command = command;
 	    d->invoke = invoke_pwd;
 	    d->free = free_data;
@@ -2160,9 +2641,14 @@ file_output(ErlDrvData e, char* buf, int count)
 #ifdef USE_THREADS
 	if (sys_info.async_threads > 0)
 	{
-	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1);
+	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + 
+			      FILENAME_CHARSIZE);
 	
-	    strcpy(d->b, name);
+	    FILENAME_COPY(d->b, name);
+#ifdef USE_VM_PROBES
+	    dt_s1 = d->b;
+	    dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE;
+#endif
 	    d->dir_handle = NULL;
 	    d->command = command;
 	    d->invoke = invoke_readdir;
@@ -2175,33 +2661,83 @@ file_output(ErlDrvData e, char* buf, int count)
 	else   
 #endif
 	{
-	    char resbuf[RESBUFSIZE+1];
+	    size_t resbufsize;
+	    size_t n = 0, total = 0;
+	    int res = 0;
+	    char resbuf[READDIR_BUFSIZE];
+
 	    EFILE_DIR_HANDLE dir_handle; /* Handle to open directory. */
 
+	    total               = READDIR_BUFSIZE;
 	    errInfo.posix_errno = 0;
-	    dir_handle = NULL;
-	    resbuf[0] = FILE_RESP_OK;
+	    dir_handle          = NULL;
+	    resbuf[0]           = FILE_RESP_LFNAME;
+
+#ifdef USE_VM_PROBES
+	    dt_s1 = name;
+	    dt_utag = name + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE;
+#endif
+	    /* Fill the buffer with multiple directory listings before sending it to the
+	     * receiving process. READDIR_CHUNKS is minimum number of files sent to the
+	     * receiver.
+	     * Format for each driver_output2:
+	     * ------------------------------------
+	     * | Type   | Len     | Filename  | ...
+	     * | 1 byte | 2 bytes | Len bytes | ...
+	     * ------------------------------------
+	     */
+
+	    do {
+		n = 1;
+		resbufsize = READDIR_BUFSIZE - n;
+
+		do {
+		    res = efile_readdir(&errInfo, name, &dir_handle, resbuf + n + 2, &resbufsize);
+
+		    if (res) {
+			put_int16((Uint16)resbufsize, resbuf + n);
+			n += 2 + resbufsize;
+			resbufsize = READDIR_BUFSIZE - n;
+		    }
+		} while( res && ((total - n - 2) >= MAXPATHLEN*FILENAME_CHARSIZE));
+
+		if (n > 1) {
+		    driver_output2(desc->port, resbuf, 1, resbuf + 1, n - 1);
+		}
+	    } while(res);
 
-	    while (efile_readdir(&errInfo, name, &dir_handle,
-				 resbuf+1, RESBUFSIZE)) {
-		int length = 1 + strlen(resbuf+1);
-		driver_output2(desc->port, resbuf, length, NULL, 0);
-	    }
 	    if (errInfo.posix_errno != 0) {
 		reply_error(desc, &errInfo);
 		return;
 	    }
+#ifdef USE_VM_PROBES
+	    if (dt_utag != NULL && dt_utag[0] == '\0') {
+                dt_utag = NULL;
+            } 
+
+	    DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag,
+		     dt_utag, command, name, dt_s2,
+		     dt_i1, dt_i2, dt_i3, dt_i4, desc->port_str);
+	    DTRACE6(efile_drv_return, dt_priv->thread_num, dt_priv->tag++, 
+		    dt_utag, command, 1, 0);
+#endif
 	    TRACE_C('R');
 	    driver_output2(desc->port, resbuf, 1, NULL, 0);
 	    return;
 	}
     case FILE_OPEN:
 	{
-	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(buf+4) + 1);
+	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(buf+4) + 
+			      FILENAME_CHARSIZE);
 	
 	    d->flags = get_int32((uchar*)buf);
 	    name = buf+4;
-	    strcpy(d->b, name);
+	    FILENAME_COPY(d->b, name);
+#ifdef USE_VM_PROBES
+	    dt_i1 = d->flags;
+	    dt_s1 = d->b;
+	    dt_utag = name + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE;
+#endif
 	    d->command = command;
 	    d->invoke = invoke_open;
 	    d->free = free_data;
@@ -2209,39 +2745,73 @@ file_output(ErlDrvData e, char* buf, int count)
 	    goto done;
 	}
 
+    case FILE_FDATASYNC:
+	{
+	    d = EF_SAFE_ALLOC(sizeof(struct t_data));
+	    
+	    d->fd = fd;
+#ifdef USE_VM_PROBES
+	    dt_utag = name;
+	    dt_i1 = fd;
+#endif
+	    d->command = command;
+	    d->invoke = invoke_fdatasync;
+	    d->free = free_data;
+	    d->level = 2;
+	    goto done;
+	}
+
     case FILE_FSYNC:
-    {
-	d = EF_SAFE_ALLOC(sizeof(struct t_data));
-	
-	d->fd = fd;
-	d->command = command;
-	d->invoke = invoke_fsync;
-	d->free = free_data;
-	d->level = 2;
-	goto done;
-    }
+	{
+	    d = EF_SAFE_ALLOC(sizeof(struct t_data));
+	    
+	    d->fd = fd;
+#ifdef USE_VM_PROBES
+	    dt_utag = name;
+	    dt_i1 = fd;
+#endif
+	    d->command = command;
+	    d->invoke = invoke_fsync;
+	    d->free = free_data;
+	    d->level = 2;
+	    goto done;
+	}
 
 
     case FILE_FSTAT: 
     case FILE_LSTAT:
-    {
-	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1);
+	{
+	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + 
+			      FILENAME_CHARSIZE);
+	    
+	    FILENAME_COPY(d->b, name);
+	    d->fd = fd;
+#ifdef USE_VM_PROBES
+	    dt_utag = name + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE;
+	    if (command == FILE_LSTAT) {
+		dt_s1 = d->b;
+	    } else {
+		dt_i1 = fd;
+	    }
+#endif
+	    d->command = command;
+	    d->invoke = invoke_flstat;
+	    d->free = free_data;
+	    d->level = 2;
+	    goto done;
+	}
 	
-	strcpy(d->b, name);
-	d->fd = fd;
-	d->command = command;
-	d->invoke = invoke_flstat;
-	d->free = free_data;
-	d->level = 2;
-	goto done;
-    }
-
     case FILE_TRUNCATE:
         {
 	    d = EF_SAFE_ALLOC(sizeof(struct t_data));
 	    
 	    d->flags = desc->flags;
 	    d->fd = fd;
+#ifdef USE_VM_PROBES
+	    dt_utag = name;
+	    dt_i1 = fd;
+	    dt_i2 = d->flags;
+#endif
 	    d->command = command;
 	    d->invoke = invoke_truncate;
 	    d->free = free_data;
@@ -2252,15 +2822,23 @@ file_output(ErlDrvData e, char* buf, int count)
     case FILE_WRITE_INFO:
 	{
 	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1
-			      + strlen(buf+21*4) + 1);
+			      + FILENAME_BYTELEN(buf + 9*4) + FILENAME_CHARSIZE);
 	    
-	    d->info.mode = get_int32(buf + 0 * 4);
-	    d->info.uid = get_int32(buf + 1 * 4);
-	    d->info.gid = get_int32(buf + 2 * 4);
-	    GET_TIME(d->info.accessTime, buf + 3 * 4);
-	    GET_TIME(d->info.modifyTime, buf + 9 * 4);
-	    GET_TIME(d->info.cTime, buf + 15 * 4);
-	    strcpy(d->b, buf+21*4);
+	    d->info.mode       = get_int32(buf +  0 * 4);
+	    d->info.uid        = get_int32(buf +  1 * 4);
+	    d->info.gid        = get_int32(buf +  2 * 4);
+	    d->info.accessTime = (time_t)((Sint64)get_int64(buf +  3 * 4));
+	    d->info.modifyTime = (time_t)((Sint64)get_int64(buf +  5 * 4));
+	    d->info.cTime      = (time_t)((Sint64)get_int64(buf +  7 * 4));
+
+	    FILENAME_COPY(d->b, buf + 9*4);
+#ifdef USE_VM_PROBES
+	    dt_i1              = d->info.mode;
+	    dt_i2              = d->info.uid;
+	    dt_i3              = d->info.gid;
+	    dt_s1 = d->b;
+	    dt_utag = buf + 9 * 4 + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE;
+#endif
 	    d->command = command;
 	    d->invoke = invoke_write_info;
 	    d->free = free_data;
@@ -2272,7 +2850,11 @@ file_output(ErlDrvData e, char* buf, int count)
 	{
 	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + RESBUFSIZE + 1);
 	
-	    strcpy(d->b, name);
+	    FILENAME_COPY(d->b, name);
+#ifdef USE_VM_PROBES
+	    dt_s1 = d->b;
+	    dt_utag = name + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE;
+#endif
 	    d->command = command;
 	    d->invoke = invoke_readlink;
 	    d->free = free_data;
@@ -2281,28 +2863,38 @@ file_output(ErlDrvData e, char* buf, int count)
 	}
 
     case FILE_ALTNAME:
-    {
-	d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + RESBUFSIZE + 1);
-	strcpy(d->b, name);
-	d->command = command;
-	d->invoke = invoke_altname;
-	d->free = free_data;
-	d->level = 2;
-	goto done;
-    }
+	{
+	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + RESBUFSIZE + 1);
+	    FILENAME_COPY(d->b, name);
+#ifdef USE_VM_PROBES
+	    dt_s1 = d->b;
+	    dt_utag = name + FILENAME_BYTELEN(d->b) + FILENAME_CHARSIZE;
+#endif
+	    d->command = command;
+	    d->invoke = invoke_altname;
+	    d->free = free_data;
+	    d->level = 2;
+	    goto done;
+	}
 
 
     case FILE_LINK:
 	{
 	    char* new_name;
+	    int namelen = FILENAME_BYTELEN(name) + FILENAME_CHARSIZE;
 
-	    new_name = name+strlen(name)+1;
+	    new_name = name+namelen;
 	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1
-			      + strlen(name) + 1
-			      + strlen(new_name) + 1);
+			      + namelen
+			      + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE);
 	
-	    strcpy(d->b, name);
-	    strcpy(d->b + strlen(name) + 1, new_name);
+	    FILENAME_COPY(d->b, name);
+	    FILENAME_COPY(d->b + namelen, new_name);
+#ifdef USE_VM_PROBES
+	    dt_s1 = d->b;
+	    dt_s2 = d->b + namelen;
+	    dt_utag = buf + namelen + FILENAME_BYTELEN(dt_s2) + FILENAME_CHARSIZE;
+#endif
 	    d->flags = desc->flags;
 	    d->fd = fd;
 	    d->command = command;
@@ -2315,14 +2907,20 @@ file_output(ErlDrvData e, char* buf, int count)
     case FILE_SYMLINK:
 	{
 	    char* new_name;
+	    int namelen = FILENAME_BYTELEN(name) + FILENAME_CHARSIZE;
 
-	    new_name = name+strlen(name)+1;
+	    new_name = name+namelen;
 	    d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1
-			      + strlen(name) + 1
-			      + strlen(new_name) + 1);
+			      + namelen
+			      + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE);
 	
-	    strcpy(d->b, name);
-	    strcpy(d->b + strlen(name) + 1, new_name);
+	    FILENAME_COPY(d->b, name);
+	    FILENAME_COPY(d->b + namelen, new_name);
+#ifdef USE_VM_PROBES
+	    dt_s1 = d->b;
+	    dt_s2 = d->b + namelen;
+	    dt_utag = buf + namelen + FILENAME_BYTELEN(dt_s2) + FILENAME_CHARSIZE;
+#endif
 	    d->flags = desc->flags;
 	    d->fd = fd;
 	    d->command = command;
@@ -2332,6 +2930,28 @@ file_output(ErlDrvData e, char* buf, int count)
 	    goto done;
 	}
 
+    case FILE_FADVISE:
+    {
+        d = EF_SAFE_ALLOC(sizeof(struct t_data));
+
+        d->fd = fd;
+        d->command = command;
+        d->invoke = invoke_fadvise;
+        d->free = free_data;
+        d->level = 2;
+        d->c.fadvise.offset = get_int64((uchar*) buf);
+        d->c.fadvise.length = get_int64(((uchar*) buf) + sizeof(Sint64));
+        d->c.fadvise.advise = get_int32(((uchar*) buf) + 2 * sizeof(Sint64));
+#ifdef USE_VM_PROBES
+        dt_i1 = d->fd;
+        dt_i2 = d->c.fadvise.offset;
+        dt_i3 = d->c.fadvise.length;
+        dt_i4 = d->c.fadvise.advise;
+        dt_utag = buf + 3 * sizeof(Sint64);
+#endif
+        goto done;
+    }
+
     }
 
     /*
@@ -2342,6 +2962,22 @@ file_output(ErlDrvData e, char* buf, int count)
 
  done:
     if (d) {
+#ifdef USE_VM_PROBES
+	d->sched_i1 = dt_priv->thread_num;
+	d->sched_i2 = dt_priv->tag;
+	d->sched_utag[0] = '\0';
+	if (dt_utag != NULL) {
+	    if (dt_utag[0] == '\0') {
+		dt_utag = NULL;
+	    } else {
+		strncpy(d->sched_utag, dt_utag, sizeof(d->sched_utag) - 1);
+		d->sched_utag[sizeof(d->sched_utag) - 1] = '\0';
+	    }
+	}
+	DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag++,
+		 dt_utag, command, dt_s1, dt_s2,
+		 dt_i1, dt_i2, dt_i3, dt_i4, desc->port_str);
+#endif
 	cq_enq(desc, d);
     }
 }
@@ -2352,16 +2988,33 @@ file_output(ErlDrvData e, char* buf, int count)
 static void 
 file_flush(ErlDrvData e) {
     file_descriptor *desc = (file_descriptor *)e;
+#ifdef DEBUG
     int r;
+#endif
+#ifdef  USE_VM_PROBES
+    dt_private *dt_priv = get_dt_private(dt_driver_io_worker_base);
+#endif
 
     TRACE_C('f');
 
-    r = flush_write(desc, NULL);
+#ifdef HAVE_SENDFILE
+    flush_sendfile(desc, NULL);
+#endif
+
+#ifdef DEBUG
+    r = 
+#endif
+         flush_write(desc, NULL
+#ifdef USE_VM_PROBES
+		     , dt_priv, (desc->d == NULL) ? NULL : desc->d->sched_utag
+#endif
+		     );
     /* Only possible reason for bad return value is ENOMEM, and 
      * there is nobody to tell...
      */
+#ifdef DEBUG
     ASSERT(r == 0); 
-    r = 0; /* Avoiding warning */
+#endif
     cq_execute(desc);
 }
 
@@ -2370,16 +3023,23 @@ file_flush(ErlDrvData e) {
 /*********************************************************************
  * Driver entry point -> control
  */
-static int 
+static ErlDrvSSizeT
 file_control(ErlDrvData e, unsigned int command, 
-			 char* buf, int len, char **rbuf, int rlen) {
+	     char* buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen) {
+    /*
+     *  warning: variable ‘desc’ set but not used 
+     *  [-Wunused-but-set-variable]
+     *  ... no kidding ...
+     *
+     *
     file_descriptor *desc = (file_descriptor *)e;
     switch (command) {
     default:
 	return 0;
-    } /* switch (command) */
+    } 
     ASSERT(0);
-    desc = NULL; /* XXX Avoid warning while empty switch */
+    desc = NULL; 
+    */
     return 0;
 }
 
@@ -2390,6 +3050,9 @@ static void
 file_timeout(ErlDrvData e) {
     file_descriptor *desc = (file_descriptor *)e;
     enum e_timer timer_state = desc->timer_state;
+#ifdef  USE_VM_PROBES
+    dt_private *dt_priv = get_dt_private(dt_driver_io_worker_base);
+#endif
 
     TRACE_C('t');
 
@@ -2404,12 +3067,18 @@ file_timeout(ErlDrvData e) {
 	driver_async(desc->port, KEY(desc), desc->invoke, desc->d, desc->free);
 	break;
     case timer_write: {
-	int r = flush_write(desc, NULL);
+#ifdef DEBUG
+	int r = 
+#endif
+	         flush_write(desc, NULL
+#ifdef USE_VM_PROBES
+			     , dt_priv, (desc->d == NULL) ? NULL : desc->d->sched_utag
+#endif
+			     );
 	/* Only possible reason for bad return value is ENOMEM, and 
 	 * there is nobody to tell...
 	 */
 	ASSERT(r == 0); 
-	r = 0; /* Avoiding warning */
 	cq_execute(desc);
     } break;
     } /* case */
@@ -2426,6 +3095,14 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
     char command;
     int p, q;
     int err;
+    struct t_data *d = NULL;
+#ifdef USE_VM_PROBES
+    Sint64 dt_i1 = 0, dt_i2 = 0, dt_i3 = 0;
+    Sint64 dt_i4 = 0;
+    char *dt_utag = NULL;
+    char *dt_s1 = NULL;
+    dt_private *dt_priv = get_dt_private(dt_driver_io_worker_base);
+#endif
 
     TRACE_C('v');
 
@@ -2445,18 +3122,19 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
     switch (command) {
 
     case FILE_CLOSE: {
+#ifdef USE_VM_PROBES
+	dt_utag = EV_CHAR_P(ev, p, q);
+#endif
 	flush_read(desc);
-	if (flush_write_check_error(desc, &err) < 0) {
+	if (flush_write_check_error(desc, &err
+#ifdef USE_VM_PROBES
+				    , dt_priv, dt_utag
+#endif
+				    ) < 0) {
 	    reply_posix_error(desc, err);
 	    goto done;
 	}
-	if (ev->size != 1) {
-	    /* Wrong command length */
-	    reply_posix_error(desc, EINVAL);
-	    goto done;
-	}
 	if (desc->fd != FILE_FD_INVALID) {
-	    struct t_data *d;
 	    if (! (d = EF_ALLOC(sizeof(struct t_data)))) {
 		reply_posix_error(desc, ENOMEM);
 	    } else {
@@ -2464,6 +3142,10 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 		d->reply = !0;
 		d->fd = desc->fd;
 		d->flags = desc->flags;
+#ifdef USE_VM_PROBES
+		dt_i1 = d->fd;
+		dt_i2 = d->flags;
+#endif
 		d->invoke = invoke_close;
 		d->free = free_data;
 		d->level = 2;
@@ -2479,8 +3161,21 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
     case FILE_READ: {
 	Uint32 sizeH, sizeL;
 	size_t size, alloc_size;
-	struct t_data *d;
-	if (flush_write_check_error(desc, &err) < 0) {
+
+	if (!EV_GET_UINT32(ev, &sizeH, &p, &q)
+	    || !EV_GET_UINT32(ev, &sizeL, &p, &q)) {
+	    /* Wrong buffer length to contain the read count */
+	    reply_posix_error(desc, EINVAL);
+	    goto done;
+	}
+#ifdef USE_VM_PROBES
+	dt_utag = EV_CHAR_P(ev, p, q);
+#endif
+	if (flush_write_check_error(desc, &err
+#ifdef USE_VM_PROBES
+				    , dt_priv, dt_utag
+#endif
+				    ) < 0) {
 	    reply_posix_error(desc, err);
 	    goto done;
 	}
@@ -2488,19 +3183,16 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	if (desc->read_bufsize == 0 && desc->read_binp != NULL && desc->read_size > 0) {
 	    /* We have allocated a buffer for line mode but should not really have a 
 	       read-ahead buffer... */
-	    if (lseek_flush_read(desc, &err) < 0) {
+	    if (lseek_flush_read(desc, &err
+#ifdef USE_VM_PROBES
+				 , dt_priv, dt_utag
+#endif
+				 ) < 0) {
 		reply_posix_error(desc, err);
 		goto done;
 	    }
 	}
 #endif
-	if (ev->size != 1+8
-	    || !EV_GET_UINT32(ev, &sizeH, &p, &q)
-	    || !EV_GET_UINT32(ev, &sizeL, &p, &q)) {
-	    /* Wrong buffer length to contain the read count */
-	    reply_posix_error(desc, EINVAL);
-	    goto done;
-	}
 #if SIZEOF_SIZE_T == 4
 	if (sizeH != 0) {
 	    reply_posix_error(desc, EINVAL);
@@ -2576,6 +3268,11 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	d->c.read.bin_offset = desc->read_offset + desc->read_size;
 	d->c.read.bin_size = desc->read_binp->orig_size - d->c.read.bin_offset;
 	d->c.read.size = size;
+#ifdef USE_VM_PROBES
+	dt_i1 = d->fd;
+	dt_i2 = d->flags;
+	dt_i3 = d->c.read.size;
+#endif
 	driver_binary_inc_refc(d->c.read.binp);
 	d->invoke = invoke_read;
 	d->free = free_read;
@@ -2593,12 +3290,22 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	 *    allocated binary + dealing with offsets and lengts are done in file_async ready
 	 *    for this OP.
 	 */
-	struct t_data *d;
-	if (flush_write_check_error(desc, &err) < 0) {
+#ifdef USE_VM_PROBES
+	dt_utag = EV_CHAR_P(ev, p, q);
+#endif
+	if (flush_write_check_error(desc, &err
+#ifdef USE_VM_PROBES
+				    , dt_priv, dt_utag
+#endif
+				    ) < 0) {
 	    reply_posix_error(desc, err);
 	    goto done;
 	}
-	if (ev->size != 1) {
+	if (ev->size != 1
+#ifdef USE_VM_PROBES
+	    + FILENAME_BYTELEN(dt_utag) + FILENAME_CHARSIZE
+#endif
+	    ) {
 	    /* Wrong command length */
 	    reply_posix_error(desc, EINVAL);
 	    goto done;
@@ -2654,8 +3361,16 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	d->c.read_line.binp = desc->read_binp;
 	d->c.read_line.read_offset = desc->read_offset;
 	d->c.read_line.read_size = desc->read_size;
+#ifdef USE_VM_PROBES
+	dt_i1 = d->fd;
+	dt_i2 = d->flags;
+	dt_i3 = d->c.read_line.read_offset;
+#endif
 #if !ALWAYS_READ_LINE_AHEAD
 	d->c.read_line.read_ahead = (desc->read_bufsize > 0);
+#ifdef USE_VM_PROBES
+	dt_i4 = d->c.read_line.read_ahead;
+#endif
 #endif 
 	driver_binary_inc_refc(d->c.read.binp);
 	d->invoke = invoke_read_line;
@@ -2663,10 +3378,22 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	d->level = 1;
 	cq_enq(desc, d);
     } goto done;
-    case FILE_WRITE: {
-	int skip = 1;
-	int size = ev->size - skip;
-	if (lseek_flush_read(desc, &err) < 0) {
+    case FILE_WRITE: { /* Dtrace: The dtrace user tag is not last in message, 
+			  but follows the message tag directly. 
+			  This is handled specially in prim_file.erl */
+	ErlDrvSizeT skip = 1;
+	ErlDrvSizeT size = ev->size - skip;
+
+#ifdef USE_VM_PROBES
+	dt_utag = EV_CHAR_P(ev, p, q);
+	skip += FILENAME_BYTELEN(dt_utag) + FILENAME_CHARSIZE;
+	size = ev->size - skip;
+#endif
+	if (lseek_flush_read(desc, &err
+#ifdef USE_VM_PROBES
+			     , dt_priv, dt_utag
+#endif
+			     ) < 0) {
 	    reply_posix_error(desc, err);
 	    goto done;
 	}
@@ -2674,7 +3401,7 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	    reply_posix_error(desc, EBADF);
 	    goto done;
 	}
-	if (size <= 0) {
+	if (size == 0) {
 	    reply_Uint(desc, size);
 	    goto done;
 	}
@@ -2693,7 +3420,11 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 		driver_set_timer(desc->port, desc->write_delay);
 	    }
 	} else {
-	    if (async_write(desc, &err, !0, size) != 0) {
+	    if ((d = async_write(desc, &err, !0, size
+#ifdef USE_VM_PROBES
+				 , &dt_i1, &dt_i2, &dt_i3
+#endif
+				 )) == NULL) {
 		MUTEX_UNLOCK(desc->q_mtx);
 		reply_posix_error(desc, err);
 		goto done;
@@ -2703,24 +3434,49 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	}
     } goto done; /* case FILE_WRITE */
 
-    case FILE_PWRITEV: {
+    case FILE_PWRITEV: { /* Dtrace: The dtrace user tag is not last in message, 
+			   but follows the message tag directly. 
+			   This is handled specially in prim_file.erl */
 	Uint32 i, j, n; 
 	size_t total;
-	struct t_data *d;
-	if (lseek_flush_read(desc, &err) < 0) {
-	    reply_Uint_posix_error(desc, 0, err);
-	    goto done;
-	}
-	if (flush_write_check_error(desc, &err) < 0) {
-	    reply_Uint_posix_error(desc, 0, err);
-	    goto done;
+#ifdef USE_VM_PROBES
+	char dt_tmp;
+	int dt_utag_bytes = 1;
+
+	dt_utag = EV_CHAR_P(ev, p, q);
+	/* This will work for UTF-8, but not for UTF-16 - extra reminder here */
+#ifdef FILENAMES_16BIT 
+#error 16bit characters in filenames and dtrace in combination is not supported.
+#endif
+	while (EV_GET_CHAR(ev, &dt_tmp, &p, &q) && dt_tmp != '\0') {
+	    dt_utag_bytes++;
 	}
+#endif
 	if (ev->size < 1+4
+#ifdef USE_VM_PROBES
+	    + dt_utag_bytes
+#endif
 	    || !EV_GET_UINT32(ev, &n, &p, &q)) {
 	    /* Buffer too short to contain even the number of pos/size specs */
 	    reply_Uint_posix_error(desc, 0, EINVAL);
 	    goto done;
 	}
+	if (lseek_flush_read(desc, &err
+#ifdef USE_VM_PROBES
+			     , dt_priv, dt_utag
+#endif
+			     ) < 0) {
+	    reply_Uint_posix_error(desc, 0, err);
+	    goto done;
+	}
+	if (flush_write_check_error(desc, &err
+#ifdef USE_VM_PROBES
+				    , dt_priv, dt_utag
+#endif
+				    ) < 0) {
+	    reply_Uint_posix_error(desc, 0, err);
+	    goto done;
+	}
 	if (n == 0) {
 	    /* Trivial case - nothing to write */
 	    if (ev->size != 1+4) {
@@ -2730,7 +3486,11 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	    }
 	    goto done;
 	}
-	if (ev->size < 1+4+8*(2*n)) {
+	if (ev->size < 1+4+8*(2*n)
+#ifdef USE_VM_PROBES
+	    + dt_utag_bytes
+#endif
+	    ) {
 	    /* Buffer too short to contain even the pos/size specs */
 	    reply_Uint_posix_error(desc, 0, EINVAL);
 	    goto done;
@@ -2745,6 +3505,10 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	d->reply = !0;
 	d->fd = desc->fd;
 	d->flags = desc->flags;
+#ifdef USE_VM_PROBES
+	dt_i1 = d->fd;
+	dt_i2 = d->flags;
+#endif
 	d->c.pwritev.port = desc->port;
 	d->c.pwritev.q_mtx = desc->q_mtx;
 	d->c.pwritev.n = n;
@@ -2782,13 +3546,20 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	    }
 	}
 	d->c.pwritev.size = total;
+#ifdef USE_VM_PROBES
+	dt_i3 = d->c.pwritev.size;
+#endif
 	d->c.pwritev.free_size = 0;
 	if (j == 0) {
 	    /* Trivial case - nothing to write */
 	    EF_FREE(d);
 	    reply_Uint(desc, 0);
 	} else {
-	    size_t skip = 1 + 4 + 8*(2*n);
+	    ErlDrvSizeT skip = 1 + 4 + 8 * (2*n) 
+#ifdef USE_VM_PROBES
+		+ dt_utag_bytes
+#endif
+		;
 	    if (skip + total != ev->size) {
 		/* Actual amount of data does not match 
 		 * total of all pos/size specs
@@ -2809,27 +3580,55 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	}
     } goto done; /* case FILE_PWRITEV: */
 
-    case FILE_PREADV: {
+    case FILE_PREADV: { /* Dtrace: The dtrace user tag is not last in message, 
+			   but follows the message tag directly. 
+			   This is handled specially in prim_file.erl */
 	register void * void_ptr;
 	Uint32 i, n;
-	struct t_data *d;
 	ErlIOVec *res_ev;
-	if (lseek_flush_read(desc, &err) < 0) {
+#ifdef USE_VM_PROBES
+	char dt_tmp;
+	int dt_utag_bytes = 1;
+	/* This will work for UTF-8, but not for UTF-16 - extra reminder here */
+#ifdef FILENAMES_16BIT 
+#error 16bit characters in filenames and dtrace in combination is not supported.
+#endif
+	dt_utag = EV_CHAR_P(ev, p, q);
+	while (EV_GET_CHAR(ev, &dt_tmp, &p, &q) && dt_tmp != '\0') {
+	    dt_utag_bytes++;
+	}
+#endif
+	if (lseek_flush_read(desc, &err
+#ifdef USE_VM_PROBES
+			     , dt_priv, dt_utag
+#endif
+			     ) < 0) {
 	    reply_posix_error(desc, err);
 	    goto done;
 	}
-	if (flush_write_check_error(desc, &err) < 0) {
+	if (flush_write_check_error(desc, &err
+#ifdef USE_VM_PROBES
+				    , dt_priv, dt_utag
+#endif
+				    ) < 0) {
 	    reply_posix_error(desc, err);
 	    goto done;
 	}
 	if (ev->size < 1+8
+#ifdef USE_VM_PROBES
+	    + dt_utag_bytes
+#endif
 	    || !EV_GET_UINT32(ev, &n, &p, &q)
 	    || !EV_GET_UINT32(ev, &n, &p, &q)) {
 	    /* Buffer too short to contain even the number of pos/size specs */
 	    reply_posix_error(desc, EINVAL);
 	    goto done;
 	}
-	if (ev->size != 1+8+8*(2*n)) {
+	if (ev->size < 1+8+8*(2*n)
+#ifdef USE_VM_PROBES
+	    + dt_utag_bytes
+#endif
+	    ) {
 	    /* Buffer wrong length to contain the pos/size specs */
 	    reply_posix_error(desc, EINVAL);
 	    goto done;
@@ -2849,6 +3648,10 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	d->reply = !0;
 	d->fd = desc->fd;
 	d->flags = desc->flags;
+#ifdef USE_VM_PROBES
+	dt_i1 = d->fd;
+	dt_i2 = d->flags;
+#endif
 	d->c.preadv.n = n;
 	d->c.preadv.cnt = 0;
 	d->c.preadv.size = 0;
@@ -2876,6 +3679,9 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 #else
 	    size = ((size_t)sizeH<<32) | sizeL;
 #endif
+#ifdef USE_VM_PROBES
+	    dt_i3 += size;
+#endif
 	    if (! (res_ev->binv[i] = driver_alloc_binary(size))) {
 		reply_posix_error(desc, ENOMEM);
 		break;
@@ -2922,42 +3728,70 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
     } goto done; /* case FILE_PREADV: */
 
     case FILE_LSEEK: {
-	Sint64 offset;          /* Offset for seek */
+	Sint64 offset;		/* Offset for seek */
 	Uint32 origin;		/* Origin of seek. */
-	if (lseek_flush_read(desc, &err) < 0) {
-	    reply_posix_error(desc, err);
+
+	if (ev->size < 1+8+4
+	    || !EV_GET_UINT64(ev, &offset, &p, &q)
+	    || !EV_GET_UINT32(ev, &origin, &p, &q)) {
+	    /* Wrong length of buffer to contain offset and origin */
+	    reply_posix_error(desc, EINVAL);
 	    goto done;
 	}
-	if (flush_write_check_error(desc, &err) < 0) {
+#ifdef USE_VM_PROBES
+	dt_utag = EV_CHAR_P(ev, p, q);
+#endif
+	if (lseek_flush_read(desc, &err
+#ifdef USE_VM_PROBES
+			     , dt_priv, dt_utag
+#endif
+			     ) < 0) {
 	    reply_posix_error(desc, err);
 	    goto done;
 	}
-	if (ev->size != 1+8+4
-	    || !EV_GET_UINT64(ev, &offset, &p, &q)
-	    || !EV_GET_UINT32(ev, &origin, &p, &q)) {
-	    /* Wrong length of buffer to contain offset and origin */
-	    reply_posix_error(desc, EINVAL);
+	if (flush_write_check_error(desc, &err
+#ifdef USE_VM_PROBES
+				    , dt_priv, dt_utag
+#endif
+				    ) < 0) {
+	    reply_posix_error(desc, err);
 	    goto done;
 	}
-	if (async_lseek(desc, &err, !0, offset, origin) < 0) {
+	if ((d = async_lseek(desc, &err, !0, offset, origin
+#ifdef USE_VM_PROBES
+			     , &dt_i1, &dt_i2, &dt_i3
+#endif
+			     )) == NULL) {
 	    reply_posix_error(desc, err);
 	    goto done;
 	}
     } goto done;
 
     case FILE_READ_FILE: {
-	struct t_data *d;
+	char *filename;
 	if (ev->size < 1+1) {
 	    /* Buffer contains empty name */
 	    reply_posix_error(desc, ENOENT);
 	    goto done;
 	}
+#ifndef USE_VM_PROBES
+	/* In the dtrace case, the iov has an extra element, the dtrace utag - we will need 
+	   another test to see that
+	   the filename is in a single buffer: */
 	if (ev->size-1 != ev->iov[q].iov_len-p) {
 	    /* Name not in one single buffer */
 	    reply_posix_error(desc, EINVAL);
 	    goto done;
 	}
-	d = EF_ALLOC(sizeof(struct t_data) + ev->size);
+#else
+	if (((byte *)ev->iov[q].iov_base)[ev->iov[q].iov_len-1] != '\0') {
+	    /* Name not in one single buffer */
+	    reply_posix_error(desc, EINVAL);
+	    goto done;
+	}	
+#endif
+	filename = EV_CHAR_P(ev, p, q);
+	d = EF_ALLOC(sizeof(struct t_data) -1 + FILENAME_BYTELEN(filename) + FILENAME_CHARSIZE);
 	if (! d) {
 	    reply_posix_error(desc, ENOMEM);
 	    goto done;
@@ -2965,8 +3799,21 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	d->command = command;
 	d->reply = !0;
 	/* Copy name */
-	memcpy(d->c.read_file.name, EV_CHAR_P(ev, p, q), ev->size-1);
-	d->c.read_file.name[ev->size-1] = '\0';
+	FILENAME_COPY(d->b, filename);
+#ifdef USE_VM_PROBES
+	{
+	    char dt_tmp;
+
+	    /* This will work for UTF-8, but not for UTF-16 - extra reminder here */
+#ifdef FILENAMES_16BIT 
+#error 16bit characters in filenames and dtrace in combination is not supported.
+#endif
+	    while (EV_GET_CHAR(ev, &dt_tmp, &p, &q) && dt_tmp != '\0') 
+		;
+	    dt_s1 = d->b;
+	    dt_utag = EV_CHAR_P(ev, p, q);
+	}
+#endif
 	d->c.read_file.binp = NULL;
 	d->invoke = invoke_read_file;
 	d->free = free_read_file;
@@ -2986,7 +3833,6 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	char mode;
 	Sint64 hdr_offset;
 	Uint32 max_size;
-	struct t_data *d;
 	ErlIOVec *res_ev;
 	int vsize;
 	if (! EV_GET_CHAR(ev, &mode, &p, &q)) {
@@ -2998,14 +3844,6 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	    reply_posix_error(desc, EINVAL);
 	    goto done;
 	}
-	if (lseek_flush_read(desc, &err) < 0) {
-	    reply_posix_error(desc, err);
-	    goto done;
-	}
-	if (flush_write_check_error(desc, &err) < 0) {
-	    reply_posix_error(desc, err);
-	    goto done;
-	}
 	if (ev->size < 1+1+8+4
 	    || !EV_GET_UINT64(ev, &hdr_offset, &p, &q)
 	    || !EV_GET_UINT32(ev, &max_size, &p, &q)) {
@@ -3014,6 +3852,25 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	    reply_posix_error(desc, EINVAL);
 	    goto done;
 	}
+#ifdef USE_VM_PROBES
+	dt_utag = EV_CHAR_P(ev, p, q);
+#endif
+	if (lseek_flush_read(desc, &err
+#ifdef USE_VM_PROBES
+			     , dt_priv, dt_utag
+#endif
+			     ) < 0) {
+	    reply_posix_error(desc, err);
+	    goto done;
+	}
+	if (flush_write_check_error(desc, &err
+#ifdef USE_VM_PROBES
+				    , dt_priv, dt_utag
+#endif
+				    ) < 0) {
+	    reply_posix_error(desc, err);
+	    goto done;
+	}
 	/* Create the thread data structure with the contained ErlIOVec 
 	 * and corresponding binaries for the response 
 	 */
@@ -3030,6 +3887,12 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	d->flags = desc->flags;
 	d->c.preadv.offsets[0] = hdr_offset;
 	d->c.preadv.size = max_size;
+#ifdef USE_VM_PROBES
+	dt_i1 = d->fd;
+	dt_i2 = d->flags;
+	dt_i3 = d->c.preadv.offsets[0];
+	dt_i4 = d->c.preadv.size;
+#endif
 	res_ev = &d->c.preadv.eiov;
 	/* XXX possible alignment problems here for weird machines */
 	res_ev->iov = void_ptr = d + 1;
@@ -3044,16 +3907,24 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 
     case FILE_SETOPT: {
 	char opt;
+
 	if (ev->size < 1+1
 	    || !EV_GET_CHAR(ev, &opt, &p, &q)) {
 	    /* Buffer too short to contain even the option type */
 	    reply_posix_error(desc, EINVAL);
 	    goto done;
 	}
+#ifdef USE_VM_PROBES
+	dt_i1 = opt;
+	dt_utag = EV_CHAR_P(ev, p, q);
+#endif
 	switch (opt) {
 	case FILE_OPT_DELAYED_WRITE: {
 	    Uint32 sizeH, sizeL, delayH, delayL;
 	    if (ev->size != 1+1+4*sizeof(Uint32)
+#ifdef USE_VM_PROBES
+		+ FILENAME_BYTELEN(dt_utag) + FILENAME_CHARSIZE
+#endif
 		|| !EV_GET_UINT32(ev, &sizeH, &p, &q)
 		|| !EV_GET_UINT32(ev, &sizeL, &p, &q)
 		|| !EV_GET_UINT32(ev, &delayH, &p, &q)
@@ -3080,12 +3951,18 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 #else
 	    desc->write_delay = ((unsigned long)delayH << 32) | delayL;
 #endif
+#ifdef USE_VM_PROBES
+	    dt_i2 = desc->write_delay;
+#endif
 	    TRACE_C('K');
 	    reply_ok(desc);
 	} goto done;
 	case FILE_OPT_READ_AHEAD: {
 	    Uint32 sizeH, sizeL;
 	    if (ev->size != 1+1+2*sizeof(Uint32)
+#ifdef USE_VM_PROBES
+		+ FILENAME_BYTELEN(dt_utag)+FILENAME_CHARSIZE
+#endif
 		|| !EV_GET_UINT32(ev, &sizeH, &p, &q)
 		|| !EV_GET_UINT32(ev, &sizeL, &p, &q)) {
 		/* Buffer has wrong length to contain the option values */
@@ -3101,6 +3978,9 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 #else
 	    desc->read_bufsize = ((size_t)sizeH << 32) | sizeL;
 #endif
+#ifdef USE_VM_PROBES
+	    dt_i2 = desc->read_bufsize;
+#endif
 	    TRACE_C('K');
 	    reply_ok(desc);
 	} goto done;
@@ -3109,14 +3989,97 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
 	    goto done;
 	} /* case FILE_OPT_DELAYED_WRITE: */
     } ASSERT(0); goto done; /* case FILE_SETOPT: */
-    
+
+    case FILE_SENDFILE: {
+
+#ifdef HAVE_SENDFILE
+        struct t_data *d;
+	Uint32 out_fd, offsetH, offsetL, hd_len, tl_len;
+	Uint64 nbytes;
+	char flags;
+
+	if (ev->size < 1 + 7 * sizeof(Uint32) + sizeof(char)
+		|| !EV_GET_UINT32(ev, &out_fd, &p, &q)
+		|| !EV_GET_CHAR(ev, &flags, &p, &q)
+		|| !EV_GET_UINT32(ev, &offsetH, &p, &q)
+		|| !EV_GET_UINT32(ev, &offsetL, &p, &q)
+		|| !EV_GET_UINT64(ev, &nbytes, &p, &q)
+		|| !EV_GET_UINT32(ev, &hd_len, &p, &q)
+		|| !EV_GET_UINT32(ev, &tl_len, &p, &q)) {
+	    /* Buffer has wrong length to contain all the needed values */
+	    reply_posix_error(desc, EINVAL);
+	    goto done;
+	}
+
+	if (hd_len != 0 || tl_len != 0 || flags != 0) {
+	    // We do not allow header, trailers and/or flags right now
+	    reply_posix_error(desc, EINVAL);
+	    goto done;
+	}
+
+	d = EF_SAFE_ALLOC(sizeof(struct t_data));
+	d->fd = desc->fd;
+	d->command = command;
+	d->invoke = invoke_sendfile;
+	d->free = free_sendfile;
+	d->level = 2;
+
+	d->c.sendfile.out_fd = (int) out_fd;
+	d->c.sendfile.written = 0;
+	d->c.sendfile.port = desc->port;
+	d->c.sendfile.q_mtx = desc->q_mtx;
+
+    #if SIZEOF_OFF_T == 4
+	if (offsetH != 0) {
+	    reply_posix_error(desc, EINVAL);
+	    goto done;
+	}
+	d->c.sendfile.offset = (off_t) offsetL;
+    #else
+	d->c.sendfile.offset = ((off_t) offsetH << 32) | offsetL;
+    #endif
+
+	d->c.sendfile.nbytes = nbytes;
+
+	if (USE_THRDS_FOR_SENDFILE) {
+	    SET_BLOCKING(d->c.sendfile.out_fd);
+	} else {
+	    /**
+	     * Write a place holder to queue in order to force file_flush
+	     * to be called before the driver is closed.
+	     */
+	    char tmp[1] = "";
+	    MUTEX_LOCK(d->c.sendfile.q_mtx);
+	    if (driver_enq(d->c.sendfile.port, tmp, 1)) {
+	        MUTEX_UNLOCK(d->c.sendfile.q_mtx);
+		reply_posix_error(desc, ENOMEM);
+		goto done;
+	    }
+	    MUTEX_UNLOCK(d->c.sendfile.q_mtx);
+	}
+
+	cq_enq(desc, d);
+#else
+	reply_posix_error(desc, ENOTSUP);
+#endif
+	goto done;
+        } /* case FILE_SENDFILE: */
+
     } /* switch(command) */
-    
-    if (lseek_flush_read(desc, &err) < 0) {
+
+    if (lseek_flush_read(desc, &err
+#ifdef USE_VM_PROBES
+			 , dt_priv, dt_utag
+#endif
+			 ) < 0) {
 	reply_posix_error(desc, err);
 	goto done;
     }
-    if (flush_write_check_error(desc, &err) < 0) {
+    if (flush_write_check_error(desc, &err
+#ifdef USE_VM_PROBES
+				, dt_priv, dt_utag
+#endif
+				) < 0) {
 	reply_posix_error(desc, err);
 	goto done;
     } else {
@@ -3134,5 +4097,50 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) {
     }
 
  done:
+    if (d != NULL) {
+#ifdef USE_VM_PROBES
+	/*
+	 * If d == NULL, then either:
+	 *    1). There was an error of some sort, or
+	 *    2). The command given to us is actually implemented
+	 *	  by file_output() instead.
+	 *
+	 * Case #1 is probably a TODO item, perhaps?
+	 * Case #2 we definitely don't want to activate a probe.
+	 */
+	d->sched_i1 = dt_priv->thread_num;
+	d->sched_i2 = dt_priv->tag;
+	d->sched_utag[0] = '\0';
+	if (dt_utag != NULL) {
+	    if (dt_utag[0] == '\0') {
+                dt_utag = NULL;
+            } else {
+		strncpy(d->sched_utag, dt_utag, sizeof(d->sched_utag) - 1);
+		d->sched_utag[sizeof(d->sched_utag) - 1] = '\0';
+	    }
+	}
+	DTRACE11(efile_drv_entry, dt_priv->thread_num, dt_priv->tag++,
+		 dt_utag, command, dt_s1, NULL, dt_i1, dt_i2, dt_i3, dt_i4,
+		 desc->port_str);
+#endif
+    }
     cq_execute(desc);
 }
+
+#ifdef  USE_VM_PROBES
+dt_private *
+get_dt_private(int base)
+{
+    dt_private *dt_priv = (dt_private *) pthread_getspecific(dt_driver_key);
+
+    if (dt_priv == NULL) {
+	dt_priv = EF_SAFE_ALLOC(sizeof(dt_private));
+	erts_mtx_lock(&dt_driver_mutex);
+	dt_priv->thread_num = (base + dt_driver_idnum++);
+	erts_mtx_unlock(&dt_driver_mutex);
+	dt_priv->tag = 0;
+	pthread_setspecific(dt_driver_key, dt_priv);
+    }
+    return dt_priv;
+}
+#endif  /* USE_VM_PROBES */
diff --git a/erts/emulator/drivers/common/erl_efile.h b/erts/emulator/drivers/common/erl_efile.h
index 9aa941e550..69ad02633c 100644
--- a/erts/emulator/drivers/common/erl_efile.h
+++ b/erts/emulator/drivers/common/erl_efile.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /*
@@ -32,7 +32,8 @@
 #define EFILE_MODE_READ_WRITE 	3
 #define EFILE_MODE_APPEND	4
 #define EFILE_COMPRESSED 	8
-#define EFILE_NO_TRUNCATE      16 /* Special for reopening on VxWorks */
+#define EFILE_MODE_EXCL        16
+#define EFILE_NO_TRUNCATE      32 /* Special for reopening on VxWorks */
 
 /*
  * Seek modes for efile_seek().
@@ -58,6 +59,19 @@
 #define FA_WRITE 	1
 #define FA_READ		2
 
+/* Some OS'es (i.e. Windows) has filenames in wide charaqcters. That requires special handling */
+/* Note that we do *not* honor alignment in the communication to the OS specific driver, */
+/* which is not a problem on x86, but might be on other platforms. The OS specific efile */
+/* implementation is expected to align if needed */
+#ifdef __WIN32__
+#define FILENAMES_16BIT 1
+#endif
+
+// We use sendfilev if it exist on solaris
+#if !defined(HAVE_SENDFILE) && defined(HAVE_SENDFILEV)
+#define HAVE_SENDFILE
+#endif
+
 /*
  * An handle to an open directory.  To be cast to the correct type
  * in the system-dependent directory functions.
@@ -76,14 +90,15 @@ typedef struct _Efile_error {
 /*
  * This structure contains date and time.
  */
-typedef struct _Efile_time {
-    unsigned year;		/* (4 digits). */
-    unsigned month;		/* (1..12). */
-    unsigned day;		/* (1..31). */
-    unsigned hour;		/* (0..23). */
-    unsigned minute;		/* (0..59). */
-    unsigned second;		/* (0..59). */
-} Efile_time;
+
+//typedef struct _Efile_time {
+//    unsigned year;		/* (4 digits). */
+//    unsigned month;		/* (1..12). */
+//    unsigned day;		/* (1..31). */
+//    unsigned hour;		/* (0..23). */
+//    unsigned minute;		/* (0..59). */
+//    unsigned second;		/* (0..59). */
+//} Efile_time;
 
 
 /*
@@ -102,13 +117,26 @@ typedef struct _Efile_info {
     Uint32 inode;		/* Inode number. */
     Uint32 uid;			/* User id of owner. */
     Uint32 gid;			/* Group id of owner. */
-    Efile_time accessTime;	/* Last time the file was accessed. */
-    Efile_time modifyTime;	/* Last time the file was modified. */
-    Efile_time cTime;		/* Creation time (Windows) or last
+    time_t accessTime;		/* Last time the file was accessed. */
+    time_t modifyTime;		/* Last time the file was modified. */
+    time_t cTime;		/* Creation time (Windows) or last
 				 * inode change (Unix).
 				 */
 } Efile_info;
 
+
+#ifdef HAVE_SENDFILE
+/*
+ * Describes the structure of headers/trailers for sendfile
+ */
+struct t_sendfile_hdtl {
+    SysIOVec *headers;
+    int hdr_cnt;
+    SysIOVec *trailers;
+    int trl_cnt;
+};
+#endif /* HAVE_SENDFILE */
+
 /*
  * Functions.
  */
@@ -122,10 +150,11 @@ int efile_getdcwd(Efile_error* errInfo, int drive,
 		  char* buffer, size_t size);
 int efile_readdir(Efile_error* errInfo, char* name, 
 		  EFILE_DIR_HANDLE* dir_handle,
-		  char* buffer, size_t size);
+		  char* buffer, size_t *size);
 int efile_openfile(Efile_error* errInfo, char* name, int flags,
 		   int* pfd, Sint64* pSize);
 void efile_closefile(int fd);
+int efile_fdatasync(Efile_error* errInfo, int fd);
 int efile_fsync(Efile_error* errInfo, int fd);
 int efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo,
 		   char *name, int info_for_link);
@@ -133,7 +162,7 @@ int efile_write_info(Efile_error* errInfo, Efile_info* pInfo, char *name);
 int efile_write(Efile_error* errInfo, int flags, int fd, 
 		char* buf, size_t count);
 int efile_writev(Efile_error* errInfo, int flags, int fd, 
-		 SysIOVec* iov, int iovcnt, size_t size);
+		 SysIOVec* iov, int iovcnt);
 int efile_read(Efile_error* errInfo, int flags, int fd, 
 	       char* buf, size_t count, size_t* pBytesRead);
 int efile_seek(Efile_error* errInfo, int fd, 
@@ -150,3 +179,9 @@ int efile_altname(Efile_error* errInfo, char *name,
 int efile_link(Efile_error* errInfo, char* old, char* new);
 int efile_symlink(Efile_error* errInfo, char* old, char* new);
 int efile_may_openfile(Efile_error* errInfo, char *name);
+int efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, Sint64 length,
+		  int advise);
+#ifdef HAVE_SENDFILE
+int efile_sendfile(Efile_error* errInfo, int in_fd, int out_fd,
+		      off_t *offset, Uint64 *nbytes, struct t_sendfile_hdtl *hdtl);
+#endif /* HAVE_SENDFILE */
diff --git a/erts/emulator/drivers/common/gzio.c b/erts/emulator/drivers/common/gzio.c
index 801bc61d4d..a9303d55bc 100644
--- a/erts/emulator/drivers/common/gzio.c
+++ b/erts/emulator/drivers/common/gzio.c
@@ -27,8 +27,11 @@
 #endif
 
 #ifdef __WIN32__
+#ifndef HAVE_CONFLICTING_FREAD_DECLARATION
 #define HAVE_CONFLICTING_FREAD_DECLARATION
 #endif
+#define FILENAMES_16BIT 1
+#endif
 
 #ifdef STDC
 #  define zstrerror(errnum) strerror(errnum)
@@ -102,6 +105,40 @@ local uLong  getLong      OF((gz_stream *s));
 # define ERTS_GZREAD(File, Buf, Count) fread((Buf), 1, (Count), (File))
 #endif
 
+/*
+ * Ripped from efile_drv.c
+ */
+
+#ifdef FILENAMES_16BIT
+#  define FILENAME_BYTELEN(Str) filename_len_16bit(Str)
+#  define FILENAME_COPY(To,From) filename_cpy_16bit((To),(From)) 
+#  define FILENAME_CHARSIZE 2
+
+   static int filename_len_16bit(const char *str) 
+   {
+       const char *p = str;
+       while(*p != '\0' || p[1] != '\0') {
+	   p += 2;
+       }
+       return (p - str);
+   }
+
+   static void filename_cpy_16bit(char *to, const char *from) 
+   {
+       while(*from != '\0' || from[1] != '\0') {
+	   *to++ = *from++;
+	   *to++ = *from++;
+       }
+       *to++ = *from++;
+       *to++ = *from++;
+   }
+
+#else
+#  define FILENAME_BYTELEN(Str) strlen(Str)
+#  define FILENAME_COPY(To,From) strcpy(To,From) 
+#  define FILENAME_CHARSIZE 1
+#endif
+
 /* ===========================================================================
      Opens a gzip (.gz) file for reading or writing. The mode parameter
    is as in fopen ("rb" or "wb"). The file is given either by file descriptor
@@ -144,11 +181,11 @@ local gzFile gz_open (path, mode)
     s->position = 0;
     s->destroy = destroy;
 
-    s->path = (char*)ALLOC(strlen(path)+1);
+    s->path = (char*)ALLOC(FILENAME_BYTELEN(path)+FILENAME_CHARSIZE);
     if (s->path == NULL) {
         return s->destroy(s), (gzFile)Z_NULL;
     }
-    strcpy(s->path, path); /* do this early for debugging */
+    FILENAME_COPY(s->path, path); /* do this early for debugging */
 
     s->mode = '\0';
     do {
@@ -194,7 +231,22 @@ local gzFile gz_open (path, mode)
     s->stream.avail_out = Z_BUFSIZE;
 
     errno = 0;
-#ifdef UNIX
+#if defined(FILENAMES_16BIT)
+    {
+	char wfmode[160];
+	int i=0,j;
+	for(j=0;fmode[j] != '\0';++j) {
+	    wfmode[i++]=fmode[j];
+	    wfmode[i++]='\0';
+	}
+	wfmode[i++] = '\0';
+	wfmode[i++] = '\0';
+	s->file = F_OPEN(path, wfmode);
+	if (s->file == NULL) {
+	    return s->destroy(s), (gzFile)Z_NULL;
+	}
+    }
+#elif defined(UNIX)
     if (s->mode == 'r') {
 	s->file = open(path, O_RDONLY);
     } else {
@@ -582,6 +634,7 @@ erts_gzseek(gzFile file, int offset, int whence)
     while (s->position < pos) {
 	char buf[512];
 	int n;
+	int save_pos = s->position;
 
 	n = pos - s->position;
 	if (n > sizeof(buf))
@@ -593,6 +646,7 @@ erts_gzseek(gzFile file, int offset, int whence)
 	    memset(buf, '\0', n);
 	    erts_gzwrite(file, buf, n);
 	}
+	if (save_pos == s->position) break;
     }
 
     return s->position;
diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c
index 2bff5bd4f7..76a9b55179 100644
--- a/erts/emulator/drivers/common/inet_drv.c
+++ b/erts/emulator/drivers/common/inet_drv.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1997-2011. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1997-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -41,15 +41,22 @@
 #define STRINGIFY_1(b) IDENTITY(#b)
 #define STRINGIFY(a) STRINGIFY_1(a)
 
-#ifndef _OSE_
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
 #ifdef HAVE_SYS_UIO_H
 #include <sys/uio.h>
 #endif
-#endif
 
+#ifdef HAVE_NET_IF_DL_H
+#include <net/if_dl.h>
+#endif
+#ifdef HAVE_IFADDRS_H
+#include <ifaddrs.h>
+#endif
+#ifdef HAVE_NETPACKET_PACKET_H
+#include <netpacket/packet.h>
+#endif
 
 /* All platforms fail on malloc errors. */
 #define FATAL_MALLOC
@@ -57,6 +64,28 @@
 
 #include "erl_driver.h"
 
+/* The IS_SOCKET_ERROR macro below is used for portability reasons. While
+   POSIX specifies that errors from socket-related system calls should be
+   indicated with a -1 return value, some users have experienced non-Windows
+   OS kernels that return negative values other than -1. While one can argue
+   that such kernels are technically broken, comparing against values less
+   than 0 covers their out-of-spec return values without imposing incorrect
+   semantics on systems that manage to correctly return -1 for errors, thus
+   increasing Erlang's portability.
+*/
+#ifdef __WIN32__
+#define IS_SOCKET_ERROR(val) ((val) == SOCKET_ERROR)
+#else
+#define IS_SOCKET_ERROR(val) ((val) < 0)
+#endif
+
+#ifdef __WIN32__
+#define LLU "%I64u"
+#else
+#define LLU "%llu"
+#endif
+typedef unsigned long long llu_t;
+
 #ifdef __WIN32__
 #define  STRNCASECMP strncasecmp
 
@@ -66,8 +95,21 @@
 #include <winsock2.h>
 #endif
 #include <windows.h>
+#include <Ws2tcpip.h>   /* NEED VC 6.0 or higher */
+
+/* Visual studio 2008+: NTDDI_VERSION needs to be set for iphlpapi.h
+   to define the right structures. It needs to be set to WINXP (or LONGHORN)
+   for IPV6 to work and it's set lower by default, so we need to change it. */
+#ifdef HAVE_SDKDDKVER_H
+#  include <sdkddkver.h>
+#  ifdef NTDDI_VERSION 
+#    undef NTDDI_VERSION
+#  endif
+#  define NTDDI_VERSION NTDDI_WINXP
+#endif
+
+#include <iphlpapi.h>
 
-#include <Ws2tcpip.h>   /* NEED VC 6.0 !!! */
 
 #undef WANT_NONBLOCKING
 #include "sys.h"
@@ -75,6 +117,77 @@
 #undef EWOULDBLOCK
 #undef ETIMEDOUT
 
+#ifdef EINPROGRESS
+#undef EINPROGRESS
+#endif
+#ifdef EALREADY
+#undef EALREADY
+#endif
+#ifdef ENOTSOCK
+#undef ENOTSOCK
+#endif
+#ifdef EDESTADDRREQ
+#undef EDESTADDRREQ
+#endif
+#ifdef EMSGSIZE
+#undef EMSGSIZE
+#endif
+#ifdef EPROTOTYPE
+#undef EPROTOTYPE
+#endif
+#ifdef ENOPROTOOPT
+#undef ENOPROTOOPT
+#endif
+#ifdef EPROTONOSUPPORT
+#undef EPROTONOSUPPORT
+#endif
+#ifdef EOPNOTSUPP
+#undef EOPNOTSUPP
+#endif
+#ifdef EAFNOSUPPORT
+#undef EAFNOSUPPORT
+#endif
+#ifdef EADDRINUSE
+#undef EADDRINUSE
+#endif
+#ifdef EADDRNOTAVAIL
+#undef EADDRNOTAVAIL
+#endif
+#ifdef ENETDOWN
+#undef ENETDOWN
+#endif
+#ifdef ENETUNREACH
+#undef ENETUNREACH
+#endif
+#ifdef ENETRESET
+#undef ENETRESET
+#endif
+#ifdef ECONNABORTED
+#undef ECONNABORTED
+#endif
+#ifdef ECONNRESET
+#undef ECONNRESET
+#endif
+#ifdef ENOBUFS
+#undef ENOBUFS
+#endif
+#ifdef EISCONN
+#undef EISCONN
+#endif
+#ifdef ENOTCONN
+#undef ENOTCONN
+#endif
+#ifdef ECONNREFUSED
+#undef ECONNREFUSED
+#endif
+#ifdef ELOOP
+#undef ELOOP
+#endif
+#ifdef EHOSTUNREACH
+#undef EHOSTUNREACH
+#endif
+
+
 #define HAVE_MULTICAST_SUPPORT
 
 #define ERRNO_BLOCK             WSAEWOULDBLOCK
@@ -186,18 +299,8 @@ static unsigned long one_value = 1;
 #include <netdb.h>
 #endif
 
-#ifndef _OSE_
 #include <sys/socket.h>
 #include <netinet/in.h>
-#else
-/* datatypes and macros from Solaris socket.h */
-struct  linger {
-        int     l_onoff;                /* option on/off */
-        int     l_linger;               /* linger time */
-};
-#define SO_OOBINLINE    0x0100          /* leave received OOB data in line */
-#define SO_LINGER       0x0080          /* linger on close if data present */
-#endif
 
 #ifdef VXWORKS
 #include <rpc/rpctypes.h>
@@ -206,12 +309,10 @@ struct  linger {
 #include <rpc/types.h>
 #endif
 
-#ifndef _OSE_
 #include <netinet/tcp.h>
 #include <arpa/inet.h>
-#endif
 
-#if (!defined(VXWORKS) && !defined(_OSE_))
+#if (!defined(VXWORKS))
 #include <sys/param.h>
 #ifdef HAVE_ARPA_NAMESER_H
 #include <arpa/nameser.h>
@@ -226,33 +327,11 @@ struct  linger {
 #include <sys/ioctl.h>
 #endif
 
-#ifndef _OSE_
 #include <net/if.h>
-#else
-#define IFF_MULTICAST 0x00000800
-#endif
-
-#ifdef _OSE_
-#include "inet.h"
-#include "ineterr.h"
-#include "ose_inet_drv.h"
-#include "nameser.h" 
-#include "resolv.h"
-#define SET_ASYNC(s) setsockopt((s), SOL_SOCKET, SO_OSEEVENT, (&(s)), sizeof(int))
-
-extern void select_release(void);
-
-#endif /* _OSE_ */
-
-/* Solaris headers, only to be used with SFK */
-#ifdef _OSE_SFK_
-#include <ctype.h>
-#include <string.h>
-#endif
 
 /* SCTP support -- currently for UNIX platforms only: */
 #undef HAVE_SCTP
-#if (!defined(VXWORKS) && !defined(_OSE_) && !defined(__WIN32__) && defined(HAVE_SCTP_H))
+#if (!defined(VXWORKS) && !defined(__WIN32__) && defined(HAVE_SCTP_H))
 
 #include <netinet/sctp.h>
 
@@ -279,6 +358,57 @@ extern void select_release(void);
 #     define    SCTP_EOF        MSG_EOF
 #endif
 
+/* More Solaris 10 fixes: */
+#if ! HAVE_DECL_SCTP_CLOSED && HAVE_DECL_SCTPS_IDLE
+#    define SCTP_CLOSED SCTPS_IDLE
+#    undef HAVE_DECL_SCTP_CLOSED
+#    define HAVE_DECL_SCTP_CLOSED 1
+#endif
+#if ! HAVE_DECL_SCTP_BOUND && HAVE_DECL_SCTPS_BOUND
+#    define SCTP_BOUND SCTPS_BOUND
+#    undef HAVE_DECL_SCTP_BOUND
+#    define HAVE_DECL_SCTP_BOUND 1
+#endif
+#if ! HAVE_DECL_SCTP_LISTEN && HAVE_DECL_SCTPS_LISTEN
+#    define SCTP_LISTEN SCTPS_LISTEN
+#    undef HAVE_DECL_SCTP_LISTEN
+#    define HAVE_DECL_SCTP_LISTEN 1
+#endif
+#if ! HAVE_DECL_SCTP_COOKIE_WAIT && HAVE_DECL_SCTPS_COOKIE_WAIT
+#    define SCTP_COOKIE_WAIT SCTPS_COOKIE_WAIT
+#    undef HAVE_DECL_SCTP_COOKIE_WAIT
+#    define HAVE_DECL_SCTP_COOKIE_WAIT 1
+#endif
+#if ! HAVE_DECL_SCTP_COOKIE_ECHOED && HAVE_DECL_SCTPS_COOKIE_ECHOED
+#    define SCTP_COOKIE_ECHOED SCTPS_COOKIE_ECHOED
+#    undef HAVE_DECL_SCTP_COOKIE_ECHOED
+#    define HAVE_DECL_SCTP_COOKIE_ECHOED 1
+#endif
+#if ! HAVE_DECL_SCTP_ESTABLISHED && HAVE_DECL_SCTPS_ESTABLISHED
+#    define SCTP_ESTABLISHED SCTPS_ESTABLISHED
+#    undef HAVE_DECL_SCTP_ESTABLISHED
+#    define HAVE_DECL_SCTP_ESTABLISHED 1
+#endif
+#if ! HAVE_DECL_SCTP_SHUTDOWN_PENDING && HAVE_DECL_SCTPS_SHUTDOWN_PENDING
+#    define SCTP_SHUTDOWN_PENDING SCTPS_SHUTDOWN_PENDING
+#    undef HAVE_DECL_SCTP_SHUTDOWN_PENDING
+#    define HAVE_DECL_SCTP_SHUTDOWN_PENDING 1
+#endif
+#if ! HAVE_DECL_SCTP_SHUTDOWN_SENT && HAVE_DECL_SCTPS_SHUTDOWN_SENT
+#    define SCTP_SHUTDOWN_SENT SCTPS_SHUTDOWN_SENT
+#    undef HAVE_DECL_SCTP_SHUTDOWN_SENT
+#    define HAVE_DECL_SCTP_SHUTDOWN_SENT 1
+#endif
+#if ! HAVE_DECL_SCTP_SHUTDOWN_RECEIVED && HAVE_DECL_SCTPS_SHUTDOWN_RECEIVED
+#    define SCTP_SHUTDOWN_RECEIVED SCTPS_SHUTDOWN_RECEIVED
+#    undef HAVE_DECL_SCTP_SHUTDOWN_RECEIVED
+#    define HAVE_DECL_SCTP_SHUTDOWN_RECEIVED 1
+#endif
+#if ! HAVE_DECL_SCTP_SHUTDOWN_ACK_SENT && HAVE_DECL_SCTPS_SHUTDOWN_ACK_SENT
+#    define SCTP_SHUTDOWN_ACK_SENT SCTPS_SHUTDOWN_ACK_SENT
+#    undef HAVE_DECL_SCTP_SHUTDOWN_ACK_SENT
+#    define HAVE_DECL_SCTP_SHUTDOWN_ACK_SENT 1
+#endif
 /* New spelling in lksctp 2.6.22 or maybe even earlier:
  *  adaption -> adaptation
  */
@@ -293,12 +423,13 @@ extern void select_release(void);
 #     define sctp_adaptation_layer_event sctp_adaption_layer_event
 #endif
 
-static void *h_libsctp = NULL;
 #ifdef __GNUC__
 static typeof(sctp_bindx) *p_sctp_bindx = NULL;
+static typeof(sctp_peeloff) *p_sctp_peeloff = NULL;
 #else
 static int (*p_sctp_bindx)(int sd, struct sockaddr *addrs,
 			   int addrcnt, int flags) = NULL;
+static int (*p_sctp_peeloff)(int sd, sctp_assoc_t assoc_id) = NULL;
 #endif
 
 #endif /* SCTP supported */
@@ -315,7 +446,7 @@ static int (*p_sctp_bindx)(int sd, struct sockaddr *addrs,
 #define DEBUGF(X) printf X
 #endif
 
-#if !defined(__WIN32__) && !defined(HAVE_STRNCASECMP)
+#if !defined(HAVE_STRNCASECMP)
 #define STRNCASECMP my_strncasecmp
 
 static int my_strncasecmp(const char *s1, const char *s2, size_t n)
@@ -335,6 +466,7 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n)
 #define INVALID_SOCKET -1
 #define INVALID_EVENT  -1
 #define SOCKET_ERROR   -1
+
 #define SOCKET int
 #define HANDLE long int
 #define FD_READ    ERL_DRV_READ
@@ -362,20 +494,6 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n)
 #define sock_htons(x)               htons((x))
 #define sock_htonl(x)               htonl((x))
 
-#ifdef _OSE_
-#define sock_accept(s, addr, len)   ose_inet_accept((s), (addr), (len))
-#define sock_send(s,buf,len,flag)   ose_inet_send((s),(buf),(len),(flag))
-#define sock_sendto(s,buf,blen,flag,addr,alen) \
-                ose_inet_sendto((s),(buf),(blen),(flag),(addr),(alen))
-#define sock_sendv(s, vec, size, np, flag) \
-		(*(np) = ose_inet_sendv((s), (SysIOVec*)(vec), (size)))
-#define sock_open(af, type, proto)  ose_inet_socket((af), (type), (proto))
-#define sock_close(s)               ose_inet_close((s))
-#define sock_hostname(buf, len)     ose_gethostname((buf), (len))
-#define sock_getservbyname(name,proto) ose_getservbyname((name), (proto))
-#define sock_getservbyport(port,proto) ose_getservbyport((port), (proto))
-
-#else
 #define sock_accept(s, addr, len)   accept((s), (addr), (len))
 #define sock_send(s,buf,len,flag)   send((s),(buf),(len),(flag))
 #define sock_sendto(s,buf,blen,flag,addr,alen) \
@@ -391,7 +509,6 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n)
 #define sock_hostname(buf, len)     gethostname((buf), (len))
 #define sock_getservbyname(name,proto) getservbyname((name), (proto))
 #define sock_getservbyport(port,proto) getservbyport((port), (proto))
-#endif /* _OSE_ */
 
 #define sock_recv(s,buf,len,flag)   recv((s),(buf),(len),(flag))
 #define sock_recvfrom(s,buf,blen,flag,addr,alen) \
@@ -402,15 +519,11 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n)
 #define sock_create_event(d)        ((d)->s) /* return file descriptor */
 #define sock_close_event(e)                  /* do nothing */
 
-#ifdef _OSE_
-#define inet_driver_select(port, e, mode, on) \
-                                    ose_inet_select(port, e, mode, on)
-#else
 #define inet_driver_select(port, e, mode, on) \
                                     driver_select(port, e, mode | (on?ERL_DRV_USE:0), on)
-#endif /* _OSE_ */
 
 #define sock_select(d, flags, onoff) do { \
+        ASSERT(!(d)->is_ignored); \
         (d)->event_mask = (onoff) ? \
                  ((d)->event_mask | (flags)) : \
                  ((d)->event_mask & ~(flags)); \
@@ -433,6 +546,19 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n)
 			     (((unsigned char*) (s))[1] << 8) | \
 			     (((unsigned char*) (s))[0]))
 
+
+#ifdef VALGRIND
+#  include <valgrind/memcheck.h>   
+#else
+#  define VALGRIND_MAKE_MEM_DEFINED(ptr,size)
+#endif
+
+/*
+  Magic errno value used locally for return of {error, system_limit}
+  - the emulator definition of SYSTEM_LIMIT is not available here.
+*/
+#define INET_ERRNO_SYSTEM_LIMIT  (15 << 8)
+
 /*----------------------------------------------------------------------------
 ** Interface constants.
 ** 
@@ -445,7 +571,7 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n)
 #define INET_AF_ANY         3 /* INADDR_ANY or IN6ADDR_ANY_INIT */
 #define INET_AF_LOOPBACK    4 /* INADDR_LOOPBACK or IN6ADDR_LOOPBACK_INIT */
 
-/* INET_REQ_GETTYPE enumeration */
+/* open and INET_REQ_GETTYPE enumeration */
 #define INET_TYPE_STREAM    1
 #define INET_TYPE_DGRAM     2
 #define INET_TYPE_SEQPACKET 3
@@ -501,17 +627,22 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n)
 #define INET_REQ_IFGET         22
 #define INET_REQ_IFSET         23
 #define INET_REQ_SUBSCRIBE     24
+#define INET_REQ_GETIFADDRS    25
+#define INET_REQ_ACCEPT        26
+#define INET_REQ_LISTEN        27
+#define INET_REQ_IGNOREFD      28
+
 /* TCP requests */
-#define TCP_REQ_ACCEPT         40
-#define TCP_REQ_LISTEN         41
+/* #define TCP_REQ_ACCEPT         40 MOVED */
+/* #define TCP_REQ_LISTEN         41 MERGED */
 #define TCP_REQ_RECV           42
 #define TCP_REQ_UNRECV         43
 #define TCP_REQ_SHUTDOWN       44
-#define TCP_REQ_MULTI_OP       45
 /* UDP and SCTP requests */
 #define PACKET_REQ_RECV        60 /* Common for UDP and SCTP         */
-#define SCTP_REQ_LISTEN	       61 /* Different from TCP; not for UDP */
+/* #define SCTP_REQ_LISTEN       61 MERGED Different from TCP; not for UDP */
 #define SCTP_REQ_BINDX	       62 /* Multi-home SCTP bind            */
+#define SCTP_REQ_PEELOFF       63
 
 /* INET_REQ_SUBSCRIBE sub-requests */
 #define INET_SUBS_EMPTY_OUT_Q  1
@@ -525,7 +656,7 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n)
 /* *_REQ_* replies */
 #define INET_REP_ERROR       0
 #define INET_REP_OK          1
-#define INET_REP_SCTP        2
+#define INET_REP             2
 
 /* INET_REQ_SETOPTS and INET_REQ_GETOPTS options */
 #define INET_OPT_REUSEADDR  0   /* enable/disable local address reuse */
@@ -646,10 +777,14 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n)
 ** End of interface constants.
 **--------------------------------------------------------------------------*/
 
-#define INET_STATE_CLOSED    0
-#define INET_STATE_OPEN      (INET_F_OPEN)
-#define INET_STATE_BOUND     (INET_STATE_OPEN | INET_F_BOUND)
-#define INET_STATE_CONNECTED (INET_STATE_BOUND | INET_F_ACTIVE)
+#define INET_STATE_CLOSED          (0)
+#define INET_STATE_OPEN            (INET_F_OPEN)
+#define INET_STATE_BOUND           (INET_STATE_OPEN | INET_F_BOUND)
+#define INET_STATE_CONNECTED       (INET_STATE_BOUND | INET_F_ACTIVE)
+#define INET_STATE_LISTENING       (INET_STATE_BOUND | INET_F_LISTEN)
+#define INET_STATE_CONNECTING      (INET_STATE_BOUND | INET_F_CON)
+#define INET_STATE_ACCEPTING       (INET_STATE_LISTENING | INET_F_ACC)
+#define INET_STATE_MULTI_ACCEPTING (INET_STATE_ACCEPTING | INET_F_MULTI_CLIENT)
 
 #define IS_OPEN(d) \
  (((d)->state & INET_F_OPEN) == INET_F_OPEN)
@@ -666,15 +801,12 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n)
 #define IS_BUSY(d) \
   (((d)->state & INET_F_BUSY) == INET_F_BUSY)
 
+#define INET_MAX_OPT_BUFFER (64*1024)
+
 #define INET_DEF_BUFFER     1460        /* default buffer size */
 #define INET_MIN_BUFFER     1           /* internal min buffer */
-#define INET_MAX_BUFFER     (1024*64)   /* internal max buffer */
 
-/* Note: INET_HIGH_WATERMARK MUST be less than 2*INET_MAX_BUFFER */
 #define INET_HIGH_WATERMARK (1024*8) /* 8k pending high => busy  */
-/* Note: INET_LOW_WATERMARK MUST be less than INET_MAX_BUFFER and
-** less than INET_HIGH_WATERMARK
-*/
 #define INET_LOW_WATERMARK  (1024*4) /* 4k pending => allow more */
 
 #define INET_INFINITY  0xffffffff  /* infinity value */
@@ -687,6 +819,11 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n)
 /* Max interface name */
 #define INET_IFNAMSIZ          16
 
+/* INET Ignore states */
+#define INET_IGNORE_NONE 0
+#define INET_IGNORE_READ 1
+#define INET_IGNORE_WRITE 1 << 1
+
 /* Max length of Erlang Term Buffer (for outputting structured terms):  */
 #ifdef  HAVE_SCTP
 #define PACKET_ERL_DRV_TERM_DATA_LEN  512
@@ -695,7 +832,7 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n)
 #endif
 
 
-#define BIN_REALLOC_LIMIT(x)  (((x)*3)/4)  /* 75% */
+#define BIN_REALLOC_MARGIN(x)  ((x)/4)  /* 25% */
 
 /* The general purpose sockaddr */
 typedef union {
@@ -826,20 +963,13 @@ typedef struct {
     double send_avg;            /* average packet size sent */
 
     subs_list empty_out_q_subs; /* Empty out queue subscribers */
+    int is_ignored;             /* if a fd is ignored by the inet_drv.
+				   This flag should be set to true when
+				   the fd is used outside of inet_drv. */
 } inet_descriptor;
 
 
 
-#define TCP_STATE_CLOSED     INET_STATE_CLOSED
-#define TCP_STATE_OPEN       (INET_F_OPEN)
-#define TCP_STATE_BOUND      (TCP_STATE_OPEN | INET_F_BOUND)
-#define TCP_STATE_CONNECTED  (TCP_STATE_BOUND | INET_F_ACTIVE)
-#define TCP_STATE_LISTEN     (TCP_STATE_BOUND | INET_F_LISTEN)
-#define TCP_STATE_CONNECTING (TCP_STATE_BOUND | INET_F_CON)
-#define TCP_STATE_ACCEPTING  (TCP_STATE_LISTEN | INET_F_ACC)
-#define TCP_STATE_MULTI_ACCEPTING (TCP_STATE_ACCEPTING | INET_F_MULTI_CLIENT)
-
-
 #define TCP_MAX_PACKET_SIZE 0x4000000  /* 64 M */
 
 #define MAX_VSIZE 16		/* Max number of entries allowed in an I/O
@@ -848,13 +978,14 @@ typedef struct {
 
 static int tcp_inet_init(void);
 static void tcp_inet_stop(ErlDrvData);
-static void tcp_inet_command(ErlDrvData, char*, int);
+static void tcp_inet_command(ErlDrvData, char*, ErlDrvSizeT);
 static void tcp_inet_commandv(ErlDrvData, ErlIOVec*);
 static void tcp_inet_flush(ErlDrvData drv_data);
 static void tcp_inet_drv_input(ErlDrvData, ErlDrvEvent);
 static void tcp_inet_drv_output(ErlDrvData data, ErlDrvEvent event);
 static ErlDrvData tcp_inet_start(ErlDrvPort, char* command);
-static int tcp_inet_ctl(ErlDrvData, unsigned int, char*, int, char**, int);
+static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData, unsigned int,
+				 char*, ErlDrvSizeT, char**, ErlDrvSizeT);
 static void tcp_inet_timeout(ErlDrvData);
 static void tcp_inet_process_exit(ErlDrvData, ErlDrvMonitor *); 
 static void inet_stop_select(ErlDrvEvent, void*); 
@@ -895,25 +1026,19 @@ static struct erl_drv_entry tcp_inet_driver_entry =
     inet_stop_select
 };
 
-#define PACKET_STATE_CLOSED     INET_STATE_CLOSED
-#define PACKET_STATE_OPEN       (INET_F_OPEN)
-#define PACKET_STATE_BOUND      (PACKET_STATE_OPEN  | INET_F_BOUND)
-#define SCTP_STATE_LISTEN	(PACKET_STATE_BOUND | INET_F_LISTEN)
-#define SCTP_STATE_CONNECTING   (PACKET_STATE_BOUND | INET_F_CON)
-#define PACKET_STATE_CONNECTED  (PACKET_STATE_BOUND | INET_F_ACTIVE)
 
 
 static int        packet_inet_init(void);
 static void       packet_inet_stop(ErlDrvData);
-static void       packet_inet_command(ErlDrvData, char*, int);
+static void       packet_inet_command(ErlDrvData, char*, ErlDrvSizeT);
 static void       packet_inet_drv_input(ErlDrvData data, ErlDrvEvent event);
 static void	  packet_inet_drv_output(ErlDrvData data, ErlDrvEvent event);
 static ErlDrvData udp_inet_start(ErlDrvPort, char* command);
 #ifdef HAVE_SCTP
 static ErlDrvData sctp_inet_start(ErlDrvPort, char* command);
 #endif
-static int        packet_inet_ctl(ErlDrvData, unsigned int, char*, 
-				  int, char**, int);
+static ErlDrvSSizeT packet_inet_ctl(ErlDrvData, unsigned int, char*,
+				    ErlDrvSizeT, char**, ErlDrvSizeT);
 static void       packet_inet_timeout(ErlDrvData);
 #ifdef __WIN32__
 static void       packet_inet_event(ErlDrvData, ErlDrvEvent);
@@ -1007,7 +1132,7 @@ typedef struct {
 } tcp_descriptor;
 
 /* send function */
-static int tcp_send(tcp_descriptor* desc, char* ptr, int len);
+static int tcp_send(tcp_descriptor* desc, char* ptr, ErlDrvSizeT len);
 static int tcp_sendv(tcp_descriptor* desc, ErlIOVec* ev);
 static int tcp_recv(tcp_descriptor* desc, int request_len);
 static int tcp_deliver(tcp_descriptor* desc, int len);
@@ -1018,6 +1143,9 @@ static int tcp_inet_input(tcp_descriptor* desc, HANDLE event);
 typedef struct {
     inet_descriptor inet;   /* common data structure (DON'T MOVE) */
     int read_packets;       /* Number of packets to read per invocation */
+    int i_bufsz;            /* current input buffer size */
+    ErlDrvBinary* i_buf;    /* current binary buffer */
+    char* i_ptr;            /* current pos in buf */
 } udp_descriptor;
 
 
@@ -1071,7 +1199,7 @@ static ErlDrvTermData am_tos;
 
 
 static int inet_init(void);
-static int ctl_reply(int, char*, int, char**, int);
+static ErlDrvSSizeT ctl_reply(int, char*, ErlDrvSizeT, char**, ErlDrvSizeT);
 
 struct erl_drv_entry inet_driver_entry = 
 {
@@ -1081,11 +1209,27 @@ struct erl_drv_entry inet_driver_entry =
     NULL, /* output */
     NULL, /* ready_input */
     NULL, /* ready_output */
-    "inet"
+    "inet",
+    NULL,
+    NULL, /* handle */
+    NULL, /* control */
+    NULL, /* timeout */
+    NULL, /* outputv */
+    NULL, /* ready_async */
+    NULL, /* flush */
+    NULL, /* call */
+    NULL, /* event */
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
+    NULL,
 };
 
 /* XXX: is this a driver interface function ??? */
-extern void erl_exit(int n, char*, _DOTS_);
+void erl_exit(int n, char*, ...);
 
 /*
  * Malloc wrapper,
@@ -1095,7 +1239,7 @@ extern void erl_exit(int n, char*, _DOTS_);
 
 #ifdef FATAL_MALLOC
 
-static void *alloc_wrapper(size_t size){
+static void *alloc_wrapper(ErlDrvSizeT size){
     void *ret = driver_alloc(size);
     if(ret == NULL) 
 	erl_exit(1,"Out of virtual memory in malloc (%s)", __FILE__);
@@ -1103,7 +1247,7 @@ static void *alloc_wrapper(size_t size){
 }
 #define ALLOC(X) alloc_wrapper(X)
 
-static void *realloc_wrapper(void *current, size_t size){
+static void *realloc_wrapper(void *current, ErlDrvSizeT size){
     void *ret = driver_realloc(current,size);
     if(ret == NULL) 
 	erl_exit(1,"Out of virtual memory in realloc (%s)", __FILE__);
@@ -1285,139 +1429,136 @@ static int load_ip_and_port
 	LOAD_ATOM((spec), (i), (flag) ? am_true : am_false);
 #endif /* HAVE_SCTP */
 
+/* Assume a cache line size of 64 bytes */
+#define INET_DRV_CACHE_LINE_SIZE ((ErlDrvUInt) 64)
+#define INET_DRV_CACHE_LINE_MASK (INET_DRV_CACHE_LINE_SIZE - 1)
+
 /*
 ** Binary Buffer Managment
 ** We keep a stack of usable buffers 
 */
-#define BUFFER_STACK_SIZE 16
+#define BUFFER_STACK_SIZE 14
+#define BUFFER_STACK_MAX_MEM_SIZE (1024*1024)
 
-static erts_smp_spinlock_t inet_buffer_stack_lock;
-static ErlDrvBinary* buffer_stack[BUFFER_STACK_SIZE];
-static int buffer_stack_pos = 0;
+ErlDrvTSDKey buffer_stack_key;
 
+typedef struct {
+    int mem_size;
+    int pos;
+    ErlDrvBinary* stk[BUFFER_STACK_SIZE];
+} InetDrvBufStkBase;
 
-/*
- * XXX
- * The erts_smp_spin_* functions should not be used by drivers (but this
- * driver is special). Replace when driver locking api has been implemented.
- * /rickard
- */
-#define BUFSTK_LOCK	erts_smp_spin_lock(&inet_buffer_stack_lock);
-#define BUFSTK_UNLOCK	erts_smp_spin_unlock(&inet_buffer_stack_lock);
+typedef struct {
+    InetDrvBufStkBase buf;
+    char align[(((sizeof(InetDrvBufStkBase) - 1) / INET_DRV_CACHE_LINE_SIZE) + 1)
+	       * INET_DRV_CACHE_LINE_SIZE];
+} InetDrvBufStk;
+
+static InetDrvBufStk *get_bufstk(void)
+{
+    InetDrvBufStk *bs = erl_drv_tsd_get(buffer_stack_key);
+    if (bs)
+	return bs;
+    bs = driver_alloc(sizeof(InetDrvBufStk)
+		      + INET_DRV_CACHE_LINE_SIZE - 1);
+    if (!bs)
+	return NULL;
+    if ((((ErlDrvUInt) bs) & INET_DRV_CACHE_LINE_MASK) != 0)
+	bs = ((InetDrvBufStk *)
+	      ((((ErlDrvUInt) bs) & ~INET_DRV_CACHE_LINE_MASK)
+	       + INET_DRV_CACHE_LINE_SIZE));
+    erl_drv_tsd_set(buffer_stack_key, bs);
+    bs->buf.pos = 0;
+    bs->buf.mem_size = 0;
 
-#ifdef DEBUG
-static int tot_buf_allocated = 0;  /* memory in use for i_buf */
-static int tot_buf_stacked = 0;   /* memory on stack */
-static int max_buf_allocated = 0; /* max allocated */
-
-#define COUNT_BUF_ALLOC(sz) do { \
-  BUFSTK_LOCK; \
-  tot_buf_allocated += (sz); \
-  if (tot_buf_allocated > max_buf_allocated) \
-    max_buf_allocated = tot_buf_allocated; \
-  BUFSTK_UNLOCK; \
-} while(0)
-
-#define COUNT_BUF_FREE(sz) do { \
- BUFSTK_LOCK; \
- tot_buf_allocated -= (sz); \
- BUFSTK_UNLOCK; \
- } while(0)
-
-#define COUNT_BUF_STACK(sz) do { \
- BUFSTK_LOCK; \
- tot_buf_stacked += (sz); \
- BUFSTK_UNLOCK; \
- } while(0)
+    ASSERT(bs == erl_drv_tsd_get(buffer_stack_key));
 
-#else
+    return bs;
+}
 
-#define COUNT_BUF_ALLOC(sz)
-#define COUNT_BUF_FREE(sz)
-#define COUNT_BUF_STACK(sz)
+static ErlDrvBinary* alloc_buffer(ErlDrvSizeT minsz)
+{
+    InetDrvBufStk *bs = get_bufstk();
 
-#endif
+    DEBUGF(("alloc_buffer: "LLU"\r\n", (llu_t)minsz));
 
-static ErlDrvBinary* alloc_buffer(long minsz)
-{
-    ErlDrvBinary* buf = NULL;
+    if (bs && bs->buf.pos > 0) {
+	long size;
+	ErlDrvBinary* buf = bs->buf.stk[--bs->buf.pos];
+	size = buf->orig_size;
+	bs->buf.mem_size -= size;
+	ASSERT(0 <= bs->buf.mem_size
+	       && bs->buf.mem_size <= BUFFER_STACK_MAX_MEM_SIZE);
+	if (size >= minsz)
+	    return buf;
 
-    BUFSTK_LOCK;
+	driver_free_binary(buf);
+    }
 
-    DEBUGF(("alloc_buffer: sz = %ld, tot = %d, max = %d\r\n", 
-	    minsz, tot_buf_allocated, max_buf_allocated));
+    ASSERT(!bs || bs->buf.pos != 0 || bs->buf.mem_size == 0);
 
-    if (buffer_stack_pos > 0) {
-	int origsz;
+    return driver_alloc_binary(minsz);
+}
 
-	buf = buffer_stack[--buffer_stack_pos];
-	origsz = buf->orig_size;
-	BUFSTK_UNLOCK;
-	COUNT_BUF_STACK(-origsz);
-	if (origsz < minsz) {
-	    if ((buf = driver_realloc_binary(buf, minsz)) == NULL)
-		return NULL;
-	    COUNT_BUF_ALLOC(buf->orig_size - origsz);
+/*#define CHECK_DOUBLE_RELEASE 1*/
+#ifdef CHECK_DOUBLE_RELEASE
+static void
+check_double_release(InetDrvBufStk *bs, ErlDrvBinary* buf)
+{
+#ifdef __GNUC__
+#warning CHECK_DOUBLE_RELEASE is enabled, this is a custom build emulator
+#endif
+    int i;
+    for (i = 0; i < bs->buf.pos; ++i) {
+	if (bs->buf.stk[i] == buf) {
+	    erl_exit(ERTS_ABORT_EXIT,
+		     "Multiple buffer release in inet_drv, this "
+		     "is a bug, save the core and send it to "
+		     "[email protected]!");
 	}
     }
-    else {
-	BUFSTK_UNLOCK;
-	if ((buf = driver_alloc_binary(minsz)) == NULL)
-	    return NULL;
-	COUNT_BUF_ALLOC(buf->orig_size);
-    }
-    return buf;
 }
+#endif
 
-/*
-** Max buffer memory "cached" BUFFER_STACK_SIZE * INET_MAX_BUFFER
-** (16 * 64k ~ 1M)
-*/
-/*#define CHECK_DOUBLE_RELEASE 1*/
 static void release_buffer(ErlDrvBinary* buf)
 {
+    InetDrvBufStk *bs;
+    long size;
+
     DEBUGF(("release_buffer: %ld\r\n", (buf==NULL) ? 0 : buf->orig_size));
-    if (buf == NULL)
+
+    if (!buf)
 	return;
-    BUFSTK_LOCK;
-    if ((buf->orig_size > INET_MAX_BUFFER) || 
-	(buffer_stack_pos >= BUFFER_STACK_SIZE)) {
-	BUFSTK_UNLOCK;
-	COUNT_BUF_FREE(buf->orig_size);
+
+    size = buf->orig_size;
+
+    if (size > BUFFER_STACK_MAX_MEM_SIZE)
+	goto free_binary;
+
+    bs = get_bufstk();
+    if (!bs
+	|| (bs->buf.mem_size + size > BUFFER_STACK_MAX_MEM_SIZE)
+	|| (bs->buf.pos >= BUFFER_STACK_SIZE)) {
+    free_binary:
 	driver_free_binary(buf);
     }
     else {
 #ifdef CHECK_DOUBLE_RELEASE
-#ifdef __GNUC__
-#warning CHECK_DOUBLE_RELEASE is enabled, this is a custom build emulator
+	check_double_release(bs, buf);
 #endif
-	int i;
-	for (i = 0; i < buffer_stack_pos; ++i) {
-	    if (buffer_stack[i] == buf) {
-		erl_exit(1,"Multiple buffer release in inet_drv, this is a "
-			 "bug, save the core and send it to "
-			 "[email protected]!");
-	    }
-	}
-#endif
-	buffer_stack[buffer_stack_pos++] = buf;
-	BUFSTK_UNLOCK;
-	COUNT_BUF_STACK(buf->orig_size);
+	ASSERT(bs->buf.pos != 0 || bs->buf.mem_size == 0);
+
+	bs->buf.mem_size += size;
+	bs->buf.stk[bs->buf.pos++] = buf;
+
+	ASSERT(0 <= bs->buf.mem_size
+	       && bs->buf.mem_size <= BUFFER_STACK_MAX_MEM_SIZE);
     }
 }
 
-static ErlDrvBinary* realloc_buffer(ErlDrvBinary* buf, long newsz)
+static ErlDrvBinary* realloc_buffer(ErlDrvBinary* buf, ErlDrvSizeT newsz)
 {
-    ErlDrvBinary* bin;
-#ifdef DEBUG
-    long orig_size =  buf->orig_size;
-#endif
-
-    if ((bin = driver_realloc_binary(buf,newsz)) != NULL) {
-	COUNT_BUF_ALLOC(newsz - orig_size);
-	;
-    }
-    return bin;
+    return driver_realloc_binary(buf, newsz);
 }
 
 /* use a TRICK, access the refc field to see if any one else has
@@ -1431,10 +1572,8 @@ static void free_buffer(ErlDrvBinary* buf)
     if (buf != NULL) {
 	if (driver_binary_get_refc(buf) == 1)
 	    release_buffer(buf);
-	else {
-	    COUNT_BUF_FREE(buf->orig_size);
+	else
 	    driver_free_binary(buf);
-	}
     }
 }
 
@@ -1446,8 +1585,9 @@ static ErlDrvData dummy_start(ErlDrvPort port, char* command)
     return (ErlDrvData)port;
 }
 
-static int dummy_ctl(ErlDrvData data, unsigned int cmd, char* buf, int len,
-		     char** rbuf, int rsize)
+static ErlDrvSSizeT dummy_ctl(ErlDrvData data, unsigned int cmd,
+			      char* buf, ErlDrvSizeT len, char** rbuf,
+			      ErlDrvSizeT rsize)
 {
     static char error[] = "no_winsock2";
 
@@ -1455,7 +1595,7 @@ static int dummy_ctl(ErlDrvData data, unsigned int cmd, char* buf, int len,
     return ctl_reply(INET_REP_ERROR, error, sizeof(error), rbuf, rsize);
 }
 
-static void dummy_command(ErlDrvData data, char* buf, int len)
+static void dummy_command(ErlDrvData data, char* buf, ErlDrvSizeT len)
 {
 }
 
@@ -1511,8 +1651,20 @@ static struct erl_drv_entry dummy_sctp_driver_entry =
 
 #endif
 
+/* return lowercase string form of errno value */
+static char *errno_str(int err)
+{
+    switch (err) {
+    case INET_ERRNO_SYSTEM_LIMIT:
+	return "system_limit";
+    default:
+	return erl_errno_id(err);
+    }
+}
+
 /* general control reply function */
-static int ctl_reply(int rep, char* buf, int len, char** rbuf, int rsize)
+static ErlDrvSSizeT ctl_reply(int rep, char* buf, ErlDrvSizeT len,
+			      char** rbuf, ErlDrvSizeT rsize)
 {
     char* ptr;
 
@@ -1528,18 +1680,14 @@ static int ctl_reply(int rep, char* buf, int len, char** rbuf, int rsize)
 }
 
 /* general control error reply function */
-static int ctl_error(int err, char** rbuf, int rsize)
+static ErlDrvSSizeT ctl_error(int err, char** rbuf, ErlDrvSizeT rsize)
 {
-    char response[256];		/* Response buffer. */
-    char* s;
-    char* t;
+    char* s = errno_str(err);
 
-    for (s = erl_errno_id(err), t = response; *s; s++, t++)
-	*t = tolower(*s);
-    return ctl_reply(INET_REP_ERROR, response, t-response, rbuf, rsize);
+    return ctl_reply(INET_REP_ERROR, s, strlen(s), rbuf, rsize);
 }
 
-static int ctl_xerror(char* xerr, char** rbuf, int rsize)
+static ErlDrvSSizeT ctl_xerror(char* xerr, char** rbuf, ErlDrvSizeT rsize)
 {
     int n = strlen(xerr);
     return ctl_reply(INET_REP_ERROR, xerr, n, rbuf, rsize);
@@ -1548,14 +1696,7 @@ static int ctl_xerror(char* xerr, char** rbuf, int rsize)
 
 static ErlDrvTermData error_atom(int err)
 {
-    char errstr[256];
-    char* s;
-    char* t;
-
-    for (s = erl_errno_id(err), t = errstr; *s; s++, t++)
-	*t = tolower(*s);
-    *t = '\0';
-    return driver_mk_atom(errstr);
+    return driver_mk_atom(errno_str(err));
 }
 
 
@@ -1788,7 +1929,6 @@ send_async_error(ErlDrvPort port, ErlDrvTermData Port, int Ref,
 			LOAD_INT_CNT + 2*LOAD_TUPLE_CNT];
     int i = 0;
     
-    i = 0;
     i = LOAD_ATOM(spec, i, am_inet_async);
     i = LOAD_PORT(spec, i, Port);
     i = LOAD_INT(spec, i, Ref);
@@ -1878,6 +2018,26 @@ static int inet_reply_ok(inet_descriptor* desc)
     return driver_send_term(desc->port, caller, spec, i);    
 }
 
+#ifdef HAVE_SCTP
+static int inet_reply_ok_port(inet_descriptor* desc, ErlDrvTermData dport)
+{
+    ErlDrvTermData spec[2*LOAD_ATOM_CNT + 2*LOAD_PORT_CNT + 2*LOAD_TUPLE_CNT];
+    ErlDrvTermData caller = desc->caller;
+    int i = 0;
+
+    i = LOAD_ATOM(spec, i, am_inet_reply);
+    i = LOAD_PORT(spec, i, desc->dport);
+    i = LOAD_ATOM(spec, i, am_ok);
+    i = LOAD_PORT(spec, i, dport);
+    i = LOAD_TUPLE(spec, i, 2);
+    i = LOAD_TUPLE(spec, i, 3);
+    ASSERT(i == sizeof(spec)/sizeof(*spec));
+
+    desc->caller = 0;
+    return driver_send_term(desc->port, caller, spec, i);
+}
+#endif
+
 /* send:
 **   {inet_reply, S, {error, Reason}} 
 */
@@ -1975,7 +2135,7 @@ static int http_response_inetdrv(void *arg, int major, int minor,
     tcp_descriptor* desc = (tcp_descriptor*) arg;
     int i = 0;
     ErlDrvTermData spec[27];
-    ErlDrvTermData caller;
+    ErlDrvTermData caller = ERL_DRV_NIL;
     
     if (desc->inet.active == INET_PASSIVE) {
         /* {inet_async,S,Ref,{ok,{http_response,Version,Status,Phrase}}} */
@@ -2068,7 +2228,7 @@ http_request_inetdrv(void* arg, const http_atom_t* meth, const char* meth_ptr,
     tcp_descriptor* desc = (tcp_descriptor*) arg;
     int i = 0;
     ErlDrvTermData spec[43];
-    ErlDrvTermData caller;
+    ErlDrvTermData caller = ERL_DRV_NIL;
     
     if (desc->inet.active == INET_PASSIVE) {
         /* {inet_async, S, Ref, {ok,{http_request,Meth,Uri,Version}}} */
@@ -2119,7 +2279,7 @@ http_header_inetdrv(void* arg, const http_atom_t* name, const char* name_ptr,
     tcp_descriptor* desc = (tcp_descriptor*) arg;
     int i = 0;
     ErlDrvTermData spec[26];
-    ErlDrvTermData caller;
+    ErlDrvTermData caller = ERL_DRV_NIL;
     
     if (desc->inet.active == INET_PASSIVE) {
         /* {inet_async,S,Ref,{ok,{http_header,Bit,Name,IValue,Value}} */
@@ -2208,7 +2368,7 @@ static int http_error_inetdrv(void* arg, const char* buf, int len)
   ErlDrvTermData spec[19];
 
   if (desc->inet.active == INET_PASSIVE) {
-    /* {inet_async,S,Ref,{error,{http_error,Line}}} */
+    /* {inet_async,S,Ref,{ok,{http_error,Line}}} */
     int req;
     int aid;
     ErlDrvTermData caller;
@@ -2218,7 +2378,7 @@ static int http_error_inetdrv(void* arg, const char* buf, int len)
     i = LOAD_ATOM(spec, i,  am_inet_async);
     i = LOAD_PORT(spec, i,  desc->inet.dport);
     i = LOAD_INT(spec, i,   aid);
-    i = LOAD_ATOM(spec, i,  am_error);
+    i = LOAD_ATOM(spec, i,  am_ok);
     i = LOAD_ATOM(spec, i,  am_http_error);
     i = http_load_string(desc, spec, i, buf, len);
     i = LOAD_TUPLE(spec, i, 2);
@@ -2248,7 +2408,7 @@ int ssl_tls_inetdrv(void* arg, unsigned type, unsigned major, unsigned minor,
     tcp_descriptor* desc = (tcp_descriptor*) arg;
     int i = 0;
     ErlDrvTermData spec[28];
-    ErlDrvTermData caller;
+    ErlDrvTermData caller = ERL_DRV_NIL;
     ErlDrvBinary* bin;
     int ret;
 
@@ -2416,14 +2576,19 @@ static ErlDrvTermData   am_sctp_rtoinfo, /* Option names */
     am_active,                         am_inactive,
     
     /* For #sctp_status{}: */
-    am_empty,                          am_closed,
+#    if HAVE_DECL_SCTP_EMPTY
+    am_empty,
+#    endif
+#    if HAVE_DECL_SCTP_BOUND
+    am_bound,
+#    endif
+#    if HAVE_DECL_SCTP_LISTEN
+    am_listen,
+#    endif
     am_cookie_wait,                    am_cookie_echoed,
     am_established,                    am_shutdown_pending,
     am_shutdown_sent,                  am_shutdown_received,
     am_shutdown_ack_sent;
-    /* Not yet implemented in the Linux kernel:
-    ** am_bound,                          am_listen;
-    */
 
 /*
 ** Parsing of "sctp_sndrcvinfo": ancillary data coming with received msgs.
@@ -2692,7 +2857,8 @@ static int sctp_parse_async_event
 #	    ifdef HAVE_STRUCT_SCTP_REMOTE_ERROR_SRE_DATA
 	    chunk = (char*) (&(sptr->sre_data));
 #	    else
-	    chunk = ((char*)sptr) + sizeof(*sptr);
+	    chunk = ((char*) &(sptr->sre_assoc_id))
+		+ sizeof(sptr->sre_assoc_id);
 #	    endif
 	    chlen = sptr->sre_length  - (chunk - (char *)sptr);
 	    i = sctp_parse_error_chunk(spec, i, chunk, chlen);
@@ -2743,7 +2909,8 @@ static int sctp_parse_async_event
 #	    ifdef HAVE_STRUCT_SCTP_SEND_FAILED_SSF_DATA
 	    chunk = (char*) (&(sptr->ssf_data));
 #	    else
-	    chunk = ((char*)sptr) + sizeof(*sptr);
+	    chunk = ((char*) &(sptr->ssf_assoc_id))
+		+ sizeof(sptr->ssf_assoc_id);
 #	    endif
 	    chlen = sptr->ssf_length - (chunk - (char*) sptr);
 	    choff = chunk - bin->orig_bytes;
@@ -2957,6 +3124,7 @@ static int tcp_message(inet_descriptor* desc, const char* buf, int len)
     int i = 0;
 
     DEBUGF(("tcp_message(%ld): len = %d\r\n", (long)desc->port, len));    
+    /* XXX fprintf(stderr,"tcp_message send.\r\n"); */
 
     i = LOAD_ATOM(spec, i, am_tcp);
     i = LOAD_PORT(spec, i, desc->dport);
@@ -3417,8 +3585,15 @@ static void inet_init_sctp(void) {
     INIT_ATOM(inactive);
     
     /* For #sctp_status{}: */
+#    if HAVE_DECL_SCTP_EMPTY
     INIT_ATOM(empty);
-    INIT_ATOM(closed);
+#    endif
+#    if HAVE_DECL_SCTP_BOUND
+    INIT_ATOM(bound);
+#    endif
+#    if HAVE_DECL_SCTP_LISTEN
+    INIT_ATOM(listen);
+#    endif
     INIT_ATOM(cookie_wait);
     INIT_ATOM(cookie_echoed);
     INIT_ATOM(established);
@@ -3426,10 +3601,6 @@ static void inet_init_sctp(void) {
     INIT_ATOM(shutdown_sent);
     INIT_ATOM(shutdown_received);
     INIT_ATOM(shutdown_ack_sent);
-    /* Not yet implemented in the Linux kernel:
-    ** INIT_ATOM(bound);
-    ** INIT_ATOM(listen);
-    */
 }
 #endif /* HAVE_SCTP */
 
@@ -3438,20 +3609,14 @@ static int inet_init()
     if (!sock_init())
 	goto error;
 
-    buffer_stack_pos = 0;
-
-    erts_smp_spinlock_init(&inet_buffer_stack_lock, "inet_buffer_stack_lock");
+    if (0 != erl_drv_tsd_key_create("inet_buffer_stack_key", &buffer_stack_key))
+	goto error;
 
     ASSERT(sizeof(struct in_addr) == 4);
 #   if defined(HAVE_IN6) && defined(AF_INET6)
     ASSERT(sizeof(struct in6_addr) == 16);
 #   endif
 
-#ifdef DEBUG
-    tot_buf_allocated = 0;
-    max_buf_allocated = 0;
-    tot_buf_stacked = 0;
-#endif
     INIT_ATOM(ok);
     INIT_ATOM(tcp);
     INIT_ATOM(udp);
@@ -3480,29 +3645,40 @@ static int inet_init()
     INIT_ATOM(scheme);
 
     /* add TCP, UDP and SCTP drivers */
-#ifdef _OSE_
-    add_ose_tcp_drv_entry(&tcp_inet_driver_entry);
-    add_ose_udp_drv_entry(&udp_inet_driver_entry);
-#else
     add_driver_entry(&tcp_inet_driver_entry);
     add_driver_entry(&udp_inet_driver_entry);
-#  ifdef HAVE_SCTP
+#ifdef HAVE_SCTP
     /* Check the size of SCTP AssocID -- currently both this driver and the
        Erlang part require 32 bit: */
     ASSERT(sizeof(sctp_assoc_t)==ASSOC_ID_LEN);
-#   ifndef LIBSCTP
-#     error LIBSCTP not defined
-#   endif
-    if (erts_sys_ddll_open_noext(STRINGIFY(LIBSCTP), &h_libsctp, NULL) == 0) {
-	void *ptr;
-	if (erts_sys_ddll_sym(h_libsctp, "sctp_bindx", &ptr) == 0) {
-	    p_sctp_bindx = ptr;
-	    inet_init_sctp();
-	    add_driver_entry(&sctp_inet_driver_entry);
+#   if defined(HAVE_SCTP_BINDX) && defined (HAVE_SCTP_PEELOFF)
+    p_sctp_bindx = sctp_bindx;
+    p_sctp_peeloff = sctp_peeloff;
+    inet_init_sctp();
+    add_driver_entry(&sctp_inet_driver_entry);
+#   else
+#       ifndef LIBSCTP
+#           error LIBSCTP not defined
+#       endif
+    {
+	static void *h_libsctp = NULL;
+
+	if (erts_sys_ddll_open_noext(STRINGIFY(LIBSCTP), &h_libsctp, NULL)
+	    == 0) {
+	    void *ptr;
+	    if (erts_sys_ddll_sym(h_libsctp, "sctp_bindx", &ptr) == 0) {
+		p_sctp_bindx = ptr;
+		inet_init_sctp();
+		add_driver_entry(&sctp_inet_driver_entry);
+		if (erts_sys_ddll_sym(h_libsctp, "sctp_peeloff", &ptr) == 0) {
+		    p_sctp_peeloff = ptr;
+		}
+	    }
 	}
     }
-#  endif
-#endif /* _OSE_ */
+#   endif
+#endif
+
     /* remove the dummy inet driver */
     remove_driver_entry(&inet_driver_entry);
     return 0;
@@ -3522,7 +3698,8 @@ static int inet_init()
 ** and is set to actual length of dst on return
 ** return NULL on error and ptr after port address on success
 */
-static char* inet_set_address(int family, inet_address* dst, char* src, int* len)
+static char* inet_set_address(int family, inet_address* dst,
+			      char* src, ErlDrvSizeT* len)
 {
     short port;
 
@@ -3558,7 +3735,7 @@ static char* inet_set_address(int family, inet_address* dst, char* src, int* len
 ** src = [TAG,P1,P0,X1,X2,...] when TAG = INET_AF_INET | INET_AF_INET6
 */
 static char *inet_set_faddress(int family, inet_address* dst,
-			       char *src, int* len) {
+			       char *src, ErlDrvSizeT* len) {
     int tag;
     
     if (*len < 1) return NULL;
@@ -3673,7 +3850,13 @@ static void desc_close(inet_descriptor* desc)
 	desc->forced_events = 0;
 	desc->send_would_block = 0;
 #endif
-	driver_select(desc->port, (ErlDrvEvent)(long)desc->event, ERL_DRV_USE, 0);
+	// We should close the fd here, but the other driver might still
+	// be selecting on it.
+	if (!desc->is_ignored)
+	    driver_select(desc->port,(ErlDrvEvent)(long)desc->event, 
+			  ERL_DRV_USE, 0);
+	else
+	  inet_stop_select((ErlDrvEvent)(long)desc->event,NULL);
 	desc->event = INVALID_EVENT; /* closed by stop_select callback */
 	desc->s = INVALID_SOCKET;
 	desc->event_mask = 0;
@@ -3716,8 +3899,8 @@ static int erl_inet_close(inet_descriptor* desc)
 }
 
 
-static int inet_ctl_open(inet_descriptor* desc, int domain, int type, 
-			 char** rbuf, int rsize)
+static ErlDrvSSizeT inet_ctl_open(inet_descriptor* desc, int domain, int type,
+				  char** rbuf, ErlDrvSizeT rsize)
 {
     if (desc->state != INET_STATE_CLOSED)
 	return ctl_xerror(EXBADSEQ, rbuf, rsize);
@@ -3737,15 +3920,17 @@ static int inet_ctl_open(inet_descriptor* desc, int domain, int type,
 
 
 /* as inet_open but pass in an open socket (MUST BE OF RIGHT TYPE) */
-static int inet_ctl_fdopen(inet_descriptor* desc, int domain, int type,
-			   SOCKET s, char** rbuf, int rsize)
+static ErlDrvSSizeT inet_ctl_fdopen(inet_descriptor* desc, int domain, int type,
+				    SOCKET s, char** rbuf, ErlDrvSizeT rsize)
 {
     inet_address name;
     unsigned int sz = sizeof(name);
 
     /* check that it is a socket and that the socket is bound */
-    if (sock_name(s, (struct sockaddr*) &name, &sz) == SOCKET_ERROR)
+    if (IS_SOCKET_ERROR(sock_name(s, (struct sockaddr*) &name, &sz)))
 	return ctl_error(sock_errno(), rbuf, rsize);
+    if (name.sa.sa_family != domain)
+	return ctl_error(EINVAL, rbuf, rsize);
     desc->s = s;
     if ((desc->event = sock_create_event(desc)) == INVALID_EVENT)
 	return ctl_error(sock_errno(), rbuf, rsize);
@@ -3756,7 +3941,7 @@ static int inet_ctl_fdopen(inet_descriptor* desc, int domain, int type,
     desc->state = INET_STATE_BOUND; /* assume bound */
     if (type == SOCK_STREAM) { /* check if connected */
 	sz = sizeof(name);
-	if (sock_peer(s, (struct sockaddr*) &name, &sz) != SOCKET_ERROR)
+	if (!IS_SOCKET_ERROR(sock_peer(s, (struct sockaddr*) &name, &sz)))
 	    desc->state = INET_STATE_CONNECTED;
     }
 
@@ -3862,50 +4047,95 @@ do { if ((end)-(ptr) < (n)) goto error; } while(0)
 static char* sockaddr_to_buf(struct sockaddr* addr, char* ptr, char* end)
 {
     if (addr->sa_family == AF_INET || addr->sa_family == 0) {
-	struct in_addr a;
-	buf_check(ptr,end,sizeof(struct in_addr));
-	a = ((struct sockaddr_in*) addr)->sin_addr;
-	sys_memcpy(ptr, (char*)&a, sizeof(struct in_addr));
-	return ptr + sizeof(struct in_addr);
+	struct in_addr *p = &(((struct sockaddr_in*) addr)->sin_addr);
+	buf_check(ptr, end, 1 + sizeof(struct in_addr));
+	*ptr = INET_AF_INET;
+	sys_memcpy(ptr+1, (char*)p, sizeof(struct in_addr));
+	return ptr + 1 + sizeof(struct in_addr);
     }
 #if defined(HAVE_IN6) && defined(AF_INET6)
     else if (addr->sa_family == AF_INET6) {
-	struct in6_addr a;
-	buf_check(ptr,end,sizeof(struct in6_addr));
-	a = ((struct sockaddr_in6*) addr)->sin6_addr;
-	sys_memcpy(ptr, (char*)&a, sizeof(struct in6_addr));
-	return ptr + sizeof(struct in6_addr);
+	struct in6_addr *p = &(((struct sockaddr_in6*) addr)->sin6_addr);
+	buf_check(ptr, end, 1 + sizeof(struct in6_addr));
+	*ptr = INET_AF_INET6;
+	sys_memcpy(ptr+1, (char*)p, sizeof(struct in6_addr));
+	return ptr + 1 + sizeof(struct in6_addr);
+    }
+#endif
+#if defined(AF_LINK)
+    else if (addr->sa_family == AF_LINK) {
+	struct sockaddr_dl *sdl_p = (struct sockaddr_dl*) addr;
+	buf_check(ptr, end, 2 + sdl_p->sdl_alen);
+	put_int16(sdl_p->sdl_alen, ptr); ptr += 2;
+	sys_memcpy(ptr, sdl_p->sdl_data + sdl_p->sdl_nlen, sdl_p->sdl_alen);
+	return ptr + sdl_p->sdl_alen;
     }
 #endif
+#if defined(AF_PACKET) && defined(HAVE_NETPACKET_PACKET_H)
+    else if(addr->sa_family == AF_PACKET) {
+	struct sockaddr_ll *sll_p = (struct sockaddr_ll*) addr;
+	buf_check(ptr, end, 2 + sll_p->sll_halen);
+	put_int16(sll_p->sll_halen, ptr); ptr += 2;
+	sys_memcpy(ptr, sll_p->sll_addr, sll_p->sll_halen);
+	return ptr + sll_p->sll_halen;
+    }
+#endif
+    return ptr;
  error:
     return NULL;
-
 }
 
 static char* buf_to_sockaddr(char* ptr, char* end, struct sockaddr* addr)
 {
-    buf_check(ptr,end,sizeof(struct in_addr));
-    sys_memcpy((char*) &((struct sockaddr_in*)addr)->sin_addr, ptr,
-	       sizeof(struct in_addr));
-    addr->sa_family = AF_INET;
-    return ptr +  sizeof(struct in_addr);
-
+    buf_check(ptr,end,1);
+    switch (*ptr++) {
+    case INET_AF_INET: {
+	struct in_addr *p = &((struct sockaddr_in*)addr)->sin_addr;
+	buf_check(ptr,end,sizeof(struct in_addr));
+	sys_memcpy((char*) p, ptr, sizeof(struct in_addr));
+	addr->sa_family = AF_INET;
+	return ptr + sizeof(struct in_addr);
+    }
+#if defined(HAVE_IN6) && defined(AF_INET6)
+    case INET_AF_INET6: {
+	struct in6_addr *p = &((struct sockaddr_in6*)addr)->sin6_addr;
+	buf_check(ptr,end,sizeof(struct in6_addr));
+	sys_memcpy((char*) p, ptr, sizeof(struct in6_addr));
+	addr->sa_family = AF_INET6;
+	return ptr + sizeof(struct in6_addr);
+    }
+#endif
+    }
  error:
     return NULL;
 }
 
 
+#if defined (IFF_POINTOPOINT)
+#define IFGET_FLAGS(cflags) IFGET_FLAGS_P2P(cflags, IFF_POINTOPOINT)
+#elif defined IFF_POINTTOPOINT
+#define IFGET_FLAGS(cflags) IFGET_FLAGS_P2P(cflags, IFF_POINTTOPOINT)
+#endif
+
+#define IFGET_FLAGS_P2P(cflags, iff_ptp)				\
+    ((((cflags) & IFF_UP) ? INET_IFF_UP : 0) |				\
+     (((cflags) & IFF_BROADCAST) ? INET_IFF_BROADCAST : 0) |		\
+     (((cflags) & IFF_LOOPBACK) ? INET_IFF_LOOPBACK : 0) |		\
+     (((cflags) & iff_ptp) ? INET_IFF_POINTTOPOINT : 0) |		\
+     (((cflags) & IFF_UP) ? INET_IFF_RUNNING : 0) |  /* emulate running ? */ \
+     (((cflags) & IFF_MULTICAST) ? INET_IFF_MULTICAST : 0))
 
 #if defined(__WIN32__) && defined(SIO_GET_INTERFACE_LIST)
 
-static int inet_ctl_getiflist(inet_descriptor* desc, char** rbuf, int rsize)
+static ErlDrvSSizeT inet_ctl_getiflist(inet_descriptor* desc,
+				       char** rbuf, ErlDrvSizeT rsize)
 {
     char ifbuf[BUFSIZ];
     char sbuf[BUFSIZ];
     char* sptr;
     INTERFACE_INFO* ifp;
     DWORD len;
-    int n;
+    ErlDrvSizeT n;
     int err;
 
     ifp = (INTERFACE_INFO*) ifbuf;
@@ -3932,12 +4162,11 @@ static int inet_ctl_getiflist(inet_descriptor* desc, char** rbuf, int rsize)
     return ctl_reply(INET_REP_OK, sbuf, sptr - sbuf, rbuf, rsize);
 }
 
-
 /* input is an ip-address in string format i.e A.B.C.D 
 ** scan the INTERFACE_LIST to get the options 
 */
-static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len,
-			  char** rbuf, int rsize)
+static ErlDrvSSizeT inet_ctl_ifget(inet_descriptor* desc, char* buf,
+				   ErlDrvSizeT len, char** rbuf, ErlDrvSizeT rsize)
 {
     char ifbuf[BUFSIZ];
     int  n;
@@ -3949,7 +4178,7 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len,
     INTERFACE_INFO* ifp;
     long namaddr;
 
-    if ((len == 0) || ((namlen = buf[0]) > len))
+    if ((len == 0) || ((namlen = get_int8(buf)) > len))
 	goto error;
     if (parse_addr(buf+1, namlen, &namaddr) < 0)
 	goto error;
@@ -4018,27 +4247,12 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len,
 	    break;
 
 	case INET_IFOPT_FLAGS: {
-	    long eflags = 0;
 	    int flags = ifp->iiFlags;
 	    /* just enumerate the interfaces (no names) */
 
-	    /* translate flags */
-	    if (flags & IFF_UP)
-		eflags |= INET_IFF_UP;
-	    if (flags & IFF_BROADCAST)
-		eflags |= INET_IFF_BROADCAST;
-	    if (flags & IFF_LOOPBACK)
-		eflags |= INET_IFF_LOOPBACK;
-	    if (flags & IFF_POINTTOPOINT)
-		eflags |= INET_IFF_POINTTOPOINT;
-	    if (flags & IFF_UP) /* emulate runnign ? */
-		eflags |= INET_IFF_RUNNING;
-	    if (flags & IFF_MULTICAST)
-		eflags |= INET_IFF_MULTICAST;
-
 	    buf_check(sptr, s_end, 5);
 	    *sptr++ = INET_IFOPT_FLAGS;
-	    put_int32(eflags, sptr);
+	    put_int32(IFGET_FLAGS(flags), sptr);
 	    sptr += 4;
 	    break;
 	}
@@ -4053,13 +4267,13 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len,
 }
 
 /* not supported */
-static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len,
-			  char** rbuf, int rsize)
+static ErlDrvSSizeT inet_ctl_ifset(inet_descriptor* desc,
+				   char* buf, ErlDrvSizeT len,
+				   char** rbuf, ErlDrvSizeT rsize)
 {
     return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
 }
 
-
 #elif defined(SIOCGIFCONF) && defined(SIOCSIFFLAGS)
 /* cygwin has SIOCGIFCONF but not SIOCSIFFLAGS (Nov 2002) */
 
@@ -4070,72 +4284,111 @@ static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len,
 #define SIZEA(p) (sizeof (p))
 #endif
 
-
-static int inet_ctl_getiflist(inet_descriptor* desc, char** rbuf, int rsize)
-{
-    struct ifconf ifc;
-    struct ifreq *ifr;
-    char *buf;
-    int buflen, ifc_len, i;
-    char *sbuf, *sp;
-    
-    /* Courtesy of Per Bergqvist and W. Richard Stevens */
-    
-    ifc_len = 0;
-    buflen = 100 * sizeof(struct ifreq);
-    buf = ALLOC(buflen);
+static int get_ifconf(SOCKET s, struct ifconf *ifcp) {
+    int ifc_len = 0;
+    int buflen = 100 * sizeof(struct ifreq);
+    char *buf = ALLOC(buflen);
 
     for (;;) {
-	ifc.ifc_len = buflen;
-	ifc.ifc_buf = buf;
-	if (ioctl(desc->s, SIOCGIFCONF, (char *)&ifc) < 0) {
+	ifcp->ifc_len = buflen;
+	ifcp->ifc_buf = buf;
+	if (ioctl(s, SIOCGIFCONF, (char *)ifcp) < 0) {
 	    int res = sock_errno();
 	    if (res != EINVAL || ifc_len) {
 		FREE(buf);
-		return ctl_error(res, rbuf, rsize);
+		return -1;
 	    }
 	} else {
-	    if (ifc.ifc_len == ifc_len) break; /* buf large enough */
-	    ifc_len = ifc.ifc_len;
+	    if (ifcp->ifc_len == ifc_len) break; /* buf large enough */
+	    ifc_len = ifcp->ifc_len;
 	}
 	buflen += 10 * sizeof(struct ifreq);
 	buf = (char *)REALLOC(buf, buflen);
     }
-    
-    sp = sbuf = ALLOC(ifc_len+1);
+    return 0;
+}
+
+static void free_ifconf(struct ifconf *ifcp) {
+    FREE(ifcp->ifc_buf);
+}
+
+static ErlDrvSSizeT inet_ctl_getiflist(inet_descriptor* desc,
+				       char** rbuf, ErlDrvSizeT rsize)
+{
+    struct ifconf ifc;
+    struct ifreq *ifrp;
+    char *sbuf, *sp;
+    ErlDrvSizeT i;
+
+    /* Courtesy of Per Bergqvist and W. Richard Stevens */
+
+    if (get_ifconf(desc->s, &ifc) < 0) {
+	return ctl_error(sock_errno(), rbuf, rsize);
+    }
+
+    sp = sbuf = ALLOC(ifc.ifc_len+1);
     *sp++ = INET_REP_OK;
     i = 0;
     for (;;) {
-	int n;
-	
-	ifr = (struct ifreq *) VOIDP(buf + i);
-	n = sizeof(ifr->ifr_name) + SIZEA(ifr->ifr_addr);
-	if (n < sizeof(*ifr)) n = sizeof(*ifr);
-	if (i+n > ifc_len) break;
+	ErlDrvSizeT n;
+
+	ifrp = (struct ifreq *) VOIDP(ifc.ifc_buf + i);
+	n = sizeof(ifrp->ifr_name) + SIZEA(ifrp->ifr_addr);
+	if (n < sizeof(*ifrp)) n = sizeof(*ifrp);
+	if (i+n > ifc.ifc_len) break;
 	i += n;
-	
-	switch (ifr->ifr_addr.sa_family) {
+
+	switch (ifrp->ifr_addr.sa_family) {
 #if defined(HAVE_IN6) && defined(AF_INET6)
 	case AF_INET6:
 #endif
 	case AF_INET:
-	    ASSERT(sp+IFNAMSIZ+1 < sbuf+buflen+1)
-	    strncpy(sp, ifr->ifr_name, IFNAMSIZ);
+	    ASSERT(sp+IFNAMSIZ+1 < sbuf+ifc.ifc_len+1)
+	    strncpy(sp, ifrp->ifr_name, IFNAMSIZ);
 	    sp[IFNAMSIZ] = '\0';
 	    sp += strlen(sp), ++sp;
 	}
-	
-	if (i >= ifc_len) break;
+
+	if (i >= ifc.ifc_len) break;
     }
-    FREE(buf);
+    free_ifconf(&ifc);
     *rbuf = sbuf;
     return sp - sbuf;
 }
 
+#ifdef HAVE_LIBDLPI_H
+#include <libdlpi.h>
+static int hwaddr_libdlpi_lookup(const char *ifnm,
+                                 uchar_t *addr, size_t *alen)
+{
+    dlpi_handle_t handle;
+    dlpi_info_t linkinfo;
+    int ret = -1;
 
+    if (dlpi_open(ifnm, &handle, 0) != DLPI_SUCCESS) {
+        return -1;
+    }
+
+    if (dlpi_get_physaddr(handle, DL_CURR_PHYS_ADDR,
+                          addr, alen) == DLPI_SUCCESS &&
+        dlpi_info(handle, &linkinfo, 0) == DLPI_SUCCESS)
+    {
+        ret = 0;
+    }
+
+    dlpi_close(handle);
+    return ret;
+}
+#endif
+
+/* FIXME: temporary hack */
+#ifndef IFHWADDRLEN
+#define IFHWADDRLEN 6
+#endif
 
-static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len,
-			  char** rbuf, int rsize)
+static ErlDrvSSizeT inet_ctl_ifget(inet_descriptor* desc,
+				   char* buf, ErlDrvSizeT len,
+				   char** rbuf, ErlDrvSizeT rsize)
 {
     char sbuf[BUFSIZ];
     char* sptr;
@@ -4143,11 +4396,11 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len,
     struct ifreq ifreq;
     int namlen;
 
-    if ((len == 0) || ((namlen = buf[0]) > len))
+    if ((len == 0) || ((namlen = get_int8(buf)) > len))
 	goto error;
     sys_memset(ifreq.ifr_name, '\0', IFNAMSIZ);
     sys_memcpy(ifreq.ifr_name, buf+1, 
-	       (namlen > IFNAMSIZ) ? IFNAMSIZ : namlen);
+	       (namlen >= IFNAMSIZ) ? IFNAMSIZ-1 : namlen);
     buf += (namlen+1);
     len -= (namlen+1);
     sptr = sbuf;
@@ -4164,14 +4417,72 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len,
 	    break;
 
 	case INET_IFOPT_HWADDR: {
-#ifdef SIOCGIFHWADDR
+#ifdef HAVE_LIBDLPI_H
+	    /*
+	    ** OpenSolaris have SIGCGIFHWADDR, but no ifr_hwaddr member..
+	    ** The proper way to get the mac address would be to
+	    ** use libdlpi...
+	    */
+	    uchar_t addr[DLPI_PHYSADDR_MAX];
+	    size_t alen = sizeof(addr);
+
+	    if (hwaddr_libdlpi_lookup(ifreq.ifr_name, addr, &alen) == 0) {
+		buf_check(sptr, s_end, 1+2+alen);
+		*sptr++ = INET_IFOPT_HWADDR;
+		put_int16(alen, sptr);
+                    sptr += 2;
+                    sys_memcpy(sptr, addr, alen);
+                    sptr += alen;
+	    }
+#elif defined(SIOCGIFHWADDR) && defined(HAVE_STRUCT_IFREQ_IFR_HWADDR)
 	    if (ioctl(desc->s, SIOCGIFHWADDR, (char *)&ifreq) < 0)
 		break;
-	    buf_check(sptr, s_end, 1+IFHWADDRLEN);
+	    buf_check(sptr, s_end, 1+2+IFHWADDRLEN);
 	    *sptr++ = INET_IFOPT_HWADDR;
+	    put_int16(IFHWADDRLEN, sptr); sptr += 2;
 	    /* raw memcpy (fix include autoconf later) */
 	    sys_memcpy(sptr, (char*)(&ifreq.ifr_hwaddr.sa_data), IFHWADDRLEN);
 	    sptr += IFHWADDRLEN;
+#elif defined(SIOCGENADDR) && defined(HAVE_STRUCT_IFREQ_IFR_ENADDR)
+	    if (ioctl(desc->s, SIOCGENADDR, (char *)&ifreq) < 0)
+		break;
+	    buf_check(sptr, s_end, 1+2+sizeof(ifreq.ifr_enaddr));
+	    *sptr++ = INET_IFOPT_HWADDR;
+	    put_int16(sizeof(ifreq.ifr_enaddr), sptr); sptr += 2;
+	    /* raw memcpy (fix include autoconf later) */
+	    sys_memcpy(sptr, (char*)(&ifreq.ifr_enaddr),
+		       sizeof(ifreq.ifr_enaddr));
+	    sptr += sizeof(ifreq.ifr_enaddr);
+#elif defined(HAVE_GETIFADDRS) && defined(AF_LINK)
+	    struct ifaddrs *ifa, *ifp;
+	    struct sockaddr_dl *sdlp;
+	    int found = 0;
+
+	    if (getifaddrs(&ifa) == -1)
+		goto error;
+
+	    for (ifp = ifa; ifp; ifp = ifp->ifa_next) {
+		if ((ifp->ifa_addr->sa_family == AF_LINK) &&
+		    (sys_strcmp(ifp->ifa_name, ifreq.ifr_name) == 0)) {
+		    found = 1;
+		    break;
+		}
+	    }
+
+	    if (found == 0) {
+		freeifaddrs(ifa);
+		break;
+	    }
+	    sdlp = (struct sockaddr_dl *)ifp->ifa_addr;
+
+	    buf_check(sptr, s_end, 1+2+sdlp->sdl_alen);
+	    *sptr++ = INET_IFOPT_HWADDR;
+	    put_int16(sdlp->sdl_alen, sptr); sptr += 2;
+	    sys_memcpy(sptr,
+		       sdlp->sdl_data + sdlp->sdl_nlen,
+		       sdlp->sdl_alen);
+	    freeifaddrs(ifa);
+	    sptr += sdlp->sdl_alen;
 #endif
 	    break;
 	}
@@ -4248,29 +4559,15 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len,
 
 	case INET_IFOPT_FLAGS: {
 	    int flags;
-	    int eflags = 0;
 
 	    if (ioctl(desc->s, SIOCGIFFLAGS, (char*)&ifreq) < 0)
 		flags = 0;
 	    else
 		flags = ifreq.ifr_flags;
-	    /* translate flags */
-	    if (flags & IFF_UP)
-		eflags |= INET_IFF_UP;
-	    if (flags & IFF_BROADCAST)
-		eflags |= INET_IFF_BROADCAST;
-	    if (flags & IFF_LOOPBACK)
-		eflags |= INET_IFF_LOOPBACK;	
-	    if (flags & IFF_POINTOPOINT)
-		eflags |= INET_IFF_POINTTOPOINT;
-	    if (flags & IFF_RUNNING)
-		eflags |= INET_IFF_RUNNING;
-	    if (flags & IFF_MULTICAST)
-		eflags |= INET_IFF_MULTICAST;
 
 	    buf_check(sptr, s_end, 5);
 	    *sptr++ = INET_IFOPT_FLAGS;
-	    put_int32(eflags, sptr);
+	    put_int32(IFGET_FLAGS(flags), sptr);
 	    sptr += 4;
 	    break;
 	}
@@ -4284,23 +4581,20 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len,
     return ctl_error(EINVAL, rbuf, rsize);
 }
 
-/* FIXME: temporary hack */
-#ifndef IFHWADDRLEN
-#define IFHWADDRLEN 6
-#endif
 
-static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len,
-			  char** rbuf, int rsize)
+static ErlDrvSSizeT inet_ctl_ifset(inet_descriptor* desc,
+				   char* buf, ErlDrvSizeT len,
+				   char** rbuf, ErlDrvSizeT rsize)
 {
     struct ifreq ifreq;
     int namlen;
     char* b_end = buf + len;
 
-    if ((len == 0) || ((namlen = buf[0]) > len))
+    if ((len == 0) || ((namlen = get_int8(buf)) > len))
 	goto error;
     sys_memset(ifreq.ifr_name, '\0', IFNAMSIZ);
     sys_memcpy(ifreq.ifr_name, buf+1, 
-	       (namlen > IFNAMSIZ) ? IFNAMSIZ : namlen);
+	       (namlen >= IFNAMSIZ) ? IFNAMSIZ-1 : namlen);
     buf += (namlen+1);
     len -= (namlen+1);
 
@@ -4312,17 +4606,22 @@ static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len,
 	    (void) ioctl(desc->s, SIOCSIFADDR, (char*)&ifreq);
 	    break;
 
-	case INET_IFOPT_HWADDR: 
-	    buf_check(buf, b_end, IFHWADDRLEN);
+	case INET_IFOPT_HWADDR: {
+	    unsigned int hwalen;
+	    buf_check(buf, b_end, 2);
+	    hwalen = get_int16(buf); buf += 2;
+	    buf_check(buf, b_end, hwalen);
 #ifdef SIOCSIFHWADDR
 	    /* raw memcpy (fix include autoconf later) */
-	    sys_memcpy((char*)(&ifreq.ifr_hwaddr.sa_data), buf, IFHWADDRLEN);
+	    sys_memset((char*)(&ifreq.ifr_hwaddr.sa_data),
+		       '\0', sizeof(ifreq.ifr_hwaddr.sa_data));
+	    sys_memcpy((char*)(&ifreq.ifr_hwaddr.sa_data), buf, hwalen);
 
 	    (void) ioctl(desc->s, SIOCSIFHWADDR, (char *)&ifreq);
 #endif
-	    buf += IFHWADDRLEN;
+	    buf += hwalen;
 	    break;
-
+	}
 
 	case INET_IFOPT_BROADADDR:
 #ifdef SIOCSIFBRDADDR
@@ -4406,27 +4705,583 @@ static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len,
 #else
 
 
-static int inet_ctl_getiflist(inet_descriptor* desc, char** rbuf, int rsize)
+static ErlDrvSSizeT inet_ctl_getiflist(inet_descriptor* desc,
+				       char** rbuf, ErlDrvSizeT rsize)
 {
     return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
 }
 
 
-static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len,
-			  char** rbuf, int rsize)
+static ErlDrvSSizeT inet_ctl_ifget(inet_descriptor* desc,
+				   char* buf, ErlDrvSizeT len,
+				   char** rbuf, ErlDrvSizeT rsize)
 {
     return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
 }
 
 
-static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len,
-			  char** rbuf, int rsize)
+static ErlDrvSSizeT inet_ctl_ifset(inet_descriptor* desc,
+				   char* buf, ErlDrvSizeT len,
+				   char** rbuf, ErlDrvSizeT rsize)
 {
     return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
 }
 
 #endif
 
+
+
+#if defined(__WIN32__) || defined(HAVE_GETIFADDRS)
+/* Latin-1 to utf8 */
+
+static int utf8_len(const char *c, int m) {
+    int l;
+    for (l = 0;  m;  c++, l++, m--) {
+	if (*c == '\0') break;
+	if ((*c & 0x7f) != *c) l++;
+    }
+    return l;
+}
+
+static void utf8_encode(const char *c, int m, char *p) {
+    for (;  m;  c++, m--) {
+	if (*c == '\0') break;
+	if ((*c & 0x7f) != *c) {
+	    *p++ = (char) (0xC0 | (0x03 & (*c >> 6)));
+	    *p++ = (char) (0x80 | (0x3F & *c));
+	} else {
+	    *p++ = (char) *c;
+	}
+    }
+}
+#endif
+
+#if defined(__WIN32__)
+
+static void set_netmask_bytes(char *c, int len, int pref_len) {
+    int i, m;
+    for (i = 0, m = pref_len >> 3;  i < m && i < len;  i++) c[i] = '\xFF';
+    if (i < len) c[i++] = 0xFF << (8 - (pref_len & 7));
+    for (;  i < len;  i++) c[i] = '\0';
+}
+
+
+int eq_masked_bytes(char *a, char *b, int pref_len) {
+    int i, m;
+    for (i = 0, m = pref_len >> 3;  i < m;  i++) {
+	if (a[i] != b[i]) return 0;
+    }
+    m = pref_len & 7;
+    if (m) {
+	m = 0xFF & (0xFF << (8 - m));
+	if ((a[i] & m) != (b[i] & m)) return 0;
+    }
+    return !0;
+}
+
+static ErlDrvSSizeT inet_ctl_getifaddrs(inet_descriptor* desc_p,
+					char **rbuf_pp, ErlDrvSizeT rsize)
+{
+    int i;
+    DWORD ret, n;
+    IP_INTERFACE_INFO *info_p;
+    MIB_IPADDRTABLE *ip_addrs_p;
+    IP_ADAPTER_ADDRESSES *ip_adaddrs_p, *ia_p;
+
+    char *buf_p;
+    char *buf_alloc_p;
+    ErlDrvSizeT buf_size = 512;
+#   define BUF_ENSURE(Size)						\
+    do {								\
+	int NEED_, GOT_ = buf_p - buf_alloc_p;				\
+	NEED_ = GOT_ + (Size);						\
+	if (NEED_ > buf_size) {						\
+	    buf_size = NEED_ + 512;					\
+	    buf_alloc_p = REALLOC(buf_alloc_p, buf_size);		\
+	    buf_p = buf_alloc_p + GOT_;					\
+	}								\
+    } while(0)
+#   define SOCKADDR_TO_BUF(opt, sa)					\
+    do {								\
+	if (sa) {							\
+	    char *P_;							\
+	    *buf_p++ = (opt);						\
+	    while (! (P_ = sockaddr_to_buf((sa), buf_p,			\
+					   buf_alloc_p+buf_size))) {    \
+		int GOT_ = buf_p - buf_alloc_p;				\
+		buf_size += 512;					\
+		buf_alloc_p = REALLOC(buf_alloc_p, buf_size);		\
+		buf_p = buf_alloc_p + GOT_;				\
+	    }								\
+	    if (P_ == buf_p) {						\
+		buf_p--;						\
+	    } else {							\
+		buf_p = P_;						\
+	    }								\
+	}								\
+    } while (0)
+
+    {
+	/* Try GetAdaptersAddresses, if it is available */
+	unsigned long ip_adaddrs_size = 16 * 1024;
+	ULONG family = AF_UNSPEC;
+	ULONG flags =
+	    GAA_FLAG_INCLUDE_PREFIX | GAA_FLAG_SKIP_ANYCAST |
+	    GAA_FLAG_SKIP_DNS_SERVER | GAA_FLAG_SKIP_FRIENDLY_NAME |
+	    GAA_FLAG_SKIP_MULTICAST;
+	ULONG (WINAPI *fpGetAdaptersAddresses)
+	    (ULONG, ULONG, PVOID, PIP_ADAPTER_ADDRESSES, PULONG);
+	HMODULE iphlpapi = GetModuleHandle("iphlpapi");
+	fpGetAdaptersAddresses = (void *)
+	    (iphlpapi ?
+		GetProcAddress(iphlpapi, "GetAdaptersAddresses") :
+		NULL);
+	if (fpGetAdaptersAddresses) {
+	    ip_adaddrs_p = ALLOC(ip_adaddrs_size);
+	    for (i = 17;  i;  i--) {
+		ret = fpGetAdaptersAddresses(
+		    family, flags, NULL, ip_adaddrs_p, &ip_adaddrs_size);
+		ip_adaddrs_p = REALLOC(ip_adaddrs_p, ip_adaddrs_size);
+		if (ret == NO_ERROR) break;
+		if (ret == ERROR_BUFFER_OVERFLOW) continue;
+		i = 0;
+	    }
+	    if (! i) {
+		FREE(ip_adaddrs_p);
+		ip_adaddrs_p = NULL;
+	    }
+	} else ip_adaddrs_p = NULL;
+    }
+
+    {
+	/* Load the IP_INTERFACE_INFO table (only IPv4 interfaces),
+	 * reliable source of interface names on XP
+	 */
+	unsigned long info_size = 4 * 1024;
+	info_p = ALLOC(info_size);
+	for (i = 17;  i;  i--) {
+	    ret = GetInterfaceInfo(info_p, &info_size);
+	    info_p = REALLOC(info_p, info_size);
+	    if (ret == NO_ERROR) break;
+	    if (ret == ERROR_INSUFFICIENT_BUFFER) continue;
+	    i = 0;
+	}
+	if (! i) {
+	    FREE(info_p);
+	    info_p = NULL;
+	}
+    }
+
+    if (! ip_adaddrs_p) {
+	/* If GetAdaptersAddresses gave nothing we fall back to
+	 * MIB_IPADDRTABLE (only IPv4 interfaces)
+	 */
+	unsigned long ip_addrs_size = 16 * sizeof(*ip_addrs_p);
+	ip_addrs_p = ALLOC(ip_addrs_size);
+	for (i = 17;  i;  i--) {
+	    ret = GetIpAddrTable(ip_addrs_p, &ip_addrs_size, FALSE);
+	    ip_addrs_p = REALLOC(ip_addrs_p, ip_addrs_size);
+	    if (ret == NO_ERROR) break;
+	    if (ret == ERROR_INSUFFICIENT_BUFFER) continue;
+	    i = 0;
+	}
+	if (! i) {
+	    if (info_p) FREE(info_p);
+	    FREE(ip_addrs_p);
+	    return ctl_reply(INET_REP_OK, NULL, 0, rbuf_pp, rsize);
+	}
+    } else ip_addrs_p = NULL;
+
+    buf_p = buf_alloc_p = ALLOC(buf_size);
+    *buf_p++ = INET_REP_OK;
+
+    /* Iterate over MIB_IPADDRTABLE or IP_ADAPTER_ADDRESSES */
+    for (ia_p = NULL, ip_addrs_p ? ((void *)(i = 0)) : (ia_p = ip_adaddrs_p);
+	 ip_addrs_p ? (i < ip_addrs_p->dwNumEntries) : (ia_p != NULL);
+	 ip_addrs_p ? ((void *)(i++)) : (ia_p = ia_p->Next)) {
+	MIB_IPADDRROW *ipaddrrow_p = NULL;
+	DWORD flags = INET_IFF_MULTICAST;
+	DWORD index = 0;
+	WCHAR *wname_p = NULL;
+	MIB_IFROW ifrow;
+
+	if (ip_addrs_p) {
+	    ipaddrrow_p = ip_addrs_p->table + i;
+	    index = ipaddrrow_p->dwIndex;
+	} else {
+	    index = ia_p->IfIndex;
+	    if (ia_p->Flags & IP_ADAPTER_NO_MULTICAST) {
+		flags &= ~INET_IFF_MULTICAST;
+	    }
+	}
+index:
+	if (! index) goto done;
+	sys_memzero(&ifrow, sizeof(ifrow));
+	ifrow.dwIndex = index;
+	if (GetIfEntry(&ifrow) != NO_ERROR) break;
+	/* Find the interface name - first try MIB_IFROW.wzname */
+	if (ifrow.wszName[0] != 0) {
+	    wname_p = ifrow.wszName;
+	} else {
+	    /* Then try IP_ADAPTER_INDEX_MAP.Name (only IPv4 adapters) */
+	    int j;
+	    for (j = 0;  j < info_p->NumAdapters;  j++) {
+		if (info_p->Adapter[j].Index == (ULONG) ifrow.dwIndex) {
+		    if (info_p->Adapter[j].Name[0] != 0) {
+			wname_p = info_p->Adapter[j].Name;
+		    }
+		    break;
+		}
+	    }
+	}
+	if (wname_p) {
+	    int len;
+	    /* Convert interface name to UTF-8 */
+	    len =
+		WideCharToMultiByte(
+		    CP_UTF8, 0, wname_p, -1, NULL, 0, NULL, NULL);
+	    if (! len) break;
+	    BUF_ENSURE(len);
+	    WideCharToMultiByte(
+		CP_UTF8, 0, wname_p, -1, buf_p, len, NULL, NULL);
+	    buf_p += len;
+	} else {
+	    /* Found no name -
+	    * use "MIB_IFROW.dwIndex: MIB_IFROW.bDescr" as name instead */
+	    int l;
+	    l = utf8_len(ifrow.bDescr, ifrow.dwDescrLen);
+	    BUF_ENSURE(9 + l+1);
+	    buf_p +=
+		erts_sprintf(
+		    buf_p, "%lu: ", (unsigned long) ifrow.dwIndex);
+	    utf8_encode(ifrow.bDescr, ifrow.dwDescrLen, buf_p);
+	    buf_p += l;
+	    *buf_p++ = '\0';
+	}
+	/* Interface flags, often make up broadcast and multicast flags */
+	switch (ifrow.dwType) {
+	case IF_TYPE_ETHERNET_CSMACD:
+	    flags |= INET_IFF_BROADCAST;
+	    break;
+	case IF_TYPE_SOFTWARE_LOOPBACK:
+	    flags |= INET_IFF_LOOPBACK;
+	    flags &= ~INET_IFF_MULTICAST;
+	    break;
+	default:
+	    flags &= ~INET_IFF_MULTICAST;
+	    break;
+	}
+	if (ifrow.dwAdminStatus) {
+	    flags |= INET_IFF_UP;
+	    switch (ifrow.dwOperStatus) {
+	    case IF_OPER_STATUS_CONNECTING:
+		flags |= INET_IFF_POINTTOPOINT;
+		break;
+	    case IF_OPER_STATUS_CONNECTED:
+		flags |= INET_IFF_RUNNING | INET_IFF_POINTTOPOINT;
+		break;
+	    case IF_OPER_STATUS_OPERATIONAL:
+		flags |= INET_IFF_RUNNING;
+		break;
+	    }
+	}
+	BUF_ENSURE(1 + 4);
+	*buf_p++ = INET_IFOPT_FLAGS;
+	put_int32(flags, buf_p); buf_p += 4;
+	if (ipaddrrow_p) {
+	    /* Legacy implementation through GetIpAddrTable */
+	    struct sockaddr_in sin;
+	    /* IP Address */
+	    sys_memzero(&sin, sizeof(sin));
+	    sin.sin_family = AF_INET;
+	    sin.sin_addr.s_addr = ipaddrrow_p->dwAddr;
+	    BUF_ENSURE(1);
+	    /* Netmask */
+	    SOCKADDR_TO_BUF(INET_IFOPT_ADDR, (struct sockaddr *) &sin);
+	    sin.sin_addr.s_addr = ipaddrrow_p->dwMask;
+	    BUF_ENSURE(1);
+	    SOCKADDR_TO_BUF(INET_IFOPT_NETMASK, (struct sockaddr *) &sin);
+	    if (flags & INET_IFF_BROADCAST) {
+		/* Broadcast address - fake it*/
+		sin.sin_addr.s_addr = ipaddrrow_p->dwAddr;
+		sin.sin_addr.s_addr |= ~ipaddrrow_p->dwMask;
+		BUF_ENSURE(1);
+		SOCKADDR_TO_BUF(
+		    INET_IFOPT_BROADADDR, (struct sockaddr *) &sin);
+	    }
+	} else {
+	    IP_ADAPTER_UNICAST_ADDRESS *p;
+	    /* IP Address(es) */
+	    for (p = ia_p->FirstUnicastAddress;
+		p;
+		p = p->Next)
+	    {
+		IP_ADAPTER_PREFIX *q;
+		ULONG shortest_length;
+		struct sockaddr *shortest_p, *sa_p = p->Address.lpSockaddr;
+		BUF_ENSURE(1);
+		SOCKADDR_TO_BUF(INET_IFOPT_ADDR, sa_p);
+		shortest_p = NULL;
+		shortest_length = 0;
+		for (q = ia_p->FirstPrefix;
+		     q;
+		     q = q->Next) {
+		    struct sockaddr *sp_p = q->Address.lpSockaddr;
+		    if (sa_p->sa_family != sp_p->sa_family) continue;
+		    switch (sa_p->sa_family) {
+		    case AF_INET: {
+			struct sockaddr_in sin;
+			DWORD sa, sp, mask;
+			sa = ntohl((DWORD)
+				   ((struct sockaddr_in *)
+				    sa_p)->sin_addr.s_addr);
+			sp = ntohl((DWORD)
+				   ((struct sockaddr_in *)
+				    sp_p)->sin_addr.s_addr);
+			mask = 0xFFFFFFFF << (32 - q->PrefixLength);
+			if ((sa & mask) != (sp & mask)) continue;
+			if ((! shortest_p)
+			    || q->PrefixLength < shortest_length) {
+			    shortest_p = sp_p;
+			    shortest_length = q->PrefixLength;
+			}
+		    }   break;
+		    case AF_INET6: {
+			struct sockaddr_in6 sin6;
+			if (!eq_masked_bytes((char *)
+					     &((struct sockaddr_in6 *)
+					       sa_p)->sin6_addr,
+					     (char *)
+					     &((struct sockaddr_in6 *)
+					       sp_p)->sin6_addr,
+					     q->PrefixLength)) {
+			    continue;
+			}
+			if ((! shortest_p)
+			    || q->PrefixLength < shortest_length) {
+			    shortest_p = sp_p;
+			    shortest_length = q->PrefixLength;
+			}
+		    }   break;
+		    }
+		}
+		if (! shortest_p) {
+		    /* Found no shortest prefix */
+		    shortest_p = sa_p;
+		    switch (shortest_p->sa_family) {
+		    case AF_INET: {
+			/* Fall back to old classfull network addresses */
+			DWORD addr = ntohl(((struct sockaddr_in *)shortest_p)
+					   ->sin_addr.s_addr);
+			if (! (addr & 0x800000)) {
+			    /* Class A */
+			    shortest_length = 8;
+			} else if (! (addr & 0x400000)) {
+			    /* Class B */
+			    shortest_length = 16;
+			} else if (! (addr & 0x200000)) {
+			    /* Class C */
+			    shortest_length = 24;
+			} else {
+			    shortest_length = 32;
+			}
+		    }   break;
+		    case AF_INET6: {
+			/* Just play it safe */
+			shortest_length = 128;
+		    }   break;
+		    }
+		}
+		switch (shortest_p->sa_family) {
+		case AF_INET: {
+		    struct sockaddr_in sin;
+		    DWORD mask = 0xFFFFFFFF << (32 - shortest_length);
+		    sys_memzero(&sin, sizeof(sin));
+		    sin.sin_family = shortest_p->sa_family;
+		    sin.sin_addr.s_addr = htonl(mask);
+		    BUF_ENSURE(1);
+		    SOCKADDR_TO_BUF(INET_IFOPT_NETMASK,
+				    (struct sockaddr *) &sin);
+		    if (flags & INET_IFF_BROADCAST) {
+			DWORD sp =
+			    ntohl((DWORD)
+				  ((struct sockaddr_in *)shortest_p)
+				  -> sin_addr.s_addr);
+			sin.sin_addr.s_addr = htonl(sp | ~mask);
+			BUF_ENSURE(1);
+			SOCKADDR_TO_BUF(INET_IFOPT_BROADADDR,
+					(struct sockaddr *) &sin);
+		    }
+		}   break;
+		case AF_INET6: {
+		    struct sockaddr_in6 sin6;
+		    sys_memzero(&sin6, sizeof(sin6));
+		    sin6.sin6_family = shortest_p->sa_family;
+		    set_netmask_bytes((char *) &sin6.sin6_addr,
+				      16,
+				      shortest_length);
+		    BUF_ENSURE(1);
+		    SOCKADDR_TO_BUF(INET_IFOPT_NETMASK,
+				    (struct sockaddr *) &sin6);
+		}   break;
+		}
+	    }
+	}
+	if (ifrow.dwPhysAddrLen) {
+	    /* Hardware Address */
+	    BUF_ENSURE(1 + 2 + ifrow.dwPhysAddrLen);
+	    *buf_p++ = INET_IFOPT_HWADDR;
+	    put_int16(ifrow.dwPhysAddrLen, buf_p); buf_p += 2;
+	    sys_memcpy(buf_p, ifrow.bPhysAddr, ifrow.dwPhysAddrLen);
+	    buf_p += ifrow.dwPhysAddrLen;
+	}
+
+done:
+	/* That is all for this interface */
+	BUF_ENSURE(1);
+	*buf_p++ = '\0';
+	if (ia_p &&
+	    ia_p->Ipv6IfIndex &&
+	    ia_p->Ipv6IfIndex != index)
+	{
+	    /* Oops, there was an other interface for IPv6. Possible? XXX */
+	    index = ia_p->Ipv6IfIndex;
+	    goto index;
+	}
+    }
+
+    if (ip_adaddrs_p) FREE(ip_adaddrs_p);
+    if (info_p) FREE(info_p);
+    if (ip_addrs_p) FREE(ip_addrs_p);
+
+    buf_size = buf_p - buf_alloc_p;
+    buf_alloc_p = REALLOC(buf_alloc_p, buf_size);
+    /* buf_p is now unreliable */
+    *rbuf_pp = buf_alloc_p;
+    return buf_size;
+#   undef BUF_ENSURE
+}
+
+#elif defined(HAVE_GETIFADDRS)
+
+static ErlDrvSSizeT inet_ctl_getifaddrs(inet_descriptor* desc_p,
+					char **rbuf_pp, ErlDrvSizeT rsize)
+{
+    struct ifaddrs *ifa_p, *ifa_free_p;
+
+    ErlDrvSizeT buf_size;
+    char *buf_p;
+    char *buf_alloc_p;
+
+    buf_size = 512;
+    buf_alloc_p = ALLOC(buf_size);
+    buf_p = buf_alloc_p;
+#   define BUF_ENSURE(Size)						\
+    do {								\
+	int NEED_, GOT_ = buf_p - buf_alloc_p;				\
+	NEED_ = GOT_ + (Size);						\
+	if (NEED_ > buf_size) {						\
+	    buf_size = NEED_ + 512;					\
+	    buf_alloc_p = REALLOC(buf_alloc_p, buf_size);		\
+	    buf_p = buf_alloc_p + GOT_;					\
+	}								\
+    } while (0)
+#   define SOCKADDR_TO_BUF(opt, sa)				        \
+    do {    						                \
+	if (sa) {                                                       \
+	    char *P_;							\
+	    *buf_p++ = (opt);						\
+	    while (! (P_ = sockaddr_to_buf((sa), buf_p,			\
+					   buf_alloc_p+buf_size))) {	\
+		int GOT_ = buf_p - buf_alloc_p;				\
+		buf_size += 512;					\
+		buf_alloc_p = REALLOC(buf_alloc_p, buf_size);		\
+		buf_p = buf_alloc_p + GOT_;				\
+	    }								\
+	    if (P_ == buf_p) {						\
+		buf_p--;						\
+	    } else {							\
+		buf_p = P_;						\
+	    }								\
+	}                                                               \
+    } while (0)
+
+    if (getifaddrs(&ifa_p) < 0) {
+	return ctl_error(sock_errno(), rbuf_pp, rsize);
+    }
+    ifa_free_p = ifa_p;
+    *buf_p++ = INET_REP_OK;
+    for (;  ifa_p;  ifa_p = ifa_p->ifa_next) {
+	int len = utf8_len(ifa_p->ifa_name, -1);
+	BUF_ENSURE(len+1 + 1+4 + 1);
+	utf8_encode(ifa_p->ifa_name, -1, buf_p);
+	buf_p += len;
+	*buf_p++ = '\0';
+	*buf_p++ = INET_IFOPT_FLAGS;
+	put_int32(IFGET_FLAGS(ifa_p->ifa_flags), buf_p); buf_p += 4;
+	if (ifa_p->ifa_addr) {
+	    if (ifa_p->ifa_addr->sa_family == AF_INET
+#if defined(AF_INET6)
+		|| ifa_p->ifa_addr->sa_family == AF_INET6
+#endif
+		) {
+		SOCKADDR_TO_BUF(INET_IFOPT_ADDR, ifa_p->ifa_addr);
+		if (ifa_p->ifa_netmask) {
+		    BUF_ENSURE(1);
+		    SOCKADDR_TO_BUF(INET_IFOPT_NETMASK, ifa_p->ifa_netmask);
+		}
+		if (ifa_p->ifa_dstaddr &&
+		    (ifa_p->ifa_flags & IFF_POINTOPOINT)) {
+		    BUF_ENSURE(1);
+		    SOCKADDR_TO_BUF(INET_IFOPT_DSTADDR, ifa_p->ifa_dstaddr);
+		} else if (ifa_p->ifa_broadaddr &&
+			   (ifa_p->ifa_flags & IFF_BROADCAST)) {
+		    BUF_ENSURE(1);
+		    SOCKADDR_TO_BUF(INET_IFOPT_BROADADDR, ifa_p->ifa_broadaddr);
+		}
+	    }
+#if defined(AF_LINK) || defined(AF_PACKET)
+	    else if (
+#if defined(AF_LINK)
+		     ifa_p->ifa_addr->sa_family == AF_LINK
+#else
+		     0
+#endif
+#if defined(AF_PACKET)
+		     || ifa_p->ifa_addr->sa_family == AF_PACKET
+#endif
+		     ) {
+		char *bp = buf_p;
+		BUF_ENSURE(1);
+		SOCKADDR_TO_BUF(INET_IFOPT_HWADDR, ifa_p->ifa_addr);
+		if (buf_p - bp < 4) buf_p = bp; /* Empty hwaddr */
+	    }
+#endif
+	}
+	BUF_ENSURE(1);
+	*buf_p++ = '\0';
+    }
+    buf_size = buf_p - buf_alloc_p;
+    buf_alloc_p = REALLOC(buf_alloc_p, buf_size);
+    /* buf_p is now unreliable */
+    freeifaddrs(ifa_free_p);
+    *rbuf_pp = buf_alloc_p;
+    return buf_size;
+#   undef BUF_ENSURE
+}
+
+#else
+
+static ErlDrvSSizeT inet_ctl_getifaddrs(inet_descriptor* desc_p,
+					char **rbuf_pp, ErlDrvSizeT rsize)
+{
+    return ctl_error(ENOTSUP, rbuf_pp, rsize);
+}
+
+#endif
+
+
+
 #ifdef VXWORKS
 /*
 ** THIS is a terrible creature, a bug in the TCP part
@@ -4469,9 +5324,17 @@ static STATUS wrap_sockopt(STATUS (*function)() /* Yep, no parameter
 }
 #endif
 
+/* Per H @ Tail-f: The original code here had problems that possibly
+   only occur if you abuse it for non-INET sockets, but anyway:
+   a) If the getsockopt for SO_PRIORITY or IP_TOS failed, the actual
+      requested setsockopt was never even attempted.
+   b) If {get,set}sockopt for one of IP_TOS and SO_PRIORITY failed,
+      but ditto for the other worked and that was actually the requested
+      option, failure was still reported to erlang.                  */
+
 #if  defined(IP_TOS) && defined(SOL_IP) && defined(SO_PRIORITY)
 static int setopt_prio_tos_trick
-	   (int fd, int proto, int type, char* arg_ptr, int arg_sz)
+	(int fd, int proto, int type, char* arg_ptr, int arg_sz, int propagate)
 {
     /* The relations between SO_PRIORITY, TOS and other options
        is not what you (or at least I) would expect...:
@@ -4484,6 +5347,8 @@ static int setopt_prio_tos_trick
     int          tmp_ival_prio;
     int          tmp_ival_tos;
     int          res;
+    int          res_prio;
+    int          res_tos;
 #ifdef HAVE_SOCKLEN_T
 	    socklen_t
 #else
@@ -4492,28 +5357,35 @@ static int setopt_prio_tos_trick
 		tmp_arg_sz_prio = sizeof(tmp_ival_prio),
 		tmp_arg_sz_tos  = sizeof(tmp_ival_tos);
 
-    res = sock_getopt(fd, SOL_SOCKET, SO_PRIORITY,
+    res_prio = sock_getopt(fd, SOL_SOCKET, SO_PRIORITY,
 		      (char *) &tmp_ival_prio, &tmp_arg_sz_prio);
-    if (res == 0) {
-	res = sock_getopt(fd, SOL_IP, IP_TOS, 
+    res_tos = sock_getopt(fd, SOL_IP, IP_TOS, 
 		      (char *) &tmp_ival_tos, &tmp_arg_sz_tos);
-	if (res == 0) {
 	    res = sock_setopt(fd, proto, type, arg_ptr, arg_sz);
 	    if (res == 0) {
 		if (type != SO_PRIORITY) {
-		    if (type != IP_TOS) {
-			res = sock_setopt(fd, 
+	    if (type != IP_TOS && res_tos == 0) {
+		res_tos = sock_setopt(fd, 
 					  SOL_IP, 
 					  IP_TOS,
 					  (char *) &tmp_ival_tos, 
 					  tmp_arg_sz_tos);
+		if (propagate)
+		    res = res_tos;
 		    }
-		    if (res == 0) {
-			res =  sock_setopt(fd, 
+	    if (res == 0 && res_prio == 0) {
+		res_prio = sock_setopt(fd, 
 					   SOL_SOCKET, 
 					   SO_PRIORITY,
 					   (char *) &tmp_ival_prio, 
 					   tmp_arg_sz_prio);
+		if (propagate) {		
+		    /* Some kernels set a SO_PRIORITY by default that you are not permitted to reset,
+		       silently ignore this error condition */
+		    if (res_prio != 0 && sock_errno() == EPERM) {
+			res = 0;
+		    } else {
+			res = res_prio;
 		    }
 		}
 	    }
@@ -4555,6 +5427,7 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
     if (IS_SCTP(desc))
 	return sctp_set_opts(desc, ptr, len);
 #endif
+    /* XXX { int i; for(i=0;i<len;++i) fprintf(stderr,"0x%02X, ", (unsigned) ptr[i]); fprintf(stderr,"\r\n");} */
 
     while(len >= 5) {
 	opt = *ptr++;
@@ -4588,8 +5461,7 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
 	case INET_LOPT_BUFFER:
 	    DEBUGF(("inet_set_opts(%ld): s=%d, BUFFER=%d\r\n",
 		    (long)desc->port, desc->s, ival));
-	    if (ival > INET_MAX_BUFFER)  ival = INET_MAX_BUFFER;
-	    else if (ival < INET_MIN_BUFFER) ival = INET_MIN_BUFFER;
+	    if (ival < INET_MIN_BUFFER) ival = INET_MIN_BUFFER;
 	    desc->bufsz = ival;
 	    continue;
 
@@ -4654,7 +5526,6 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
 	    if (desc->stype == SOCK_STREAM) {
 		tcp_descriptor* tdesc = (tcp_descriptor*) desc;
 		if (ival < 0) ival = 0;
-		else if (ival > INET_MAX_BUFFER*2) ival = INET_MAX_BUFFER*2;
 		if (tdesc->low > ival)
 		    tdesc->low = ival;
 		tdesc->high = ival;
@@ -4665,7 +5536,6 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
 	    if (desc->stype == SOCK_STREAM) {
 		tcp_descriptor* tdesc = (tcp_descriptor*) desc;
 		if (ival < 0) ival = 0;
-		else if (ival > INET_MAX_BUFFER) ival = INET_MAX_BUFFER;
 		if (tdesc->high < ival)
 		    tdesc->high = ival;
 		tdesc->low = ival;
@@ -4862,7 +5732,7 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len)
 	    return -1;
 	}
 #if  defined(IP_TOS) && defined(SOL_IP) && defined(SO_PRIORITY)
-	res = setopt_prio_tos_trick (desc->s, proto, type, arg_ptr, arg_sz);
+	res = setopt_prio_tos_trick (desc->s, proto, type, arg_ptr, arg_sz, propagate);
 #else
 	res = sock_setopt	    (desc->s, proto, type, arg_ptr, arg_sz);
 #endif
@@ -4887,10 +5757,16 @@ skip_os_setopt:
 	if (desc->active != old_active)
 	    sock_select(desc, (FD_READ|FD_CLOSE), (desc->active>0));
 
+	/* XXX: UDP sockets could also trigger immediate read here NIY */
 	if ((desc->stype==SOCK_STREAM) && desc->active) {
 	    if (!old_active || (desc->htype != old_htype)) {
 		/* passive => active change OR header type change in active mode */
-		return 1;
+		/* Return > 1 if only active changed to INET_ONCE -> direct read if
+		   header type is unchanged. */
+		/* XXX fprintf(stderr,"desc->htype == %d, old_htype == %d, 
+		   desc->active == %d, old_active == %d\r\n",(int)desc->htype, 
+		   (int) old_htype, (int) desc->active, (int) old_active );*/
+		return 1+(desc->htype == old_htype && desc->active == INET_ONCE);
 	    }
 	    return 0;
 	}
@@ -5011,9 +5887,6 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 	case INET_LOPT_BUFFER:
 	    desc->bufsz  = get_int32(curr);		curr += 4;
 
-            if (desc->bufsz > INET_MAX_BUFFER)
-		desc->bufsz = INET_MAX_BUFFER;
-            else
 	    if (desc->bufsz < INET_MIN_BUFFER)
 		desc->bufsz = INET_MIN_BUFFER;
 	    res = 0;	  /* This does not affect the kernel buffer size */
@@ -5076,8 +5949,8 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 	}
 	case INET_OPT_LINGER:
 	{
-	    CHKLEN(curr, ASSOC_ID_LEN + 2 + 4);
-	    arg.lin.l_onoff  = get_int16 (curr);  curr += 2;
+	    CHKLEN(curr, 2*4);
+	    arg.lin.l_onoff  = get_int32 (curr);  curr += 4;
 	    arg.lin.l_linger = get_int32 (curr);  curr += 4;
 
 	    proto   = SOL_SOCKET;
@@ -5208,7 +6081,7 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 	case SCTP_OPT_PRIMARY_ADDR:
 	case SCTP_OPT_SET_PEER_PRIMARY_ADDR:
 	{
-	    int alen;
+	    ErlDrvSizeT alen;
 	    char *after;
 	    
 	    CHKLEN(curr, ASSOC_ID_LEN);
@@ -5250,13 +6123,16 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 	}
 	case SCTP_OPT_PEER_ADDR_PARAMS:
 	{
-	    int alen;
+	    ErlDrvSizeT alen;
 	    char *after;
 #	    ifdef HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_FLAGS
 	    int eflags, cflags, hb_enable, hb_disable,
-		pmtud_enable, pmtud_disable,
+		pmtud_enable, pmtud_disable;
+#	    ifdef HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_SACKDELAY
+	    int
 		sackdelay_enable, sackdelay_disable;
 #           endif
+#           endif
 	    
 	    CHKLEN(curr, ASSOC_ID_LEN);
 	    arg.pap.spp_assoc_id = GET_ASSOC_ID(curr);	curr += ASSOC_ID_LEN;
@@ -5305,12 +6181,15 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 	    if (pmtud_enable)			cflags |= SPP_PMTUD_ENABLE;
 	    if (pmtud_disable)			cflags |= SPP_PMTUD_DISABLE;
 
+#	    ifdef HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_SACKDELAY
+	    /* The followings are missing in FreeBSD 7.1 */
 	    sackdelay_enable =eflags& SCTP_FLAG_SACDELAY_ENABLE;
 	    sackdelay_disable=eflags& SCTP_FLAG_SACDELAY_DISABLE;
 	    if (sackdelay_enable && sackdelay_disable)
 		return -1;
 	    if (sackdelay_enable)		cflags |= SPP_SACKDELAY_ENABLE;
 	    if (sackdelay_disable)		cflags |= SPP_SACKDELAY_DISABLE;
+#           endif
 
 	    arg.pap.spp_flags  = cflags;
 #	    endif
@@ -5331,6 +6210,7 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 	    type    = SCTP_DEFAULT_SEND_PARAM;
 	    arg_ptr = (char*) (&arg.sri);
 	    arg_sz  = sizeof  ( arg.sri);
+	    VALGRIND_MAKE_MEM_DEFINED(arg_ptr, arg_sz); /*suppress "uninitialised bytes"*/
 	    break;
 	}
 	case SCTP_OPT_EVENTS:
@@ -5374,7 +6254,7 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 	    proto   = IPPROTO_SCTP;
 	    type    = SCTP_DELAYED_ACK_TIME;
 	    arg_ptr = (char*) (&arg.av);
-	    arg_sz  = sizeof  ( arg.es);
+	    arg_sz  = sizeof  ( arg.av);
 	    break;
 	}
 #	endif
@@ -5389,7 +6269,7 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 	    return -1;
 	}
 #if  defined(IP_TOS) && defined(SOL_IP) && defined(SO_PRIORITY)
-	res = setopt_prio_tos_trick (desc->s, proto, type, arg_ptr, arg_sz);
+	res = setopt_prio_tos_trick (desc->s, proto, type, arg_ptr, arg_sz, 1);
 #else
 	res = sock_setopt	    (desc->s, proto, type, arg_ptr, arg_sz);
 #endif
@@ -5420,8 +6300,9 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len)
 ** ptr should point to a buffer with 9*len +1 to be safe!!
 */
 
-static int inet_fill_opts(inet_descriptor* desc,
-			  char* buf, int len, char** dest, int destlen)
+static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc,
+				   char* buf, ErlDrvSizeT len,
+				   char** dest, ErlDrvSizeT destlen)
 {
     int type;
     int proto;
@@ -5431,8 +6312,8 @@ static int inet_fill_opts(inet_descriptor* desc,
     char* arg_ptr;
     unsigned int arg_sz;
     char *ptr = NULL;
-    int dest_used = 0;
-    int dest_allocated = destlen;
+    ErlDrvSizeT dest_used = 0;
+    ErlDrvSizeT dest_allocated = destlen;
     char *orig_dest = *dest;
 
     /* Ptr is a name parameter */ 
@@ -5447,8 +6328,8 @@ static int inet_fill_opts(inet_descriptor* desc,
 
 #define PLACE_FOR(Size,Ptr)						   \
     do {								   \
-	int need = dest_used + (Size);					   \
-	if (need > INET_MAX_BUFFER) {					   \
+	ErlDrvSizeT need = dest_used + (Size);					   \
+	if (need > INET_MAX_OPT_BUFFER) {				   \
 	    RETURN_ERROR();						   \
 	}								   \
 	if (need > dest_allocated) {					   \
@@ -5468,7 +6349,7 @@ static int inet_fill_opts(inet_descriptor* desc,
     /* Ptr is a name parameter */ 
 #define TRUNCATE_TO(Size,Ptr)				\
     do {						\
-	int new_need = ((Ptr) - (*dest)) + (Size);	\
+	ErlDrvSizeT new_need = ((Ptr) - (*dest)) + (Size);	\
 	if (new_need > dest_used) {			\
 	    erl_exit(1,"Internal error in inet_drv, "	\
 		     "miscalculated buffer size");	\
@@ -5672,7 +6553,7 @@ static int inet_fill_opts(inet_descriptor* desc,
 		buf += 4;
 		data_provided = (int) *buf++;
 		arg_sz = get_int32(buf);
-		if (arg_sz > INET_MAX_BUFFER) {	
+		if (arg_sz > INET_MAX_OPT_BUFFER) {
 		    RETURN_ERROR();
 		}
 		buf += 4;
@@ -5687,8 +6568,8 @@ static int inet_fill_opts(inet_descriptor* desc,
 		    buf += arg_sz;
 		    len -= arg_sz;
 		}
-		if (sock_getopt(desc->s,proto,type,arg_ptr,&arg_sz) == 
-		    SOCKET_ERROR) {
+		if (IS_SOCKET_ERROR(sock_getopt(desc->s,proto,type,
+						arg_ptr,&arg_sz))) {
 		    TRUNCATE_TO(0,ptr); 
 		    continue;
 		}
@@ -5705,7 +6586,7 @@ static int inet_fill_opts(inet_descriptor* desc,
 	    RETURN_ERROR();
 	}
 	/* We have 5 bytes allocated to ptr */
-	if (sock_getopt(desc->s,proto,type,arg_ptr,&arg_sz) == SOCKET_ERROR) {
+	if (IS_SOCKET_ERROR(sock_getopt(desc->s,proto,type,arg_ptr,&arg_sz))) {
 	    TRUNCATE_TO(0,ptr);
 	    continue;
 	}
@@ -5758,8 +6639,9 @@ static int load_paddrinfo (ErlDrvTermData * spec, int i,
 /*
 **  "sctp_fill_opts":   Returns {ok, Results}, or an error:
 */
-static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen,
-			  char** dest, int destlen)
+static ErlDrvSSizeT sctp_fill_opts(inet_descriptor* desc,
+				   char* buf, ErlDrvSizeT buflen,
+				   char** dest, ErlDrvSizeT destlen)
 {
     /* In contrast to the generic "inet_fill_opts", the output here is
        represented by tuples/records, which are formed in the "spec":
@@ -5786,7 +6668,7 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen,
 		     "miscalculated buffer size");              \
 	}                                                       \
 	need = (Index) + (N);                                   \
-	if (need > INET_MAX_BUFFER/sizeof(ErlDrvTermData)) {    \
+	if (need > INET_MAX_OPT_BUFFER/sizeof(ErlDrvTermData)) {\
 	    RETURN_ERROR((Spec), -ENOMEM);                      \
 	}                                                       \
 	if (need > spec_allocated) {                            \
@@ -6138,7 +7020,7 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen,
 		      2*LOAD_ATOM_CNT + LOAD_INT_CNT + 2*LOAD_TUPLE_CNT);
 	    i = LOAD_ATOM	(spec, i, am_sctp_adaptation_layer);
 	    i = LOAD_ATOM	(spec, i, am_sctp_setadaptation);
-	    i = LOAD_INT	(spec, i, ad.ssb_adaptation_ind);
+	    i = LOAD_INT	(spec, i, sock_ntohl(ad.ssb_adaptation_ind));
 	    i = LOAD_TUPLE	(spec, i, 2);
 	    i = LOAD_TUPLE	(spec, i, 2);
 	    break;
@@ -6149,7 +7031,7 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen,
 	    unsigned int             sz = sizeof(ap);
 	    int                      n;
 	    char                    *after;
-	    int                      alen;
+	    ErlDrvSizeT              alen;
 	    
 	    if (buflen < ASSOC_ID_LEN) RETURN_ERROR(spec, -EINVAL);
 	    ap.spp_assoc_id = GET_ASSOC_ID(buf);
@@ -6211,13 +7093,15 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen,
 	    
 	    if (ap.spp_flags & SPP_PMTUD_DISABLE)
 		{ i = LOAD_ATOM (spec, i, am_pmtud_disable);         n++; }
-	    
+#	    ifdef HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_SACKDELAY
+	    /* SPP_SACKDELAY_* not in FreeBSD 7.1 */
 	    if (ap.spp_flags & SPP_SACKDELAY_ENABLE)
 		{ i = LOAD_ATOM (spec, i, am_sackdelay_enable);      n++; }
 	    
 	    if (ap.spp_flags & SPP_SACKDELAY_DISABLE)
 		{ i = LOAD_ATOM (spec, i, am_sackdelay_disable);     n++; }
 #	    endif
+#	    endif
 	    
 	    PLACE_FOR(spec, i,
 		      LOAD_NIL_CNT + LOAD_LIST_CNT + 2*LOAD_TUPLE_CNT);
@@ -6237,6 +7121,10 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen,
 	    struct       sctp_sndrcvinfo sri;
 	    unsigned int sz  = sizeof(sri);
 	    
+	    if (buflen < ASSOC_ID_LEN) RETURN_ERROR(spec, -EINVAL);
+	    sri.sinfo_assoc_id = GET_ASSOC_ID(buf);
+	    buf += ASSOC_ID_LEN;
+	    buflen -= ASSOC_ID_LEN;
 	    if (sock_getopt(desc->s, IPPROTO_SCTP, SCTP_DEFAULT_SEND_PARAM,
 			    &sri, &sz) < 0) continue;
 	    /* Fill in the response: */
@@ -6275,7 +7163,7 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen,
 	    break;
 	}
 	/* The following option is not available in Solaris 10: */
-#	ifdef SCTP_DELAYED_ACK_TIME
+#	if HAVE_DECL_SCTP_DELAYED_ACK_TIME
 	case SCTP_OPT_DELAYED_ACK_TIME:
 	{
 	    struct       sctp_assoc_value av;
@@ -6322,7 +7210,7 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen,
 	    switch(st.sstat_state)
 	    {
             /*  SCTP_EMPTY is not supported on SOLARIS10: */
-#	    ifdef SCTP_EMPTY
+#	    if HAVE_DECL_SCTP_EMPTY
 	    case SCTP_EMPTY:
 		i = LOAD_ATOM	(spec, i, am_empty);
 		break;
@@ -6330,14 +7218,16 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen,
 	    case SCTP_CLOSED:
 		i = LOAD_ATOM   (spec, i, am_closed);
 		break;
-	    /* The following states are not supported by Linux Kernel SCTP yet:
+#           if HAVE_DECL_SCTP_BOUND
 	    case SCTP_BOUND:
 		i = LOAD_ATOM	(spec, i, am_bound);
 		break;
+#           endif
+#           if HAVE_DECL_SCTP_LISTEN
 	    case SCTP_LISTEN:
 		i = LOAD_ATOM	(spec, i, am_listen);
 		break;
-	    */
+#           endif
 	    case SCTP_COOKIE_WAIT:
 		i = LOAD_ATOM	(spec, i, am_cookie_wait);
 		break;
@@ -6381,7 +7271,7 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen,
 	    struct sctp_paddrinfo  pai;
 	    unsigned int           sz = sizeof(pai);
 	    char                  *after;
-	    int                    alen;
+	    ErlDrvSizeT            alen;
 	    
 	    if (buflen < ASSOC_ID_LEN) RETURN_ERROR(spec, -EINVAL);
 	    pai.spinfo_assoc_id = GET_ASSOC_ID(buf);
@@ -6408,7 +7298,7 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen,
 	default:
 	    RETURN_ERROR(spec, -EINVAL); /* No more valid options */
 	}
-	/* If we get here one result has been succesfully loaded */
+	/* If we get here one result has been successfully loaded */
 	length ++;
     }
     if (buflen != 0) RETURN_ERROR(spec, -EINVAL); /* Optparam mismatch */
@@ -6428,7 +7318,7 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen,
     driver_send_term(desc->port, driver_caller(desc->port), spec, i);
     FREE(spec);
 
-    (*dest)[0] = INET_REP_SCTP;
+    (*dest)[0] = INET_REP;
     return 1;   /* Response length */
 #   undef PLACE_FOR
 #   undef RETURN_ERROR
@@ -6438,7 +7328,8 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen,
 /* fill statistics reply, op codes from src and result in dest
 ** dst area must be a least 5*len + 1 bytes
 */
-static int inet_fill_stat(inet_descriptor* desc, char* src, int len, char* dst)
+static ErlDrvSSizeT inet_fill_stat(inet_descriptor* desc,
+				   char* src, ErlDrvSizeT len, char* dst)
 {
     unsigned long val;
     int op;
@@ -6471,7 +7362,7 @@ static int inet_fill_stat(inet_descriptor* desc, char* src, int len, char* dst)
 	    val = (unsigned long) desc->send_avg;
 	    break;
 	case INET_STAT_SEND_PND:  
-	    val = driver_sizeq(desc->port); 
+	    val = (unsigned long) driver_sizeq(desc->port);
 	    break;
 	case INET_STAT_RECV_OCT:
 	    put_int32(desc->recv_oct[1], dst);   /* write high 32bit */
@@ -6516,7 +7407,8 @@ send_empty_out_q_msgs(inet_descriptor* desc)
 /* subscribe and fill subscription reply, op codes from src and
 ** result in dest dst area must be a least 5*len + 1 bytes
 */
-static int inet_subscribe(inet_descriptor* desc, char* src, int len, char* dst)
+static ErlDrvSSizeT inet_subscribe(inet_descriptor* desc,
+				   char* src, ErlDrvSizeT len, char* dst)
 {
     unsigned long val;
     int op;
@@ -6603,6 +7495,8 @@ static ErlDrvData inet_start(ErlDrvPort port, int size, int protocol)
 
     sys_memzero((char *)&desc->remote,sizeof(desc->remote));
 
+    desc->is_ignored = 0;
+
     return (ErlDrvData)desc;
 }
 
@@ -6614,14 +7508,14 @@ static ErlDrvData inet_start(ErlDrvPort port, int size, int protocol)
 /*
 ** common TCP/UDP/SCTP control command
 */
-static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
-		    char** rbuf, int rsize)
+static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf,
+			     ErlDrvSizeT len, char** rbuf, ErlDrvSizeT rsize)
 {
     switch (cmd) {
 
     case INET_REQ_GETSTAT: {
 	  char* dst;
-	  int i;
+	  ErlDrvSizeT i;
 	  int dstlen = 1;  /* Reply code */
 
 	  for (i = 0; i < len; i++) {
@@ -6632,7 +7526,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 	      }
 	  }
 	  DEBUGF(("inet_ctl(%ld): GETSTAT\r\n", (long) desc->port)); 
-	  if (dstlen > INET_MAX_BUFFER) /* sanity check */
+	  if (dstlen > INET_MAX_OPT_BUFFER) /* sanity check */
 	      return 0;
 	  if (dstlen > rsize) {
 	      if ((dst = (char*) ALLOC(dstlen)) == NULL)
@@ -6648,7 +7542,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 	  char* dst;
 	  int dstlen = 1 /* Reply code */ + len*5;
 	  DEBUGF(("inet_ctl(%ld): INET_REQ_SUBSCRIBE\r\n", (long) desc->port)); 
-	  if (dstlen > INET_MAX_BUFFER) /* sanity check */
+	  if (dstlen > INET_MAX_OPT_BUFFER) /* sanity check */
 	      return 0;
 	  if (dstlen > rsize) {
 	      if ((dst = (char*) ALLOC(dstlen)) == NULL)
@@ -6661,7 +7555,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
       }
 
     case INET_REQ_GETOPTS: {    /* get options */
-	int replen;
+	ErlDrvSSizeT replen;
 	DEBUGF(("inet_ctl(%ld): GETOPTS\r\n", (long)desc->port)); 
 #ifdef HAVE_SCTP
         if (IS_SCTP(desc))
@@ -6683,6 +7577,13 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 	return inet_ctl_getiflist(desc, rbuf, rsize);
     }
 
+    case INET_REQ_GETIFADDRS: {
+	DEBUGF(("inet_ctl(%ld): GETIFADDRS\r\n", (long)desc->port));
+	if (!IS_OPEN(desc))
+	    return ctl_xerror(EXBADPORT, rbuf, rsize);
+	return inet_ctl_getifaddrs(desc, rbuf, rsize);
+    }
+
     case INET_REQ_IFGET: {
 	DEBUGF(("inet_ctl(%ld): IFGET\r\n", (long)desc->port)); 	
 	if (!IS_OPEN(desc))
@@ -6699,17 +7600,27 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 
     case INET_REQ_SETOPTS:  {   /* set options */
 	DEBUGF(("inet_ctl(%ld): SETOPTS\r\n", (long)desc->port)); 
+	/* XXX fprintf(stderr,"inet_ctl(%ld): SETOPTS (len = %d)\r\n", (long)desc->port,(int) len); */
 	switch(inet_set_opts(desc, buf, len)) {
 	case -1: 
 	    return ctl_error(EINVAL, rbuf, rsize);
 	case 0: 
 	    return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
-	default:  /* active/passive change!! */
+	case 1:
 	    /*
 	     * Let's hope that the descriptor really is a tcp_descriptor here.
 	     */
+	    /* fprintf(stderr,"Triggered tcp_deliver by setopt.\r\n"); */
 	    tcp_deliver((tcp_descriptor *) desc, 0);
 	    return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
+	default:  
+	    /* fprintf(stderr,"Triggered tcp_recv by setopt.\r\n"); */
+	    /*
+	     * Same as above, but active changed to once w/o header type
+	     * change, so try a read instead of just deliver. 
+	     */
+	    tcp_recv((tcp_descriptor *) desc, 0);
+	    return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
 	}
     }
 
@@ -6770,7 +7681,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 	if (len != 0)
 	    return ctl_error(EINVAL, rbuf, rsize);
 
-	if (sock_hostname(tbuf, MAXHOSTNAMELEN) == SOCKET_ERROR)
+	if (IS_SOCKET_ERROR(sock_hostname(tbuf, MAXHOSTNAMELEN)))
 	    return ctl_error(sock_errno(), rbuf, rsize);
 	return ctl_reply(INET_REP_OK, tbuf, strlen(tbuf), rbuf, rsize);
     }
@@ -6787,7 +7698,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 	    return ctl_error(ENOTCONN, rbuf, rsize);
 	if ((ptr = desc->peer_ptr) == NULL) {
 	    ptr = &peer;
-	    if (sock_peer(desc->s, (struct sockaddr*)ptr,&sz) == SOCKET_ERROR)
+	    if (IS_SOCKET_ERROR(sock_peer(desc->s, (struct sockaddr*)ptr,&sz)))
 		return ctl_error(sock_errno(), rbuf, rsize);
 	}
 	if (inet_get_address(desc->sfamily, tbuf, ptr, &sz) < 0)
@@ -6824,7 +7735,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 
 	if ((ptr = desc->name_ptr) == NULL) {
 	    ptr = &name;
-	    if (sock_name(desc->s, (struct sockaddr*)ptr, &sz) == SOCKET_ERROR)
+	    if (IS_SOCKET_ERROR(sock_name(desc->s, (struct sockaddr*)ptr, &sz)))
 		return ctl_error(sock_errno(), rbuf, rsize);
 	}
 	if (inet_get_address(desc->sfamily, tbuf, ptr, &sz) < 0)
@@ -6863,7 +7774,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 	if (inet_set_address(desc->sfamily, &local, buf, &len) == NULL)
 	    return ctl_error(EINVAL, rbuf, rsize);
 
-	if (sock_bind(desc->s,(struct sockaddr*) &local, len) == SOCKET_ERROR)
+	if (IS_SOCKET_ERROR(sock_bind(desc->s,(struct sockaddr*) &local, len)))
 	    return ctl_error(sock_errno(), rbuf, rsize);
 
 	desc->state = INET_STATE_BOUND;
@@ -6878,6 +7789,33 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 	return ctl_reply(INET_REP_OK, tbuf, 2, rbuf, rsize);
     }
 
+    case INET_REQ_IGNOREFD: {
+      DEBUGF(("inet_ctl(%ld): IGNOREFD, IGNORED = %d\r\n",
+	      (long)desc->port,(int)*buf));
+
+      /*
+       * FD can only be ignored for connected TCP connections for now,
+       * possible to add UDP and SCTP support if needed.
+       */
+      if (!IS_CONNECTED(desc))
+	  return ctl_error(ENOTCONN, rbuf, rsize);
+
+      if (!desc->stype == SOCK_STREAM)
+	  return ctl_error(EINVAL, rbuf, rsize);
+
+      if (*buf == 1 && !desc->is_ignored) {
+	  sock_select(desc, (FD_READ|FD_WRITE|FD_CLOSE|ERL_DRV_USE_NO_CALLBACK), 0);
+	  desc->is_ignored = INET_IGNORE_READ;
+      } else if (*buf == 0 && desc->is_ignored) {
+	  int flags = (FD_READ|FD_CLOSE|((desc->is_ignored & INET_IGNORE_WRITE)?FD_WRITE:0));
+	  desc->is_ignored = INET_IGNORE_NONE;
+	  sock_select(desc, flags, 1);
+      } else
+	  return ctl_error(EINVAL, rbuf, rsize);
+
+      return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
+    }
+
 #ifndef VXWORKS
 
     case INET_REQ_GETSERVBYNAME: { /* L1 Name-String L2 Proto-String */
@@ -6890,13 +7828,13 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 
 	if (len < 2)
 	    return ctl_error(EINVAL, rbuf, rsize);
-	n = buf[0]; buf++; len--;
+	n = get_int8(buf); buf++; len--;
 	if (n >= len) /* the = sign makes the test inklude next length byte */
 	    return ctl_error(EINVAL, rbuf, rsize);
 	memcpy(namebuf, buf, n);
 	namebuf[n] = '\0';
 	len -= n; buf += n;
-	n = buf[0]; buf++; len--;
+	n = get_int8(buf); buf++; len--;
 	if (n > len)
 	    return ctl_error(EINVAL, rbuf, rsize);
 	memcpy(protobuf, buf, n);
@@ -6919,7 +7857,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 	port = get_int16(buf);
 	port = sock_htons(port);
 	buf += 2;
-	n = buf[0]; buf++; len -= 3;
+	n = get_int8(buf); buf++; len -= 3;
 	if (n > len)
 	    return ctl_error(EINVAL, rbuf, rsize);
 	memcpy(protobuf, buf, n);
@@ -6938,7 +7876,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len,
 }
 
 /* update statistics on output packets */
-static void inet_output_count(inet_descriptor* desc, int len)
+static void inet_output_count(inet_descriptor* desc, ErlDrvSizeT len)
 {
     unsigned long n = desc->send_cnt + 1;
     unsigned long t = desc->send_oct[0] + len;
@@ -6958,7 +7896,7 @@ static void inet_output_count(inet_descriptor* desc, int len)
 }
 
 /* update statistics on input packets */
-static void inet_input_count(inet_descriptor* desc, int len)
+static void inet_input_count(inet_descriptor* desc, ErlDrvSizeT len)
 {
     unsigned long n = desc->recv_cnt + 1;
     unsigned long t = desc->recv_oct[0] + len;
@@ -7081,7 +8019,7 @@ static void tcp_clear_input(tcp_descriptor* desc)
 static void tcp_clear_output(tcp_descriptor* desc)
 {
     ErlDrvPort ix  = desc->inet.port;
-    int qsz = driver_sizeq(ix);
+    ErlDrvSizeT qsz = driver_sizeq(ix);
 
     driver_deq(ix, qsz);
     send_empty_out_q_msgs(INETP(desc));
@@ -7139,7 +8077,7 @@ static ErlDrvData tcp_inet_start(ErlDrvPort port, char* args)
 
 /* Copy a descriptor, by creating a new port with same settings
  * as the descriptor desc.
- * return NULL on error (ENFILE no ports avail)
+ * return NULL on error (SYSTEM_LIMIT no ports avail)
  */
 static tcp_descriptor* tcp_inet_copy(tcp_descriptor* desc,SOCKET s,
 				     ErlDrvTermData owner, int* err)
@@ -7178,7 +8116,7 @@ static tcp_descriptor* tcp_inet_copy(tcp_descriptor* desc,SOCKET s,
     /* The new port will be linked and connected to the original caller */
     port = driver_create_port(port, owner, "tcp_inet", (ErlDrvData) copy_desc);
     if ((long)port == -1) {
-	*err = ENFILE;
+	*err = INET_ERRNO_SYSTEM_LIMIT;
 	FREE(copy_desc);
 	return NULL;
     }
@@ -7196,22 +8134,22 @@ static tcp_descriptor* tcp_inet_copy(tcp_descriptor* desc,SOCKET s,
 static void tcp_close_check(tcp_descriptor* desc)
 {
     /* XXX:PaN - multiple clients to handle! */
-    if (desc->inet.state == TCP_STATE_ACCEPTING) {
+    if (desc->inet.state == INET_STATE_ACCEPTING) {
 	inet_async_op *this_op = desc->inet.opt;
 	sock_select(INETP(desc), FD_ACCEPT, 0);
-	desc->inet.state = TCP_STATE_LISTEN;
+	desc->inet.state = INET_STATE_LISTENING;
 	if (this_op != NULL) {
 	    driver_demonitor_process(desc->inet.port, &(this_op->monitor));
 	}
 	async_error_am(INETP(desc), am_closed);
     } 
-    else if (desc->inet.state == TCP_STATE_MULTI_ACCEPTING) {
+    else if (desc->inet.state == INET_STATE_MULTI_ACCEPTING) {
 	int id,req;
 	ErlDrvTermData caller;
 	ErlDrvMonitor monitor;
 
 	sock_select(INETP(desc), FD_ACCEPT, 0);
-	desc->inet.state = TCP_STATE_LISTEN;
+	desc->inet.state = INET_STATE_LISTENING;
 	while (deq_multi_op(desc,&id,&req,&caller,NULL,&monitor) == 0) {
 	    driver_demonitor_process(desc->inet.port, &monitor);
 	    send_async_error(desc->inet.port, desc->inet.dport, id, caller, am_closed);
@@ -7219,10 +8157,10 @@ static void tcp_close_check(tcp_descriptor* desc)
 	clean_multi_timers(&(desc->mtd), desc->inet.port);
     }
 
-    else if (desc->inet.state == TCP_STATE_CONNECTING) {
+    else if (desc->inet.state == INET_STATE_CONNECTING) {
 	async_error_am(INETP(desc), am_closed);
     }
-    else if (desc->inet.state == TCP_STATE_CONNECTED) {
+    else if (desc->inet.state == INET_STATE_CONNECTED) {
 	async_error_am_all(INETP(desc), am_closed);
     }
 }
@@ -7249,45 +8187,69 @@ static void tcp_inet_stop(ErlDrvData e)
     
 
 /* TCP requests from Erlang */
-static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
-			char** rbuf, int rsize)
+static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd,
+				 char* buf, ErlDrvSizeT len,
+				 char** rbuf, ErlDrvSizeT rsize)
 {
     tcp_descriptor* desc = (tcp_descriptor*)e;
+
     switch(cmd) {
-    case INET_REQ_OPEN:   /* open socket and return internal index */
+    case INET_REQ_OPEN: { /* open socket and return internal index */
+	int domain;
 	DEBUGF(("tcp_inet_ctl(%ld): OPEN\r\n", (long)desc->inet.port));
-	if ((len == 1) && (buf[0] == INET_AF_INET))
-	    return
-		inet_ctl_open(INETP(desc), AF_INET, SOCK_STREAM, rbuf, rsize);
+	if (len != 2) return ctl_error(EINVAL, rbuf, rsize);
+	switch(buf[0]) {
+	case INET_AF_INET:
+	    domain = AF_INET;
+	    break;
 #if defined(HAVE_IN6) && defined(AF_INET6)
-        else if ((len == 1) && (buf[0] == INET_AF_INET6))
-	    return
-		inet_ctl_open(INETP(desc), AF_INET6, SOCK_STREAM, rbuf, rsize);
+	case INET_AF_INET6:
+	    domain = AF_INET6;
+	    break;
 #else
-	else if ((len == 1) && (buf[0] == INET_AF_INET6))
-	    return ctl_xerror("eafnosupport",rbuf,rsize);
+	case INET_AF_INET6:
+	    return ctl_xerror("eafnosupport", rbuf, rsize);
+	    break;
 #endif
-	else
+	default:
 	    return ctl_error(EINVAL, rbuf, rsize);
+	}
+	if (buf[1] != INET_TYPE_STREAM) return ctl_error(EINVAL, rbuf, rsize);
+	return inet_ctl_open(INETP(desc), domain, SOCK_STREAM, rbuf, rsize);
+	break;
+    }
 
-    case INET_REQ_FDOPEN:   /* pass in an open socket */
-	DEBUGF(("tcp_inet_ctl(%ld): FDOPEN\r\n", (long)desc->inet.port)); 
-	if ((len == 5) && (buf[0] == INET_AF_INET))
-	    return inet_ctl_fdopen(INETP(desc), AF_INET, SOCK_STREAM,
-				   (SOCKET) get_int32(buf+1), rbuf, rsize);
+    case INET_REQ_FDOPEN: {  /* pass in an open socket */
+	int domain;
+	DEBUGF(("tcp_inet_ctl(%ld): FDOPEN\r\n", (long)desc->inet.port));
+	if (len != 6) return ctl_error(EINVAL, rbuf, rsize);
+	switch(buf[0]) {
+	case INET_AF_INET:
+	    domain = AF_INET;
+	    break;
 #if defined(HAVE_IN6) && defined(AF_INET6)
-        else if ((len == 5) && (buf[0] == INET_AF_INET6))
-	    return inet_ctl_fdopen(INETP(desc), AF_INET6, SOCK_STREAM,
-				   (SOCKET) get_int32(buf+1), rbuf, rsize);
+	case INET_AF_INET6:
+	    domain = AF_INET6;
+	    break;
+#else
+	case INET_AF_INET6:
+	    return ctl_xerror("eafnosupport", rbuf, rsize);
+	    break;
 #endif
-	else
+	default:
 	    return ctl_error(EINVAL, rbuf, rsize);
+	}
+	if (buf[1] != INET_TYPE_STREAM) return ctl_error(EINVAL, rbuf, rsize);
+	return inet_ctl_fdopen(INETP(desc), domain, SOCK_STREAM,
+			       (SOCKET) get_int32(buf+2), rbuf, rsize);
+	break;
+    }
 
-    case TCP_REQ_LISTEN: { /* argument backlog */
+    case INET_REQ_LISTEN: { /* argument backlog */
 
 	int backlog;
 	DEBUGF(("tcp_inet_ctl(%ld): LISTEN\r\n", (long)desc->inet.port)); 
-	if (desc->inet.state == TCP_STATE_CLOSED)
+	if (desc->inet.state == INET_STATE_CLOSED)
 	    return ctl_xerror(EXBADPORT, rbuf, rsize);
 	if (!IS_OPEN(INETP(desc)))
 	    return ctl_xerror(EXBADPORT, rbuf, rsize);
@@ -7296,9 +8258,9 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	if (len != 2)
 	    return ctl_error(EINVAL, rbuf, rsize);
 	backlog = get_int16(buf);
-	if (sock_listen(desc->inet.s, backlog) == SOCKET_ERROR)
+	if (IS_SOCKET_ERROR(sock_listen(desc->inet.s, backlog)))
 	    return ctl_error(sock_errno(), rbuf, rsize);
-	desc->inet.state = TCP_STATE_LISTEN;
+	desc->inet.state = INET_STATE_LISTENING;
 	return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
     }
 
@@ -7330,17 +8292,17 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	
 	code = sock_connect(desc->inet.s, 
 			    (struct sockaddr*) &desc->inet.remote, len);
-	if ((code == SOCKET_ERROR) && 
+	if (IS_SOCKET_ERROR(code) &&
 		((sock_errno() == ERRNO_BLOCK) ||  /* Winsock2 */
 		 (sock_errno() == EINPROGRESS))) {	/* Unix & OSE!! */
           sock_select(INETP(desc), FD_CONNECT, 1);
-	    desc->inet.state = TCP_STATE_CONNECTING;
+	    desc->inet.state = INET_STATE_CONNECTING;
 	    if (timeout != INET_INFINITY)
 		driver_set_timer(desc->inet.port, timeout);
 	    enq_async(INETP(desc), tbuf, INET_REQ_CONNECT);
 	}
 	else if (code == 0) { /* ok we are connected */
-	    desc->inet.state = TCP_STATE_CONNECTED;
+	    desc->inet.state = INET_STATE_CONNECTED;
 	    if (desc->inet.active)
 		sock_select(INETP(desc), (FD_READ|FD_CLOSE), 1);
 	    enq_async(INETP(desc), tbuf, INET_REQ_CONNECT);
@@ -7352,7 +8314,7 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	return ctl_reply(INET_REP_OK, tbuf, 2, rbuf, rsize);
     }
 
-    case TCP_REQ_ACCEPT: {  /* do async accept */
+    case INET_REQ_ACCEPT: {  /* do async accept */
 	char tbuf[2];
 	unsigned timeout;
 	inet_address remote;
@@ -7362,19 +8324,19 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	DEBUGF(("tcp_inet_ctl(%ld): ACCEPT\r\n", (long)desc->inet.port)); 
 	/* INPUT: Timeout(4) */
 
-	if ((desc->inet.state != TCP_STATE_LISTEN && desc->inet.state != TCP_STATE_ACCEPTING &&
-	     desc->inet.state != TCP_STATE_MULTI_ACCEPTING) || len != 4) {
+	if ((desc->inet.state != INET_STATE_LISTENING && desc->inet.state != INET_STATE_ACCEPTING &&
+	     desc->inet.state != INET_STATE_MULTI_ACCEPTING) || len != 4) {
 	    return ctl_error(EINVAL, rbuf, rsize);
 	}
 
 	timeout = get_int32(buf);
 
-	if (desc->inet.state == TCP_STATE_ACCEPTING) {
-	    unsigned long time_left;
-	    int oid;
-	    ErlDrvTermData ocaller;
-	    int oreq;
-	    unsigned otimeout;
+	if (desc->inet.state == INET_STATE_ACCEPTING) {
+	    unsigned long time_left = 0;
+	    int oid = 0;
+	    ErlDrvTermData ocaller = ERL_DRV_NIL;
+	    int oreq = 0;
+	    unsigned otimeout = 0;
 	    ErlDrvTermData caller = driver_caller(desc->inet.port);
 	    MultiTimerData *mtd = NULL,*omtd = NULL;
 	    ErlDrvMonitor monitor, omonitor;
@@ -7398,10 +8360,10 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 		mtd = add_multi_timer(&(desc->mtd), desc->inet.port, caller, 
 				      timeout, &tcp_inet_multi_timeout);
 	    }
-	    enq_multi_op(desc, tbuf, TCP_REQ_ACCEPT, caller, mtd, &monitor);
-	    desc->inet.state = TCP_STATE_MULTI_ACCEPTING;
+	    enq_multi_op(desc, tbuf, INET_REQ_ACCEPT, caller, mtd, &monitor);
+	    desc->inet.state = INET_STATE_MULTI_ACCEPTING;
 	    return ctl_reply(INET_REP_OK, tbuf, 2, rbuf, rsize);
-	} else if (desc->inet.state == TCP_STATE_MULTI_ACCEPTING) {
+	} else if (desc->inet.state == INET_STATE_MULTI_ACCEPTING) {
 	    ErlDrvTermData caller = driver_caller(desc->inet.port);
 	    MultiTimerData *mtd = NULL;
 	    ErlDrvMonitor monitor;
@@ -7413,7 +8375,7 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 		mtd = add_multi_timer(&(desc->mtd), desc->inet.port, caller, 
 				      timeout, &tcp_inet_multi_timeout);
 	    }
-	    enq_multi_op(desc, tbuf, TCP_REQ_ACCEPT, caller, mtd, &monitor);
+	    enq_multi_op(desc, tbuf, INET_REQ_ACCEPT, caller, mtd, &monitor);
 	    return ctl_reply(INET_REP_OK, tbuf, 2, rbuf, rsize);
  	} else {
 	    n = sizeof(desc->inet.remote);
@@ -7425,8 +8387,8 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 					       &monitor) != 0) { 
 			return ctl_xerror("noproc", rbuf, rsize);
 		    }
-		    enq_async_w_tmo(INETP(desc), tbuf, TCP_REQ_ACCEPT, timeout, &monitor);
-		    desc->inet.state = TCP_STATE_ACCEPTING;
+		    enq_async_w_tmo(INETP(desc), tbuf, INET_REQ_ACCEPT, timeout, &monitor);
+		    desc->inet.state = INET_STATE_ACCEPTING;
 		    sock_select(INETP(desc),FD_ACCEPT,1);
 		    if (timeout != INET_INFINITY) {
 			driver_set_timer(desc->inet.port, timeout);
@@ -7453,8 +8415,8 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 		driver_select(accept_desc->inet.port, accept_desc->inet.event, 
 			      ERL_DRV_READ, 1);
 #endif
-		accept_desc->inet.state = TCP_STATE_CONNECTED;
-		enq_async(INETP(desc), tbuf, TCP_REQ_ACCEPT);
+		accept_desc->inet.state = INET_STATE_CONNECTED;
+		enq_async(INETP(desc), tbuf, INET_REQ_ACCEPT);
 		async_ok_port(INETP(desc), accept_desc->inet.dport);
 	    }
 	    return ctl_reply(INET_REP_OK, tbuf, 2, rbuf, rsize);
@@ -7496,13 +8458,14 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	if (enq_async(INETP(desc), tbuf, TCP_REQ_RECV) < 0)
 	    return ctl_error(EALREADY, rbuf, rsize);
 
-	if (tcp_recv(desc, n) == 0) {
+	if (INETP(desc)->is_ignored || tcp_recv(desc, n) == 0) {
 	    if (timeout == 0)
 		async_error_am(INETP(desc), am_timeout);
 	    else {
 		if (timeout != INET_INFINITY)
-		    driver_set_timer(desc->inet.port, timeout); 
-		sock_select(INETP(desc),(FD_READ|FD_CLOSE),1);
+		    driver_set_timer(desc->inet.port, timeout);
+		if (!INETP(desc)->is_ignored)
+		    sock_select(INETP(desc),(FD_READ|FD_CLOSE),1);
 	    }
 	}
 	return ctl_reply(INET_REP_OK, tbuf, 2, rbuf, rsize);
@@ -7517,7 +8480,6 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	    tcp_deliver(desc, 0);
 	return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
     }
-#ifndef _OSE_
     case TCP_REQ_SHUTDOWN: {
 	int how;
 	DEBUGF(("tcp_inet_ctl(%ld): FDOPEN\r\n", (long)desc->inet.port)); 
@@ -7534,7 +8496,6 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	    return ctl_error(sock_errno(), rbuf, rsize);
 	}
     }
-#endif
     default:
 	DEBUGF(("tcp_inet_ctl(%ld): %u\r\n", (long)desc->inet.port, cmd)); 
 	return inet_ctl(INETP(desc), cmd, buf, len, rbuf, rsize);
@@ -7562,7 +8523,7 @@ static void tcp_inet_timeout(ErlDrvData e)
 	    (long)desc->inet.port, desc->inet.s)); 
     if ((state & INET_F_MULTI_CLIENT)) { /* Multi-client always means multi-timers */
 	fire_multi_timers(&(desc->mtd), desc->inet.port, e);
-    } else if ((state & TCP_STATE_CONNECTED) == TCP_STATE_CONNECTED) {
+    } else if ((state & INET_STATE_CONNECTED) == INET_STATE_CONNECTED) {
 	if (desc->busy_on_send) {
 	    ASSERT(IS_BUSY(INETP(desc)));
 	    desc->inet.caller = desc->inet.busy_caller;
@@ -7582,20 +8543,20 @@ static void tcp_inet_timeout(ErlDrvData e)
 	    async_error_am(INETP(desc), am_timeout);
 	}
     }
-    else if ((state & TCP_STATE_CONNECTING) == TCP_STATE_CONNECTING) {
+    else if ((state & INET_STATE_CONNECTING) == INET_STATE_CONNECTING) {
 	/* assume connect timeout */
 	/* close the socket since it's not usable (see man pages) */
 	erl_inet_close(INETP(desc));
 	async_error_am(INETP(desc), am_timeout);
     }
-    else if ((state & TCP_STATE_ACCEPTING) == TCP_STATE_ACCEPTING) {
+    else if ((state & INET_STATE_ACCEPTING) == INET_STATE_ACCEPTING) {
 	inet_async_op *this_op = desc->inet.opt;
 	/* timer is set on accept */
 	sock_select(INETP(desc), FD_ACCEPT, 0);
 	if (this_op != NULL) {
 	    driver_demonitor_process(desc->inet.port, &(this_op->monitor));
 	}
-	desc->inet.state = TCP_STATE_LISTEN;
+	desc->inet.state = INET_STATE_LISTENING;
 	async_error_am(INETP(desc), am_timeout);
     }
     DEBUGF(("tcp_inet_timeout(%ld) }\r\n", (long)desc->inet.port)); 
@@ -7613,7 +8574,7 @@ static void tcp_inet_multi_timeout(ErlDrvData e, ErlDrvTermData caller)
     driver_demonitor_process(desc->inet.port, &monitor);
     if (desc->multi_first == NULL) {
 	sock_select(INETP(desc),FD_ACCEPT,0);
-	desc->inet.state = TCP_STATE_LISTEN; /* restore state */
+	desc->inet.state = INET_STATE_LISTENING; /* restore state */
     }
     send_async_error(desc->inet.port, desc->inet.dport, id, caller, am_timeout);
 }
@@ -7629,7 +8590,7 @@ static void tcp_inet_multi_timeout(ErlDrvData e, ErlDrvTermData caller)
 ** but distribution still uses the tcp_inet_command!!
 */
 
-static void tcp_inet_command(ErlDrvData e, char *buf, int len)
+static void tcp_inet_command(ErlDrvData e, char *buf, ErlDrvSizeT len)
 {
     tcp_descriptor* desc = (tcp_descriptor*)e;
     desc->inet.caller = driver_caller(desc->inet.port);
@@ -7679,7 +8640,7 @@ static void tcp_inet_process_exit(ErlDrvData e, ErlDrvMonitor *monitorp)
     ErlDrvTermData who = driver_get_monitored_process(desc->inet.port,monitorp);
     int state = desc->inet.state;
 
-    if ((state & TCP_STATE_MULTI_ACCEPTING) == TCP_STATE_MULTI_ACCEPTING) {
+    if ((state & INET_STATE_MULTI_ACCEPTING) == INET_STATE_MULTI_ACCEPTING) {
 	int id,req;
 	MultiTimerData *timeout;
 	if (remove_multi_op(desc, &id, &req, who, &timeout, NULL) != 0) {
@@ -7690,15 +8651,15 @@ static void tcp_inet_process_exit(ErlDrvData e, ErlDrvMonitor *monitorp)
 	}
 	if (desc->multi_first == NULL) {
 	    sock_select(INETP(desc),FD_ACCEPT,0);
-	    desc->inet.state = TCP_STATE_LISTEN; /* restore state */
+	    desc->inet.state = INET_STATE_LISTENING; /* restore state */
 	}
-    } else if ((state & TCP_STATE_ACCEPTING) == TCP_STATE_ACCEPTING) {
+    } else if ((state & INET_STATE_ACCEPTING) == INET_STATE_ACCEPTING) {
 	int did,drid; 
 	ErlDrvTermData dcaller;
 	deq_async(INETP(desc), &did, &dcaller, &drid);
 	driver_cancel_timer(desc->inet.port);
 	sock_select(INETP(desc),FD_ACCEPT,0);
-	desc->inet.state = TCP_STATE_LISTEN; /* restore state */
+	desc->inet.state = INET_STATE_LISTENING; /* restore state */
     }
 } 
 
@@ -7848,8 +8809,15 @@ static int tcp_remain(tcp_descriptor* desc, int* len)
     else if (tlen == 0) { /* need unknown more */
         *len = 0;
         if (nsz == 0) {
-            if (nfill == n)
-                goto error;
+            if (nfill == n) {
+                if (desc->inet.psize != 0 && desc->inet.psize > nfill) {
+                    if (tcp_expand_buffer(desc, desc->inet.psize) < 0)
+                        return -1;
+                    return desc->inet.psize;
+                }
+                else
+                    goto error;
+            }
             DEBUGF((" => restart more=%d\r\n", nfill - n));
             return nfill - n;
         }
@@ -7888,32 +8856,29 @@ static int tcp_deliver(tcp_descriptor* desc, int len)
     }
 
     while (len > 0) {
-	int code = 0;
+	int code;
 
 	inet_input_count(INETP(desc), len);
 
 	/* deliver binary? */
 	if (len*4 >= desc->i_buf->orig_size*3) { /* >=75% */
+	    code = tcp_reply_binary_data(desc, desc->i_buf,
+					 (desc->i_ptr_start -
+					  desc->i_buf->orig_bytes),
+					 len);
+	    if (code < 0)
+		return code;
+
 	    /* something after? */
 	    if (desc->i_ptr_start + len == desc->i_ptr) { /* no */
-		code = tcp_reply_binary_data(desc, desc->i_buf,
-					     (desc->i_ptr_start -
-					      desc->i_buf->orig_bytes),
-					     len);
 		tcp_clear_input(desc);
 	    }
 	    else { /* move trail to beginning of a new buffer */
-		ErlDrvBinary* bin;
+		ErlDrvBinary* bin = alloc_buffer(desc->i_bufsz);
 		char* ptr_end = desc->i_ptr_start + len;
 		int sz = desc->i_ptr - ptr_end;
 
-		bin = alloc_buffer(desc->i_bufsz);
 		memcpy(bin->orig_bytes, ptr_end, sz);
-
-		code = tcp_reply_binary_data(desc, desc->i_buf,
-					     (desc->i_ptr_start-
-					      desc->i_buf->orig_bytes),
-					     len);
 		free_buffer(desc->i_buf);
 		desc->i_buf = bin;
 		desc->i_ptr_start = desc->i_buf->orig_bytes;
@@ -7925,22 +8890,22 @@ static int tcp_deliver(tcp_descriptor* desc, int len)
 	    code = tcp_reply_data(desc, desc->i_ptr_start, len);
 	    /* XXX The buffer gets thrown away on error  (code < 0)    */
 	    /* Windows needs workaround for this in tcp_inet_event...  */
+	    if (code < 0)
+		return code;
 	    desc->i_ptr_start += len;
 	    if (desc->i_ptr_start == desc->i_ptr)
 		tcp_clear_input(desc);
 	    else
 		desc->i_remain = 0;
-
 	}
 
-	if (code < 0)
-	    return code;
-
 	count++;
 	len = 0;
 
 	if (!desc->inet.active) {
-	    driver_cancel_timer(desc->inet.port);
+	    if (!desc->busy_on_send) {
+		driver_cancel_timer(desc->inet.port);
+	    }
 	    sock_select(INETP(desc),(FD_READ|FD_CLOSE),0);
 	    if (desc->i_buf != NULL)
 		tcp_restart_input(desc);
@@ -8008,7 +8973,7 @@ static int tcp_recv(tcp_descriptor* desc, int request_len)
 
     n = sock_recv(desc->inet.s, desc->i_ptr, nread, 0);
 
-    if (n == SOCKET_ERROR) {
+    if (IS_SOCKET_ERROR(n)) {
 	int err = sock_errno();
 	if (err == ECONNRESET) {
 	    DEBUGF((" => detected close (connreset)\r\n"));
@@ -8238,8 +9203,8 @@ static void tcp_inet_event(ErlDrvData e, ErlDrvEvent event)
 
 
 /* socket has input:
-** 1. TCP_STATE_ACCEPTING  => non block accept ? 
-** 2. TCP_STATE_CONNECTED => read input
+** 1. INET_STATE_ACCEPTING  => non block accept ?
+** 2. INET_STATE_CONNECTED => read input
 */
 static int tcp_inet_input(tcp_descriptor* desc, HANDLE event)
 {
@@ -8247,8 +9212,10 @@ static int tcp_inet_input(tcp_descriptor* desc, HANDLE event)
 #ifdef DEBUG
     long port = (long) desc->inet.port;  /* Used after driver_exit() */
 #endif
+    ASSERT(!INETP(desc)->is_ignored);
     DEBUGF(("tcp_inet_input(%ld) {s=%d\r\n", port, desc->inet.s));
-    if (desc->inet.state == TCP_STATE_ACCEPTING) {
+    /* XXX fprintf(stderr,"tcp_inet_input(%ld) {s=%d}\r\n",(long) desc->inet.port, desc->inet.s); */
+    if (desc->inet.state == INET_STATE_ACCEPTING) {
 	SOCKET s;
 	unsigned int len;
 	inet_address remote;
@@ -8263,7 +9230,7 @@ static int tcp_inet_input(tcp_descriptor* desc, HANDLE event)
 	}
 
 	sock_select(INETP(desc),FD_ACCEPT,0);
-	desc->inet.state = TCP_STATE_LISTEN; /* restore state */
+	desc->inet.state = INET_STATE_LISTENING; /* restore state */
 
 	if (this_op != NULL) {
 	    driver_demonitor_process(desc->inet.port, &(this_op->monitor));
@@ -8303,11 +9270,11 @@ static int tcp_inet_input(tcp_descriptor* desc, HANDLE event)
 	    driver_select(accept_desc->inet.port, accept_desc->inet.event, 
 			  ERL_DRV_READ, 1);
 #endif
-	    accept_desc->inet.state = TCP_STATE_CONNECTED;
+	    accept_desc->inet.state = INET_STATE_CONNECTED;
 	    ret =  async_ok_port(INETP(desc), accept_desc->inet.dport);
 	    goto done;
 	}
-    } else if (desc->inet.state == TCP_STATE_MULTI_ACCEPTING) {
+    } else if (desc->inet.state == INET_STATE_MULTI_ACCEPTING) {
 	SOCKET s;
 	unsigned int len;
 	inet_address remote;
@@ -8319,7 +9286,7 @@ static int tcp_inet_input(tcp_descriptor* desc, HANDLE event)
 	int times = 0;
 #endif
 
-	while (desc->inet.state == TCP_STATE_MULTI_ACCEPTING) {
+	while (desc->inet.state == INET_STATE_MULTI_ACCEPTING) {
 	    len = sizeof(desc->inet.remote);
 	    s = sock_accept(desc->inet.s, (struct sockaddr*) &remote, &len);
 	    
@@ -8339,7 +9306,7 @@ static int tcp_inet_input(tcp_descriptor* desc, HANDLE event)
 	    
 	    if (desc->multi_first == NULL) {
 		sock_select(INETP(desc),FD_ACCEPT,0);
-		desc->inet.state = TCP_STATE_LISTEN; /* restore state */
+		desc->inet.state = INET_STATE_LISTENING; /* restore state */
 	    }
 	    
 	    if (timeout != NULL) {
@@ -8370,7 +9337,7 @@ static int tcp_inet_input(tcp_descriptor* desc, HANDLE event)
 		driver_select(accept_desc->inet.port, accept_desc->inet.event, 
 			      ERL_DRV_READ, 1);
 #endif
-		accept_desc->inet.state = TCP_STATE_CONNECTED;
+		accept_desc->inet.state = INET_STATE_CONNECTED;
 		ret =  send_async_ok_port(desc->inet.port, desc->inet.dport, 
 					  id, caller, accept_desc->inet.dport);
 	    }
@@ -8453,12 +9420,12 @@ static int tcp_send_error(tcp_descriptor* desc, int err)
 */
 static int tcp_sendv(tcp_descriptor* desc, ErlIOVec* ev)
 {
-    int sz;
+    ErlDrvSizeT sz;
     char buf[4];
-    int h_len;
-    int n;
+    ErlDrvSizeT h_len;
+    ssize_t n;
     ErlDrvPort ix = desc->inet.port;
-    int len = ev->size;
+    ErlDrvSizeT len = ev->size;
 
      switch(desc->inet.htype) {
      case TCP_PB_1:
@@ -8506,12 +9473,16 @@ static int tcp_sendv(tcp_descriptor* desc, ErlIOVec* ev)
     else {
 	int vsize = (ev->vsize > MAX_VSIZE) ? MAX_VSIZE : ev->vsize;
 	
-	DEBUGF(("tcp_sendv(%ld): s=%d, about to send %d,%d bytes\r\n",
-		(long)desc->inet.port, desc->inet.s, h_len, len));
-	if (desc->tcp_add_flags & TCP_ADDF_DELAY_SEND) {
+	DEBUGF(("tcp_sendv(%ld): s=%d, about to send "LLU","LLU" bytes\r\n",
+		(long)desc->inet.port, desc->inet.s, (llu_t)h_len, (llu_t)len));
+
+	if (INETP(desc)->is_ignored) {
+	    INETP(desc)->is_ignored |= INET_IGNORE_WRITE;
+	    n = 0;
+	} else if (desc->tcp_add_flags & TCP_ADDF_DELAY_SEND) {
 	    n = 0;
-	} else if (sock_sendv(desc->inet.s, ev->iov, vsize, &n, 0) 
-		   == SOCKET_ERROR) {
+	} else if (IS_SOCKET_ERROR(sock_sendv(desc->inet.s, ev->iov,
+					      vsize, &n, 0))) {
 	    if ((sock_errno() != ERRNO_BLOCK) && (sock_errno() != EINTR)) {
 		int err = sock_errno();
 		DEBUGF(("tcp_sendv(%ld): s=%d, "
@@ -8529,14 +9500,17 @@ static int tcp_sendv(tcp_descriptor* desc, ErlIOVec* ev)
 	    return 0;
 	}
 	else {
-	    DEBUGF(("tcp_sendv(%ld): s=%d, only sent %d/%d of %d/%d bytes/items\r\n", 
-			(long)desc->inet.port, desc->inet.s, n, vsize, ev->size, ev->vsize));
+	    DEBUGF(("tcp_sendv(%ld): s=%d, only sent "
+		    LLU"/%d of "LLU"/%d bytes/items\r\n",
+		    (long)desc->inet.port, desc->inet.s,
+		    (llu_t)n, vsize, (llu_t)ev->size, ev->vsize));
 	}
 
 	DEBUGF(("tcp_sendv(%ld): s=%d, Send failed, queuing\r\n", 
 		(long)desc->inet.port, desc->inet.s));
 	driver_enqv(ix, ev, n); 
-	sock_select(INETP(desc),(FD_WRITE|FD_CLOSE), 1);
+	if (!INETP(desc)->is_ignored)
+	    sock_select(INETP(desc),(FD_WRITE|FD_CLOSE), 1);
     }
     return 0;
 }
@@ -8544,7 +9518,7 @@ static int tcp_sendv(tcp_descriptor* desc, ErlIOVec* ev)
 /*
 ** Send non blocking data
 */
-static int tcp_send(tcp_descriptor* desc, char* ptr, int len)
+static int tcp_send(tcp_descriptor* desc, char* ptr, ErlDrvSizeT len)
 {
     int sz;
     char buf[4];
@@ -8599,12 +9573,15 @@ static int tcp_send(tcp_descriptor* desc, char* ptr, int len)
 	iov[1].iov_base = ptr;
 	iov[1].iov_len = len;
 
-	DEBUGF(("tcp_send(%ld): s=%d, about to send %d,%d bytes\r\n",
-		(long)desc->inet.port, desc->inet.s, h_len, len));
-	if (desc->tcp_add_flags & TCP_ADDF_DELAY_SEND) {
+	DEBUGF(("tcp_send(%ld): s=%d, about to send "LLU","LLU" bytes\r\n",
+		(long)desc->inet.port, desc->inet.s, (llu_t)h_len, (llu_t)len));
+	if (INETP(desc)->is_ignored) {
+	    INETP(desc)->is_ignored |= INET_IGNORE_WRITE;
+	    n = 0;
+	} else if (desc->tcp_add_flags & TCP_ADDF_DELAY_SEND) {
 	    sock_send(desc->inet.s, buf, 0, 0);
 	    n = 0;
-	} else 	if (sock_sendv(desc->inet.s,iov,2,&n,0) == SOCKET_ERROR) {
+	} else 	if (IS_SOCKET_ERROR(sock_sendv(desc->inet.s,iov,2,&n,0))) {
 	    if ((sock_errno() != ERRNO_BLOCK) && (sock_errno() != EINTR)) {
 		int err = sock_errno();
 		DEBUGF(("tcp_send(%ld): s=%d,sock_sendv(size=2) errno = %d\r\n",
@@ -8632,7 +9609,8 @@ static int tcp_send(tcp_descriptor* desc, char* ptr, int len)
 	    n -= h_len;
 	    driver_enq(ix, ptr+n, len-n);
 	}
-	sock_select(INETP(desc),(FD_WRITE|FD_CLOSE), 1);
+	if (!INETP(desc)->is_ignored)
+	    sock_select(INETP(desc),(FD_WRITE|FD_CLOSE), 1);
     }
     return 0;
 }
@@ -8648,17 +9626,18 @@ static void tcp_inet_drv_input(ErlDrvData data, ErlDrvEvent event)
 }
 
 /* socket ready for ouput:
-** 1. TCP_STATE_CONNECTING => non block connect ?
-** 2. TCP_STATE_CONNECTED  => write output
+** 1. INET_STATE_CONNECTING => non block connect ?
+** 2. INET_STATE_CONNECTED  => write output
 */
 static int tcp_inet_output(tcp_descriptor* desc, HANDLE event)
 {
     int ret = 0;
     ErlDrvPort ix = desc->inet.port;
 
+    ASSERT(!INETP(desc)->is_ignored);
     DEBUGF(("tcp_inet_output(%ld) {s=%d\r\n", 
 	    (long)desc->inet.port, desc->inet.s));
-    if (desc->inet.state == TCP_STATE_CONNECTING) {
+    if (desc->inet.state == INET_STATE_CONNECTING) {
 	sock_select(INETP(desc),FD_CONNECT,0);
 
 	driver_cancel_timer(ix);  /* posssibly cancel a timer */
@@ -8677,8 +9656,8 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event)
 	    int code = sock_peer(desc->inet.s,
 				 (struct sockaddr*) &desc->inet.remote, &sz);
 
-	    if (code == SOCKET_ERROR) {
-		desc->inet.state = TCP_STATE_BOUND;  /* restore state */
+	    if (IS_SOCKET_ERROR(code)) {
+		desc->inet.state = INET_STATE_BOUND;  /* restore state */
 		ret =  async_error(INETP(desc), sock_errno());
 		goto done;
 	    }
@@ -8691,15 +9670,15 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event)
 				   (void *)&error, &sz);
 
 	    if ((code < 0) || error) {
-		desc->inet.state = TCP_STATE_BOUND;  /* restore state */
+		desc->inet.state = INET_STATE_BOUND;  /* restore state */
 		ret = async_error(INETP(desc), error);
 		goto done;
 	    }
 	}
-#endif /* SOCKOPT_CONNECT_STAT */
+#endif /* SO_ERROR */
 #endif /* !__WIN32__ */
 
-	desc->inet.state = TCP_STATE_CONNECTED;
+	desc->inet.state = INET_STATE_CONNECTED;
 	if (desc->inet.active)
 	    sock_select(INETP(desc),(FD_READ|FD_CLOSE),1);
 	async_ok(INETP(desc));
@@ -8707,7 +9686,7 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event)
     else if (IS_CONNECTED(INETP(desc))) {
 	for (;;) {
 	    int vsize;
-	    int n;
+	    ssize_t n;
 	    SysIOVec* iov;
 
 	    if ((iov = driver_peekq(ix, &vsize)) == NULL) {
@@ -8718,7 +9697,7 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event)
 	    vsize = vsize > MAX_VSIZE ? MAX_VSIZE : vsize;
 	    DEBUGF(("tcp_inet_output(%ld): s=%d, About to send %d items\r\n", 
 		    (long)desc->inet.port, desc->inet.s, vsize));
-	    if (sock_sendv(desc->inet.s, iov, vsize, &n, 0)==SOCKET_ERROR) {
+	    if (IS_SOCKET_ERROR(sock_sendv(desc->inet.s, iov, vsize, &n, 0))) {
 		if ((sock_errno() != ERRNO_BLOCK) && (sock_errno() != EINTR)) {
 		    DEBUGF(("tcp_inet_output(%ld): sock_sendv(%d) errno = %d\r\n",
 			    (long)desc->inet.port, vsize, sock_errno()));
@@ -8799,6 +9778,59 @@ static int should_use_so_bsdcompat(void)
 #endif	/* __linux__ */
 #endif	/* HAVE_SO_BSDCOMPAT */
 
+
+
+#ifdef HAVE_SCTP
+/* Copy a descriptor, by creating a new port with same settings
+ * as the descriptor desc.
+ * return NULL on error (ENFILE no ports avail)
+ */
+static udp_descriptor* sctp_inet_copy(udp_descriptor* desc, SOCKET s, int* err)
+{
+    ErlDrvPort port = desc->inet.port;
+    udp_descriptor* copy_desc;
+
+    copy_desc = (udp_descriptor*) sctp_inet_start(port, NULL);
+
+    /* Setup event if needed */
+    if ((copy_desc->inet.s = s) != INVALID_SOCKET) {
+	if ((copy_desc->inet.event = sock_create_event(INETP(copy_desc))) ==
+	    INVALID_EVENT) {
+	    *err = sock_errno();
+	    FREE(copy_desc);
+	    return NULL;
+	}
+    }
+
+    /* Some flags must be inherited at this point */
+    copy_desc->inet.mode     = desc->inet.mode;
+    copy_desc->inet.exitf    = desc->inet.exitf;
+    copy_desc->inet.bit8f    = desc->inet.bit8f;
+    copy_desc->inet.deliver  = desc->inet.deliver;
+    copy_desc->inet.htype    = desc->inet.htype;
+    copy_desc->inet.psize    = desc->inet.psize;
+    copy_desc->inet.stype    = desc->inet.stype;
+    copy_desc->inet.sfamily  = desc->inet.sfamily;
+    copy_desc->inet.hsz      = desc->inet.hsz;
+    copy_desc->inet.bufsz    = desc->inet.bufsz;
+
+    /* The new port will be linked and connected to the caller */
+    port = driver_create_port(port, desc->inet.caller, "sctp_inet",
+			      (ErlDrvData) copy_desc);
+    if ((long)port == -1) {
+	*err = ENFILE;
+	FREE(copy_desc);
+	return NULL;
+    }
+    copy_desc->inet.port = port;
+    copy_desc->inet.dport = driver_mk_port(port);
+    *err = 0;
+    return copy_desc;
+}
+#endif
+
+
+
 static int packet_inet_init()
 {
     return 0;
@@ -8817,6 +9849,9 @@ static ErlDrvData packet_inet_start(ErlDrvPort port, char* args, int protocol)
 	return ERL_DRV_ERROR_ERRNO;
 
     desc->read_packets = INET_PACKET_POLL;
+    desc->i_bufsz = 0;
+    desc->i_buf = NULL;
+    desc->i_ptr = NULL;
     return drvd;
 }
 
@@ -8841,6 +9876,10 @@ static void packet_inet_stop(ErlDrvData e)
     */
     udp_descriptor * udesc = (udp_descriptor*) e;
     inet_descriptor* descr = INETP(udesc);
+    if (udesc->i_buf != NULL) {
+	release_buffer(udesc->i_buf);
+	udesc->i_buf = NULL;
+    }
 
     ASSERT(NO_SUBSCRIBERS(&(descr->empty_out_q_subs)));
     inet_stop(descr);
@@ -8858,28 +9897,38 @@ static int packet_error(udp_descriptor* udesc, int err)
 /*
 ** Various functions accessible via "port_control" on the Erlang side:
 */
-static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
-			   char** rbuf, int rsize)
+static ErlDrvSSizeT packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf,
+				    ErlDrvSizeT len, char** rbuf, ErlDrvSizeT rsize)
 {
-    int replen;
+    ErlDrvSSizeT replen;
     udp_descriptor * udesc = (udp_descriptor *) e;
     inet_descriptor* desc  = INETP(udesc);
     int type = SOCK_DGRAM;
-    int af;
-#ifdef HAVE_SCTP
-    if (IS_SCTP(desc)) type = SOCK_SEQPACKET;
-#endif
+    int af = AF_INET;
 
     switch(cmd) {
     case INET_REQ_OPEN:   /* open socket and return internal index */
 	DEBUGF(("packet_inet_ctl(%ld): OPEN\r\n", (long)desc->port)); 
-	if (len != 1) {
+	if (len != 2) {
 	    return ctl_error(EINVAL, rbuf, rsize);
 	}
 	switch (buf[0]) {
 	case INET_AF_INET:  af = AF_INET; break;
 #if defined(HAVE_IN6) && defined(AF_INET6)
-	case INET_AF_INET6: af = AF_INET6; break; 
+	case INET_AF_INET6: af = AF_INET6; break;
+#else
+	case INET_AF_INET6:
+	    return ctl_xerror("eafnosupport", rbuf, rsize);
+	    break;
+#endif
+	default:
+	    return ctl_error(EINVAL, rbuf, rsize);
+	}
+	switch (buf[1]) {
+	case INET_TYPE_STREAM: type = SOCK_STREAM; break;
+	case INET_TYPE_DGRAM: type = SOCK_DGRAM; break;
+#ifdef HAVE_SCTP
+	case INET_TYPE_SEQPACKET: type = SOCK_SEQPACKET; break;
 #endif
 	default:
 	    return ctl_error(EINVAL, rbuf, rsize);
@@ -8906,18 +9955,35 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	return replen;
 
 
-    case INET_REQ_FDOPEN:   /* pass in an open (and bound) socket */
+    case INET_REQ_FDOPEN: {  /* pass in an open (and bound) socket */
+	SOCKET s;
 	DEBUGF(("packet inet_ctl(%ld): FDOPEN\r\n", (long)desc->port));
-	if ((len == 5) && (buf[0] == INET_AF_INET))
-	    replen = inet_ctl_fdopen(desc, AF_INET, SOCK_DGRAM,
-				     (SOCKET)get_int32(buf+1),rbuf,rsize);
+	if (len != 6) {
+	    return ctl_error(EINVAL, rbuf, rsize);
+	}
+	switch (buf[0]) {
+	case INET_AF_INET:  af = AF_INET; break;
 #if defined(HAVE_IN6) && defined(AF_INET6)
-	else if ((len == 5) && (buf[0] == INET_AF_INET6))
-	    replen = inet_ctl_fdopen(desc, AF_INET6, SOCK_DGRAM,
-				     (SOCKET)get_int32(buf+1),rbuf,rsize);
+	case INET_AF_INET6: af = AF_INET6; break;
+#else
+	case INET_AF_INET6:
+	    return ctl_xerror("eafnosupport", rbuf, rsize);
+	    break;
 #endif
-	else
+	default:
+	    return ctl_error(EINVAL, rbuf, rsize);
+	}
+	switch (buf[1]) {
+	case INET_TYPE_STREAM: type = SOCK_STREAM; break;
+	case INET_TYPE_DGRAM: type = SOCK_DGRAM; break;
+#ifdef HAVE_SCTP
+	case INET_TYPE_SEQPACKET: type = SOCK_SEQPACKET; break;
+#endif
+	default:
 	    return ctl_error(EINVAL, rbuf, rsize);
+	}
+	s = (SOCKET)get_int32(buf+2);
+	replen = inet_ctl_fdopen(desc, af, type, s, rbuf, rsize);
 
 	if ((*rbuf)[0] != INET_REP_ERROR) {
 	    if (desc->active)
@@ -8937,13 +10003,13 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 #endif
 	}
 	return replen;
+    }
 
 
     case INET_REQ_CLOSE:
 	DEBUGF(("packet_inet_ctl(%ld): CLOSE\r\n", (long)desc->port)); 
 	erl_inet_close(desc);
 	return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
-	return 0;
 
 
     case INET_REQ_CONNECT:  {
@@ -8956,8 +10022,9 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	*/
 	int code;
 	char tbuf[2];
+#ifdef HAVE_SCTP
 	unsigned timeout;
-
+#endif
 	DEBUGF(("packet_inet_ctl(%ld): CONNECT\r\n", (long)desc->port)); 
 	
 	/* INPUT: [ Timeout(4), Port(2), Address(N) ] */
@@ -8987,16 +10054,16 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	    sock_select(desc, FD_CONNECT, 1);
 	    code = sock_connect(desc->s, &remote.sa, len);
 
-	    if ((code == SOCKET_ERROR) && (sock_errno() == EINPROGRESS)) {
+	    if (IS_SOCKET_ERROR(code) && (sock_errno() == EINPROGRESS)) {
 		/* XXX: Unix only -- WinSock would have a different cond! */
-		desc->state = SCTP_STATE_CONNECTING;
+		desc->state = INET_STATE_CONNECTING;
 		if (timeout != INET_INFINITY)
 		    driver_set_timer(desc->port, timeout);
 		enq_async(desc, tbuf, INET_REQ_CONNECT);
 	    }
 	    else if (code == 0) { /* OK we are connected */
 		sock_select(desc, FD_CONNECT, 0);
-		desc->state = PACKET_STATE_CONNECTED;
+		desc->state = INET_STATE_CONNECTED;
 		enq_async(desc, tbuf, INET_REQ_CONNECT);
 		async_ok(desc);
 	    }
@@ -9018,7 +10085,7 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	else if (len < 6)
 	    return ctl_error(EINVAL, rbuf, rsize);
 	else {
-	    timeout = get_int32(buf); /* IGNORED */
+	    /* Ignore timeout */
 	    buf += 4;
 	    len -= 4;
 	    if (inet_set_address(desc->sfamily, 
@@ -9027,7 +10094,7 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	    
 	    code = sock_connect(desc->s,
 				(struct sockaddr*) &desc->remote, len);
-	    if (code == SOCKET_ERROR) {
+	    if (IS_SOCKET_ERROR(code)) {
 		sock_connect(desc->s, (struct sockaddr*) NULL, 0);
 		desc->state &= ~INET_F_ACTIVE;
 		return ctl_error(sock_errno(), rbuf, rsize);
@@ -9042,11 +10109,11 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
     }
 
 #ifdef HAVE_SCTP
-    case SCTP_REQ_LISTEN:
+    case INET_REQ_LISTEN:
 	{	/* LISTEN is only for SCTP sockets, not UDP. This code is borrowed
 		   from the TCP section. Returns: {ok,[]} on success.
 		*/
-	    int flag;
+	    int backlog;
 	    
 	    DEBUGF(("packet_inet_ctl(%ld): LISTEN\r\n", (long)desc->port)); 
 	    if (!IS_SCTP(desc))
@@ -9056,15 +10123,14 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	    if (!IS_BOUND(desc))
 		return ctl_xerror(EXBADSEQ, rbuf, rsize);
 
-	    /* The arg is a binary value: 1:enable, 0:disable */
-	    if (len != 1)
+	    if (len != 2)
 		return ctl_error(EINVAL, rbuf, rsize);
-	    flag = get_int8(buf);
+	    backlog = get_int16(buf);
 
-	    if (sock_listen(desc->s, flag) == SOCKET_ERROR)
+	    if (IS_SOCKET_ERROR(sock_listen(desc->s, backlog)))
 		return ctl_error(sock_errno(), rbuf, rsize);
 
-	    desc->state = SCTP_STATE_LISTEN;   /* XXX: not used? */
+	    desc->state = INET_STATE_LISTENING;   /* XXX: not used? */
 	    return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
 	}
 
@@ -9089,7 +10155,7 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 		    /* List item format: Port(2), IP(4|16) -- compatible with
 		       "inet_set_address": */
 		    inet_address tmp;
-		    int  alen  = buf + len - curr;
+		    ErlDrvSizeT alen  = buf + len - curr;
 		    curr = inet_set_address(desc->sfamily, &tmp, curr, &alen);
 		    if (curr == NULL)
 			return ctl_error(EINVAL, rbuf, rsize);
@@ -9110,6 +10176,46 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 
 	    return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize);
 	}
+
+    case SCTP_REQ_PEELOFF:
+	{
+	    Uint32 assoc_id;
+	    udp_descriptor* new_udesc;
+	    int err;
+	    SOCKET new_socket;
+
+	    DEBUGF(("packet_inet_ctl(%ld): PEELOFF\r\n", (long)desc->port));
+	    if (!IS_SCTP(desc))
+		return ctl_xerror(EXBADPORT, rbuf, rsize);
+	    if (!IS_OPEN(desc))
+		return ctl_xerror(EXBADPORT, rbuf, rsize);
+	    if (!IS_BOUND(desc))
+		return ctl_xerror(EXBADSEQ, rbuf, rsize);
+	    if (! p_sctp_peeloff)
+		return ctl_error(ENOTSUP, rbuf, rsize);
+
+	    if (len != 4)
+		return ctl_error(EINVAL, rbuf, rsize);
+	    assoc_id = get_int32(buf);
+
+	    new_socket = p_sctp_peeloff(desc->s, assoc_id);
+	    if (IS_SOCKET_ERROR(new_socket)) {
+		return ctl_error(sock_errno(), rbuf, rsize);
+	    }
+
+	    desc->caller = driver_caller(desc->port);
+	    if ((new_udesc = sctp_inet_copy(udesc, new_socket, &err)) == NULL) {
+		sock_close(new_socket);
+		desc->caller = 0;
+		return ctl_error(err, rbuf, rsize);
+	    }
+	    new_udesc->inet.state = INET_STATE_CONNECTED;
+	    new_udesc->inet.stype = SOCK_STREAM;
+
+	    inet_reply_ok_port(desc, new_udesc->inet.dport);
+	    (*rbuf)[0] = INET_REP;
+	    return 1;
+	}
 #endif  /* HAVE_SCTP */
 
     case PACKET_REQ_RECV:
@@ -9129,7 +10235,7 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len,
 	    if (desc->active || (len != 8))
 		return ctl_error(EINVAL, rbuf, rsize);
 	    timeout = get_int32(buf);
-	    /* The 2nd arg, Length(4), is ignored for both UDP ans SCTP protocols,
+	    /* The 2nd arg, Length(4), is ignored for both UDP and SCTP protocols,
 	       since they are msg-oriented. */
 
 	    if (enq_async(desc, tbuf, PACKET_REQ_RECV) < 0)
@@ -9174,13 +10280,13 @@ static void packet_inet_timeout(ErlDrvData e)
 ** There is no destination address -- SCTYP send is performed over
 ** an existing association, using "sctp_sndrcvinfo" specified.
 */
-static void packet_inet_command(ErlDrvData e, char* buf, int len)
+static void packet_inet_command(ErlDrvData e, char* buf, ErlDrvSizeT len)
 {
     udp_descriptor * udesc= (udp_descriptor*) e;
     inet_descriptor* desc = INETP(udesc);
     char* ptr		  = buf;
     char* qtr;
-    int sz;
+    ErlDrvSizeT sz;
     int code;
     inet_address other;
 
@@ -9198,7 +10304,7 @@ static void packet_inet_command(ErlDrvData e, char* buf, int len)
 #ifdef HAVE_SCTP
     if (IS_SCTP(desc))
     {
-	int           data_len;
+	ErlDrvSizeT   data_len;
 	struct iovec  iov[1];		 /* For real data            */
 	struct msghdr mhdr;		 /* Message wrapper          */
 	struct sctp_sndrcvinfo *sri;     /* The actual ancilary data */
@@ -9234,6 +10340,7 @@ static void packet_inet_command(ErlDrvData e, char* buf, int len)
 	mhdr.msg_iovlen         = 1;
 	mhdr.msg_control        = cmsg.ancd;    /* For ancilary data  */
 	mhdr.msg_controllen     = cmsg.hdr.cmsg_len;
+	VALGRIND_MAKE_MEM_DEFINED(mhdr.msg_control, mhdr.msg_controllen); /*suppress "uninitialised bytes"*/
 	mhdr.msg_flags          = 0;            /* Not used with "sendmsg"   */
 	
 	/* Now do the actual sending. NB: "flags" in "sendmsg" itself are NOT
@@ -9266,7 +10373,7 @@ static void packet_inet_command(ErlDrvData e, char* buf, int len)
  check_result_code:
     /* "code" analysis is the same for both SCTP and UDP cases above: */
 #endif
-    if (code == SOCKET_ERROR) {
+    if (IS_SOCKET_ERROR(code)) {
 	int err = sock_errno();
 	inet_reply_error(desc, err);
     }
@@ -9308,12 +10415,8 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event)
 {
     inet_descriptor* desc = INETP(udesc);
     int n;
-    unsigned int len;
     inet_address other;
     char abuf[sizeof(inet_address)];  /* buffer address; enough??? */
-    int  sz;
-    char* ptr;
-    ErlDrvBinary* buf; /* binary */
     int packet_count = udesc->read_packets;
     int count = 0;     /* number of packets delivered to owner */
 #ifdef HAVE_SCTP
@@ -9324,23 +10427,39 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event)
 #endif
 
     while(packet_count--) {
-	len = sizeof(other);
-	sz = desc->bufsz;
-	/* Allocate space for message and address. NB: "bufsz" is in "desc",
-	   but the "buf" itself is allocated separately:
-	*/
-	if ((buf = alloc_buffer(sz+len)) == NULL)
-	    return packet_error(udesc, ENOMEM);
-	ptr = buf->orig_bytes + len;  /* pointer to message part */
+	unsigned int len = sizeof(other);
+
+	/* udesc->i_buf is only kept between SCTP fragments */
+	if (udesc->i_buf == NULL) {
+	    udesc->i_bufsz = desc->bufsz + len;
+	    if ((udesc->i_buf = alloc_buffer(udesc->i_bufsz)) == NULL)
+		return packet_error(udesc, ENOMEM);
+	    /* pointer to message start */
+	    udesc->i_ptr = udesc->i_buf->orig_bytes + len;
+	} else {
+	    ErlDrvBinary* tmp;
+	    int bufsz;
+	    bufsz = desc->bufsz + (udesc->i_ptr - udesc->i_buf->orig_bytes);
+	    if ((tmp = realloc_buffer(udesc->i_buf, bufsz)) == NULL) {
+		release_buffer(udesc->i_buf);
+		udesc->i_buf = NULL;
+		return packet_error(udesc, ENOMEM);
+	    } else {
+		udesc->i_ptr =
+		    tmp->orig_bytes + (udesc->i_ptr - udesc->i_buf->orig_bytes);
+		udesc->i_buf = tmp;
+		udesc->i_bufsz = bufsz;
+	    }
+	}
 
 	/* Note: On Windows NT, recvfrom() fails if the socket is connected. */
 #ifdef HAVE_SCTP
 	/* For SCTP we must use recvmsg() */
 	if (IS_SCTP(desc)) {
-	    iov->iov_base = ptr; /* Data will come here    */
-	    iov->iov_len  = sz;	 /* Remaining buffer space */
+	    iov->iov_base = udesc->i_ptr; /* Data will come here    */
+	    iov->iov_len = desc->bufsz; /* Remaining buffer space */
 	    
-	    mhdr.msg_name	= &other;  /* Peer addr comes into "other" */
+	    mhdr.msg_name	= &other; /* Peer addr comes into "other" */
 	    mhdr.msg_namelen	= len;
 	    mhdr.msg_iov	= iov;
 	    mhdr.msg_iovlen	= 1;
@@ -9350,40 +10469,28 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event)
 	    
 	    /* Do the actual SCTP receive: */
 	    n = sock_recvmsg(desc->s, &mhdr, 0);
+	    len = mhdr.msg_namelen;
 	    goto check_result;
 	}
 #endif
 	/* Use recv() instead on connected sockets. */
 	if ((desc->state & INET_F_ACTIVE)) {
-	    n = sock_recv(desc->s, ptr, sz, 0);
+	    n = sock_recv(desc->s, udesc->i_ptr, desc->bufsz, 0);
 	    other = desc->remote;
+	    goto check_result;
 	}
-	else
-	    n = sock_recvfrom(desc->s, ptr, sz, 0, &other.sa, &len);
-
-#ifdef HAVE_SCTP
+	n = sock_recvfrom(desc->s, udesc->i_ptr, desc->bufsz,
+			  0, &other.sa, &len);
     check_result:
-#endif
 	/* Analyse the result: */
-	if (n == SOCKET_ERROR
-#ifdef HAVE_SCTP
-	    || (short_recv = (IS_SCTP(desc) && !(mhdr.msg_flags & MSG_EOR)))
-	    /* NB: here we check for EOR not being set -- this is an error as
-	       well, we don't support partial msgs:
-	    */
-#endif
-	    ) {
+	if (IS_SOCKET_ERROR(n)) {
 	    int err = sock_errno();
-	    release_buffer(buf);
 	    if (err != ERRNO_BLOCK) {
+		/* real error */
+		release_buffer(udesc->i_buf);
+		udesc->i_buf = NULL;
 		if (!desc->active) {
-#ifdef HAVE_SCTP
-		    if (short_recv)
-			async_error_am(desc, am_short_recv);
-		    else
-#else
-			async_error(desc, err);
-#endif
+		    async_error(desc, err);
 		    driver_cancel_timer(desc->port);
 		    sock_select(desc,FD_READ,0);
 		}
@@ -9391,46 +10498,72 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event)
 		    /* This is for an active desc only: */
 		    packet_error_message(udesc, err);
 		}
+		return count;
 	    }
-	    else if (!desc->active)
+	    /* would block error - try again */
+	    if (!desc->active
+#ifdef HAVE_SCTP
+		|| short_recv
+#endif
+		) {
 		sock_select(desc,FD_READ,1);
+	    }
 	    return count;		/* strange, not ready */
 	}
-	else {
-	    int offs;
-	    int nsz;
+
+#ifdef HAVE_SCTP
+	if (IS_SCTP(desc) && (short_recv = !(mhdr.msg_flags & MSG_EOR))) {
+	    /* SCTP non-final message fragment */
+	    inet_input_count(desc, n);
+	    udesc->i_ptr += n;
+	    continue; /* wait for more fragments */
+	}
+#endif
+
+	{
+	    /* message received */
 	    int code;
-	    unsigned int alen = len;
 	    void * extra = NULL;
+	    char * ptr;
+	    int nsz;
 
 	    inet_input_count(desc, n);
-	    inet_get_address(desc->sfamily, abuf, &other, &alen);
-	    /* Copy formatted address to the buffer allocated; "alen" is the
-	       actual length which must be <= than the original reserved "len".
+	    udesc->i_ptr += n;
+	    inet_get_address(desc->sfamily, abuf, &other, &len);
+	    /* Copy formatted address to the buffer allocated; "len" is the
+	       actual length which must be <= than the original reserved.
 	       This means that the addr + data in the buffer are contiguous,
-	       but they may start not at the "orig_bytes", but with some "offs"
-	       from them:
+	       but they may start not at the "orig_bytes", instead at "ptr":
 	    */
-	    ASSERT (alen <= len);
-	    sys_memcpy(ptr - alen, abuf, alen); 
-	    ptr -= alen;
-	    nsz  = n + alen;              /* nsz = data + address */
-	    offs = ptr - buf->orig_bytes; /* initial pointer offset */
+	    ASSERT (len <= sizeof(other));
+	    ptr = udesc->i_buf->orig_bytes + sizeof(other) - len;
+	    sys_memcpy(ptr, abuf, len);
+
+	    nsz = udesc->i_ptr - ptr;
 
 	    /* Check if we need to reallocate binary */
-	    if ((desc->mode == INET_MODE_BINARY) &&
-		(desc->hsz < n) && (nsz < BIN_REALLOC_LIMIT(sz))) {
+	    if ((desc->mode == INET_MODE_BINARY)
+		&& (desc->hsz < (nsz - len))
+		&& (nsz + BIN_REALLOC_MARGIN(desc->bufsz) < udesc->i_bufsz)) {
 		ErlDrvBinary* tmp;
-		if ((tmp = realloc_buffer(buf,nsz+offs)) != NULL)
-		    buf = tmp;
+		int bufsz;
+		bufsz = udesc->i_ptr - udesc->i_buf->orig_bytes;
+		if ((tmp = realloc_buffer(udesc->i_buf, bufsz)) != NULL) {
+		    udesc->i_buf = tmp;
+		    udesc->i_bufsz = bufsz;
+		    udesc->i_ptr = NULL;  /* not used from here */
+		}
 	    }
 #ifdef HAVE_SCTP
 	    if (IS_SCTP(desc)) extra = &mhdr;
 #endif
 	    /* Actual parsing and return of the data received, occur here: */
-	    code = packet_reply_binary_data(desc, (unsigned int)alen,
-					    buf, offs, nsz, extra);
-	    free_buffer(buf);
+	    code = packet_reply_binary_data(desc, len, udesc->i_buf,
+					    (sizeof(other) - len),
+					    nsz,
+					    extra);
+	    free_buffer(udesc->i_buf);
+	    udesc->i_buf = NULL;
 	    if (code < 0)
 		return count;
 	    count++;
@@ -9440,7 +10573,17 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event)
 		return count;  /* passive mode (read one packet only) */
 	    }
 	}
+    } /*  while(packet_count--) { */
+
+    /* we ran out of tries (packet_count) either on an active socket
+     * that got that many messages or an SCTP socket that got that
+     * many message fragments but still not the final
+     */
+#ifdef HAVE_SCTP
+    if (short_recv) {
+	sock_select(desc, FD_READ, 1);
     }
+#endif
     return count;
 }
 
@@ -9450,7 +10593,7 @@ static void packet_inet_drv_output(ErlDrvData e, ErlDrvEvent event)
 }
 
 /* UDP/SCTP socket ready for output:
-**	This is a Back-End for Non-Block SCTP Connect (SCTP_STATE_CONNECTING)
+**	This is a Back-End for Non-Block SCTP Connect (INET_STATE_CONNECTING)
 */
 static int packet_inet_output(udp_descriptor* udesc, HANDLE event)
 {
@@ -9461,7 +10604,7 @@ static int packet_inet_output(udp_descriptor* udesc, HANDLE event)
     DEBUGF(("packet_inet_output(%ld) {s=%d\r\n", 
 	    (long)desc->port, desc->s));
 
-    if (desc->state == SCTP_STATE_CONNECTING) {
+    if (desc->state == INET_STATE_CONNECTING) {
 	sock_select(desc, FD_CONNECT, 0);
 
 	driver_cancel_timer(ix);  /* posssibly cancel a timer */
@@ -9480,8 +10623,8 @@ static int packet_inet_output(udp_descriptor* udesc, HANDLE event)
 	    int code = sock_peer(desc->s,
 				 (struct sockaddr*) &desc->remote, &sz);
 
-	    if (code == SOCKET_ERROR) {
-		desc->state = PACKET_STATE_BOUND;  /* restore state */
+	    if (IS_SOCKET_ERROR(code)) {
+		desc->state = INET_STATE_BOUND;  /* restore state */
 		ret =  async_error(desc, sock_errno());
 		goto done;
 	    }
@@ -9494,15 +10637,15 @@ static int packet_inet_output(udp_descriptor* udesc, HANDLE event)
 				   (void *)&error, &sz);
 
 	    if ((code < 0) || error) {
-		desc->state = PACKET_STATE_BOUND;  /* restore state */
+		desc->state = INET_STATE_BOUND;  /* restore state */
 		ret = async_error(desc, error);
 		goto done;
 	    }
 	}
-#endif /* SOCKOPT_CONNECT_STAT */
+#endif /* SO_ERROR */
 #endif /* !__WIN32__ */
 
-	desc->state = PACKET_STATE_CONNECTED;
+	desc->state = INET_STATE_CONNECTED;
 	async_ok(desc);
     }
     else {
@@ -9909,7 +11052,7 @@ int erts_sock_connect(erts_sock_t socket, byte *ip_addr, int len, Uint16 port)
 {
     SOCKET s = (SOCKET) socket;
     char buf[2 + 4];
-    int blen = 6;
+    ErlDrvSizeT blen = 6;
     inet_address addr;
 
     if (socket == ERTS_SOCK_INVALID_SOCKET || len != 4)
@@ -9921,23 +11064,26 @@ int erts_sock_connect(erts_sock_t socket, byte *ip_addr, int len, Uint16 port)
     if (!inet_set_address(AF_INET, &addr, buf, &blen))
 	return 0;
 
-    if (SOCKET_ERROR == sock_connect(s,
+    if (IS_SOCKET_ERROR(sock_connect(s,
 				     (struct sockaddr *) &addr,
-				     sizeof(struct sockaddr_in)))
+				     sizeof(struct sockaddr_in))))
 	return 0;
     return 1;
 }
 
 Sint erts_sock_send(erts_sock_t socket, const void *buf, Sint len)
 {
-    return (Sint) sock_send((SOCKET) socket, buf, (size_t) len, 0);
+    Sint result = (Sint) sock_send((SOCKET) socket, buf, (size_t) len, 0);
+    if (IS_SOCKET_ERROR(result))
+	return SOCKET_ERROR;
+    return result;
 }
 
 
 int erts_sock_gethostname(char *buf, int bufsz)
 {
-    if (sock_hostname(buf, bufsz) == SOCKET_ERROR)
-	return -1;
+    if (IS_SOCKET_ERROR(sock_hostname(buf, bufsz)))
+	return SOCKET_ERROR;
     return 0;
 }
 
diff --git a/erts/emulator/drivers/common/ram_file_drv.c b/erts/emulator/drivers/common/ram_file_drv.c
index 4a39a156e6..a109e40333 100644
--- a/erts/emulator/drivers/common/ram_file_drv.c
+++ b/erts/emulator/drivers/common/ram_file_drv.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1997-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -35,6 +35,7 @@
 #define RAM_FILE_TRUNCATE      14
 #define RAM_FILE_PREAD         17
 #define RAM_FILE_PWRITE        18
+#define RAM_FILE_FDATASYNC     19
 
 /* other operations */
 #define RAM_FILE_GET           30
@@ -45,6 +46,8 @@
 #define RAM_FILE_UUENCODE      35  /* uuencode file */
 #define RAM_FILE_UUDECODE      36  /* uudecode file */
 #define RAM_FILE_SIZE          37  /* get file size */
+#define RAM_FILE_ADVISE        38  /* predeclare the access
+                                    * pattern for file data */
 /* possible new operations include:
    DES_ENCRYPT
    DES_DECRYPT
@@ -95,7 +98,7 @@ typedef unsigned char uchar;
 static ErlDrvData rfile_start(ErlDrvPort, char*);
 static int rfile_init(void);
 static void rfile_stop(ErlDrvData);
-static void rfile_command(ErlDrvData, char*, int);
+static void rfile_command(ErlDrvData, char*, ErlDrvSizeT);
 
 
 struct erl_drv_entry ram_file_driver_entry = {
@@ -105,7 +108,23 @@ struct erl_drv_entry ram_file_driver_entry = {
     rfile_command,
     NULL,
     NULL,
-    "ram_file_drv"
+    "ram_file_drv",
+    NULL,
+    NULL, /* handle */
+    NULL, /* control */
+    NULL, /* timeout */
+    NULL, /* outputv */
+    NULL, /* ready_async */
+    NULL, /* flush */
+    NULL, /* call */
+    NULL, /* event */
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
+    NULL,
 };
 
 /* A File is represented as a array of bytes, this array is
@@ -118,9 +137,9 @@ typedef struct ram_file {
     int flags;          /* flags read/write */
     ErlDrvBinary* bin;  /* binary to hold binary file */
     char* buf;          /* buffer start (in binary) */
-    int size;           /* buffer size (allocated) */
-    int cur;            /* current position in buffer */
-    int end;            /* end position in buffer */
+    ErlDrvSSizeT size;   /* buffer size (allocated) */
+    ErlDrvSSizeT cur;    /* current position in buffer */
+    ErlDrvSSizeT end;            /* end position in buffer */
 } RamFile;
 
 #ifdef LOADABLE
@@ -208,7 +227,7 @@ static int reply(RamFile *f, int ok, int err)
     return 0;
 }
 
-static int numeric_reply(RamFile *f, int result)
+static int numeric_reply(RamFile *f, ErlDrvSSizeT result)
 {
     char tmp[5];
 
@@ -228,7 +247,8 @@ static int numeric_reply(RamFile *f, int result)
 
 /* install bin as the new binary reset all pointer */
 
-static void ram_file_set(RamFile *f, ErlDrvBinary *bin, int bsize, int len)
+static void ram_file_set(RamFile *f, ErlDrvBinary *bin,
+			 ErlDrvSSizeT bsize, ErlDrvSSizeT len)
 {
     f->size = bsize;
     f->buf = bin->orig_bytes;
@@ -237,9 +257,9 @@ static void ram_file_set(RamFile *f, ErlDrvBinary *bin, int bsize, int len)
     f->bin = bin;
 }
 
-static int ram_file_init(RamFile *f, char *buf, int count, int *error)
+static int ram_file_init(RamFile *f, char *buf, ErlDrvSSizeT count, int *error)
 {
-    int bsize;
+    ErlDrvSSizeT bsize;
     ErlDrvBinary* bin;
     
     if (count < 0) {
@@ -265,9 +285,9 @@ static int ram_file_init(RamFile *f, char *buf, int count, int *error)
     return count;
 }
 
-static int ram_file_expand(RamFile *f, int size, int *error)
+static ErlDrvSSizeT ram_file_expand(RamFile *f, ErlDrvSSizeT size, int *error)
 {
-    int bsize;
+    ErlDrvSSizeT bsize;
     ErlDrvBinary* bin;
     
     if (size < 0) {
@@ -295,10 +315,10 @@ static int ram_file_expand(RamFile *f, int size, int *error)
 }
 
 
-static int ram_file_write(RamFile *f, char *buf, int len, 
-			  int *location, int *error)
+static ErlDrvSSizeT ram_file_write(RamFile *f, char *buf, ErlDrvSSizeT len,
+			  ErlDrvSSizeT *location, int *error)
 {
-    int cur = f->cur;
+    ErlDrvSSizeT cur = f->cur;
     
     if (!(f->flags & RAM_FILE_MODE_WRITE)) {
 	*error = EBADF;
@@ -319,11 +339,11 @@ static int ram_file_write(RamFile *f, char *buf, int len,
     return len;
 }
 
-static int ram_file_read(RamFile *f, int len, ErlDrvBinary **bp, 
-			 int *location, int *error)
+static ErlDrvSSizeT ram_file_read(RamFile *f, ErlDrvSSizeT len, ErlDrvBinary **bp,
+			 ErlDrvSSizeT *location, int *error)
 {
     ErlDrvBinary* bin;
-    int cur = f->cur;
+    ErlDrvSSizeT cur = f->cur;
     
     if (!(f->flags & RAM_FILE_MODE_READ)) {
 	*error = EBADF;
@@ -349,9 +369,10 @@ static int ram_file_read(RamFile *f, int len, ErlDrvBinary **bp,
     return len;
 }
 
-static int ram_file_seek(RamFile *f, int offset, int whence, int *error)
+static ErlDrvSSizeT ram_file_seek(RamFile *f, ErlDrvSSizeT offset, int whence,
+				 int *error)
 {
-    int pos;
+    ErlDrvSSizeT pos;
 
     if (f->flags == 0) {
 	*error = EBADF;
@@ -386,13 +407,13 @@ static int ram_file_seek(RamFile *f, int offset, int whence, int *error)
 
 static int ram_file_uuencode(RamFile *f)
 {
-    int code_len = UULINE(UNIX_LINE);
-    int len = f->end;
-    int usize = 4*((len+2)/3) + 2*((len+code_len-1)/code_len) + 2;    
+    ErlDrvSSizeT code_len = UULINE(UNIX_LINE);
+    ErlDrvSSizeT len = f->end;
+    ErlDrvSSizeT usize = 4*((len+2)/3) + 2*((len+code_len-1)/code_len) + 2;
     ErlDrvBinary* bin;
     uchar* inp;
     uchar* outp;
-    int count = 0;
+    ErlDrvSSizeT count = 0;
 
     if ((bin = driver_alloc_binary(usize)) == NULL)
 	return error_reply(f, ENOMEM);
@@ -444,8 +465,8 @@ static int ram_file_uuencode(RamFile *f)
 
 static int ram_file_uudecode(RamFile *f)
 {
-    int len = f->end;
-    int usize = ( (len+3) / 4 ) * 3;
+    ErlDrvSSizeT len = f->end;
+    ErlDrvSSizeT usize = ( (len+3) / 4 ) * 3;
     ErlDrvBinary* bin;
     uchar* inp;
     uchar* outp;
@@ -507,7 +528,7 @@ static int ram_file_uudecode(RamFile *f)
 
 static int ram_file_compress(RamFile *f)
 {
-    int size = f->end;
+    ErlDrvSSizeT size = f->end;
     ErlDrvBinary* bin;
 
     if ((bin = erts_gzdeflate_buffer(f->buf, size)) == NULL) {
@@ -525,7 +546,7 @@ static int ram_file_compress(RamFile *f)
 
 static int ram_file_uncompress(RamFile *f)
 {
-    int size = f->end;
+    ErlDrvSSizeT size = f->end;
     ErlDrvBinary* bin;
 
     if ((bin = erts_gzinflate_buffer(f->buf, size)) == NULL) {
@@ -538,15 +559,15 @@ static int ram_file_uncompress(RamFile *f)
 }
 
 
-static void rfile_command(ErlDrvData e, char* buf, int count)
+static void rfile_command(ErlDrvData e, char* buf, ErlDrvSizeT count)
 {
     RamFile* f = (RamFile*)e;
     int error = 0;
     ErlDrvBinary* bin;
     char header[5];     /* result code + count */
-    int offset;
-    int origin;		/* Origin of seek. */
-    int n;
+    ErlDrvSSizeT offset;
+    ErlDrvSSizeT origin;		/* Origin of seek. */
+    ErlDrvSSizeT n;
 
     count--;
     switch(*(uchar*)buf++) {
@@ -558,6 +579,13 @@ static void rfile_command(ErlDrvData e, char* buf, int count)
 	    numeric_reply(f, 0); /* 0 is not used */
 	break;
 
+    case RAM_FILE_FDATASYNC:
+	if (f->flags == 0)
+	    error_reply(f, EBADF);
+	else
+	    reply(f, 1, 0);
+	break;
+
     case RAM_FILE_FSYNC:
 	if (f->flags == 0)
 	    error_reply(f, EBADF);
@@ -685,6 +713,13 @@ static void rfile_command(ErlDrvData e, char* buf, int count)
     case RAM_FILE_UUDECODE:   /* uudecode file */
 	ram_file_uudecode(f);
 	break;
+
+    case RAM_FILE_ADVISE:
+	if (f->flags == 0)
+	    error_reply(f, EBADF);
+	else
+	    reply(f, 1, 0);
+	break;
     }
     /*
      * Ignore anything else -- let the caller hang.
diff --git a/erts/emulator/drivers/common/zlib_drv.c b/erts/emulator/drivers/common/zlib_drv.c
index 723efeaa13..60394b610b 100644
--- a/erts/emulator/drivers/common/zlib_drv.c
+++ b/erts/emulator/drivers/common/zlib_drv.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -64,8 +64,9 @@
 static int zlib_init(void);
 static ErlDrvData zlib_start(ErlDrvPort port, char* buf);
 static void zlib_stop(ErlDrvData e);
-static int zlib_ctl(ErlDrvData drv_data, unsigned int command, char *buf, 
-		    int len, char **rbuf, int rlen);
+static void zlib_flush(ErlDrvData e);
+static ErlDrvSSizeT zlib_ctl(ErlDrvData drv_data, unsigned int command, char *buf,
+			     ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen);
 static void zlib_outputv(ErlDrvData drv_data, ErlIOVec *ev);
 
 ErlDrvEntry zlib_driver_entry = {
@@ -82,7 +83,7 @@ ErlDrvEntry zlib_driver_entry = {
     NULL,                           /* timeout */
     zlib_outputv,
     NULL,                           /* read_async */
-    NULL,                           /* flush */
+    zlib_flush,
     NULL,                           /* call */
     NULL,                           /* event */
     ERL_DRV_EXTENDED_MARKER,
@@ -115,7 +116,7 @@ typedef struct {
 static int zlib_inflate(ZLibData* d, int flush);
 static int zlib_deflate(ZLibData* d, int flush);
 
-#if defined(_OSE_) || defined(__WIN32__)
+#if defined(__WIN32__)
 static int i32(char* buf)
 #else
 static inline int i32(char* buf)
@@ -162,12 +163,12 @@ static char* zlib_reason(int code, int* err)
 }
 
 
-static int zlib_return(int code, char** rbuf, int rlen)
+static ErlDrvSSizeT zlib_return(int code, char** rbuf, ErlDrvSizeT rlen)
 {
     int msg_code = 0; /* 0=ok, 1=error */
     char* dst = *rbuf;
     char* src;
-    int len = 0;
+    ErlDrvSizeT len = 0;
 
     src = zlib_reason(code, &msg_code);
     *dst++ = msg_code;
@@ -182,7 +183,8 @@ static int zlib_return(int code, char** rbuf, int rlen)
     return len;
 }
 
-static int zlib_value2(int msg_code, int value, char** rbuf, int rlen)
+static ErlDrvSSizeT zlib_value2(int msg_code, int value,
+				char** rbuf, ErlDrvSizeT rlen)
 {
     char* dst = *rbuf;
 
@@ -197,7 +199,7 @@ static int zlib_value2(int msg_code, int value, char** rbuf, int rlen)
     return 5;
 }
 
-static int zlib_value(int value, char** rbuf, int rlen)
+static ErlDrvSSizeT zlib_value(int value, char** rbuf, ErlDrvSizeT rlen)
 {
     return zlib_value2(2, value, rbuf, rlen);
 }
@@ -409,8 +411,15 @@ static void zlib_stop(ErlDrvData e)
     driver_free(d);
 }
 
-static int zlib_ctl(ErlDrvData drv_data, unsigned int command, char *buf, 
-		    int len, char **rbuf, int rlen)
+static void zlib_flush(ErlDrvData drv_data)
+{
+    ZLibData* d = (ZLibData*) drv_data;
+
+    driver_deq(d->port, driver_sizeq(d->port));
+}
+
+static ErlDrvSSizeT zlib_ctl(ErlDrvData drv_data, unsigned int command, char *buf,
+			     ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen)
 {
     ZLibData* d = (ZLibData*)drv_data;
     int res;
diff --git a/erts/emulator/drivers/unix/mem_drv.c b/erts/emulator/drivers/unix/mem_drv.c
deleted file mode 100644
index 1417ca1121..0000000000
--- a/erts/emulator/drivers/unix/mem_drv.c
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-
-/* Purpose: Access to elib memory statistics */
-
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-
-#include "sys.h"
-#include "erl_driver.h"
-#include "elib_stat.h"
-
-#define MAP_BUF_SIZE    1000   /* Max map size */
-#define HISTO_BUF_SIZE  100    /* Max histogram buckets */
-
-static ErlDrvData mem_start(ErlDrvPort);
-static int mem_init(void); 
-static void mem_stop(ErlDrvData);
-static void mem_command(ErlDrvData, char*, int);
-
-const struct driver_entry mem_driver_entry = {
-    mem_init,
-    mem_start,
-    mem_stop,
-    mem_command,
-    NULL,
-    NULL,
-    "mem_drv"
-};
-
-static int mem_init(void)
-{
-    return 0;
-}
-
-static ErlDrvData mem_start(ErlDrvPort port, char* buf)
-{
-    return (ErlDrvData)port;
-}
-
-static void mem_stop(ErlDrvData port)
-{
-}
-
-void putint32(p, v)
-byte* p; int v;
-{
-    p[0] = (v >> 24) & 0xff;
-    p[1] = (v >> 16) & 0xff;
-    p[2] = (v >> 8) & 0xff;
-    p[3] = (v) & 0xff;
-}
-
-int getint16(p)
-byte* p;
-{
-    return (p[0] << 8) | p[1];
-}
-
-/*
-** Command:
-**   m L1 L0  -> a heap map of length L1*256 + L0 is returned
-**   s        -> X3 X2 X1 X0 Y3 Y2 Y1 Y0 Z3 Z2 Z1 Z0
-**               X == Total heap size bytes
-**               Y == Total free bytes
-**               Z == Size of largest free block in bytes
-**
-**   h L1 L0 B0 -> Generate a logarithm historgram base B with L buckets
-**   l L1 L0 S0 -> Generate a linear histogram with step S with L buckets
-*/
-unsigned char outbuf[HISTO_BUF_SIZE*2*4];
-
-static void mem_command(ErlDrvData port, char* buf, int count)
-{
-    if ((count == 1) && buf[0] == 's') {
-	struct elib_stat info;
-	char v[3*4];
-
-	elib_stat(&info);
-
-	putint32(v, info.mem_total*4);
-	putint32(v+4, info.mem_free*4);
-	putint32(v+8, info.max_free*4);
-	driver_output((ErlDrvPort)port, v, 12);
-	return;
-    }
-    else if ((count == 3) && buf[0] == 'm') {
-	char w[MAP_BUF_SIZE];
-	int n = getint16(buf+1);
-
-	if (n > MAP_BUF_SIZE)
-	    n = MAP_BUF_SIZE;
-	elib_heap_map(w, n);
-	driver_output((ErlDrvPort)port, w, n);
-	return;
-    }
-    else if ((count == 4) && (buf[0] == 'h' || buf[0] == 'l')) {
-	unsigned long vf[HISTO_BUF_SIZE];
-	unsigned long va[HISTO_BUF_SIZE];
-	int n = getint16(buf+1);
-	int base = (unsigned char) buf[3];
-
-	if (n >= HISTO_BUF_SIZE)
-	    n = HISTO_BUF_SIZE;
-	if (buf[0] == 'l')
-	    base = -base;
-	if (elib_histo(vf, va, n, base) < 0) {
-	    driver_failure((ErlDrvPort)port, -1);
-	    return;
-	}
-	else {
-	    char* p = outbuf;
-	    int i;
-
-	    for (i = 0; i < n; i++) {
-		putint32(p, vf[i]);
-		p += 4;
-	    }
-	    for (i = 0; i < n; i++) {
-		putint32(p, va[i]);
-		p += 4;
-	    }
-	    driver_output((ErlDrvPort)port, outbuf, n*8);
-	}
-	return;
-    }
-    driver_failure((ErlDrvPort)port, -1);
-}
diff --git a/erts/emulator/drivers/unix/ttsl_drv.c b/erts/emulator/drivers/unix/ttsl_drv.c
index 4cd54c073f..b29f80a8ba 100644
--- a/erts/emulator/drivers/unix/ttsl_drv.c
+++ b/erts/emulator/drivers/unix/ttsl_drv.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -106,7 +106,7 @@ static int lpos;                /* The current "cursor position" in the line buf
 
 /* Main interface functions. */
 static void ttysl_stop(ErlDrvData);
-static void ttysl_from_erlang(ErlDrvData, char*, int);
+static void ttysl_from_erlang(ErlDrvData, char*, ErlDrvSizeT);
 static void ttysl_from_tty(ErlDrvData, ErlDrvEvent);
 static void ttysl_stop_select(ErlDrvEvent, void*);
 static Sint16 get_sint16(char*);
@@ -141,7 +141,8 @@ static void update_cols(void);
 static int tty_init(int,int,int,int);
 static int tty_set(int);
 static int tty_reset(int);
-static int ttysl_control(ErlDrvData, unsigned int, char *, int, char **, int);
+static ErlDrvSSizeT ttysl_control(ErlDrvData, unsigned int,
+				  char *, ErlDrvSizeT, char **, ErlDrvSizeT);
 #ifdef ERTS_NOT_USED
 static RETSIGTYPE suspend(int);
 #endif
@@ -242,7 +243,7 @@ static ErlDrvData ttysl_start(ErlDrvPort port, char* buf)
 #ifndef HAVE_TERMCAP
     return ERL_DRV_ERROR_GENERAL;
 #else
-    char *s, *t, c, *l;
+    char *s, *t, *l;
     int canon, echo, sig;	/* Terminal characteristics */
     int flag;
     extern int using_oldshell; /* set this to let the rest of erts know */
@@ -262,7 +263,6 @@ static ErlDrvData ttysl_start(ErlDrvPort port, char* buf)
 	s++;
 	/* Find end of this argument (start of next) and insert NUL. */
 	if ((t = strchr(s, ' '))) {
-	    c = *t;
 	    *t = '\0';
 	}
 	if ((flag = ((*s == '+') ? 1 : ((*s == '-') ? -1 : 0)))) {
@@ -314,7 +314,7 @@ static ErlDrvData ttysl_start(ErlDrvPort port, char* buf)
     sys_sigset(SIGCONT, cont);
     sys_sigset(SIGWINCH, winch);
 
-    driver_select(port, (ErlDrvEvent)(Uint)ttysl_fd, ERL_DRV_READ|ERL_DRV_USE, 1);
+    driver_select(port, (ErlDrvEvent)(UWord)ttysl_fd, ERL_DRV_READ|ERL_DRV_USE, 1);
     ttysl_port = port;
 
     /* we need to know this when we enter the break handler */
@@ -346,13 +346,13 @@ static void ttysl_get_window_size(Uint32 *width, Uint32 *height)
     *height = DEF_HEIGHT;
 }
     
-static int ttysl_control(ErlDrvData drv_data,
-			 unsigned int command,
-			 char *buf, int len,
-			 char **rbuf, int rlen)
+static ErlDrvSSizeT ttysl_control(ErlDrvData drv_data,
+				  unsigned int command,
+				  char *buf, ErlDrvSizeT len,
+				  char **rbuf, ErlDrvSizeT rlen)
 {
     char resbuff[2*sizeof(Uint32)];
-    int res_size;
+    ErlDrvSizeT res_size;
     switch (command) {
     case CTRL_OP_GET_WINSIZE:
 	{
@@ -394,7 +394,7 @@ static void ttysl_stop(ErlDrvData ttysl_data)
 	stop_lbuf();
 	stop_termcap();
 	tty_reset(ttysl_fd);
-	driver_select(ttysl_port, (ErlDrvEvent)(Uint)ttysl_fd, ERL_DRV_READ|ERL_DRV_USE, 0);
+	driver_select(ttysl_port, (ErlDrvEvent)(UWord)ttysl_fd, ERL_DRV_READ|ERL_DRV_USE, 0);
 	sys_sigset(SIGCONT, SIG_DFL);
 	sys_sigset(SIGWINCH, SIG_DFL);
     }
@@ -634,14 +634,14 @@ static int check_buf_size(byte *s, int n)
 }
 
 
-static void ttysl_from_erlang(ErlDrvData ttysl_data, char* buf, int count)
+static void ttysl_from_erlang(ErlDrvData ttysl_data, char* buf, ErlDrvSizeT count)
 {
     if (lpos > MAXSIZE) 
 	put_chars((byte*)"\n", 1);
 
     switch (buf[0]) {
     case OP_PUTC:
-	DEBUGLOG(("OP: Putc(%d)",count-1));
+	DEBUGLOG(("OP: Putc(%lu)",(unsigned long) count-1));
 	if (check_buf_size((byte*)buf+1, count-1) == 0)
 	    return; 
 	put_chars((byte*)buf+1, count-1);
@@ -685,7 +685,7 @@ static void ttysl_from_tty(ErlDrvData ttysl_data, ErlDrvEvent fd)
 	utf8buf_size = 0;
     }
     
-    if ((i = read((int)(Sint)fd, (char *) p, left)) >= 0) {
+    if ((i = read((int)(SWord)fd, (char *) p, left)) >= 0) {
 	if (p != b) {
 	    i += (p - b);
 	}
diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c
index d395b68691..b250bac4dc 100644
--- a/erts/emulator/drivers/unix/unix_efile.c
+++ b/erts/emulator/drivers/unix/unix_efile.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1997-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /*
@@ -33,17 +33,9 @@
 #include <sys/types.h>
 #include <sys/uio.h>
 #endif
-
-#ifdef _OSE_
-#include "efs.h"
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-
-#ifdef _OSE_SFK_
-#include <string.h>
+#if defined(HAVE_SENDFILE) && (defined(__linux__) || (defined(__sun) && defined(__SVR4)))
+#include <sys/sendfile.h>
 #endif
-#endif /* _OSE_ */
 
 #if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__)
 #define DARWIN 1
@@ -88,23 +80,6 @@ extern STATUS copy(char *, char *);
  * Macros for testing file types.
  */
 
-#ifdef _OSE_
-
-#define ISDIR(st) S_ISDIR(((st).st_mode))
-#define ISREG(st) S_ISREG(((st).st_mode))
-#define ISDEV(st) (S_ISCHR(((st).st_mode)) || S_ISBLK(((st).st_mode)))
-#define ISLNK(st) S_ISLNK(((st).st_mode))
-#ifdef NO_UMASK
-#define FILE_MODE (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)
-#define DIR_MODE  (S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)
-#else
-#define FILE_MODE (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)
-#define DIR_MODE  (S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IWGRP | S_IXGRP | S_IROTH | \
-                   S_IWOTH | S_IXOTH)
-#endif 
-
-#else  /* !_OSE_ */
-
 #define ISDIR(st) (((st).st_mode & S_IFMT) == S_IFDIR)
 #define ISREG(st) (((st).st_mode & S_IFMT) == S_IFREG)
 #define ISDEV(st) \
@@ -118,8 +93,6 @@ extern STATUS copy(char *, char *);
 #define DIR_MODE  0777
 #endif
 
-#endif /* _OSE_ */
-
 #ifdef VXWORKS /* Currently only used on vxworks */
 
 #define EF_ALLOC(S)		driver_alloc((S))
@@ -128,14 +101,14 @@ extern STATUS copy(char *, char *);
 #define EF_SAFE_REALLOC(P, S)	ef_safe_realloc((P), (S))
 #define EF_FREE(P)		do { if((P)) driver_free((P)); } while(0)
 
-extern void erl_exit(int n, char *fmt, _DOTS_);
+void erl_exit(int n, char *fmt, ...);
 
 static void *ef_safe_alloc(Uint s)
 {
     void *p = EF_ALLOC(s);
     if (!p) erl_exit(1,
-		     "unix efile drv: Can't allocate %d bytes of memory\n",
-		     s);
+		     "unix efile drv: Can't allocate %lu bytes of memory\n",
+		     (unsigned long)s);
     return p;
 }
 
@@ -145,8 +118,8 @@ static void *ef_safe_realloc(void *op, Uint s)
 {
     void *p = EF_REALLOC(op, s);
     if (!p) erl_exit(1,
-		     "unix efile drv: Can't reallocate %d bytes of memory\n",
-		     s);
+		     "unix efile drv: Can't reallocate %lu bytes of memory\n",
+		     (unsigned long)s);
     return p;
 }
 
@@ -157,7 +130,7 @@ static void *ef_safe_realloc(void *op, Uint s)
     (s[0] == '.' && (s[1] == '\0' || (s[1] == '.' && s[2] == '\0')))
 
 #ifdef VXWORKS
-static FUNCTION(int, vxworks_to_posix, (int vx_errno));
+static int vxworks_to_posix(int vx_errno);
 #endif
 
 /*
@@ -176,7 +149,7 @@ static FUNCTION(int, vxworks_to_posix, (int vx_errno));
 #define CHECK_PATHLEN(X,Y) /* Nothing */
 #endif
 
-static FUNCTION(int, check_error, (int result, Efile_error* errInfo));
+static int check_error(int result, Efile_error* errInfo);
 
 static int
 check_error(int result, Efile_error *errInfo)
@@ -361,15 +334,6 @@ path_size(char *pathname)
     
 #endif /* VXWORKS */
 
-#ifdef _OSE_
-static int 
-ose_enotsup(Efile_error *errInfo) 
-{
-    errInfo->posix_errno = errInfo->os_errno = ENOTSUP;
-    return 0;
-}
-#endif /* _OSE_ */
-
 int
 efile_mkdir(Efile_error* errInfo,	/* Where to return error codes. */
 	    char* name)			/* Name of directory to create. */
@@ -446,18 +410,12 @@ efile_delete_file(Efile_error* errInfo,	/* Where to return error codes. */
 		  char* name)		/* Name of file to delete. */
 {
     CHECK_PATHLEN(name,errInfo);
-#ifdef _OSE_
-    if (remove(name) == 0) {
-	return 1;
-    }
-#else
     if (unlink(name) == 0) {
 	return 1;
     }
     if (errno == EISDIR) {	/* Linux sets the wrong error code. */
 	errno = EPERM;
     }
-#endif
     return check_error(-1, errInfo);
 }
 
@@ -524,7 +482,7 @@ efile_rename(Efile_error* errInfo,	/* Where to return error codes. */
     if (errno == ENOTEMPTY) {
 	errno = EEXIST;
     }
-#if defined (sparc) && !defined(VXWORKS) && !defined(_OSE_)
+#if defined (sparc) && !defined(VXWORKS)
     /*
      * SunOS 4.1.4 reports overwriting a non-empty directory with a
      * directory as EINVAL instead of EEXIST (first rule out the correct
@@ -632,7 +590,8 @@ efile_readdir(Efile_error* errInfo,	/* Where to return error codes. */
 						   open directory.*/
 	      char* buffer,		/* Pointer to buffer for 
 					   one filename. */
-	      size_t size)		/* Size of buffer. */
+	      size_t *size)		/* in-out Size of buffer, length
+					   of name. */
 {
     DIR *dp;			/* Pointer to directory structure. */
     struct dirent* dirp;	/* Pointer to directory entry. */
@@ -664,7 +623,8 @@ efile_readdir(Efile_error* errInfo,	/* Where to return error codes. */
 	if (IS_DOT_OR_DOTDOT(dirp->d_name))
 	    continue;
 	buffer[0] = '\0';
-	strncat(buffer, dirp->d_name, size-1);
+	strncat(buffer, dirp->d_name, (*size)-1);
+	*size = strlen(dirp->d_name);
 	return 1;
     }
 }
@@ -751,6 +711,9 @@ efile_openfile(Efile_error* errInfo,	/* Where to return error codes. */
 #endif
     }
 
+    if (flags & EFILE_MODE_EXCL) {
+	mode |= O_EXCL;
+    }
 
 #ifdef VXWORKS
     if (*name != '/') {
@@ -819,6 +782,17 @@ efile_closefile(int fd)
 }
 
 int
+efile_fdatasync(Efile_error *errInfo, /* Where to return error codes. */
+	    int fd)               /* File descriptor for file to sync data. */
+{
+#ifdef HAVE_FDATASYNC
+    return check_error(fdatasync(fd), errInfo);
+#else
+    return efile_fsync(errInfo, fd);
+#endif
+}
+
+int
 efile_fsync(Efile_error *errInfo, /* Where to return error codes. */
 	    int fd)               /* File descriptor for file to sync. */
 {
@@ -842,7 +816,6 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo,
 	       char* name, int info_for_link)
 {
     struct stat statbuf;	/* Information about the file */
-    struct tm *timep;		/* Broken-apart filetime. */
     int result;
 
 #ifdef VXWORKS
@@ -855,7 +828,7 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo,
     CHECK_PATHLEN(name, errInfo);
 
     if (info_for_link) {
-#if (defined(VXWORKS) || defined(_OSE_))
+#if (defined(VXWORKS))
 	result = stat(name, &statbuf);
 #else
 	result = lstat(name, &statbuf);
@@ -909,44 +882,17 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo,
     else
 	pInfo->type = FT_OTHER;
 
-#if defined(HAVE_LOCALTIME_R) || defined(VXWORKS)
-    {
-	/* Use the reentrant version of localtime() */
-	static struct tm local_tm;
-#define localtime(a) (localtime_r((a), &local_tm), &local_tm)
-#endif
-
-
-#define GET_TIME(dst, src) \
-    timep = localtime(&statbuf.src); \
-    (dst).year = timep->tm_year+1900; \
-    (dst).month = timep->tm_mon+1; \
-    (dst).day = timep->tm_mday; \
-    (dst).hour = timep->tm_hour; \
-    (dst).minute = timep->tm_min; \
-    (dst).second = timep->tm_sec
-
-    GET_TIME(pInfo->accessTime, st_atime);
-    GET_TIME(pInfo->modifyTime, st_mtime);
-    GET_TIME(pInfo->cTime, st_ctime);
+    pInfo->accessTime   = statbuf.st_atime;
+    pInfo->modifyTime   = statbuf.st_mtime;
+    pInfo->cTime        = statbuf.st_ctime;
 
-#undef GET_TIME
-
-#if defined(HAVE_LOCALTIME_R) || defined(VXWORKS)
-    }
-#endif
-
-    pInfo->mode = statbuf.st_mode;
-    pInfo->links = statbuf.st_nlink;
+    pInfo->mode         = statbuf.st_mode;
+    pInfo->links        = statbuf.st_nlink;
     pInfo->major_device = statbuf.st_dev;
-#ifdef _OSE_
-    pInfo->minor_device = 0;
-#else
     pInfo->minor_device = statbuf.st_rdev;
-#endif
-    pInfo->inode = statbuf.st_ino;
-    pInfo->uid = statbuf.st_uid;
-    pInfo->gid = statbuf.st_gid;
+    pInfo->inode        = statbuf.st_ino;
+    pInfo->uid          = statbuf.st_uid;
+    pInfo->gid          = statbuf.st_gid;
 
     return 1;
 }
@@ -954,6 +900,8 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo,
 int
 efile_write_info(Efile_error *errInfo, Efile_info *pInfo, char *name)
 {
+    struct utimbuf tval;
+
     CHECK_PATHLEN(name, errInfo);
 
 #ifdef VXWORKS
@@ -989,11 +937,9 @@ efile_write_info(Efile_error *errInfo, Efile_info *pInfo, char *name)
      * you don't try to chown a file to someone besides youself.
      */
 
-#ifndef _OSE_    
     if (chown(name, pInfo->uid, pInfo->gid) && errno != EPERM) {
 	return check_error(-1, errInfo);
     }
-#endif
 
     if (pInfo->mode != -1) {
 	mode_t newMode = pInfo->mode & (S_ISUID | S_ISGID |
@@ -1008,41 +954,18 @@ efile_write_info(Efile_error *errInfo, Efile_info *pInfo, char *name)
 
 #endif /* !VXWORKS */
 
-#ifndef _OSE_
-
-    if (pInfo->accessTime.year != -1 && pInfo->modifyTime.year != -1) {
-	struct utimbuf tval;
-	struct tm timebuf;
-
-#define MKTIME(tb, ts) \
-    timebuf.tm_year = ts.year-1900; \
-    timebuf.tm_mon = ts.month-1; \
-    timebuf.tm_mday = ts.day; \
-    timebuf.tm_hour = ts.hour; \
-    timebuf.tm_min = ts.minute; \
-    timebuf.tm_sec = ts.second; \
-    timebuf.tm_isdst = -1; \
-    if ((tb = mktime(&timebuf)) == (time_t) -1) { \
-       errno = EINVAL; \
-       return check_error(-1, errInfo); \
-    }
+    tval.actime  = pInfo->accessTime;
+    tval.modtime = pInfo->modifyTime;
 
-        MKTIME(tval.actime, pInfo->accessTime);
-	MKTIME(tval.modtime, pInfo->modifyTime);
-#undef MKTIME
-	
 #ifdef VXWORKS
-	/* VxWorks' utime doesn't work when the file is a nfs mounted
-	 * one, don't report error if utime fails.
-	 */
-	utime(name, &tval);
-	return 1;
+    /* VxWorks' utime doesn't work when the file is a nfs mounted
+     * one, don't report error if utime fails.
+     */
+    utime(name, &tval);
+    return 1;
 #else
-	return check_error(utime(name, &tval), errInfo);
+    return check_error(utime(name, &tval), errInfo);
 #endif
-    }
-#endif /* !_OSE_ */
-    return 1;
 }
 
 
@@ -1081,13 +1004,11 @@ efile_writev(Efile_error* errInfo,   /* Where to return error codes */
 				      * opened */
 	     int fd,                 /* File descriptor to write to */
 	     SysIOVec* iov,          /* Vector of buffer structs.
-				      * The structs are unchanged 
-				      * after the call */
-	     int iovcnt,             /* Number of structs in vector */
-	     size_t size)            /* Number of bytes to write */
+				      * The structs may be changed i.e.
+				      * due to incomplete writes */
+	     int iovcnt)             /* Number of structs in vector */
 {
     int cnt = 0;                     /* Buffers so far written */
-    int p = 0;                       /* Position in next buffer */
 
     ASSERT(iovcnt >= 0);
     
@@ -1098,66 +1019,47 @@ efile_writev(Efile_error* errInfo,   /* Where to return error codes */
 #endif
 
     while (cnt < iovcnt) {
+	if ((! iov[cnt].iov_base) || (iov[cnt].iov_len <= 0)) {
+	    /* Empty buffer - skip */
+	    cnt++;
+	} else { /* Non-empty buffer */
+	    ssize_t w;                   /* Bytes written in this call */
 #ifdef HAVE_WRITEV
-	int w;                       /* Bytes written in this call */
-	int b = iovcnt - cnt;        /* Buffers to write */
-	if (b > MAXIOV)
-	    b = MAXIOV;
-	if (iov[cnt].iov_base && iov[cnt].iov_len > 0) {
-	    if (b == 1) {
-		/* Degenerated io vector */
-		do {
-		    w = write(fd, iov[cnt].iov_base + p, iov[cnt].iov_len - p);
-		} while (w < 0 && errno == EINTR);
-	    } else {
-		/* Non-empty vector first.
-		 * Adjust pos in first buffer in case of 
-		 * previous incomplete writev */
-		iov[cnt].iov_base += p;
-		iov[cnt].iov_len  -= p;
+	    int b = iovcnt - cnt;        /* Buffers to write */
+	    /* Use as many buffers as MAXIOV allows */
+	    if (b > MAXIOV)
+		b = MAXIOV;
+	    if (b > 1) {
 		do {
 		    w = writev(fd, &iov[cnt], b);
 		} while (w < 0 && errno == EINTR);
-		iov[cnt].iov_base -= p;
-		iov[cnt].iov_len  += p;
-	    }
-	    if (w < 0)
-		return check_error(-1, errInfo);
-	} else {
-	    /* Empty vector first - skip */
-	    cnt++;
-	    continue;
-	}
-	ASSERT(w >= 0);
-	/* Move forward to next vector to write */
-	for (; cnt < iovcnt; cnt++) {
-	    if (iov[cnt].iov_base && iov[cnt].iov_len > 0) {
-		if (w < iov[cnt].iov_len)
-		    break;
-		else
-		    w -= iov[cnt].iov_len;
-	    }
-	}
-	ASSERT(w >= 0);
-	p = w > 0 ? w : 0; /* Skip p bytes next writev */
-#else /* #ifdef HAVE_WRITEV */
-	if (iov[cnt].iov_base && iov[cnt].iov_len > 0) {
-	    /* Non-empty vector */
-	    int w;                   /* Bytes written in this call */
-	    while (p < iov[cnt].iov_len) {
-		do {
-		    w = write(fd, iov[cnt].iov_base + p, iov[cnt].iov_len - p);
-		} while (w < 0 && errno == EINTR);
-		if (w < 0)
-		    return check_error(-1, errInfo);
-		p += w;
+	    } else
+		/* Degenerated io vector - use regular write */
+#endif
+		{
+		    do {
+			w = write(fd, iov[cnt].iov_base, iov[cnt].iov_len);
+		    } while (w < 0 && errno == EINTR);
+		    ASSERT(w <= iov[cnt].iov_len);
+		}
+	    if (w < 0) return check_error(-1, errInfo);
+	    /* Move forward to next buffer to write */
+	    for (; cnt < iovcnt && w > 0; cnt++) {
+		if (iov[cnt].iov_base && iov[cnt].iov_len > 0) {
+		    if (w < iov[cnt].iov_len) {
+			/* Adjust the buffer for next write */
+			iov[cnt].iov_len -= w;
+			iov[cnt].iov_base += w;
+			w = 0;
+			break;
+		    } else {
+			w -= iov[cnt].iov_len;
+		    }
+		}
 	    }
-	}
-	cnt++;
-	p = 0;
-#endif /* #ifdef HAVE_WRITEV */
+	    ASSERT(w == 0);
+	} /* else Non-empty buffer */
     } /* while (cnt< iovcnt) */
-    size = 0; /* Avoid compiler warning */
     return 1;
 }
 
@@ -1451,9 +1353,6 @@ efile_truncate_file(Efile_error* errInfo, int *fd, int flags)
 int
 efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size)
 {
-#ifdef _OSE_
-    return ose_enotsup(errInfo);
-#else
 #ifdef VXWORKS
     return vxworks_enotsup(errInfo);
 #else
@@ -1466,7 +1365,6 @@ efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size)
     buffer[len] = '\0';
     return 1;
 #endif
-#endif
 }
 
 int
@@ -1479,27 +1377,142 @@ efile_altname(Efile_error* errInfo, char* name, char* buffer, size_t size)
 int
 efile_link(Efile_error* errInfo, char* old, char* new)
 {
-#ifdef _OSE_
-    return ose_enotsup(errInfo);
-#else
 #ifdef VXWORKS
     return vxworks_enotsup(errInfo);
 #else
     return check_error(link(old, new), errInfo);
 #endif
-#endif
 }
 
 int
 efile_symlink(Efile_error* errInfo, char* old, char* new)
 {
-#ifdef _OSE_
-    return ose_enotsup(errInfo);
-#else
 #ifdef VXWORKS
     return vxworks_enotsup(errInfo);
 #else
     return check_error(symlink(old, new), errInfo);
 #endif
+}
+
+int
+efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset,
+		  Sint64 length, int advise)
+{
+#ifdef HAVE_POSIX_FADVISE
+    return check_error(posix_fadvise(fd, offset, length, advise), errInfo);
+#else
+    return check_error(0, errInfo);
+#endif
+}
+
+#ifdef HAVE_SENDFILE
+// For some reason the maximum size_t cannot be used as the max size
+// 3GB seems to work on all platforms
+#define SENDFILE_CHUNK_SIZE ((1UL << 30) -1)
+
+/*
+ * sendfile: The implementation of the sendfile system call varies
+ * a lot on different *nix platforms so to make the api similar in all
+ * we have to emulate some things in linux and play with variables on
+ * bsd/darwin.
+ *
+ * All of the calls will split a command which tries to send more than
+ * SENDFILE_CHUNK_SIZE of data at once.
+ *
+ * On platforms where *nbytes of 0 does not mean the entire file, this is
+ * simulated.
+ *
+ * It could be possible to implement header/trailer in sendfile. Though
+ * you would have to emulate it in linux and on BSD/Darwin some complex
+ * calculations have to be made when using a non blocking socket to figure
+ * out how much of the header/file/trailer was sent in each command.
+ *
+ * The semantics of the API is this:
+ * Return value: 1 if all data was sent and the function does not need to
+ * be called again. 0 if an error occures OR if there is more data which
+ * has to be sent (EAGAIN or EINTR will be set appropriately)
+ *
+ * The amount of data written in a call is returned through nbytes.
+ *
+ */
+
+int
+efile_sendfile(Efile_error* errInfo, int in_fd, int out_fd,
+	       off_t *offset, Uint64 *nbytes, struct t_sendfile_hdtl* hdtl)
+{
+    Uint64 written = 0;
+#if defined(__linux__)
+    ssize_t retval;
+    do {
+      // check if *nbytes is 0 or greater than chunk size
+      if (*nbytes == 0 || *nbytes > SENDFILE_CHUNK_SIZE)
+	retval = sendfile(out_fd, in_fd, offset, SENDFILE_CHUNK_SIZE);
+      else
+	retval = sendfile(out_fd, in_fd, offset, *nbytes);
+      if (retval > 0) {
+	written += retval;
+	*nbytes -= retval;
+      }
+    } while (retval == SENDFILE_CHUNK_SIZE);
+    if (written != 0) {
+      // -1 is not returned by the linux API so we have to simulate it
+      retval = -1;
+      errno = EAGAIN;
+    }
+#elif defined(__sun) && defined(__SVR4) && defined(HAVE_SENDFILEV)
+    ssize_t retval;
+    size_t len;
+    sendfilevec_t fdrec;
+    fdrec.sfv_fd = in_fd;
+    fdrec.sfv_flag = 0;
+    do {
+      fdrec.sfv_off = *offset;
+      len = 0;
+      // check if *nbytes is 0 or greater than chunk size
+      if (*nbytes == 0 || *nbytes > SENDFILE_CHUNK_SIZE)
+	fdrec.sfv_len = SENDFILE_CHUNK_SIZE;
+      else
+	fdrec.sfv_len = *nbytes;
+      retval = sendfilev(out_fd, &fdrec, 1, &len);
+      if (retval != -1 || errno == EAGAIN || errno == EINTR) {
+        *offset += len;
+	*nbytes -= len;
+	written += len;
+      }
+    } while (len == SENDFILE_CHUNK_SIZE);
+#elif defined(DARWIN)
+    int retval;
+    off_t len;
+    do {
+      // check if *nbytes is 0 or greater than chunk size
+      if(*nbytes > SENDFILE_CHUNK_SIZE)
+	len = SENDFILE_CHUNK_SIZE;
+      else
+	len = *nbytes;
+      retval = sendfile(in_fd, out_fd, *offset, &len, NULL, 0);
+      if (retval != -1 || errno == EAGAIN || errno == EINTR) {
+        *offset += len;
+	*nbytes -= len;
+	written += len;
+      }
+    } while (len == SENDFILE_CHUNK_SIZE);
+#elif defined(__FreeBSD__) || defined(__DragonFly__)
+    off_t len;
+    int retval;
+    do {
+      if (*nbytes > SENDFILE_CHUNK_SIZE)
+	retval = sendfile(in_fd, out_fd, *offset, SENDFILE_CHUNK_SIZE,
+			  NULL, &len, 0);
+      else
+	retval = sendfile(in_fd, out_fd, *offset, *nbytes, NULL, &len, 0);
+      if (retval != -1 || errno == EAGAIN || errno == EINTR) {
+	*offset += len;
+	*nbytes -= len;
+	written += len;
+      }
+    } while(len == SENDFILE_CHUNK_SIZE);
 #endif
+    *nbytes = written;
+    return check_error(retval, errInfo);
 }
+#endif /* HAVE_SENDFILE */
diff --git a/erts/emulator/drivers/win32/mem_drv.c b/erts/emulator/drivers/win32/mem_drv.c
deleted file mode 100644
index fa7c46eca8..0000000000
--- a/erts/emulator/drivers/win32/mem_drv.c
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-/* Purpose: Access to elib memory statistics */
-
-#include "sys.h"
-#include "erl_driver.h"
-#include "elib_stat.h"
-
-#define MAP_BUF_SIZE    1000   /* Max map size */
-#define HISTO_BUF_SIZE  100    /* Max histogram buckets */
-
-static ErlDrvData mem_start(ErlDrvPort, char*);
-static int mem_init(void); 
-static void mem_stop(ErlDrvData);
-static void mem_command(ErlDrvData);
-
-ErlDrvEntry mem_driver_entry = {
-    mem_init,
-    mem_start,
-    mem_stop,
-    mem_command,
-    NULL,
-    NULL,
-    "mem_drv"
-};
-
-static int mem_init(void)
-{
-    return 0;
-}
-
-static ErlDrvData mem_start(ErlDrvPort port, char* buf)
-{
-    return (ErlDrvData)port;
-}
-
-static void mem_stop(ErlDrvData port)
-{
-}
-
-void putint32(p, v)
-byte* p; int v;
-{
-    p[0] = (v >> 24) & 0xff;
-    p[1] = (v >> 16) & 0xff;
-    p[2] = (v >> 8) & 0xff;
-    p[3] = (v) & 0xff;
-}
-
-int getint16(p)
-byte* p;
-{
-    return (p[0] << 8) | p[1];
-}
-
-/*
-** Command:
-**   m L1 L0  -> a heap map of length L1*256 + L0 is returned
-**   s        -> X3 X2 X1 X0 Y3 Y2 Y1 Y0 Z3 Z2 Z1 Z0
-**               X == Total heap size bytes
-**               Y == Total free bytes
-**               Z == Size of largest free block in bytes
-**
-**   h L1 L0 B0 -> Generate a logarithm histogram base B with L buckets
-**   l L1 L0 S0 -> Generate a linear histogram with step S with L buckets
-*/
-unsigned char outbuf[HISTO_BUF_SIZE*2*4];
-
-static void mem_command(ErlDrvData port, char* buf, int count)
-{
-    if ((count == 1) && buf[0] == 's') {
-	struct elib_stat info;
-	char v[3*4];
-
-	elib_stat(&info);
-
-	putint32(v, info.mem_total*4);
-	putint32(v+4, info.mem_free*4);
-	putint32(v+8, info.max_free*4);
-	driver_output((ErlDrvPort)port, v, 12);
-	return;
-    }
-    else if ((count == 3) && buf[0] == 'm') {
-	char w[MAP_BUF_SIZE];
-	int n = getint16(buf+1);
-
-	if (n > MAP_BUF_SIZE)
-	    n = MAP_BUF_SIZE;
-	elib_heap_map(w, n);
-	driver_output((ErlDrvPort)port, w, n);
-	return;
-    }
-    else if ((count == 4) && (buf[0] == 'h' || buf[0] == 'l')) {
-	unsigned long vf[HISTO_BUF_SIZE];
-	unsigned long va[HISTO_BUF_SIZE];
-	int n = getint16(buf+1);
-	int base = (unsigned char) buf[3];
-
-	if (n >= HISTO_BUF_SIZE)
-	    n = HISTO_BUF_SIZE;
-	if (buf[0] == 'l')
-	    base = -base;
-	if (elib_histo(vf, va, n, base) < 0) {
-	    driver_failure((ErlDrvPort)port, -1);
-	    return;
-	}
-	else {
-	    char* p = outbuf;
-	    int i;
-
-	    for (i = 0; i < n; i++) {
-		putint32(p, vf[i]);
-		p += 4;
-	    }
-	    for (i = 0; i < n; i++) {
-		putint32(p, va[i]);
-		p += 4;
-	    }
-	    driver_output((ErlDrvPort)port, outbuf, n*8);
-	}
-	return;
-    }
-    driver_failure((ErlDrvPort)port, -1);
-}
-
diff --git a/erts/emulator/drivers/win32/registry_drv.c b/erts/emulator/drivers/win32/registry_drv.c
index 05fd2ea55f..1fad34e380 100644
--- a/erts/emulator/drivers/win32/registry_drv.c
+++ b/erts/emulator/drivers/win32/registry_drv.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -84,7 +84,7 @@ static int maperror(DWORD error);
 static int reg_init(void);
 static ErlDrvData reg_start(ErlDrvPort, char*);
 static void reg_stop(ErlDrvData);
-static void reg_from_erlang(ErlDrvData, char*, int);
+static void reg_from_erlang(ErlDrvData, char*, ErlDrvSizeT);
 
 struct erl_drv_entry registry_driver_entry = {
     reg_init,
@@ -95,10 +95,21 @@ struct erl_drv_entry registry_driver_entry = {
     NULL,
     "registry__drv__",
     NULL,
+    NULL, /* handle */
+    NULL, /* control */
+    NULL, /* timeout */
+    NULL, /* outputv */
+    NULL, /* ready_async */
+    NULL, /* flush */
+    NULL, /* call */
+    NULL, /* event */
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
     NULL,
     NULL,
     NULL,
-    NULL
 };
 
 static int
@@ -158,7 +169,7 @@ reg_stop(ErlDrvData clientData)
 }
 
 static void
-reg_from_erlang(ErlDrvData clientData, char* buf, int count)
+reg_from_erlang(ErlDrvData clientData, char* buf, ErlDrvSizeT count)
 {
     RegPort* rp = (RegPort *) clientData;
     int cmd;
@@ -301,7 +312,7 @@ reg_from_erlang(ErlDrvData clientData, char* buf, int count)
 		buf = (char *) &dword;
 		ASSERT(count == 4);
 	    }
-	    result = RegSetValueEx(rp->hkey, name, 0, type, buf, count);
+	    result = RegSetValueEx(rp->hkey, name, 0, type, buf, (DWORD)count);
 	    reply(rp, result);
 	}
 	break;
diff --git a/erts/emulator/drivers/win32/ttsl_drv.c b/erts/emulator/drivers/win32/ttsl_drv.c
index fd88dafd34..1a74d21e99 100644
--- a/erts/emulator/drivers/win32/ttsl_drv.c
+++ b/erts/emulator/drivers/win32/ttsl_drv.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -21,6 +21,9 @@
  * smart line for output.
  */
 
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
 #include "sys.h"
 #include <ctype.h>
 #include <stdlib.h>
@@ -80,8 +83,9 @@ int lpos;                       /* The current "cursor position" in the line buf
 static int ttysl_init(void);
 static ErlDrvData ttysl_start(ErlDrvPort, char*);
 static void ttysl_stop(ErlDrvData);
-static int ttysl_control(ErlDrvData, unsigned int, char *, int, char **, int);
-static void ttysl_from_erlang(ErlDrvData, char*, int);
+static ErlDrvSSizeT ttysl_control(ErlDrvData, unsigned int,
+				  char *, ErlDrvSizeT, char **, ErlDrvSizeT);
+static void ttysl_from_erlang(ErlDrvData, char*, ErlDrvSizeT);
 static void ttysl_from_tty(ErlDrvData, ErlDrvEvent);
 static Sint16 get_sint16(char *s);
 
@@ -117,7 +121,19 @@ struct erl_drv_entry ttsl_driver_entry = {
     NULL,
     NULL,
     ttysl_control,
-    NULL
+    NULL, /* timeout */
+    NULL, /* outputv */
+    NULL, /* ready_async */
+    NULL, /* flush */
+    NULL, /* call */
+    NULL, /* event */
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
+    NULL,
 };
 
 static int utf8_mode = 0;
@@ -151,13 +167,13 @@ static void ttysl_get_window_size(Uint32 *width, Uint32 *height)
 }
     
 
-static int ttysl_control(ErlDrvData drv_data,
+static ErlDrvSSizeT ttysl_control(ErlDrvData drv_data,
 			 unsigned int command,
-			 char *buf, int len,
-			 char **rbuf, int rlen)
+			 char *buf, ErlDrvSizeT len,
+			 char **rbuf, ErlDrvSizeT rlen)
 {
     char resbuff[2*sizeof(Uint32)];
-    int res_size;
+    ErlDrvSizeT res_size;
     switch (command) {
     case CTRL_OP_GET_WINSIZE:
 	{
@@ -173,7 +189,7 @@ static int ttysl_control(ErlDrvData drv_data,
 	res_size = 1;
 	break;
     case CTRL_OP_SET_UNICODE_STATE:
-	if (len > 0) {
+	if (len != 0) {
 	    int m = (int) *buf;
 	    *resbuff = (utf8_mode) ? 1 : 0;
 	    res_size = 1;
@@ -435,14 +451,14 @@ static int check_buf_size(byte *s, int n)
 }
 
 
-static void ttysl_from_erlang(ErlDrvData ttysl_data, char* buf, int count)
+static void ttysl_from_erlang(ErlDrvData ttysl_data, char* buf, ErlDrvSizeT count)
 {
     if (lpos > MAXSIZE) 
 	put_chars((byte*)"\n", 1);
 
     switch (buf[0]) {
     case OP_PUTC:
-	DEBUGLOG(("OP: Putc(%d)",count-1));
+	DEBUGLOG(("OP: Putc(%I64u)",(unsigned long long)count-1));
 	if (check_buf_size((byte*)buf+1, count-1) == 0)
 	    return; 
 	put_chars((byte*)buf+1, count-1);
diff --git a/erts/emulator/drivers/win32/win_con.c b/erts/emulator/drivers/win32/win_con.c
index 2202ca655f..6b45b92cbe 100644
--- a/erts/emulator/drivers/win32/win_con.c
+++ b/erts/emulator/drivers/win32/win_con.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -21,6 +21,9 @@
 #define _UNICODE 1
 #include <tchar.h>
 #include <stdio.h>
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
 #include "sys.h"
 #include <windowsx.h>
 #include "resource.h"
@@ -34,6 +37,23 @@
 #define REALLOC(X,Y) realloc(X,Y)
 #define FREE(X) free(X)
 
+#if SIZEOF_VOID_P == 8
+#define WIN64 1
+#ifndef GCL_HBRBACKGROUND
+#define GCL_HBRBACKGROUND GCLP_HBRBACKGROUND
+#endif
+#define DIALOG_PROC_RET INT_PTR
+#define CF_HOOK_RET INT_PTR
+#define CC_HOOK_RET INT_PTR
+#define OFN_HOOK_RET INT_PTR
+#else
+#define DIALOG_PROC_RET BOOL
+#define CF_HOOK_RET UINT
+#define CC_HOOK_RET UINT
+#define OFN_HOOK_RET UINT
+#endif
+
+
 #ifndef STATE_SYSTEM_INVISIBLE
 /* Mingw problem with oleacc.h and WIN32_LEAN_AND_MEAN */
 #define STATE_SYSTEM_INVISIBLE 0x00008000
@@ -150,7 +170,7 @@ static TCHAR *erlang_window_title = TEXT("Erlang");
 static unsigned __stdcall ConThreadInit(LPVOID param);
 static LRESULT CALLBACK ClientWndProc(HWND hwnd, UINT iMsg, WPARAM wParam, LPARAM lParam);
 static LRESULT CALLBACK FrameWndProc(HWND hwnd, UINT iMsg, WPARAM wParam, LPARAM lParam);
-static BOOL CALLBACK AboutDlgProc(HWND hDlg, UINT iMsg, WPARAM wParam, LPARAM lParam);
+static DIALOG_PROC_RET CALLBACK AboutDlgProc(HWND hDlg, UINT iMsg, WPARAM wParam, LPARAM lParam);
 static ScreenLine_t *ConNewLine(void);
 static void DeleteTopLine(void);
 static void ensure_line_below(void);
@@ -704,6 +724,18 @@ FrameWndProc(HWND hwnd, UINT iMsg, WPARAM wParam, LPARAM lParam)
 	}
         write_inbuf(&c, 1);
 	return 0;
+    case WM_MOUSEWHEEL:
+      {
+	int delta = GET_WHEEL_DELTA_WPARAM(wParam);
+	if (delta < 0) {
+	  PostMessage(hClientWnd, WM_VSCROLL, MAKELONG(SB_THUMBTRACK,
+						       (iVscrollPos + 5)),0);
+	} else {
+	  WORD pos = ((iVscrollPos - 5) < 0) ? 0 : (iVscrollPos - 5);
+	  PostMessage(hClientWnd, WM_VSCROLL, MAKELONG(SB_THUMBTRACK,pos),0);
+	}
+      return 0;
+      }
     case WM_CHAR:
 	c = (TCHAR)wParam;
         write_inbuf(&c,1);
@@ -1596,7 +1628,7 @@ OnEditSelAll(HWND hwnd)
     InvalidateRect(hwnd, NULL, TRUE);
 }
 
-UINT APIENTRY CFHookProc(HWND hDlg,UINT iMsg,WPARAM wParam,LPARAM lParam)
+CF_HOOK_RET APIENTRY CFHookProc(HWND hDlg,UINT iMsg,WPARAM wParam,LPARAM lParam)
 {
     /* Hook procedure for font dialog box */
     HWND hOwner;
@@ -1614,11 +1646,11 @@ UINT APIENTRY CFHookProc(HWND hDlg,UINT iMsg,WPARAM wParam,LPARAM lParam)
         OffsetRect(&rc, -rcDlg.right, -rcDlg.bottom); 
         SetWindowPos(hDlg,HWND_TOP,rcOwner.left + (rc.right / 2), 
 		     rcOwner.top + (rc.bottom / 2),0,0,SWP_NOSIZE); 
-        return 1;
+        return (CF_HOOK_RET) 1;
     default:
         break;
     }
-    return 0;			/* Let the default procedure process the message */
+    return (CF_HOOK_RET) 0; /* Let the default procedure process the message */
 }
 
 static BOOL
@@ -1693,7 +1725,7 @@ ConSetFont(HWND hwnd)
     InvalidateRect(hwnd, NULL, TRUE);
 }
 
-UINT APIENTRY
+CC_HOOK_RET APIENTRY
 CCHookProc(HWND hDlg,UINT iMsg,WPARAM wParam,LPARAM lParam)
 {
     /* Hook procedure for choose color dialog box */
@@ -1712,11 +1744,11 @@ CCHookProc(HWND hDlg,UINT iMsg,WPARAM wParam,LPARAM lParam)
         OffsetRect(&rc, -rcDlg.right, -rcDlg.bottom); 
         SetWindowPos(hDlg,HWND_TOP,rcOwner.left + (rc.right / 2), 
 		     rcOwner.top + (rc.bottom / 2),0,0,SWP_NOSIZE); 
-        return 1;
+        return (CC_HOOK_RET) 1;
     default:
         break;
     }
-    return 0;			/* Let the default procedure process the message */
+    return (CC_HOOK_RET) 0;			/* Let the default procedure process the message */
 }
 
 void ConChooseColor(HWND hwnd)
@@ -1746,7 +1778,8 @@ void ConChooseColor(HWND hwnd)
     }    
 }
 
-UINT APIENTRY OFNHookProc(HWND hwndDlg,UINT iMsg,WPARAM wParam,LPARAM lParam)
+OFN_HOOK_RET APIENTRY OFNHookProc(HWND hwndDlg,UINT iMsg,
+				  WPARAM wParam,LPARAM lParam)
 {
     /* Hook procedure for open file dialog box */
     HWND hOwner,hDlg;
@@ -1765,11 +1798,11 @@ UINT APIENTRY OFNHookProc(HWND hwndDlg,UINT iMsg,WPARAM wParam,LPARAM lParam)
         OffsetRect(&rc, -rcDlg.right, -rcDlg.bottom); 
         SetWindowPos(hDlg,HWND_TOP,rcOwner.left + (rc.right / 2), 
 		     rcOwner.top + (rc.bottom / 2),0,0,SWP_NOSIZE); 
-        return 1;
+        return (OFN_HOOK_RET) 1;
     default:
         break;
     }
-    return 0;			/* the let default procedure process the message */
+    return (OFN_HOOK_RET) 0; /* the let default procedure process the message */
 }
 
 static void
@@ -1921,7 +1954,7 @@ write_outbuf(TCHAR *data, int num_chars)
     return num_chars;
 }
 
-BOOL CALLBACK AboutDlgProc(HWND hDlg, UINT iMsg, WPARAM wParam, LPARAM lParam)
+DIALOG_PROC_RET CALLBACK AboutDlgProc(HWND hDlg, UINT iMsg, WPARAM wParam, LPARAM lParam)
 {
     HWND hOwner;
     RECT rc,rcOwner,rcDlg;
@@ -1941,17 +1974,17 @@ BOOL CALLBACK AboutDlgProc(HWND hDlg, UINT iMsg, WPARAM wParam, LPARAM lParam)
 		     rcOwner.top + (rc.bottom / 2),0,0,SWP_NOSIZE); 
         SetDlgItemText(hDlg, ID_VERSIONSTRING,
 		       TEXT("Erlang emulator version ") TEXT(ERLANG_VERSION)); 
-        return TRUE;
+        return (DIALOG_PROC_RET) TRUE;
     case WM_COMMAND:
         switch (LOWORD(wParam)) {
         case IDOK:
         case IDCANCEL:
             EndDialog(hDlg,0);
-            return TRUE;
+            return (DIALOG_PROC_RET) TRUE;
         }
         break;
     }
-    return FALSE;
+    return (DIALOG_PROC_RET) FALSE;
 }
 
 static void
@@ -2105,7 +2138,7 @@ AddToCmdHistory(void)
     }
 }
 
-static TBBUTTON tbb[] =
+/*static TBBUTTON tbb[] =
 {
     0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP,    0, 0, 0, 0,
     0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP,    0, 0, 0, 0,
@@ -2137,6 +2170,39 @@ static TBBUTTON tbb[] =
     2,	        IDMENU_FONT,	TBSTATE_ENABLED, TBSTYLE_AUTOSIZE, 0, 0, 0, 0,
     3,	        IDMENU_ABOUT,	TBSTATE_ENABLED, TBSTYLE_AUTOSIZE, 0, 0, 0, 0,
     0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP,    0, 0, 0, 0,
+    };*/
+static TBBUTTON tbb[] =
+{
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP},
+  {0,          IDMENU_COPY,    TBSTATE_ENABLED, TBSTYLE_AUTOSIZE},
+  {1,	       IDMENU_PASTE,   TBSTATE_ENABLED, TBSTYLE_AUTOSIZE},
+  {2,	       IDMENU_FONT,    TBSTATE_ENABLED, TBSTYLE_AUTOSIZE},
+  {3,	       IDMENU_ABOUT,   TBSTATE_ENABLED, TBSTYLE_AUTOSIZE},
+  {0,          0,              TBSTATE_ENABLED, TBSTYLE_SEP} 
 };
 
 static TBADDBITMAP tbbitmap =
@@ -2144,6 +2210,17 @@ static TBADDBITMAP tbbitmap =
     HINST_COMMCTRL, IDB_STD_SMALL_COLOR,
 };
 
+#ifdef HARDDEBUG
+/* For really hard GUI startup debugging, place DEBUGBOX() macros in code
+   and get modal message boxes with the line number. */
+static void debug_box(int line) {
+  TCHAR buff[1024];
+  swprintf(buff,1024,TEXT("DBG:%d"),line);
+  MessageBox(NULL,buff,TEXT("DBG"),MB_OK|MB_APPLMODAL);
+}
+
+#define DEBUGBOX() debug_box(__LINE__)  
+#endif
 
 static HWND
 InitToolBar(HWND hwndParent) 
@@ -2157,7 +2234,6 @@ InitToolBar(HWND hwndParent)
     COLORMAP colorMap;
     colorMap.from = RGB(192, 192, 192);
     colorMap.to = backgroundColor;
-
     /* Create toolbar window with tooltips */
     hwndTB = CreateWindowEx(0,TOOLBARCLASSNAME,(TCHAR *)NULL,
 			    WS_CHILD|CCS_TOP|WS_CLIPSIBLINGS|TBSTYLE_TOOLTIPS,
@@ -2168,9 +2244,10 @@ InitToolBar(HWND hwndParent)
     tbbitmap.hInst = NULL;
     tbbitmap.nID   = (UINT) CreateMappedBitmap(beam_module, 1,0, &colorMap, 1);
     SendMessage(hwndTB, TB_ADDBITMAP, (WPARAM) 4, 
-		(WPARAM) &tbbitmap); 
+		(LPARAM) &tbbitmap); 
+
     SendMessage(hwndTB,TB_ADDBUTTONS, (WPARAM) 30,
-		(LPARAM) (LPTBBUTTON) tbb); 
+		(LPARAM) tbb); 
     if (toolbarVisible)
 	ShowWindow(hwndTB, SW_SHOW); 
 
diff --git a/erts/emulator/drivers/win32/win_efile.c b/erts/emulator/drivers/win32/win_efile.c
index 89aaad31da..606fa1d7de 100644
--- a/erts/emulator/drivers/win32/win_efile.c
+++ b/erts/emulator/drivers/win32/win_efile.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /*
@@ -21,32 +21,55 @@
  */
 
 #include <windows.h>
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
 #include "sys.h"
 #include <ctype.h>
-
+#include <wchar.h>
 #include "erl_efile.h"
 
 /*
  * Microsoft-specific function to map a WIN32 error code to a Posix errno.
  */
 
-#define ISSLASH(a)  ((a) == '\\' || (a) == '/')
+#define ISSLASH(a)  ((a) == L'\\' || (a) == L'/')
 
 #define ISDIR(st) (((st).st_mode&S_IFMT) == S_IFDIR)
 #define ISREG(st) (((st).st_mode&S_IFMT) == S_IFREG)
 
 #define IS_DOT_OR_DOTDOT(s) \
-    (s[0] == '.' && (s[1] == '\0' || (s[1] == '.' && s[2] == '\0')))
+    ((s)[0] == L'.' && ((s)[1] == L'\0' || ((s)[1] == L'.' && (s)[2] == L'\0')))
 
 #ifndef INVALID_FILE_ATTRIBUTES
 #define INVALID_FILE_ATTRIBUTES ((DWORD) 0xFFFFFFFF)
 #endif
 
+#define TICKS_PER_SECOND (10000000ULL)
+#define EPOCH_DIFFERENCE (11644473600LL)
+
+#define FILETIME_TO_EPOCH(epoch, ft) \
+    do { \
+	ULARGE_INTEGER ull; \
+	ull.LowPart  = (ft).dwLowDateTime; \
+	ull.HighPart = (ft).dwHighDateTime; \
+	(epoch) = ((ull.QuadPart / TICKS_PER_SECOND) - EPOCH_DIFFERENCE); \
+    } while(0)
+
+#define EPOCH_TO_FILETIME(ft, epoch) \
+    do { \
+	ULARGE_INTEGER ull; \
+	ull.QuadPart = (((epoch) + EPOCH_DIFFERENCE) * TICKS_PER_SECOND); \
+	(ft).dwLowDateTime = ull.LowPart; \
+	(ft).dwHighDateTime = ull.HighPart; \
+    } while(0)
+
+
 static int check_error(int result, Efile_error* errInfo);
 static int set_error(Efile_error* errInfo);
-static int IsRootUNCName(const char* path);
-static int extract_root(char* name);
-static unsigned short dos_to_posix_mode(int attr, const char *name);
+static int is_root_unc_name(const WCHAR *path);
+static int extract_root(WCHAR *name);
+static unsigned short dos_to_posix_mode(int attr, const WCHAR *name);
 
 static int errno_map(DWORD last_error) {
 
@@ -127,6 +150,8 @@ static int errno_map(DWORD last_error) {
 	return EBUSY;
     case ERROR_NO_PROC_SLOTS:
 	return EAGAIN;
+    case ERROR_CANT_RESOLVE_FILENAME:
+	return EMLINK;
     case ERROR_ARENA_TRASHED:
     case ERROR_INVALID_BLOCK:
     case ERROR_BAD_ENVIRONMENT:
@@ -196,27 +221,26 @@ win_writev(Efile_error* errInfo,
 
 
 int
-efile_mkdir(errInfo, name)
-Efile_error* errInfo;		/* Where to return error codes. */
-char* name;			/* Name of directory to create. */
+efile_mkdir(Efile_error* errInfo,	/* Where to return error codes. */
+	    char* name)			/* Name of directory to create. */
 {
-    return check_error(mkdir(name), errInfo);
+    return check_error(_wmkdir((WCHAR *) name), errInfo);
 }
 
 int
-efile_rmdir(errInfo, name)
-Efile_error* errInfo;		/* Where to return error codes. */
-char* name;			/* Name of directory to delete. */
+efile_rmdir(Efile_error* errInfo,	/* Where to return error codes. */
+	    char* name)			/* Name of directory to delete. */
 {
     OSVERSIONINFO os;
     DWORD attr;
+    WCHAR *wname = (WCHAR *) name;
 
-    if (RemoveDirectory(name) != FALSE) {
+    if (RemoveDirectoryW(wname) != FALSE) {
 	return 1;
     }
     errno = errno_map(GetLastError());
     if (errno == EACCES) {
-	attr = GetFileAttributes(name);
+	attr = GetFileAttributesW(wname);
 	if (attr != (DWORD) -1) {
 	    if ((attr & FILE_ATTRIBUTE_DIRECTORY) == 0) {
 		/* 
@@ -238,21 +262,21 @@ char* name;			/* Name of directory to delete. */
 	    GetVersionEx(&os);
 	    if (os.dwPlatformId == VER_PLATFORM_WIN32_WINDOWS) {
 		HANDLE handle;
-		WIN32_FIND_DATA data;
-		char buffer[2*MAX_PATH];
+		WIN32_FIND_DATAW data;
+		WCHAR buffer[2*MAX_PATH];
 		int len;
 
-		len = strlen(name);
-		strcpy(buffer, name);
-		if (buffer[0] && buffer[len-1] != '\\' && buffer[len-1] != '/') {
-		    strcat(buffer, "\\");
+		len = wcslen(wname);
+		wcscpy(buffer, wname);
+		if (buffer[0] && buffer[len-1] != L'\\' && buffer[len-1] != L'/') {
+		    wcscat(buffer, L"\\");
 		}
-		strcat(buffer, "*.*");
-		handle = FindFirstFile(buffer, &data);
+		wcscat(buffer, L"*.*");
+		handle = FindFirstFileW(buffer, &data);
 		if (handle != INVALID_HANDLE_VALUE) {
 		    while (1) {
-			if ((strcmp(data.cFileName, ".") != 0)
-				&& (strcmp(data.cFileName, "..") != 0)) {
+			if ((wcscmp(data.cFileName, L".") != 0)
+				&& (wcscmp(data.cFileName, L"..") != 0)) {
 			    /*
 			     * Found something in this directory.
 			     */
@@ -260,7 +284,7 @@ char* name;			/* Name of directory to delete. */
 			    errno = EEXIST;
 			    break;
 			}
-			if (FindNextFile(handle, &data) == FALSE) {
+			if (FindNextFileW(handle, &data) == FALSE) {
 			    break;
 			}
 		    }
@@ -284,19 +308,19 @@ char* name;			/* Name of directory to delete. */
 }
 
 int
-efile_delete_file(errInfo, name)
-Efile_error* errInfo;		/* Where to return error codes. */
-char* name;			/* Name of file to delete. */
+efile_delete_file(Efile_error* errInfo,		/* Where to return error codes. */
+		  char* name)			/* Name of file to delete. */
 {
     DWORD attr;
+    WCHAR *wname = (WCHAR *) name;
 
-    if (DeleteFile(name) != FALSE) {
+    if (DeleteFileW(wname) != FALSE) {
 	return 1;
     }
 
     errno = errno_map(GetLastError());
     if (errno == EACCES) {
-        attr = GetFileAttributes(name);
+        attr = GetFileAttributesW(wname);
 	if (attr != (DWORD) -1) {
 	    if (attr & FILE_ATTRIBUTE_DIRECTORY) {
 		/*
@@ -308,7 +332,7 @@ char* name;			/* Name of file to delete. */
 	    }
 	}
     } else if (errno == ENOENT) {
-        attr = GetFileAttributes(name);
+        attr = GetFileAttributesW(wname);
 	if (attr != (DWORD) -1) {
 	    if (attr & FILE_ATTRIBUTE_DIRECTORY) {
 	    	/*
@@ -362,20 +386,21 @@ char* name;			/* Name of file to delete. */
  */
 
 int
-efile_rename(errInfo, src, dst)
-Efile_error* errInfo;		/* Where to return error codes. */
-char* src;			/* Original name. */
-char* dst;			/* New name. */
+efile_rename(Efile_error* errInfo,	/* Where to return error codes. */
+	     char* src,			/* Original name. */
+	     char* dst)			/* New name. */
 {
     DWORD srcAttr, dstAttr;
+    WCHAR *wsrc = (WCHAR *) src;
+    WCHAR *wdst = (WCHAR *) dst;
     
-    if (MoveFile(src, dst) != FALSE) {
+    if (MoveFileW(wsrc, wdst) != FALSE) {
 	return 1;
     }
 
     errno = errno_map(GetLastError());
-    srcAttr = GetFileAttributes(src);
-    dstAttr = GetFileAttributes(dst);
+    srcAttr = GetFileAttributesW(wsrc);
+    dstAttr = GetFileAttributesW(wdst);
     if (srcAttr == (DWORD) -1) {
 	srcAttr = 0;
     }
@@ -390,22 +415,22 @@ char* dst;			/* New name. */
     if (errno == EACCES) {
 	decode:
 	if (srcAttr & FILE_ATTRIBUTE_DIRECTORY) {
-	    char srcPath[MAX_PATH], dstPath[MAX_PATH];
-	    char *srcRest, *dstRest;
+	    WCHAR srcPath[MAX_PATH], dstPath[MAX_PATH];
+	    WCHAR *srcRest, *dstRest;
 	    int size;
 
-	    size = GetFullPathName(src, sizeof(srcPath), srcPath, &srcRest);
-	    if ((size == 0) || (size > sizeof(srcPath))) {
+	    size = GetFullPathNameW(wsrc, MAX_PATH, srcPath, &srcRest);
+	    if ((size == 0) || (size > MAX_PATH)) {
 		return check_error(-1, errInfo);
 	    }
-	    size = GetFullPathName(dst, sizeof(dstPath), dstPath, &dstRest);
-	    if ((size == 0) || (size > sizeof(dstPath))) {
+	    size = GetFullPathNameW(wdst, MAX_PATH, dstPath, &dstRest);
+	    if ((size == 0) || (size > MAX_PATH)) {
 		return check_error(-1, errInfo);
 	    }
 	    if (srcRest == NULL) {
-		srcRest = srcPath + strlen(srcPath);
+		srcRest = srcPath + wcslen(srcPath);
 	    }
-	    if (strnicmp(srcPath, dstPath, srcRest - srcPath) == 0) {
+	    if (_wcsnicmp(srcPath, dstPath, srcRest - srcPath) == 0) {
 		/*
 		 * Trying to move a directory into itself.
 		 */
@@ -420,14 +445,14 @@ char* dst;			/* New name. */
 	    }
 
 	    (void) extract_root(dstPath);
-	    if (dstPath[0] == '\0') {
+	    if (dstPath[0] == L'\0') {
 		/*
 		 * The filename was invalid.  (Don't know why,
 		 * but play it safe.)
 		 */
 		errno = EINVAL;
 	    }
-	    if (stricmp(srcPath, dstPath) != 0) {
+	    if (_wcsicmp(srcPath, dstPath) != 0) {
 		/*
 		 * If src is a directory and dst filesystem != src
 		 * filesystem, errno should be EXDEV.  It is very
@@ -463,14 +488,14 @@ char* dst;			/* New name. */
 		 * fails, it's because it wasn't empty.
 		 */
 
-		if (RemoveDirectory(dst)) {
+		if (RemoveDirectoryW(wdst)) {
 		    /*
 		     * Now that that empty directory is gone, we can try
 		     * renaming again.  If that fails, we'll put this empty
 		     * directory back, for completeness.
 		     */
 
-		    if (MoveFile(src, dst) != FALSE) {
+		    if (MoveFileW(wsrc, wdst) != FALSE) {
 			return 1;
 		    }
 
@@ -480,8 +505,8 @@ char* dst;			/* New name. */
 		     */
 
 		    errno = errno_map(GetLastError());
-		    CreateDirectory(dst, NULL);
-		    SetFileAttributes(dst, dstAttr);
+		    CreateDirectoryW(wdst, NULL);
+		    SetFileAttributesW(wdst, dstAttr);
 		    if (errno == EACCES) {
 			/*
 			 * Decode the EACCES to a more meaningful error.
@@ -506,17 +531,17 @@ char* dst;			/* New name. */
 		 *    put temp file back to old name.
 		 */
 
-		char tempName[MAX_PATH];
+		WCHAR tempName[MAX_PATH];
 		int result, size;
-		char *rest;
+		WCHAR *rest;
 		
-		size = GetFullPathName(dst, sizeof(tempName), tempName, &rest);
-		if ((size == 0) || (size > sizeof(tempName)) || (rest == NULL)) {
+		size = GetFullPathNameW(wdst, MAX_PATH, tempName, &rest);
+		if ((size == 0) || (size > MAX_PATH) || (rest == NULL)) {
 		    return check_error(-1, errInfo);
 		}
-		*rest = '\0';
+		*rest = L'\0';
 		result = -1;
-		if (GetTempFileName(tempName, "erlr", 0, tempName) != 0) {
+		if (GetTempFileNameW(tempName, L"erlr", 0, tempName) != 0) {
 		    /*
 		     * Strictly speaking, need the following DeleteFile and
 		     * MoveFile to be joined as an atomic operation so no
@@ -524,15 +549,15 @@ char* dst;			/* New name. */
 		     * same temp file.
 		     */
 		     
-		    DeleteFile(tempName);
-		    if (MoveFile(dst, tempName) != FALSE) {
-			if (MoveFile(src, dst) != FALSE) {
-			    SetFileAttributes(tempName, FILE_ATTRIBUTE_NORMAL);
-			    DeleteFile(tempName);
+		    DeleteFileW(tempName);
+		    if (MoveFileW(wdst, tempName) != FALSE) {
+			if (MoveFileW(wsrc, wdst) != FALSE) {
+			    SetFileAttributesW(tempName, FILE_ATTRIBUTE_NORMAL);
+			    DeleteFileW(tempName);
 			    return 1;
 			} else {
-			    DeleteFile(dst);
-			    MoveFile(tempName, dst);
+			    DeleteFileW(wdst);
+			    MoveFileW(tempName, wdst);
 			}
 		    } 
 
@@ -558,11 +583,10 @@ char* dst;			/* New name. */
 }
 
 int
-efile_chdir(errInfo, name)
-Efile_error* errInfo;		/* Where to return error codes. */
-char* name;			/* Name of directory to make current. */
+efile_chdir(Efile_error* errInfo,	/* Where to return error codes. */
+	    char* name)			/* Name of directory to make current. */
 {
-    int success = check_error(chdir(name), errInfo);
+    int success = check_error(_wchdir((WCHAR *) name), errInfo);
     if (!success && errInfo->posix_errno == EINVAL)
 	/* POSIXification of errno */
 	errInfo->posix_errno = ENOENT;
@@ -570,59 +594,65 @@ char* name;			/* Name of directory to make current. */
 }
 
 int
-efile_getdcwd(errInfo, drive, buffer, size)
-Efile_error* errInfo;		/* Where to return error codes. */
-int drive;			/* 0 - current, 1 - A, 2 - B etc. */
-char* buffer;			/* Where to return the current directory. */
-size_t size;			/* Size of buffer. */
+efile_getdcwd(Efile_error* errInfo,		/* Where to return error codes. */
+	      int drive,			/* 0 - current, 1 - A, 2 - B etc. */
+	      char* buffer,			/* Where to return the current directory. */
+	      size_t size)			/* Size of buffer. */
 {
-    if (_getdcwd(drive, buffer, size) == NULL)
+    WCHAR *wbuffer = (WCHAR *) buffer;
+    size_t wbuffer_size = size / 2; 
+    if (_wgetdcwd(drive, wbuffer, wbuffer_size) == NULL)
 	return check_error(-1, errInfo);
-    for ( ; *buffer; buffer++) 
-	if (*buffer == '\\')
-	    *buffer = '/';
+    for ( ; *wbuffer; wbuffer++) 
+	if (*wbuffer == L'\\')
+	    *wbuffer = L'/';
     return 1;
 }
 
 int
-efile_readdir(errInfo, name, dir_handle, buffer, size)
-Efile_error* errInfo;		/* Where to return error codes. */
-char* name;			/* Name of directory to open. */
-EFILE_DIR_HANDLE* dir_handle;	/* Directory handle of open directory. */
-char* buffer;			/* Pointer to buffer for one filename. */
-size_t size;			/* Size of buffer. */
+efile_readdir(Efile_error* errInfo, /* Where to return error codes. */
+	      char* name,           /* Name of directory to list */
+	      EFILE_DIR_HANDLE* dir_handle, /* Handle of opened directory or NULL */
+	      char* buffer,                 /* Buffer to put one filename in */ 
+	      size_t *size)                 /* in-out size of buffer/size of filename excluding zero
+					       termination in bytes*/
 {
     HANDLE dir;			/* Handle to directory. */
-    char wildcard[MAX_PATH];	/* Wildcard to search for. */
-    WIN32_FIND_DATA findData;	/* Data found by FindFirstFile() or FindNext(). */
+    WCHAR wildcard[MAX_PATH];	/* Wildcard to search for. */
+    WIN32_FIND_DATAW findData;	/* Data found by FindFirstFile() or FindNext(). */
+    /* Alignment is not honored, this works on x86 because of alignment fixup by processor.
+       Not perfect, but faster than alinging by hand (really) */
+    WCHAR *wname = (WCHAR *) name;
+    WCHAR *wbuffer = (WCHAR *) buffer;
 
     /*
      * First time we must setup everything.
      */
 
     if (*dir_handle == NULL) {
-	int length = strlen(name);
-	char* s;
+	int length = wcslen(wname);
+	WCHAR* s;
 
 	if (length+3 >= MAX_PATH) {
 	    errno = ENAMETOOLONG;
 	    return check_error(-1, errInfo);
 	}
 
-	strcpy(wildcard, name);
+	wcscpy(wildcard, wname);
 	s = wildcard+length-1;
-	if (*s != '/' && *s != '\\')
-	    *++s = '\\';
-	*++s = '*';
-	*++s = '\0';
-	DEBUGF(("Reading %s\n", wildcard));
-	dir = FindFirstFile(wildcard, &findData);
+	if (*s != L'/' && *s != L'\\')
+	    *++s = L'\\';
+	*++s = L'*';
+	*++s = L'\0';
+	DEBUGF(("Reading %ws\n", wildcard));
+	dir = FindFirstFileW(wildcard, &findData);
 	if (dir == INVALID_HANDLE_VALUE)
 	    return set_error(errInfo);
 	*dir_handle = (EFILE_DIR_HANDLE) dir;
 
 	if (!IS_DOT_OR_DOTDOT(findData.cFileName)) {
-	    strcpy(buffer, findData.cFileName);
+	    wcscpy(wbuffer, findData.cFileName);
+	    *size = wcslen(wbuffer)*2;
 	    return 1;
 	}
     }
@@ -635,10 +665,11 @@ size_t size;			/* Size of buffer. */
     dir = (HANDLE) *dir_handle;
 
     for (;;) {
-	if (FindNextFile(dir, &findData)) {
+	if (FindNextFileW(dir, &findData)) {
 	    if (IS_DOT_OR_DOTDOT(findData.cFileName))
 		continue;
-	    strcpy(buffer, findData.cFileName);
+	    wcscpy(wbuffer, findData.cFileName);
+	    *size = wcslen(wbuffer)*2;
 	    return 1;
 	}
 
@@ -655,17 +686,17 @@ size_t size;			/* Size of buffer. */
 }
 
 int
-efile_openfile(errInfo, name, flags, pfd, pSize)
-Efile_error* errInfo;		/* Where to return error codes. */
-char* name;			/* Name of directory to open. */
-int flags;			/* Flags to use for opening. */
-int* pfd;			/* Where to store the file descriptor. */
-Sint64* pSize;			/* Where to store the size of the file. */
+efile_openfile(Efile_error* errInfo,		/* Where to return error codes. */
+	       char* name,			/* Name of directory to open. */
+	       int flags,			/* Flags to use for opening. */
+	       int* pfd,			/* Where to store the file descriptor. */
+	       Sint64* pSize)			/* Where to store the size of the file. */
 {
     BY_HANDLE_FILE_INFORMATION fileInfo; /* File information from a handle. */
     HANDLE fd;			/* Handle to open file. */
     DWORD access;		/* Access mode: GENERIC_READ, GENERIC_WRITE. */
     DWORD crFlags;
+    WCHAR *wname = (WCHAR *) name;
 
     switch (flags & (EFILE_MODE_READ|EFILE_MODE_WRITE)) {
     case EFILE_MODE_READ:
@@ -689,7 +720,11 @@ Sint64* pSize;			/* Where to store the size of the file. */
     if (flags & EFILE_MODE_APPEND) {
 	crFlags = OPEN_ALWAYS;
     }
-    fd = CreateFile(name, access, FILE_SHARE_READ | FILE_SHARE_WRITE,
+    if (flags & EFILE_MODE_EXCL) {
+	crFlags = CREATE_NEW;
+    }
+    fd = CreateFileW(wname, access,
+		    FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
 		    NULL, crFlags, FILE_ATTRIBUTE_NORMAL, NULL);
 
     /*
@@ -707,7 +742,7 @@ Sint64* pSize;			/* Where to store the size of the file. */
 	 * to EISDIR.
 	 */
 	if (errInfo->posix_errno && 
-	    (attr = GetFileAttributes(name)) != INVALID_FILE_ATTRIBUTES && 
+	    (attr = GetFileAttributesW(wname)) != INVALID_FILE_ATTRIBUTES && 
 	    (attr & FILE_ATTRIBUTE_DIRECTORY)) {
 	    errInfo->posix_errno = EISDIR;
 	}
@@ -731,9 +766,10 @@ Sint64* pSize;			/* Where to store the size of the file. */
 
 int 
 efile_may_openfile(Efile_error* errInfo, char *name) {
+    WCHAR *wname = (WCHAR *) name;
     DWORD attr;
 
-    if ((attr = GetFileAttributes(name)) == INVALID_FILE_ATTRIBUTES) {
+    if ((attr = GetFileAttributesW(wname)) == INVALID_FILE_ATTRIBUTES) {
 	return check_error(-1, errInfo);
     }
 
@@ -742,18 +778,6 @@ efile_may_openfile(Efile_error* errInfo, char *name) {
 	return check_error(-1, errInfo);
     }
     return 1;
-#if 0
-    struct stat statbuf;
-    
-    if (stat(name, &statbuf)) {
-	return check_error(-1, errInfo);
-    }
-    if (ISDIR(statbuf)) {
-	errno = EISDIR;
-	return check_error(-1, errInfo);
-    }
-    return 1;
-#endif
 }
 
 void
@@ -764,6 +788,15 @@ int fd;				/* File descriptor for file to close. */
 }
 
 int
+efile_fdatasync(errInfo, fd)
+Efile_error* errInfo;		/* Where to return error codes. */
+int fd;				/* File descriptor for file to sync. */
+{
+    /* Not available in Windows, just call regular fsync */
+    return efile_fsync(errInfo, fd);
+}
+
+int
 efile_fsync(errInfo, fd)
 Efile_error* errInfo;		/* Where to return error codes. */
 int fd;				/* File descriptor for file to sync. */
@@ -779,16 +812,17 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo,
 	       char* orig_name, int info_for_link)
 {
     HANDLE findhandle;		/* Handle returned by FindFirstFile(). */
-    WIN32_FIND_DATA findbuf;	/* Data return by FindFirstFile(). */
-    char name[_MAX_PATH];
+    WIN32_FIND_DATAW findbuf;	/* Data return by FindFirstFile(). */
+    WCHAR name[_MAX_PATH];
     int name_len;
-    char* path;
-    char pathbuf[_MAX_PATH];
+    WCHAR *path;
+    WCHAR pathbuf[_MAX_PATH];
     int drive;			/* Drive for filename (1 = A:, 2 = B: etc). */
+    WCHAR *worig_name = (WCHAR *) orig_name; 
 
     /* Don't allow wildcards to be interpreted by system */
 
-    if (strpbrk(orig_name, "?*")) {
+    if (wcspbrk(worig_name, L"?*")) {
     enoent:
 	errInfo->posix_errno = ENOENT;
 	errInfo->os_errno = ERROR_FILE_NOT_FOUND;
@@ -800,25 +834,25 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo,
      * slash, because it causes FindFirstFile() to fail on Win95.
      */
 
-    if ((name_len = strlen(orig_name)) >= _MAX_PATH) {
+    if ((name_len = wcslen(worig_name)) >= _MAX_PATH) {
 	goto enoent;
     } else {
-	strcpy(name, orig_name);
+	wcscpy(name, worig_name);
 	if (name_len > 2 && ISSLASH(name[name_len-1]) &&
-	    name[name_len-2] != ':') {
-	    name[name_len-1] = '\0';
+	    name[name_len-2] != L':') {
+	    name[name_len-1] = L'\0';
 	}
     }
     
     /* Try to get disk from name.  If none, get current disk.  */
 
-    if (name[1] != ':') {
+    if (name[1] != L':') {
         drive = 0;
-        if (GetCurrentDirectory(sizeof(pathbuf), pathbuf) &&
-	    pathbuf[1] == ':') {
-	    drive = tolower(pathbuf[0]) - 'a' + 1;
+        if (GetCurrentDirectoryW(_MAX_PATH, pathbuf) &&
+	    pathbuf[1] == L':') {
+	    drive = towlower(pathbuf[0]) - L'a' + 1;
 	}
-    } else if (*name && name[2] == '\0') {
+    } else if (*name && name[2] == L'\0') {
 	/*
 	 * X: and nothing more is an error.
 	 */
@@ -826,15 +860,15 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo,
 	errInfo->os_errno = ERROR_FILE_NOT_FOUND;
 	return 0;
     } else
-        drive = tolower(*name) - 'a' + 1;
+        drive = towlower(*name) - L'a' + 1;
 
-    findhandle = FindFirstFile(name, &findbuf);
+    findhandle = FindFirstFileW(name, &findbuf);
     if (findhandle == INVALID_HANDLE_VALUE) {
-        if (!(strpbrk(name, "./\\") &&
-	      (path = _fullpath(pathbuf, name, _MAX_PATH)) &&
+        if (!(wcspbrk(name, L"./\\") &&
+	      (path = _wfullpath(pathbuf, name, _MAX_PATH)) &&
 	      /* root dir. ('C:\') or UNC root dir. ('\\server\share\') */
-	      ((strlen(path) == 3) || IsRootUNCName(path)) &&
-	      (GetDriveType(path) > 1)   ) ) {
+	      ((wcslen(path) == 3) || is_root_unc_name(path)) &&
+	      (GetDriveTypeW(path) > 1)   ) ) {
 	    errInfo->posix_errno = ENOENT;
 	    errInfo->os_errno = ERROR_FILE_NOT_FOUND;
 	    return 0;
@@ -847,55 +881,68 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo,
         findbuf.dwFileAttributes = FILE_ATTRIBUTE_DIRECTORY;
         findbuf.nFileSizeHigh = 0;
         findbuf.nFileSizeLow = 0;
-        findbuf.cFileName[0] = '\0';
-
-	pInfo->modifyTime.year = 1980;
-	pInfo->modifyTime.month = 1;
-	pInfo->modifyTime.day = 1;
-	pInfo->modifyTime.hour = 0;
-	pInfo->modifyTime.minute = 0;
-	pInfo->modifyTime.second = 0;
+        findbuf.cFileName[0] = L'\0';
 
-	pInfo->accessTime = pInfo->modifyTime;
+	pInfo->links = 1;
+	pInfo->cTime = pInfo->accessTime = pInfo->modifyTime = 0;
     } else {
 	SYSTEMTIME SystemTime;
         FILETIME LocalFTime;
 
-#define GET_TIME(dst, src) \
-if (!FileTimeToLocalFileTime(&findbuf.src, &LocalFTime) || \
-    !FileTimeToSystemTime(&LocalFTime, &SystemTime)) { \
-    return set_error(errInfo); \
-} \
-(dst).year = SystemTime.wYear; \
-(dst).month = SystemTime.wMonth; \
-(dst).day = SystemTime.wDay; \
-(dst).hour = SystemTime.wHour; \
-(dst).minute = SystemTime.wMinute; \
-(dst).second = SystemTime.wSecond;
+	/*first check if we are a symlink */
+	if (!info_for_link && (findbuf.dwFileAttributes &
+			       FILE_ATTRIBUTE_REPARSE_POINT)){
+	    /*
+	     * given that we know this is a symlink,
+	     we should be able to find its target */
+	    WCHAR target_name[_MAX_PATH];
+	    if (efile_readlink(errInfo, (char *) name, 
+			       (char *) target_name,256) == 1) {
+		FindClose(findhandle);
+		return efile_fileinfo(errInfo, pInfo,
+				      (char *) target_name, info_for_link);
+	    }
+	}
+
+	/* number of links: */
+	{
+	    HANDLE handle;	/* Handle returned by CreateFile() */
+	    BY_HANDLE_FILE_INFORMATION fileInfo; /* from  CreateFile() */
+	    if (handle = CreateFileW(name, GENERIC_READ, 0,NULL,
+				    OPEN_EXISTING, 0, NULL)) {
+		GetFileInformationByHandle(handle, &fileInfo);
+		pInfo->links = fileInfo.nNumberOfLinks;
+		CloseHandle(handle);
+	    } else {
+		pInfo->links = 1;
+	    }	
+	}
 
-        GET_TIME(pInfo->modifyTime, ftLastWriteTime);
+        FILETIME_TO_EPOCH(pInfo->modifyTime, findbuf.ftLastWriteTime);
 
         if (findbuf.ftLastAccessTime.dwLowDateTime == 0 &&
 	    findbuf.ftLastAccessTime.dwHighDateTime == 0) {
 	    pInfo->accessTime = pInfo->modifyTime;
 	} else {
-	    GET_TIME(pInfo->accessTime, ftLastAccessTime);
+	    FILETIME_TO_EPOCH(pInfo->accessTime, findbuf.ftLastAccessTime);
 	}
 
         if (findbuf.ftCreationTime.dwLowDateTime == 0 &&
 	    findbuf.ftCreationTime.dwHighDateTime == 0) {
 	    pInfo->cTime = pInfo->modifyTime;
 	} else {
-	    GET_TIME(pInfo->cTime, ftCreationTime);
+	    FILETIME_TO_EPOCH(pInfo->cTime ,findbuf.ftCreationTime);
 	}
-#undef GET_TIME
         FindClose(findhandle);
     }
 
     pInfo->size_low = findbuf.nFileSizeLow;
     pInfo->size_high = findbuf.nFileSizeHigh;
 	
-    if (findbuf.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+    if (info_for_link && (findbuf.dwFileAttributes &
+			  FILE_ATTRIBUTE_REPARSE_POINT))
+	pInfo->type = FT_SYMLINK;
+    else if (findbuf.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
 	pInfo->type = FT_DIRECTORY;
     else
 	pInfo->type = FT_REGULAR;
@@ -906,7 +953,6 @@ if (!FileTimeToLocalFileTime(&findbuf.src, &LocalFTime) || \
 	pInfo->access = FA_READ|FA_WRITE;
 
     pInfo->mode = dos_to_posix_mode(findbuf.dwFileAttributes, name);
-    pInfo->links = 1;
     pInfo->major_device = drive;
     pInfo->minor_device = 0;
     pInfo->inode = 0;
@@ -917,29 +963,24 @@ if (!FileTimeToLocalFileTime(&findbuf.src, &LocalFTime) || \
 }
 
 int
-efile_write_info(errInfo, pInfo, name)
-Efile_error* errInfo;
-Efile_info* pInfo;
-char* name;
+efile_write_info(Efile_error* errInfo,
+		 Efile_info* pInfo,
+		 char* name)
 {
     SYSTEMTIME timebuf;
-    FILETIME LocalFileTime;
     FILETIME ModifyFileTime;
     FILETIME AccessFileTime;
     FILETIME CreationFileTime;
     HANDLE fd;
-    FILETIME* mtime = NULL;
-    FILETIME* atime = NULL;
-    FILETIME* ctime = NULL;
     DWORD attr;
     DWORD tempAttr;
-    BOOL modifyTime = FALSE;
+    WCHAR *wname = (WCHAR *) name;
 
     /*
      * Get the attributes for the file.
      */
 
-    tempAttr = attr = GetFileAttributes((LPTSTR)name);
+    tempAttr = attr = GetFileAttributesW(wname);
     if (attr == 0xffffffff) {
 	return set_error(errInfo);
     }
@@ -957,57 +998,36 @@ char* name;
      * Construct all file times.
      */
 
-#define MKTIME(tb, ts, ptr) \
-    timebuf.wYear = ts.year; \
-    timebuf.wMonth = ts.month; \
-    timebuf.wDay = ts.day; \
-    timebuf.wHour = ts.hour; \
-    timebuf.wMinute = ts.minute; \
-    timebuf.wSecond = ts.second; \
-    timebuf.wMilliseconds = 0; \
-    if (ts.year != -1) { \
-      modifyTime = TRUE; \
-      ptr = &tb; \
-      if (!SystemTimeToFileTime(&timebuf, &LocalFileTime ) || \
-	!LocalFileTimeToFileTime(&LocalFileTime, &tb)) { \
-        errno = EINVAL; \
-	return check_error(-1, errInfo); \
-     } \
-    }
-
-    MKTIME(ModifyFileTime, pInfo->accessTime, mtime);
-    MKTIME(AccessFileTime, pInfo->modifyTime, atime);
-    MKTIME(CreationFileTime, pInfo->cTime, ctime);
-#undef MKTIME
+    EPOCH_TO_FILETIME(ModifyFileTime,   pInfo->modifyTime);
+    EPOCH_TO_FILETIME(AccessFileTime,   pInfo->accessTime);
+    EPOCH_TO_FILETIME(CreationFileTime, pInfo->cTime);
 
     /*
      * If necessary, set the file times.
      */
 
-    if (modifyTime) {
-	/*
-	 * If the has read only access, we must temporarily turn on
-	 * write access (this is necessary for native filesystems,
-	 * but not for NFS filesystems).
-	 */
+    /*
+     * If the has read only access, we must temporarily turn on
+     * write access (this is necessary for native filesystems,
+     * but not for NFS filesystems).
+     */
 
-	if (tempAttr & FILE_ATTRIBUTE_READONLY) {
-	    tempAttr &= ~FILE_ATTRIBUTE_READONLY;
-	    if (!SetFileAttributes((LPTSTR) name, tempAttr)) {
-		return set_error(errInfo);
-	    }
+    if (tempAttr & FILE_ATTRIBUTE_READONLY) {
+	tempAttr &= ~FILE_ATTRIBUTE_READONLY;
+	if (!SetFileAttributesW(wname, tempAttr)) {
+	    return set_error(errInfo);
 	}
+    }
 
-	fd = CreateFile(name, GENERIC_READ|GENERIC_WRITE,
-			FILE_SHARE_READ | FILE_SHARE_WRITE,
-			NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
-	if (fd != INVALID_HANDLE_VALUE) {
-	    BOOL result = SetFileTime(fd, ctime, atime, mtime);
-	    if (!result) {
-		return set_error(errInfo);
-	    }
-	    CloseHandle(fd);
+    fd = CreateFileW(wname, GENERIC_READ|GENERIC_WRITE,
+	    FILE_SHARE_READ | FILE_SHARE_WRITE,
+	    NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+    if (fd != INVALID_HANDLE_VALUE) {
+	BOOL result = SetFileTime(fd, &CreationFileTime, &AccessFileTime, &ModifyFileTime);
+	if (!result) {
+	    return set_error(errInfo);
 	}
+	CloseHandle(fd);
     }
 
     /*
@@ -1016,7 +1036,7 @@ char* name;
      */
 
     if (tempAttr != attr) {
-	if (!SetFileAttributes((LPTSTR) name, attr)) {
+	if (!SetFileAttributesW(wname, attr)) {
 	    return set_error(errInfo);
 	}
     }
@@ -1069,12 +1089,17 @@ char* buf;			/* Buffer to write. */
 size_t count;			/* Number of bytes to write. */
 {
     DWORD written;		/* Bytes written in last operation. */
+    OVERLAPPED overlapped;
+    OVERLAPPED* pOverlapped = NULL;
 
     if (flags & EFILE_MODE_APPEND) {
-	(void) SetFilePointer((HANDLE) fd, 0, NULL, FILE_END);
+	memset(&overlapped, 0, sizeof(overlapped));
+	overlapped.Offset = 0xffffffff;
+	overlapped.OffsetHigh = 0xffffffff;
+	pOverlapped = &overlapped;
     }
     while (count > 0) {
-	if (!WriteFile((HANDLE) fd, buf, count, &written, NULL))
+	if (!WriteFile((HANDLE) fd, buf, count, &written, pOverlapped))
 	    return set_error(errInfo);
 	buf += written;
 	count -= written;
@@ -1090,15 +1115,19 @@ efile_writev(Efile_error* errInfo,   /* Where to return error codes */
 	     SysIOVec* iov,          /* Vector of buffer structs.
 				      * The structs are unchanged 
 				      * after the call */
-	     int iovcnt,             /* Number of structs in vector */
-	     size_t size)            /* Number of bytes to write */
+	     int iovcnt)             /* Number of structs in vector */
 {
     int cnt;                         /* Buffers so far written */
+    OVERLAPPED overlapped;
+    OVERLAPPED* pOverlapped = NULL;
 
     ASSERT(iovcnt >= 0);
     
     if (flags & EFILE_MODE_APPEND) {
-	(void) SetFilePointer((HANDLE) fd, 0, NULL, FILE_END);
+	memset(&overlapped, 0, sizeof(overlapped));
+	overlapped.Offset = 0xffffffff;
+	overlapped.OffsetHigh = 0xffffffff;
+	pOverlapped = &overlapped;
     }
     for (cnt = 0; cnt < iovcnt; cnt++) {
 	if (iov[cnt].iov_base && iov[cnt].iov_len > 0) {
@@ -1110,7 +1139,7 @@ efile_writev(Efile_error* errInfo,   /* Where to return error codes */
 			       iov[cnt].iov_base + p, 
 			       iov[cnt].iov_len - p, 
 			       &w, 
-			       NULL))
+			       pOverlapped))
 		    return set_error(errInfo);
 	    }
 	}
@@ -1182,7 +1211,7 @@ int flags;
 
 
 /*
- * IsRootUNCName - returns TRUE if the argument is a UNC name specifying
+ * is_root_unc_name - returns TRUE if the argument is a UNC name specifying
  *      a root share.  That is, if it is of the form \\server\share\.
  *      This routine will also return true if the argument is of the
  *      form \\server\share (no trailing slash) but Win32 currently
@@ -1192,16 +1221,16 @@ int flags;
  */
 
 static int
-IsRootUNCName(const char* path)
+is_root_unc_name(const WCHAR *path)
 {
     /*
      * If a root UNC name, path will start with 2 (but not 3) slashes
      */
 
-    if ((strlen(path) >= 5) /* minimum string is "//x/y" */
+    if ((wcslen(path) >= 5) /* minimum string is "//x/y" */
 	&& ISSLASH(path[0]) && ISSLASH(path[1]))
     {
-        const char * p = path + 2 ;
+        const WCHAR *p = path + 2;
 
         /*
          * find the slash between the server name and share name
@@ -1244,19 +1273,19 @@ IsRootUNCName(const char* path)
  */
 
 static int
-extract_root(char* name)
+extract_root(WCHAR* name)
 {
-    int len = strlen(name);
+    int len = wcslen(name);
 
-    if (isalpha(name[0]) && name[1] == ':' && ISSLASH(name[2])) {
-	int c = name[3];
-	name[3] = '\0';
-	return c == '\0';
+    if (iswalpha(name[0]) && name[1] == L':' && ISSLASH(name[2])) {
+	WCHAR c = name[3];
+	name[3] = L'\0';
+	return c == L'\0';
     } else if (len < 5 || !ISSLASH(name[0]) || !ISSLASH(name[1])) {
 	goto error;
     } else {			/* Try to find the end of the UNC name. */
-	char* p;
-	int c;
+	WCHAR* p;
+	WCHAR c;
 
         /*
          * Find the slash between the server name and share name.
@@ -1265,7 +1294,7 @@ extract_root(char* name)
 	for (p = name + 2; *p; p++)
             if (ISSLASH(*p))
                 break;
-	if (*p == '\0')
+	if (*p == L'\0')
 	    goto error;
 
 	/*
@@ -1276,24 +1305,24 @@ extract_root(char* name)
             if (ISSLASH(*p))
                 break;
 	c = *p;
-	*p = '\0';
-	return c == '\0' || p[1] == '\0';
+	*p = L'\0';
+	return c == L'\0' || p[1] == L'\0';
     }
 
  error:
-    *name = '\0';
+    *name = L'\0';
     return 1;
 }
 
 static unsigned short
-dos_to_posix_mode(int attr, const char *name)
+dos_to_posix_mode(int attr, const WCHAR *name)
 {
     register unsigned short uxmode;
     unsigned dosmode;
-    register const char *p;
+    register const WCHAR *p;
 
     dosmode = attr & 0xff;
-    if ((p = name)[1] == ':')
+    if ((p = name)[1] == L':')
 	p += 2;
 
     /* check to see if this is a directory - note we must make a special
@@ -1302,7 +1331,7 @@ dos_to_posix_mode(int attr, const char *name)
 
     uxmode = (unsigned short)
 	(((ISSLASH(*p) && !p[1]) || (dosmode & FILE_ATTRIBUTE_DIRECTORY) ||
-	       *p == '\0') ? _S_IFDIR|_S_IEXEC : _S_IFREG);
+	       *p == L'\0') ? _S_IFDIR|_S_IEXEC : _S_IFREG);
 
     /* If attribute byte does not have read-only bit, it is read-write */
 
@@ -1311,11 +1340,11 @@ dos_to_posix_mode(int attr, const char *name)
 
     /* see if file appears to be executable - check extension of name */
 
-    if (p = strrchr(name, '.')) {
-        if (!stricmp(p, ".exe") ||
-	    !stricmp(p, ".cmd") ||
-	    !stricmp(p, ".bat") ||
-	    !stricmp(p, ".com"))
+    if (p = wcsrchr(name, L'.')) {
+        if (!_wcsicmp(p, L".exe") ||
+	    !_wcsicmp(p, L".cmd") ||
+	    !_wcsicmp(p, L".bat") ||
+	    !_wcsicmp(p, L".com"))
             uxmode |= _S_IEXEC;
     }
 
@@ -1330,6 +1359,60 @@ dos_to_posix_mode(int attr, const char *name)
 int
 efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size)
 {
+    /*
+     * load dll and see if we have CreateSymbolicLink at runtime:
+     * (Vista only)
+     */
+    HINSTANCE hModule = NULL;
+    WCHAR *wname = (WCHAR *) name;
+    WCHAR *wbuffer = (WCHAR *) buffer;
+    if ((hModule = LoadLibrary("kernel32.dll")) != NULL) {
+	typedef DWORD (WINAPI * GETFINALPATHNAMEBYHANDLEPTR)(
+							     HANDLE hFile,
+							     LPCWSTR lpFilePath,
+							     DWORD cchFilePath,
+							     DWORD dwFlags);
+
+	GETFINALPATHNAMEBYHANDLEPTR pGetFinalPathNameByHandle =
+	    (GETFINALPATHNAMEBYHANDLEPTR)GetProcAddress(hModule, "GetFinalPathNameByHandleW");
+
+	if (pGetFinalPathNameByHandle == NULL) {
+	    FreeLibrary(hModule);
+	} else {
+	    /* first check if file is a symlink; {error, einval} otherwise */
+	    DWORD fileAttributes =  GetFileAttributesW(wname);
+	    if ((fileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) {
+		BOOLEAN success = 0;
+		HANDLE h = CreateFileW(wname, GENERIC_READ, 0,NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL);
+		int len;
+		if(h != INVALID_HANDLE_VALUE) {
+		    success = pGetFinalPathNameByHandle(h, wbuffer, size,0);
+		    /* GetFinalPathNameByHandle prepends path with "\\?\": */
+		    len = wcslen(wbuffer);
+		    wmemmove(wbuffer,wbuffer+4,len-3);
+		    if (len - 4 >= 2 && wbuffer[1] == L':' && wbuffer[0] >= L'A' &&
+			wbuffer[0] <= L'Z') {
+			wbuffer[0] = wbuffer[0] + L'a' - L'A';
+		    }
+			
+		    for ( ; *wbuffer; wbuffer++) 
+			if (*wbuffer == L'\\')
+			    *wbuffer = L'/';
+		    CloseHandle(h);
+		} 
+		FreeLibrary(hModule);
+		if (success) {
+		    return 1;
+		} else {
+		    return set_error(errInfo);
+		}
+	    } else {
+		FreeLibrary(hModule);
+		errno = EINVAL;
+		return check_error(-1, errInfo);
+	    }
+	}
+    }
     errno = ENOTSUP;
     return check_error(-1, errInfo);
 }
@@ -1338,17 +1421,20 @@ efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size)
 int
 efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size)
 {
-    WIN32_FIND_DATA wfd;
+    WIN32_FIND_DATAW wfd;
     HANDLE fh;
-    char name[_MAX_PATH];
+    WCHAR name[_MAX_PATH+1];
     int name_len;
-    char* path;
-    char pathbuf[_MAX_PATH];
+    WCHAR* path;
+    WCHAR pathbuf[_MAX_PATH+1]; /* Unclear weather GetCurrentDirectory will access one char after
+				   _MAX_PATH */
+    WCHAR *worig_name = (WCHAR *) orig_name;
+    WCHAR *wbuffer = (WCHAR *) buffer;
     int drive;			/* Drive for filename (1 = A:, 2 = B: etc). */
 
     /* Don't allow wildcards to be interpreted by system */
 
-    if (strpbrk(orig_name, "?*")) {
+    if (wcspbrk(worig_name, L"?*")) {
     enoent:
 	errInfo->posix_errno = ENOENT;
 	errInfo->os_errno = ERROR_FILE_NOT_FOUND;
@@ -1360,67 +1446,114 @@ efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size)
      * slash, because it causes FindFirstFile() to fail on Win95.
      */
 
-    if ((name_len = strlen(orig_name)) >= _MAX_PATH) {
+    if ((name_len = wcslen(worig_name)) >= _MAX_PATH) {
 	goto enoent;
     } else {
-	strcpy(name, orig_name);
+	wcscpy(name, worig_name);
 	if (name_len > 2 && ISSLASH(name[name_len-1]) &&
-	    name[name_len-2] != ':') {
-	    name[name_len-1] = '\0';
+	    name[name_len-2] != L':') {
+	    name[name_len-1] = L'\0';
 	}
     }
     
     /* Try to get disk from name.  If none, get current disk.  */
 
-    if (name[1] != ':') {
+    if (name[1] != L':') {
         drive = 0;
-        if (GetCurrentDirectory(sizeof(pathbuf), pathbuf) &&
-	    pathbuf[1] == ':') {
-	    drive = tolower(pathbuf[0]) - 'a' + 1;
+        if (GetCurrentDirectoryW(_MAX_PATH, pathbuf) &&
+	    pathbuf[1] == L':') {
+	    drive = towlower(pathbuf[0]) - L'a' + 1;
 	}
-    } else if (*name && name[2] == '\0') {
+    } else if (*name && name[2] == L'\0') {
 	/*
 	 * X: and nothing more is an error.
 	 */
 	goto enoent;
     } else {
-        drive = tolower(*name) - 'a' + 1;
+        drive = towlower(*name) - L'a' + 1;
     }
-    fh = FindFirstFile(name,&wfd);
+    fh = FindFirstFileW(name,&wfd);
     if (fh == INVALID_HANDLE_VALUE) {
-        if (!(strpbrk(name, "./\\") &&
-	      (path = _fullpath(pathbuf, name, _MAX_PATH)) &&
+        if (!(wcspbrk(name, L"./\\") &&
+	      (path = _wfullpath(pathbuf, name, _MAX_PATH)) &&
 	      /* root dir. ('C:\') or UNC root dir. ('\\server\share\') */
-	      ((strlen(path) == 3) || IsRootUNCName(path)) &&
-	      (GetDriveType(path) > 1)   ) ) {
+	      ((wcslen(path) == 3) || is_root_unc_name(path)) &&
+	      (GetDriveTypeW(path) > 1)   ) ) {
 	    errno = errno_map(GetLastError());
 	    return check_error(-1, errInfo);
 	}
         /*
          * Root directories (such as C:\ or \\server\share\ are fabricated.
          */
-	strcpy(buffer,name);
+	wcscpy(wbuffer,name);
 	return 1;
     }
 	
-    strcpy(buffer,wfd.cAlternateFileName);
-    if (!*buffer) {
-	strcpy(buffer,wfd.cFileName);
+    wcscpy(wbuffer,wfd.cAlternateFileName);
+    if (!*wbuffer) {
+	wcscpy(wbuffer,wfd.cFileName);
     }
-
+    FindClose(fh);
     return 1;
 }
 
+
 int
 efile_link(Efile_error* errInfo, char* old, char* new)
 {
-    errno = ENOTSUP;
-    return check_error(-1, errInfo);
+    WCHAR *wold = (WCHAR *) old;
+    WCHAR *wnew = (WCHAR *) new;
+    if(!CreateHardLinkW(wnew, wold, NULL)) {
+	return set_error(errInfo);
+    }
+    return 1;
 }
 
 int
 efile_symlink(Efile_error* errInfo, char* old, char* new)
 {
+    /*
+     * Load dll and see if we have CreateSymbolicLink at runtime:
+     * (Vista only)
+     */
+    HINSTANCE hModule = NULL;
+    WCHAR *wold = (WCHAR *) old;
+    WCHAR *wnew = (WCHAR *) new;
+    if ((hModule = LoadLibrary("kernel32.dll")) != NULL) {
+	typedef BOOLEAN (WINAPI  * CREATESYMBOLICLINKFUNCPTR) (
+	     LPCWSTR lpSymlinkFileName,
+	     LPCWSTR lpTargetFileName,
+	     DWORD dwFlags);
+
+	CREATESYMBOLICLINKFUNCPTR pCreateSymbolicLink =
+	    (CREATESYMBOLICLINKFUNCPTR) GetProcAddress(hModule,
+						       "CreateSymbolicLinkW");
+	/* A for MBCS, W for UNICODE... char* above implies 'W'! */
+	if (pCreateSymbolicLink != NULL) {
+	    DWORD attr = GetFileAttributesW(wold);
+	    int flag = (attr != INVALID_FILE_ATTRIBUTES &&
+			attr & FILE_ATTRIBUTE_DIRECTORY) ? 1 : 0;
+	    /*  SYMBOLIC_LINK_FLAG_DIRECTORY = 1 */
+	    BOOLEAN success = pCreateSymbolicLink(wnew, wold, flag);
+	    FreeLibrary(hModule);
+
+	    if (success) {
+		return 1;
+	    } else {
+		return set_error(errInfo);
+	    }
+	} else
+	    FreeLibrary(hModule);
+    }
     errno = ENOTSUP;
     return check_error(-1, errInfo);
 }
+
+int
+efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset,
+	    Sint64 length, int advise)
+{
+    /* posix_fadvise is not available on Windows, do nothing */
+    errno = ERROR_SUCCESS;
+    return check_error(0, errInfo);
+}
diff --git a/erts/emulator/hipe/hipe_abi.txt b/erts/emulator/hipe/hipe_abi.txt
index aea30d262d..9d4726de9d 100644
--- a/erts/emulator/hipe/hipe_abi.txt
+++ b/erts/emulator/hipe/hipe_abi.txt
@@ -2,7 +2,7 @@
  %CopyrightBegin%
  %CopyrightEnd%
 
-$Id$
+
 
 HiPE ABI
 ========
@@ -62,7 +62,7 @@ exceptional condition, it puts an error code in p->freason
 and returns THE_NON_VALUE (zero, except in debug mode).
 
 If p->freason == TRAP, then the BIF redirects its call to some
-other function, given by p->def_arg_reg[].
+other function, given by p->i
 The BIF and the new callee may have different arities.
 
 The "hipe_${ARCH}_bifs.m4" macro files take care of these issues
diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c
index ff87492f4d..b5dff06987 100644
--- a/erts/emulator/hipe/hipe_amd64.c
+++ b/erts/emulator/hipe/hipe_amd64.c
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #include <stddef.h>	/* offsetof() */
 #ifdef HAVE_CONFIG_H
 #include "config.h"
@@ -334,43 +334,3 @@ void hipe_arch_print_pcb(struct hipe_process_state *p)
     U("narity     ", narity);
 #undef U
 }
-
-/*
- * XXX: The following should really be moved to a generic hipe_bifs_64 file.
- */
-
-#if 0 /* unused */
-static int term_to_Sint64(Eterm term, Sint64 *sp)
-{
-    return term_to_Sint(term, sp);
-}
-
-BIF_RETTYPE hipe_bifs_write_s64_2(BIF_ALIST_2)
-{
-    Sint64 *address;
-    Sint64 value;
-
-    address = term_to_address(BIF_ARG_1);
-    if (!address || !hipe_word64_address_ok(address))
-	BIF_ERROR(BIF_P, BADARG);
-    if (!term_to_Sint64(BIF_ARG_2, &value))
-	BIF_ERROR(BIF_P, BADARG);
-    *address = value;
-    BIF_RET(NIL);
-}
-#endif
-
-BIF_RETTYPE hipe_bifs_write_u64_2(BIF_ALIST_2)
-{
-    Uint64 *address;
-    Uint64 value;
-
-    address = term_to_address(BIF_ARG_1);
-    if (!address || !hipe_word64_address_ok(address))
-	BIF_ERROR(BIF_P, BADARG);
-    if (!term_to_Uint(BIF_ARG_2, &value))
-	BIF_ERROR(BIF_P, BADARG);
-    *address = value;
-    hipe_flush_icache_word(address);
-    BIF_RET(NIL);
-}
diff --git a/erts/emulator/hipe/hipe_amd64.h b/erts/emulator/hipe/hipe_amd64.h
index 532d47c092..bf41d238dd 100644
--- a/erts/emulator/hipe/hipe_amd64.h
+++ b/erts/emulator/hipe/hipe_amd64.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_AMD64_H
 #define HIPE_AMD64_H
 
diff --git a/erts/emulator/hipe/hipe_amd64.tab b/erts/emulator/hipe/hipe_amd64.tab
index 3787bbf23b..e039d74525 100644
--- a/erts/emulator/hipe/hipe_amd64.tab
+++ b/erts/emulator/hipe/hipe_amd64.tab
@@ -1,28 +1,24 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2004-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2004-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
-# $Id$
 # AMD64-specific atoms and bifs
 
 atom amd64
 atom handle_fp_exception
 atom inc_stack_0
 atom sse2_fnegate_mask
-
-# bif hipe_bifs:write_s64/2
-bif hipe_bifs:write_u64/2
diff --git a/erts/emulator/hipe/hipe_amd64_abi.txt b/erts/emulator/hipe/hipe_amd64_abi.txt
index 27beff4ea2..8a34bfa67f 100644
--- a/erts/emulator/hipe/hipe_amd64_abi.txt
+++ b/erts/emulator/hipe/hipe_amd64_abi.txt
@@ -2,7 +2,7 @@
  %CopyrightBegin%
  %CopyrightEnd%
 
-$Id$
+
 
 HiPE AMD64 ABI
 ==============
diff --git a/erts/emulator/hipe/hipe_amd64_asm.m4 b/erts/emulator/hipe/hipe_amd64_asm.m4
index 7f563c35d8..7c81040b8b 100644
--- a/erts/emulator/hipe/hipe_amd64_asm.m4
+++ b/erts/emulator/hipe/hipe_amd64_asm.m4
@@ -2,7 +2,7 @@ changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2004-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -17,9 +17,8 @@ changecom(`/*', `*/')dnl
  *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
+
 `#ifndef HIPE_AMD64_ASM_H
 #define HIPE_AMD64_ASM_H'
 
diff --git a/erts/emulator/hipe/hipe_amd64_bifs.m4 b/erts/emulator/hipe/hipe_amd64_bifs.m4
index f7c9604e2b..ec25c0b9b7 100644
--- a/erts/emulator/hipe/hipe_amd64_bifs.m4
+++ b/erts/emulator/hipe/hipe_amd64_bifs.m4
@@ -2,7 +2,7 @@ changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2004-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -17,29 +17,40 @@ changecom(`/*', `*/')dnl
  *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
 
 include(`hipe/hipe_amd64_asm.m4')
+#`include' "config.h"
 #`include' "hipe_literals.h"
 
+
 `#if THE_NON_VALUE == 0
 #define TEST_GOT_EXN	testq	%rax, %rax
 #else
 #define TEST_GOT_EXN	cmpq	$THE_NON_VALUE, %rax
 #endif'
 
-`#define TEST_GOT_MBUF		movq P_MBUF(P), %rdx; testq %rdx, %rdx; jnz 3f; 2:
-#define JOIN3(A,B,C)		A##B##C
-#define HANDLE_GOT_MBUF(ARITY)	3: call JOIN3(nbif_,ARITY,_gc_after_bif); jmp 2b'
+define(TEST_GOT_MBUF,`movq P_MBUF(P), %rdx	# `TEST_GOT_MBUF'
+	testq %rdx, %rdx
+	jnz 3f	
+2:')
+define(HANDLE_GOT_MBUF,`
+3:	call nbif_$1_gc_after_bif	# `HANDLE_GOT_MBUF'
+	jmp 2b')
+
+`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+#  define CALL_BIF(F)	movq $CSYM(F), P_BIF_CALLEE(P); call CSYM(hipe_debug_bif_wrapper) 
+#else
+#  define CALL_BIF(F)	call	CSYM(F)
+#endif'
 
 /*
  * standard_bif_interface_1(nbif_name, cbif_name)
  * standard_bif_interface_2(nbif_name, cbif_name)
  * standard_bif_interface_3(nbif_name, cbif_name)
+ * standard_bif_interface_0(nbif_name, cbif_name)
  *
- * Generate native interface for a BIF with 1-3 parameters and
+ * Generate native interface for a BIF with 0-3 parameters and
  * standard failure mode.
  */
 define(standard_bif_interface_1,
@@ -56,7 +67,11 @@ ASYM($1):
 
 	/* make the call on the C stack */
 	SWITCH_ERLANG_TO_C
-	call	CSYM($2)
+	pushq	%rsi
+	movq	%rsp, %rsi	/* Eterm* BIF__ARGS */
+	sub	$(8), %rsp	/* stack frame 16-byte alignment */
+	CALL_BIF($2)
+	add	$(1*8 + 8), %rsp
 	TEST_GOT_MBUF
 	SWITCH_C_TO_ERLANG
 
@@ -84,7 +99,11 @@ ASYM($1):
 
 	/* make the call on the C stack */
 	SWITCH_ERLANG_TO_C
-	call	CSYM($2)
+	pushq	%rdx
+	pushq 	%rsi
+	movq	%rsp, %rsi	/* Eterm* BIF__ARGS */
+	CALL_BIF($2)
+	add	$(2*8), %rsp
 	TEST_GOT_MBUF
 	SWITCH_C_TO_ERLANG
 
@@ -113,7 +132,13 @@ ASYM($1):
 
 	/* make the call on the C stack */
 	SWITCH_ERLANG_TO_C
-	call	CSYM($2)
+	pushq 	%rcx
+	pushq	%rdx
+	pushq	%rsi
+	movq	%rsp, %rsi	/* Eterm* BIF__ARGS */
+	sub	$(8), %rsp	/* stack frame 16-byte alignment */  
+	CALL_BIF($2)
+	add	$(3*8 + 8), %rsp
 	TEST_GOT_MBUF
 	SWITCH_C_TO_ERLANG
 
@@ -126,13 +151,7 @@ ASYM($1):
 	TYPE_FUNCTION(ASYM($1))
 #endif')
 
-/*
- * fail_bif_interface_0(nbif_name, cbif_name)
- *
- * Generate native interface for a BIF with 0 parameters and
- * standard failure mode.
- */
-define(fail_bif_interface_0,
+define(standard_bif_interface_0,
 `
 #ifndef HAVE_$1
 #`define' HAVE_$1
@@ -145,7 +164,7 @@ ASYM($1):
 
 	/* make the call on the C stack */
 	SWITCH_ERLANG_TO_C
-	call	CSYM($2)
+	CALL_BIF($2)
 	TEST_GOT_MBUF
 	SWITCH_C_TO_ERLANG
 
@@ -530,7 +549,9 @@ ASYM($1):
 /*
  * AMD64-specific primops.
  */
+#ifndef NO_FPE_SIGNALS
 noproc_primop_interface_0(nbif_handle_fp_exception, erts_restore_fpu)
+#endif /* NO_FPE_SIGNALS */
 
 /*
  * Implement gc_bif_interface_0 as nofail_primop_interface_0.
diff --git a/erts/emulator/hipe/hipe_amd64_gc.h b/erts/emulator/hipe/hipe_amd64_gc.h
index 56650901d6..c5a6fee6fe 100644
--- a/erts/emulator/hipe/hipe_amd64_gc.h
+++ b/erts/emulator/hipe/hipe_amd64_gc.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * Stack walking helpers for native stack GC procedures.
  */
 #ifndef HIPE_AMD64_GC_H
diff --git a/erts/emulator/hipe/hipe_amd64_glue.S b/erts/emulator/hipe/hipe_amd64_glue.S
index 83b7b0397b..8816906870 100644
--- a/erts/emulator/hipe/hipe_amd64_glue.S
+++ b/erts/emulator/hipe/hipe_amd64_glue.S
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2004-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -16,9 +16,7 @@
  *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
 
 #include "hipe_amd64_asm.h"
 #include "hipe_literals.h"
@@ -346,6 +344,8 @@ nbif_3_gc_after_bif:
 	subq	$(16-8), %rsp
 	movq	P, %rdi
 	movq	%rax, %rsi
+	xorl	%edx, %edx		# Pass NULL in regs
+	xorl	%ecx, %ecx		# Pass 0 in arity
 	call	CSYM(erts_gc_after_bif_call)
 	addq	$(16-8), %rsp
 	movl	$0, P_NARITY(P)		# Note: narity is a 32-bit field
@@ -400,7 +400,7 @@ nbif_3_simple_exception:
 	 * - the native heap/stack/reds registers are saved in P
 	 */
 .handle_trap:
-	movq	%rax, P_NARITY(P)
+	movl	%eax, P_NARITY(P)	# Note: narity is a 32-bit field
 	movl	$HIPE_MODE_SWITCH_RES_TRAP, %eax
 	jmp	.nosave_exit
 
diff --git a/erts/emulator/hipe/hipe_amd64_glue.h b/erts/emulator/hipe/hipe_amd64_glue.h
index c92eb842cb..36508467fa 100644
--- a/erts/emulator/hipe/hipe_amd64_glue.h
+++ b/erts/emulator/hipe/hipe_amd64_glue.h
@@ -1,24 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
+
 #ifndef HIPE_AMD64_GLUE_H
 #define HIPE_AMD64_GLUE_H
 
diff --git a/erts/emulator/hipe/hipe_amd64_primops.h b/erts/emulator/hipe/hipe_amd64_primops.h
index dcfa8be92a..55cb0eadb8 100644
--- a/erts/emulator/hipe/hipe_amd64_primops.h
+++ b/erts/emulator/hipe/hipe_amd64_primops.h
@@ -1,23 +1,25 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 PRIMOP_LIST(am_inc_stack_0, &nbif_inc_stack_0)
+#ifndef NO_FPE_SIGNALS
 PRIMOP_LIST(am_handle_fp_exception, &nbif_handle_fp_exception)
+#endif
 PRIMOP_LIST(am_sse2_fnegate_mask, &sse2_fnegate_mask)
diff --git a/erts/emulator/hipe/hipe_arch.h b/erts/emulator/hipe/hipe_arch.h
index 7803543ef1..04ed980126 100644
--- a/erts/emulator/hipe/hipe_arch.h
+++ b/erts/emulator/hipe/hipe_arch.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_ARCH_H
 #define HIPE_ARCH_H
 
diff --git a/erts/emulator/hipe/hipe_arm.c b/erts/emulator/hipe/hipe_arm.c
index b70b32947b..651d0e3a75 100644
--- a/erts/emulator/hipe/hipe_arm.c
+++ b/erts/emulator/hipe/hipe_arm.c
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #include <stddef.h>	/* offsetof() */
 #ifdef HAVE_CONFIG_H
 #include "config.h"
@@ -71,7 +71,7 @@ static struct segment {
 } curseg;
 
 #define in_area(ptr,start,nbytes)	\
-	((unsigned long)((char*)(ptr) - (char*)(start)) < (nbytes))
+	((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
 
 static void *new_code_mapping(void)
 {
@@ -181,11 +181,9 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
 	curseg.base = base;
 	curseg.code_pos = base;
 	curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES);
-#if defined(__arm__)
 	curseg.tramp_pos -= 2;
 	curseg.tramp_pos[0] = 0xE51FF004;	/* ldr pc, [pc,#-4] */
 	curseg.tramp_pos[1] = (unsigned int)&nbif_callemu;
-#endif
 
 	address = try_alloc(nrwords, nrcallees, callees, trampvec);
 	if (!address) {
@@ -214,11 +212,9 @@ static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu)
 	curseg.base = base;
 	curseg.code_pos = base;
 	curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES);
-#if defined(__arm__)
 	curseg.tramp_pos -= 2;
 	curseg.tramp_pos[0] = 0xE51FF004;	/* ldr pc, [pc,#-4] */
 	curseg.tramp_pos[1] = (unsigned int)&nbif_callemu;
-#endif
 
 	address = try_alloc(nrwords, 0, NIL, NULL);
 	if (!address) {
@@ -269,10 +265,8 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type)
 void *hipe_make_native_stub(void *beamAddress, unsigned int beamArity)
 {
     unsigned int *code;
-#if defined(__arm__)
     unsigned int *tramp_callemu;
     int callemu_offset;
-#endif
 
     /*
      * Native code calls BEAM via a stub looking as follows:
@@ -288,13 +282,6 @@ void *hipe_make_native_stub(void *beamAddress, unsigned int beamArity)
      * (Trampolines are allowed to modify r12, but they don't.)
      */
 
-#if !defined(__arm__)
-    /* verify that 'ba' can reach nbif_callemu */
-    if ((unsigned long)&nbif_callemu & ~0x01FFFFFCUL)
-	abort();
-#endif
-
-#if defined(__arm__)
     code = alloc_stub(4, &tramp_callemu);
     callemu_offset = ((int)&nbif_callemu - ((int)&code[2] + 8)) >> 2;
     if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF)) {
@@ -302,11 +289,7 @@ void *hipe_make_native_stub(void *beamAddress, unsigned int beamArity)
 	if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF))
 	    abort();
     }
-#else
-    code = alloc_stub(4, &trampoline);
-#endif
 
-#if defined(__arm__)
     /* mov r0, #beamArity */
     code[0] = 0xE3A00000 | (beamArity & 0xFF);
     /* ldr r8, [pc,#0] // beamAddress */
@@ -315,16 +298,6 @@ void *hipe_make_native_stub(void *beamAddress, unsigned int beamArity)
     code[2] = 0xEA000000 | (callemu_offset & 0x00FFFFFF);
     /* .long beamAddress */
     code[3] = (unsigned int)beamAddress;
-#else
-    /* addi r12,0,beamAddress@l */
-    code[0] = 0x39800000 | ((unsigned long)beamAddress & 0xFFFF);
-    /* addi r0,0,beamArity */
-    code[1] = 0x38000000 | (beamArity & 0x7FFF);
-    /* addis r12,r12,beamAddress@ha */
-    code[2] = 0x3D8C0000 | at_ha((unsigned long)beamAddress);
-    /* ba nbif_callemu */
-    code[3] = 0x48000002 | (unsigned long)&nbif_callemu;
-#endif
 
     hipe_flush_icache_range(code, 4*sizeof(int));
 
@@ -334,60 +307,32 @@ void *hipe_make_native_stub(void *beamAddress, unsigned int beamArity)
 static void patch_b(Uint32 *address, Sint32 offset, Uint32 AA)
 {
     Uint32 oldI = *address;
-#if defined(__arm__)
     Uint32 newI = (oldI & 0xFF000000) | (offset & 0x00FFFFFF);
-#else
-    Uint32 newI = (oldI & 0xFC000001) | ((offset & 0x00FFFFFF) << 2) | (AA & 2);
-#endif
     *address = newI;
     hipe_flush_icache_word(address);
 }
 
 int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
 {
-#if !defined(__arm__)
-    if ((Uint32)destAddress == ((Uint32)destAddress & 0x01FFFFFC)) {
-	/* The destination is in the [0,32MB[ range.
-	   We can reach it with a ba/bla instruction.
-	   This is the typical case for BIFs and primops.
-	   It's also common for trap-to-BEAM stubs (on ppc32). */
-	patch_b((Uint32*)callAddress, (Uint32)destAddress >> 2, 2);
+    Sint32 destOffset = ((Sint32)destAddress - ((Sint32)callAddress+8)) >> 2;
+    if (destOffset >= -0x800000 && destOffset <= 0x7FFFFF) {
+	/* The destination is within a [-32MB,+32MB[ range from us.
+	   We can reach it with a b/bl instruction.
+	   This is typical for nearby Erlang code. */
+	patch_b((Uint32*)callAddress, destOffset, 0);
     } else {
-#endif
-#if defined(__arm__)
-	Sint32 destOffset = ((Sint32)destAddress - ((Sint32)callAddress+8)) >> 2;
-#else
-	Sint32 destOffset = ((Sint32)destAddress - (Sint32)callAddress) >> 2;
-#endif
-	if (destOffset >= -0x800000 && destOffset <= 0x7FFFFF) {
-	    /* The destination is within a [-32MB,+32MB[ range from us.
-	       We can reach it with a b/bl instruction.
-	       This is typical for nearby Erlang code. */
-	    patch_b((Uint32*)callAddress, destOffset, 0);
-	} else {
-	    /* The destination is too distant for b/bl/ba/bla.
-	       Must do a b/bl to the trampoline. */
-#if defined(__arm__)
-	    Sint32 trampOffset = ((Sint32)trampoline - ((Sint32)callAddress+8)) >> 2;
-#else
-	    Sint32 trampOffset = ((Sint32)trampoline - (Sint32)callAddress) >> 2;
-#endif
-	    if (trampOffset >= -0x800000 && trampOffset <= 0x7FFFFF) {
-		/* Update the trampoline's address computation.
-		   (May be redundant, but we can't tell.) */
-#if defined(__arm__)
-		patch_imm32((Uint32*)trampoline+1, (Uint32)destAddress);
-#else
-		patch_li((Uint32*)trampoline, (Uint32)destAddress);
-#endif
-		/* Update this call site. */
-		patch_b((Uint32*)callAddress, trampOffset, 0);
-	    } else
-		return -1;
-	}
-#if !defined(__arm__)
+	/* The destination is too distant for b/bl.
+	   Must do a b/bl to the trampoline. */
+	Sint32 trampOffset = ((Sint32)trampoline - ((Sint32)callAddress+8)) >> 2;
+	if (trampOffset >= -0x800000 && trampOffset <= 0x7FFFFF) {
+	    /* Update the trampoline's address computation.
+	       (May be redundant, but we can't tell.) */
+	    patch_imm32((Uint32*)trampoline+1, (Uint32)destAddress);
+	    /* Update this call site. */
+	    patch_b((Uint32*)callAddress, trampOffset, 0);
+	} else
+	    return -1;
     }
-#endif
     return 0;
 }
 
diff --git a/erts/emulator/hipe/hipe_arm.h b/erts/emulator/hipe/hipe_arm.h
index 84f58a681f..19f2a986cf 100644
--- a/erts/emulator/hipe/hipe_arm.h
+++ b/erts/emulator/hipe/hipe_arm.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_ARM_H
 #define HIPE_ARM_H
 
diff --git a/erts/emulator/hipe/hipe_arm.tab b/erts/emulator/hipe/hipe_arm.tab
index 81626796a7..49b89d6748 100644
--- a/erts/emulator/hipe/hipe_arm.tab
+++ b/erts/emulator/hipe/hipe_arm.tab
@@ -1,22 +1,22 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2005-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2005-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
-# $Id$
+
 # ARM-specific atoms and bifs
 
 atom arm
diff --git a/erts/emulator/hipe/hipe_arm_abi.txt b/erts/emulator/hipe/hipe_arm_abi.txt
index 6868704d62..6778ff6663 100644
--- a/erts/emulator/hipe/hipe_arm_abi.txt
+++ b/erts/emulator/hipe/hipe_arm_abi.txt
@@ -2,7 +2,7 @@
  %CopyrightBegin%
  %CopyrightEnd%
 
-$Id$
+
 
 HiPE ARM ABI
 ================
diff --git a/erts/emulator/hipe/hipe_arm_asm.m4 b/erts/emulator/hipe/hipe_arm_asm.m4
index b9a696ffff..85dc84973d 100644
--- a/erts/emulator/hipe/hipe_arm_asm.m4
+++ b/erts/emulator/hipe/hipe_arm_asm.m4
@@ -1,25 +1,24 @@
 changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
+
 `#ifndef HIPE_ARM_ASM_H
 #define HIPE_ARM_ASM_H'
 
diff --git a/erts/emulator/hipe/hipe_arm_bifs.m4 b/erts/emulator/hipe/hipe_arm_bifs.m4
index 4d8636e711..17c013f1fb 100644
--- a/erts/emulator/hipe/hipe_arm_bifs.m4
+++ b/erts/emulator/hipe/hipe_arm_bifs.m4
@@ -1,39 +1,46 @@
 changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
 
 include(`hipe/hipe_arm_asm.m4')
+#`include' "config.h"
 #`include' "hipe_literals.h"
 
 	.text
 	.p2align 2
 
-`#define JOIN3(A,B,C)		A##B##C
-#define TEST_GOT_MBUF(ARITY)	ldr r1, [P, #P_MBUF]; cmp r1, #0; blne JOIN3(nbif_,ARITY,_gc_after_bif)'
+`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+#  define CALL_BIF(F)	ldr r14, =F; str r14, [r0, #P_BIF_CALLEE]; bl hipe_debug_bif_wrapper
+#else
+#  define CALL_BIF(F)	bl	F
+#endif'
+
+define(TEST_GOT_MBUF,`ldr r1, [P, #P_MBUF]	/* `TEST_GOT_MBUF' */
+	cmp r1, #0
+	blne nbif_$1_gc_after_bif')
 
 /*
  * standard_bif_interface_1(nbif_name, cbif_name)
  * standard_bif_interface_2(nbif_name, cbif_name)
  * standard_bif_interface_3(nbif_name, cbif_name)
+ * standard_bif_interface_0(nbif_name, cbif_name)
  *
  * Generate native interface for a BIF with 1-3 parameters and
  * standard failure mode.
@@ -50,7 +57,9 @@ $1:
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_BIF
-	bl	$2
+	str	r1, [r0, #P_ARG0]	/* Store BIF__ARGS in def_arg_reg[] */
+	add	r1, r0, #P_ARG0
+	CALL_BIF($2)
 	TEST_GOT_MBUF(1)
 
 	/* Restore registers. Check for exception. */
@@ -58,6 +67,7 @@ $1:
 	RESTORE_CONTEXT_BIF
 	beq	nbif_1_simple_exception
 	NBIF_RET(1)
+	.ltorg		/* needed by LDR in debug version of `CALL_BIF' */
 	.size	$1, .-$1
 	.type	$1, %function
 #endif')
@@ -75,7 +85,10 @@ $1:
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_BIF
-	bl	$2
+	str	r1, [r0, #P_ARG0]	/* Store BIF__ARGS in def_arg_reg[] */
+	str	r2, [r0, #P_ARG1]
+	add	r1, r0, #P_ARG0
+	CALL_BIF($2)
 	TEST_GOT_MBUF(2)
 
 	/* Restore registers. Check for exception. */
@@ -83,6 +96,7 @@ $1:
 	RESTORE_CONTEXT_BIF
 	beq	nbif_2_simple_exception
 	NBIF_RET(2)
+	.ltorg
 	.size	$1, .-$1
 	.type	$1, %function
 #endif')
@@ -101,7 +115,11 @@ $1:
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_BIF
-	bl	$2
+	str	r1, [r0, #P_ARG0]	/* Store BIF__ARGS in def_arg_reg[] */
+	str	r2, [r0, #P_ARG1]
+	str	r3, [r0, #P_ARG2]
+	add	r1, r0, #P_ARG0
+	CALL_BIF($2)
 	TEST_GOT_MBUF(3)
 
 	/* Restore registers. Check for exception. */
@@ -109,17 +127,12 @@ $1:
 	RESTORE_CONTEXT_BIF
 	beq	nbif_3_simple_exception
 	NBIF_RET(3)
+	.ltorg
 	.size	$1, .-$1
 	.type	$1, %function
 #endif')
 
-/*
- * fail_bif_interface_0(nbif_name, cbif_name)
- *
- * Generate native interface for a BIF with 0 parameters and
- * standard failure mode.
- */
-define(fail_bif_interface_0,
+define(standard_bif_interface_0,
 `
 #ifndef HAVE_$1
 #`define' HAVE_$1
@@ -130,7 +143,8 @@ $1:
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_BIF
-	bl	$2
+	/* ignore empty BIF__ARGS */
+	CALL_BIF($2)
 	TEST_GOT_MBUF(0)
 
 	/* Restore registers. Check for exception. */
@@ -138,6 +152,7 @@ $1:
 	RESTORE_CONTEXT_BIF
 	beq	nbif_0_simple_exception
 	NBIF_RET(0)
+	.ltorg
 	.size	$1, .-$1
 	.type	$1, %function
 #endif')
@@ -162,7 +177,8 @@ $1:
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_GC
-	bl	$2
+	/* ignore empty BIF__ARGS */
+	CALL_BIF($2)
 	TEST_GOT_MBUF(0)
 
 	/* Restore registers. */
@@ -184,7 +200,9 @@ $1:
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_GC
-	bl	$2
+        str     r1, [r0, #P_ARG0]       /* Store BIF__ARGS in def_arg_reg[] */
+        add     r1, r0, #P_ARG0
+	CALL_BIF($2)
 	TEST_GOT_MBUF(1)
 
 	/* Restore registers. Check for exception. */
@@ -209,7 +227,10 @@ $1:
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_GC
-	bl	$2
+        str     r1, [r0, #P_ARG0]       /* Store BIF__ARGS in def_arg_reg[] */
+        str     r2, [r0, #P_ARG1]
+        add     r1, r0, #P_ARG0
+	CALL_BIF($2)
 	TEST_GOT_MBUF(2)
 
 	/* Restore registers. Check for exception. */
diff --git a/erts/emulator/hipe/hipe_arm_gc.h b/erts/emulator/hipe/hipe_arm_gc.h
index a2a919e3d7..787c6fef3e 100644
--- a/erts/emulator/hipe/hipe_arm_gc.h
+++ b/erts/emulator/hipe/hipe_arm_gc.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * Stack walking helpers for native stack GC procedures.
  * ARM version.
  */
diff --git a/erts/emulator/hipe/hipe_arm_glue.S b/erts/emulator/hipe/hipe_arm_glue.S
index 5d626a5f69..2e2b8604a6 100644
--- a/erts/emulator/hipe/hipe_arm_glue.S
+++ b/erts/emulator/hipe/hipe_arm_glue.S
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #include "hipe_arm_asm.h"
 #include "hipe_literals.h"
 #define ASM
@@ -311,6 +311,8 @@ nbif_3_gc_after_bif:
 	str	TEMP_LR, [P, #P_NRA]
 	str	NSP, [P, #P_NSP]
 	mov	TEMP_LR, lr
+	mov	r3, #0			/* Pass 0 in arity */
+	mov	r2, #0			/* Pass NULL in regs */
 	mov	r1, r0
 	mov	r0, P
 	bl	erts_gc_after_bif_call
diff --git a/erts/emulator/hipe/hipe_arm_glue.h b/erts/emulator/hipe/hipe_arm_glue.h
index e840c3dc0f..165f73320d 100644
--- a/erts/emulator/hipe/hipe_arm_glue.h
+++ b/erts/emulator/hipe/hipe_arm_glue.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_ARM_GLUE_H
 #define HIPE_ARM_GLUE_H
 
diff --git a/erts/emulator/hipe/hipe_arm_primops.h b/erts/emulator/hipe/hipe_arm_primops.h
index a28b509eee..2a1a87b862 100644
--- a/erts/emulator/hipe/hipe_arm_primops.h
+++ b/erts/emulator/hipe/hipe_arm_primops.h
@@ -1,21 +1,21 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 PRIMOP_LIST(am_inc_stack_0, &hipe_arm_inc_stack)
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index b0abfd2310..af593229c0 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -174,8 +174,13 @@ static inline unsigned char *bytearray_lvalue(Eterm bin, Eterm idx)
 {
     Sint i;
     unsigned char *bytes;
+#ifndef DEBUG
+    ERTS_DECLARE_DUMMY(Uint bitoffs);
+    ERTS_DECLARE_DUMMY(Uint bitsize);
+#else
     Uint bitoffs;
     Uint bitsize;
+#endif
 
     if (is_not_binary(bin) ||
 	is_not_small(idx) ||
@@ -235,9 +240,15 @@ BIF_RETTYPE hipe_bifs_bitarray_2(BIF_ALIST_2)
 BIF_RETTYPE hipe_bifs_bitarray_update_3(BIF_ALIST_3)
 {
     unsigned char *bytes, bytemask;
-    Uint bitoffs, bitsize;
     Uint bitnr, bytenr;
     int set;
+#ifndef DEBUG
+    ERTS_DECLARE_DUMMY(Uint bitoffs);
+    ERTS_DECLARE_DUMMY(Uint bitsize);
+#else
+    Uint bitoffs;
+    Uint bitsize;
+#endif
 
     if (is_not_binary(BIF_ARG_1))
 	BIF_ERROR(BIF_P, BADARG);
@@ -267,8 +278,15 @@ BIF_RETTYPE hipe_bifs_bitarray_update_3(BIF_ALIST_3)
 BIF_RETTYPE hipe_bifs_bitarray_sub_2(BIF_ALIST_2)
 {
     unsigned char *bytes, bytemask;
-    Uint bitoffs, bitsize;
     Uint bitnr, bytenr;
+#ifndef DEBUG
+    ERTS_DECLARE_DUMMY(Uint bitoffs);
+    ERTS_DECLARE_DUMMY(Uint bitsize);
+#else
+    Uint bitoffs;
+    Uint bitsize;
+#endif
+
 
     if (is_not_binary(BIF_ARG_1))
 	BIF_ERROR(BIF_P, BADARG);
@@ -397,10 +415,15 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2)
     Uint nrbytes;
     void *bytes;
     void *address;
-    Uint bitoffs;
-    Uint bitsize;
     Eterm trampolines;
     Eterm *hp;
+#ifndef DEBUG
+    ERTS_DECLARE_DUMMY(Uint bitoffs);
+    ERTS_DECLARE_DUMMY(Uint bitsize);
+#else
+    Uint bitoffs;
+    Uint bitsize;
+#endif
 
     if (is_not_binary(BIF_ARG_1))
 	BIF_ERROR(BIF_P, BADARG);
@@ -440,59 +463,23 @@ BIF_RETTYPE hipe_bifs_alloc_data_2(BIF_ALIST_2)
 	 align != sizeof(long) && align != sizeof(double)))
 	BIF_ERROR(BIF_P, BADARG);
     nrbytes = unsigned_val(BIF_ARG_2);
+    if (nrbytes == 0)
+	BIF_RET(make_small(0));
     block = erts_alloc(ERTS_ALC_T_HIPE, nrbytes);
     if ((unsigned long)block & (align-1))
-	fprintf(stderr, "Yikes! erts_alloc() returned misaligned address %p\r\n", block);
+	fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte aligned\r\n",
+		__FUNCTION__, (unsigned long)nrbytes, block, (unsigned long)align);
     BIF_RET(address_to_term(block, BIF_P));
 }
 
 /*
- * Memory area for constant Erlang terms.
- *
- * These constants must not be forwarded by the gc.
- * Therefore, the gc needs to be able to distinguish between
- * collectible objects and constants. Unfortunately, an Erlang
- * process' collectible objects are scattered around in two
- * heaps and a list of message buffers, so testing "is X a
- * collectible object?" can be expensive.
- *
- * Instead, constants are placed in a single contiguous area,
- * which allows for an inexpensive "is X a constant?" test.
- *
- * XXX: Allow this area to be grown.
+ * Statistics on hipe constants: size of HiPE constants, in words.
  */
-
-/* not static, needed by garbage collector */
-Eterm *hipe_constants_start = NULL;
-Eterm *hipe_constants_next = NULL;
-static unsigned constants_avail_words = 0;
-#define CONSTANTS_BYTES	(1536*1024*sizeof(Eterm))  /* 1.5 M words */
-
-static Eterm *constants_alloc(unsigned nwords)
-{
-    Eterm *next;
-
-    /* initialise at the first call */
-    if ((next = hipe_constants_next) == NULL) {
-	next = (Eterm*)erts_alloc(ERTS_ALC_T_HIPE, CONSTANTS_BYTES);
-	hipe_constants_start = next;
-	hipe_constants_next = next;
-	constants_avail_words = CONSTANTS_BYTES / sizeof(Eterm);
-    }
-    if (nwords > constants_avail_words) {
-	fprintf(stderr, "Native code constants pool depleted!\r\n");
-	/* Must terminate immediately. erl_exit() seems to
-	   continue running some code which then SIGSEGVs. */
-	exit(1);
-    }
-    constants_avail_words -= nwords;
-    hipe_constants_next = next + nwords;
-    return next;
-}
+unsigned int hipe_constants_size = 0;
 
 BIF_RETTYPE hipe_bifs_constants_size_0(BIF_ALIST_0)
 {
-    BIF_RET(make_small(hipe_constants_next - hipe_constants_start));
+    BIF_RET(make_small(hipe_constants_size));
 }
 
 /*
@@ -523,14 +510,17 @@ static void *const_term_alloc(void *tmpl)
 {
     Eterm obj;
     Uint size;
+    Uint alloc_size;
     Eterm *hp;
     struct const_term *p;
 
     obj = (Eterm)tmpl;
     ASSERT(is_not_immed(obj));
     size = size_object(obj);
+    alloc_size = size + (offsetof(struct const_term, mem)/sizeof(Eterm));
+    hipe_constants_size += alloc_size;
 
-    p = (struct const_term*)constants_alloc(size + (offsetof(struct const_term, mem)/sizeof(Eterm)));
+    p = (struct const_term*)erts_alloc(ERTS_ALC_T_HIPE, alloc_size * sizeof(Eterm));
 
     /* I have absolutely no idea if having a private 'off_heap'
        works or not. _Some_ off_heap object is required for
@@ -1018,6 +1008,26 @@ BIF_RETTYPE hipe_conv_big_to_float(BIF_ALIST_1)
     BIF_RET(res);
 }
 
+#ifdef NO_FPE_SIGNALS
+
+/* 
+   This is the current solution to make hipe run without FPE.
+   The native code is the same except that a call to this primop
+   is made after _every_ float operation to check the result.
+   The native fcheckerror still done later will detect if an
+   "emulated" FPE has occured.
+   We use p->hipe.float_result to avoid passing a 'double' argument,
+   which has its own calling convention (on amd64 at least).
+   Simple and slow...
+*/
+void hipe_emulate_fpe(Process* p)
+{
+    if (!finite(p->hipe.float_result)) {
+	p->fp_exception = 1;
+    }
+}
+#endif
+
 #if 0 /* XXX: unused */
 /*
  * At least parts of this should be inlined in native code.
@@ -1083,10 +1093,8 @@ BIF_RETTYPE hipe_bifs_make_fun_3(BIF_ALIST_3)
     if (is_not_nil(free_vars))
 	BIF_ERROR(BIF_P, BADARG);
 
-#ifndef HYBRID /* FIND ME! */
     funp->next = MSO(BIF_P).funs;
     MSO(BIF_P).funs = funp;
-#endif
 
     BIF_RET(make_fun(funp));
 }
diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h
index ed27d5616a..c512d66f9d 100644
--- a/erts/emulator/hipe/hipe_bif0.h
+++ b/erts/emulator/hipe/hipe_bif0.h
@@ -1,22 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_bif0.h
  *
  * Compiler and linker support.
@@ -26,13 +27,9 @@
 
 extern Uint *hipe_bifs_find_pc_from_mfa(Eterm mfa);
 
-/* shared with ggc.c -- NOT an official API */
-extern Eterm *hipe_constants_start;
-extern Eterm *hipe_constants_next;
-
 extern void hipe_mfa_info_table_init(void);
 extern void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a);
-extern Eterm hipe_find_na_or_make_stub(Process*, Eterm, Eterm, Eterm);
+extern BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3);
 extern int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a);
 #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
 extern void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int a);
diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index 46c0a3d67d..ce641365e9 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -1,22 +1,22 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2001-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2001-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
-# $Id$
+#
 # HiPE level 0 bifs: compiler and linker support
 #
 # bif hipe_bifs:name/arity
@@ -140,3 +140,5 @@ atom bs_put_utf16le
 atom bs_get_utf16
 atom bs_validate_unicode
 atom bs_validate_unicode_retract
+atom emulate_fpe
+
diff --git a/erts/emulator/hipe/hipe_bif1.c b/erts/emulator/hipe/hipe_bif1.c
index 5188950e17..64de754e18 100644
--- a/erts/emulator/hipe/hipe_bif1.c
+++ b/erts/emulator/hipe/hipe_bif1.c
@@ -1,22 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_bif1.c
  *
  * Performance analysis support.
@@ -448,7 +449,7 @@ BIF_RETTYPE hipe_bifs_gc_info_0(BIF_ALIST_0)
 BIF_RETTYPE hipe_bifs_shared_gc_info_0(BIF_ALIST_0)
 {
 #ifdef __BENCHMARK__
-#if !(defined(BM_COUNTERS) && defined(HYBRID))
+#if !(defined(BM_COUNTERS))
     Uint minor_global_gc = 0;
     Uint major_global_gc = 0;
 #endif
@@ -458,17 +459,9 @@ BIF_RETTYPE hipe_bifs_shared_gc_info_0(BIF_ALIST_0)
 #endif
     Eterm *hp;
 
-#if defined(HYBRID)
-    Uint tmp_used_heap = (Uint)((BIF_P->htop - BIF_P->heap) +
-				(OLD_HTOP(BIF_P) - OLD_HEAP(BIF_P)) +
-				MBUF_SIZE(BIF_P));
-    Uint tmp_allocated_heap = (Uint)((BIF_P->hend - BIF_P->heap) +
-				     (OLD_HEND(BIF_P) - OLD_HEAP(BIF_P)) +
-				     MBUF_SIZE(BIF_P));
-#else
     Uint tmp_used_heap = 0;
     Uint tmp_allocated_heap = 0;
-#endif
+
     hp = HAlloc(BIF_P, 7);
     BIF_RET(TUPLE6(hp,
 		   make_small((uint)minor_global_gc),
@@ -485,7 +478,7 @@ BIF_RETTYPE hipe_bifs_shared_gc_info_0(BIF_ALIST_0)
 BIF_RETTYPE hipe_bifs_incremental_gc_info_0(BIF_ALIST_0)
 {
 #ifdef __BENCHMARK__
-#if !(defined(BM_COUNTERS) && defined(INCREMENTAL))
+#if !defined(BM_COUNTERS)
     Uint minor_gc_cycles = 0;
     Uint major_gc_cycles = 0;
     Uint minor_gc_stages = 0;
@@ -511,17 +504,6 @@ BIF_RETTYPE hipe_bifs_gc_info_clear_0(BIF_ALIST_0)
 #ifdef BM_COUNTERS
     minor_gc	    = 0;
     major_gc	    = 0;
-#ifdef HYBRID
-    minor_global_gc = 0;
-    major_global_gc = 0;
-    gc_in_copy	    = 0;
-#ifdef INCREMENTAL
-    minor_gc_cycles = 0;
-    major_gc_cycles = 0;
-    minor_gc_stages = 0;
-    major_gc_stages = 0;
-#endif
-#endif
 #endif
 
 #ifdef BM_HEAP_SIZES
@@ -876,22 +858,44 @@ BIF_RETTYPE hipe_bifs_misc_timer_clear_0(BIF_ALIST_0)
  * + The fallback, which is the same as {X,_} = runtime(statistics).
  */
 
+static double fallback_get_hrvtime(void)
+{
+    unsigned long ms_user;
+
+    elapsed_time_both(&ms_user, NULL, NULL, NULL);
+    return (double)ms_user;
+}
+
 #if USE_PERFCTR
 
 #include "hipe_perfctr.h"
-static int hrvtime_is_open;
-#define hrvtime_is_started()	hrvtime_is_open
+static int hrvtime_started;	/* 0: closed, +1: perfctr, -1: fallback */
+#define hrvtime_is_started()	(hrvtime_started != 0)
 
 static void start_hrvtime(void)
 {
     if (hipe_perfctr_hrvtime_open() >= 0)
-	hrvtime_is_open = 1;
+	hrvtime_started = 1;
+    else
+	hrvtime_started = -1;
 }
 
-#define get_hrvtime()		hipe_perfctr_hrvtime_get()
-#define stop_hrvtime()		hipe_perfctr_hrvtime_close()
+static void stop_hrvtime(void)
+{
+    if (hrvtime_started > 0)
+	hipe_perfctr_hrvtime_close();
+    hrvtime_started = 0;
+}
 
-#else
+static double get_hrvtime(void)
+{
+    if (hrvtime_started > 0)
+	return hipe_perfctr_hrvtime_get();
+    else
+	return fallback_get_hrvtime();
+}
+
+#else	/* !USE_PERFCTR */
 
 /*
  * Fallback, if nothing better exists.
@@ -902,15 +906,9 @@ static void start_hrvtime(void)
 #define hrvtime_is_started()	1
 #define start_hrvtime()		do{}while(0)
 #define stop_hrvtime()		do{}while(0)
+#define get_hrvtime()		fallback_get_hrvtime()
 
-static double get_hrvtime(void)
-{
-    unsigned long ms_user;
-    elapsed_time_both(&ms_user, NULL, NULL, NULL);
-    return (double)ms_user;
-}
-
-#endif	/* hrvtime support */
+#endif	/* !USE_PERFCTR */
 
 BIF_RETTYPE hipe_bifs_get_hrvtime_0(BIF_ALIST_0)
 {
@@ -918,11 +916,8 @@ BIF_RETTYPE hipe_bifs_get_hrvtime_0(BIF_ALIST_0)
     Eterm res;
     FloatDef f;
 
-    if (!hrvtime_is_started()) {
+    if (!hrvtime_is_started())
 	start_hrvtime();
-	if (!hrvtime_is_started())
-	    BIF_RET(NIL); /* arity 0 BIFs may not fail */
-    }
     f.fd = get_hrvtime();
     hp = HAlloc(BIF_P, FLOAT_SIZE_OBJECT);
     res = make_float(hp);
diff --git a/erts/emulator/hipe/hipe_bif1.h b/erts/emulator/hipe/hipe_bif1.h
index c3b607565d..89241fb835 100644
--- a/erts/emulator/hipe/hipe_bif1.h
+++ b/erts/emulator/hipe/hipe_bif1.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_bif1.h
  *
  * Performance analysis support.
diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c
index f992b758be..37a1cc193b 100644
--- a/erts/emulator/hipe/hipe_bif2.c
+++ b/erts/emulator/hipe/hipe_bif2.c
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_bif2.c
  *
  * Miscellaneous add-ons.
@@ -33,18 +33,28 @@
 #include "big.h"
 #include "hipe_debug.h"
 #include "hipe_mode_switch.h"
-#include "hipe_bif0.h" /* hipe_constants_{start,next} */
 #include "hipe_arch.h"
 #include "hipe_stack.h"
 
-BIF_RETTYPE hipe_bifs_show_estack_1(BIF_ALIST_1)
+static void proc_unlock(Process* c_p, Process* rp)
 {
+    ErtsProcLocks locks = ERTS_PROC_LOCKS_ALL;
+    if (rp == c_p) {
+	locks &= ~ERTS_PROC_LOCK_MAIN;
+    }
+    if (rp && locks) {
+	erts_smp_proc_unlock(rp, locks);
+    }
+}
+
+BIF_RETTYPE hipe_bifs_show_estack_1(BIF_ALIST_1)
+{    
     Process *rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
 				BIF_ARG_1, ERTS_PROC_LOCKS_ALL);
     if (!rp)
 	BIF_ERROR(BIF_P, BADARG);
     hipe_print_estack(rp);
-    erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL);
+    proc_unlock(BIF_P, rp);
     BIF_RET(am_true);
 }
 
@@ -55,7 +65,7 @@ BIF_RETTYPE hipe_bifs_show_heap_1(BIF_ALIST_1)
     if (!rp)
 	BIF_ERROR(BIF_P, BADARG);
     hipe_print_heap(rp);
-    erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL);
+    proc_unlock(BIF_P, rp);
     BIF_RET(am_true);
 }
 
@@ -66,7 +76,7 @@ BIF_RETTYPE hipe_bifs_show_nstack_1(BIF_ALIST_1)
     if (!rp)
 	BIF_ERROR(BIF_P, BADARG);
     hipe_print_nstack(rp);
-    erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL);
+    proc_unlock(BIF_P, rp);
     BIF_RET(am_true);
 }
 
@@ -82,7 +92,7 @@ BIF_RETTYPE hipe_bifs_show_pcb_1(BIF_ALIST_1)
     if (!rp)
 	BIF_ERROR(BIF_P, BADARG);
     hipe_print_pcb(rp);
-    erts_smp_proc_unlock(rp, ERTS_PROC_LOCKS_ALL);
+    proc_unlock(BIF_P, rp);
     BIF_RET(am_true);
 }
 
@@ -124,18 +134,6 @@ BIF_RETTYPE hipe_bifs_show_term_1(BIF_ALIST_1)
     BIF_RET(am_true);
 }
 
-BIF_RETTYPE hipe_bifs_show_literals_0(BIF_ALIST_0)
-{
-    Eterm *p;
-
-    p = hipe_constants_start;
-    for (; p < hipe_constants_next; ++p)
-	printf("0x%0*lx: 0x%0*lx\r\n",
-	       2*(int)sizeof(long), (unsigned long)p,
-	       2*(int)sizeof(long), *p);
-    BIF_RET(am_true);
-}
-
 BIF_RETTYPE hipe_bifs_in_native_0(BIF_ALIST_0)
 {
     BIF_RET(am_false);
@@ -153,18 +151,25 @@ BIF_RETTYPE hipe_bifs_modeswitch_debug_off_0(BIF_ALIST_0)
     BIF_RET(am_true);
 }
 
-/* BIFs for handling the message area */
+#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
 
-BIF_RETTYPE hipe_bifs_show_message_area_0(BIF_ALIST_0)
+BIF_RETTYPE hipe_debug_bif_wrapper(BIF_ALIST_1);
+
+#    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
+        if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN)
+#    define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \
+        if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN)
+
+BIF_RETTYPE hipe_debug_bif_wrapper(BIF_ALIST_1)
 {
-#ifdef HYBRID
-#ifdef DEBUG
-    print_message_area();
-#else
-    printf("Only available in debug compiled emulator\r\n");
-#endif
-    BIF_RET(am_true);
-#else
-    BIF_RET(am_false);
-#endif
+    typedef BIF_RETTYPE Bif(BIF_ALIST_1);
+    Bif* fp = (Bif*) (BIF_P->hipe.bif_callee);
+    BIF_RETTYPE res;
+    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(BIF_P);
+    res = (*fp)(BIF_P, BIF__ARGS);
+    ERTS_SMP_REQ_PROC_MAIN_LOCK(BIF_P);
+    return res;
 }
+
+#endif /* ERTS_ENABLE_LOCK_CHECK && ERTS_SMP */
+
diff --git a/erts/emulator/hipe/hipe_bif2.tab b/erts/emulator/hipe/hipe_bif2.tab
index d8d627e370..aac27e8bed 100644
--- a/erts/emulator/hipe/hipe_bif2.tab
+++ b/erts/emulator/hipe/hipe_bif2.tab
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 2001-2009. All Rights Reserved.
+# Copyright Ericsson AB 2001-2011. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -26,8 +26,6 @@ bif hipe_bifs:show_nstack/1
 bif hipe_bifs:nstack_used_size/0
 bif hipe_bifs:show_pcb/1
 bif hipe_bifs:show_term/1
-bif hipe_bifs:show_literals/0
 bif hipe_bifs:in_native/0
 bif hipe_bifs:modeswitch_debug_on/0
 bif hipe_bifs:modeswitch_debug_off/0
-bif hipe_bifs:show_message_area/0
diff --git a/erts/emulator/hipe/hipe_bif64.c b/erts/emulator/hipe/hipe_bif64.c
new file mode 100644
index 0000000000..baaf5af2cd
--- /dev/null
+++ b/erts/emulator/hipe/hipe_bif64.c
@@ -0,0 +1,68 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+/* hipe_bif_64.c
+ *
+ * Compiler and linker support. 64-bit specific.
+ */
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include "global.h"
+#include "error.h"
+#include "bif.h"
+#include "big.h"	/* term_to_Sint() */
+#include "hipe_arch.h"
+#include "hipe_bif0.h"
+#include "hipe_bif64.h"
+
+#if 0 /* unused */
+static int term_to_Sint64(Eterm term, Sint64 *sp)
+{
+    return term_to_Sint(term, sp);
+}
+
+BIF_RETTYPE hipe_bifs_write_s64_2(BIF_ALIST_2)
+{
+    Sint64 *address;
+    Sint64 value;
+
+    address = term_to_address(BIF_ARG_1);
+    if (!address || !hipe_word64_address_ok(address))
+	BIF_ERROR(BIF_P, BADARG);
+    if (!term_to_Sint64(BIF_ARG_2, &value))
+	BIF_ERROR(BIF_P, BADARG);
+    *address = value;
+    BIF_RET(NIL);
+}
+#endif
+
+BIF_RETTYPE hipe_bifs_write_u64_2(BIF_ALIST_2)
+{
+    Uint64 *address;
+    Uint64 value;
+
+    address = term_to_address(BIF_ARG_1);
+    if (!address || !hipe_word64_address_ok(address))
+	BIF_ERROR(BIF_P, BADARG);
+    if (!term_to_Uint(BIF_ARG_2, &value))
+	BIF_ERROR(BIF_P, BADARG);
+    *address = value;
+    hipe_flush_icache_word(address);
+    BIF_RET(NIL);
+}
diff --git a/erts/emulator/hipe/hipe_bif64.h b/erts/emulator/hipe/hipe_bif64.h
new file mode 100644
index 0000000000..6d494886ec
--- /dev/null
+++ b/erts/emulator/hipe/hipe_bif64.h
@@ -0,0 +1,26 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+/* hipe_bif64.c
+ *
+ * Compiler and linker support. 64-bit specific.
+ */
+#ifndef HIPE_BIF64_H
+#define HIPE_BIF64_H
+
+#endif /* HIPE_BIF64_H */
diff --git a/erts/emulator/zlib/Makefile b/erts/emulator/hipe/hipe_bif64.tab
index def8e1aa47..228318af39 100644
--- a/erts/emulator/zlib/Makefile
+++ b/erts/emulator/hipe/hipe_bif64.tab
@@ -1,23 +1,22 @@
-# 
+#
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 1997-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2004-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
-# %CopyrightEnd%
-# 
 #
-# Invoke with GNU make or clearmake -C gnu.
+# %CopyrightEnd%
 #
+# HiPE 64-bit specific bifs
 
-include $(ERL_TOP)/make/run_make.mk
+# bif hipe_bifs:write_s64/2
+bif hipe_bifs:write_u64/2
diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4
index c92d94ed9d..942fa0c5cb 100644
--- a/erts/emulator/hipe/hipe_bif_list.m4
+++ b/erts/emulator/hipe/hipe_bif_list.m4
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  *
  * List all non architecture-specific BIFs and primops, and
  * classify each as belonging to one of the classes below.
@@ -70,24 +70,18 @@
  ****************************************************************/
 
 /*
+ * standard_bif_interface_0(nbif_name, cbif_name)
  * standard_bif_interface_1(nbif_name, cbif_name)
  * standard_bif_interface_2(nbif_name, cbif_name)
  * standard_bif_interface_3(nbif_name, cbif_name)
  *
- * A BIF with implicit P parameter, 1-3 ordinary parameters,
+ * A BIF with implicit P parameter, 0-3 ordinary parameters,
  * which may fail.
  * HP and FCALLS may be read and updated.
  * HP_LIMIT, NSP, NSP_LIMIT, and NRA may not be accessed.
  */
 
 /*
- * fail_bif_interface_0(nbif_name, cbif_name)
- *
- * A zero-arity BIF which may fail, otherwise
- * identical to standard_bif_interface_N.
- */
-
-/*
  * nofail_primop_interface_0(nbif_name, cbif_name)
  * nofail_primop_interface_1(nbif_name, cbif_name)
  * nofail_primop_interface_2(nbif_name, cbif_name)
@@ -150,8 +144,7 @@
 /*
  * Zero-arity BIFs that can fail.
  */
-fail_bif_interface_0(nbif_memory_0, memory_0)
-fail_bif_interface_0(nbif_processes_0, processes_0)
+standard_bif_interface_0(nbif_processes_0, processes_0)
 
 /*
  * BIFs and primops that may do a GC (change heap limit and walk the native stack).
@@ -176,10 +169,10 @@ gc_bif_interface_0(nbif_hipe_bifs_nstack_used_size_0, hipe_bifs_nstack_used_size
 /*
  * Arithmetic operators called indirectly by the HiPE compiler.
  */
-standard_bif_interface_2(nbif_add_2, erts_mixed_plus)
-standard_bif_interface_2(nbif_sub_2, erts_mixed_minus)
-standard_bif_interface_2(nbif_mul_2, erts_mixed_times)
-standard_bif_interface_2(nbif_div_2, erts_mixed_div)
+standard_bif_interface_2(nbif_add_2, splus_2)
+standard_bif_interface_2(nbif_sub_2, sminus_2)
+standard_bif_interface_2(nbif_mul_2, stimes_2)
+standard_bif_interface_2(nbif_div_2, div_2)
 standard_bif_interface_2(nbif_intdiv_2, intdiv_2)
 standard_bif_interface_2(nbif_rem_2, rem_2)
 standard_bif_interface_2(nbif_bsl_2, bsl_2)
@@ -252,6 +245,10 @@ noproc_primop_interface_5(nbif_bs_put_big_integer, hipe_bs_put_big_integer)
 
 gc_bif_interface_0(nbif_check_get_msg, hipe_check_get_msg)
 
+#ifdef NO_FPE_SIGNALS
+nocons_nofail_primop_interface_0(nbif_emulate_fpe, hipe_emulate_fpe)
+#endif
+
 /*
  * SMP-specific stuff
  */
@@ -261,11 +258,6 @@ noproc_primop_interface_1(nbif_atomic_inc, hipe_atomic_inc)
 ',)dnl
 
 /*
- * Implement standard_bif_interface_0 as nofail_primop_interface_0.
- */
-define(standard_bif_interface_0,`nofail_primop_interface_0($1, $2)')
-
-/*
  * Standard BIFs.
  * BIF_LIST(ModuleAtom,FunctionAtom,Arity,CFun,Index)
  */
diff --git a/erts/emulator/hipe/hipe_debug.c b/erts/emulator/hipe/hipe_debug.c
index 548998b7b7..7ca11f8c6c 100644
--- a/erts/emulator/hipe/hipe_debug.c
+++ b/erts/emulator/hipe/hipe_debug.c
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_debug.c
  *
  * TODO:
@@ -51,7 +51,7 @@ static const char stars[2*sizeof(long)+5] = {
 
 extern Uint beam_apply[];
 
-static void print_beam_pc(Uint *pc)
+static void print_beam_pc(BeamInstr *pc)
 {
     if (pc == hipe_beam_pc_return) {
 	printf("return-to-native");
@@ -60,7 +60,7 @@ static void print_beam_pc(Uint *pc)
     } else if (pc == &beam_apply[1]) {
 	printf("normal-process-exit");
     } else {
-	Eterm *mfa = find_function_from_pc(pc);
+	BeamInstr *mfa = find_function_from_pc(pc);
 	if (mfa)
 	    erts_printf("%T:%T/%bpu + 0x%bpx",
 			mfa[0], mfa[1], mfa[2], pc - &mfa[3]);
@@ -71,7 +71,7 @@ static void print_beam_pc(Uint *pc)
 
 static void catch_slot(Eterm *pos, Eterm val)
 {
-    Uint *pc = catch_pc(val);
+    BeamInstr *pc = catch_pc(val);
     printf(" | 0x%0*lx | 0x%0*lx | CATCH 0x%0*lx (BEAM ",
 	   2*(int)sizeof(long), (unsigned long)pos,
 	   2*(int)sizeof(long), (unsigned long)val,
diff --git a/erts/emulator/hipe/hipe_debug.h b/erts/emulator/hipe/hipe_debug.h
index 3980bc8230..a28597000a 100644
--- a/erts/emulator/hipe/hipe_debug.h
+++ b/erts/emulator/hipe/hipe_debug.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_debug.h
  */
 #ifndef HIPE_DEBUG_H
diff --git a/erts/emulator/hipe/hipe_gbif_list.h b/erts/emulator/hipe/hipe_gbif_list.h
index 659f74b5e5..69dbab7ab9 100644
--- a/erts/emulator/hipe/hipe_gbif_list.h
+++ b/erts/emulator/hipe/hipe_gbif_list.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * GBIF_LIST(FunctionAtom,Arity,CFun)
  * manually maintained for now -- expand when necessary
  */
diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c
index e57e293547..07e4b8a4d6 100644
--- a/erts/emulator/hipe/hipe_gc.c
+++ b/erts/emulator/hipe/hipe_gc.c
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * GC support procedures
  */
 #ifdef HAVE_CONFIG_H
@@ -28,7 +28,6 @@
 
 #include "hipe_stack.h"
 #include "hipe_gc.h"
-#include "hipe_bif0.h"		/* for hipe_constants_{start,next} */
 
 Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
 {
@@ -47,9 +46,14 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
     char *src, *oh;
     Uint src_size, oh_size;
 
+    if (!p->hipe.nstack) {
+	ASSERT(!p->hipe.nsp && !p->hipe.nstend);
+	return n_htop;
+    }
     if (!nstack_walk_init_check(p))
 	return n_htop;
 
+    ASSERT(p->hipe.nsp && p->hipe.nstend);
     nsp = nstack_walk_nsp_begin(p);
     nsp_end = p->hipe.nstgraylim;
     if (nsp_end)
@@ -86,7 +90,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
 		if (is_boxed(gval)) {
 		    Eterm *ptr = boxed_val(gval);
 		    Eterm val = *ptr;
-		    if (IS_MOVED(val)) {
+		    if (IS_MOVED_BOXED(val)) {
 			ASSERT(is_boxed(val));
 			*nsp_i = val;
 		    } else if (in_area(ptr, src, src_size) ||
@@ -96,7 +100,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
 		} else if (is_list(gval)) {
 		    Eterm *ptr = list_val(gval);
 		    Eterm val = *ptr;
-		    if (is_non_value(val)) {
+		    if (IS_MOVED_CONS(val)) {
 			*nsp_i = ptr[1];
 		    } else if (in_area(ptr, src, src_size) ||
 			       in_area(ptr, oh, oh_size)) {
@@ -137,9 +141,14 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
     char *heap;
     Uint heap_size, mature_size;
 
+    if (!p->hipe.nstack) {
+	ASSERT(!p->hipe.nsp && !p->hipe.nstend);
+	return;
+    }
     if (!nstack_walk_init_check(p))
 	return;
 
+    ASSERT(p->hipe.nsp && p->hipe.nstend);
     nsp = nstack_walk_nsp_begin(p);
     nsp_end = p->hipe.nstgraylim;
     if (nsp_end) {
@@ -193,7 +202,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
 		if (is_boxed(gval)) {
 		    Eterm *ptr = boxed_val(gval);
 		    Eterm val = *ptr;
-		    if (IS_MOVED(val)) {
+		    if (IS_MOVED_BOXED(val)) {
 			ASSERT(is_boxed(val));
 			*nsp_i = val;
 		    } else if (in_area(ptr, heap, mature_size)) {
@@ -205,7 +214,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
 		} else if (is_list(gval)) {
 		    Eterm *ptr = list_val(gval);
 		    Eterm val = *ptr;
-		    if (is_non_value(val)) {
+		    if (IS_MOVED_CONS(val)) {
 			*nsp_i = ptr[1];
 		    } else if (in_area(ptr, heap, mature_size)) {
 			MOVE_CONS(ptr, val, old_htop, nsp_i);
@@ -228,329 +237,3 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
     }
     abort();
 }
-
-#ifdef HYBRID
-
-#ifdef INCREMENTAL
-Eterm *ma_fullsweep_nstack(Process *p, Eterm *n_htop, Eterm *n_hend)
-{
-    /* known nstack walk state */
-    Eterm *nsp;
-    Eterm *nsp_end;
-    const struct sdesc *sdesc;
-    unsigned int sdesc_size;
-    unsigned long ra;
-    unsigned int i;
-    unsigned int mask;
-    /* arch-specific nstack walk state */
-    struct nstack_walk_state walk_state;
-
-    if (!nstack_walk_init_check(p))
-	return n_htop;
-
-    nsp = nstack_walk_nsp_begin(p);
-    nsp_end = nstack_walk_nsp_end(p);
-
-    sdesc = nstack_walk_init_sdesc(p, &walk_state);
-
-    for (;;) {
-	if (nstack_walk_nsp_reached_end(nsp, nsp_end)) {
-	    if (nsp == nsp_end)
-		return n_htop;
-	    fprintf(stderr, "%s: passed end of stack\r\n", __FUNCTION__);
-	    break;
-	}
-	sdesc_size = nstack_walk_frame_size(sdesc);
-	i = 0;
-	mask = sdesc->livebits[0];
-	for (;;) {
-	    if (mask & 1) {
-		Eterm *nsp_i = nstack_walk_frame_index(nsp, i);
-		Eterm val = *nsp_i;
-		Eterm *obj_ptr = ptr_val(val);
-		switch (primary_tag(val)) {
-		  case TAG_PRIMARY_LIST:
-		    COPYMARK_CONS(obj_ptr, n_htop, nsp_i, n_hend);
-		    break;
-		  case TAG_PRIMARY_BOXED:
-		    COPYMARK_BOXED(obj_ptr, n_htop, nsp_i, n_hend);
-		    break;
-		  default:
-		    break;
-		}
-	    }
-	    if (++i >= sdesc_size)
-		break;
-	    if (i & 31)
-		mask >>= 1;
-	    else
-		mask = sdesc->livebits[i >> 5];
-	}
-	ra = nstack_walk_frame_ra(nsp, sdesc);
-	if (ra == (unsigned long)nbif_stack_trap_ra)
-	    ra = (unsigned long)p->hipe.ngra;
-	sdesc = hipe_find_sdesc(ra);
-	nsp = nstack_walk_next_frame(nsp, sdesc_size);
-    }
-    abort();
-}
-
-void ma_gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
-{
-    /* known nstack walk state */
-    Eterm *nsp;
-    Eterm *nsp_end;
-    const struct sdesc *sdesc;
-    unsigned int sdesc_size;
-    unsigned long ra;
-    unsigned int i;
-    unsigned int mask;
-    /* arch-specific nstack walk state */
-    struct nstack_walk_state walk_state;
-
-    /* ma_gensweep-specific state */
-    Eterm *low_water, *high_water, *surface;
-    Eterm *n_htop;
-    Eterm *old_htop;
-
-    if (!nstack_walk_init_check(p))
-	return;
-
-    nsp = nstack_walk_nsp_begin(p);
-    nsp_end = nstack_walk_nsp_end(p);
-
-    low_water = global_heap;
-    //high_water = global_high_water;
-    surface = global_htop;
-
-    old_htop = *ptr_old_htop;
-    n_htop = *ptr_n_htop;
-
-    sdesc = nstack_walk_init_sdesc(p, &walk_state);
-
-    for (;;) {
-	if (nstack_walk_nsp_reached_end(nsp, nsp_end)) {
-	    if (nsp == nsp_end) {
-		*ptr_old_htop = old_htop;
-		*ptr_n_htop = n_htop;
-		return;
-	    }
-	    fprintf(stderr, "%s: passed end of stack\r\n", __FUNCTION__);
-	    break;
-	}
-	sdesc_size = nstack_walk_frame_size(sdesc);
-	i = 0;
-	mask = sdesc->livebits[0];
-	for (;;) {
-	    if (mask & 1) {
-		Eterm *nsp_i = nstack_walk_frame_index(nsp, i);
-		Eterm gval = *nsp_i;
-		if (is_boxed(gval)) {
-		    Eterm *ptr = boxed_val(gval);
-		    Eterm val = *ptr;
-		    if (MY_IS_MOVED(val)) {
-			*nsp_i = val;
-		    } else if (ptr_within(ptr, low_water, high_water)) {
-			MOVE_BOXED(ptr, val, old_htop, nsp_i);
-		    } else if (ptr_within(ptr, high_water, surface)) {
-			MOVE_BOXED(ptr, val, n_htop, nsp_i);
-		    }
-		} else if (is_list(gval)) {
-		    Eterm *ptr = list_val(gval);
-		    Eterm val = *ptr;
-		    if (is_non_value(val)) {
-			*nsp_i = ptr[1];
-		    } else if (ptr_within(ptr, low_water, high_water)) {
-			MOVE_CONS(ptr, val, old_htop, nsp_i);
-		    } else if (ptr_within(ptr, high_water, surface)) {
-			MOVE_CONS(ptr, val, n_htop, nsp_i);
-		    }
-		}
-	    }
-	    if (++i >= sdesc_size)
-		break;
-	    if (i & 31)
-		mask >>= 1;
-	    else
-		mask = sdesc->livebits[i >> 5];
-	}
-	ra = nstack_walk_frame_ra(nsp, sdesc);
-	if (ra == (unsigned long)nbif_stack_trap_ra)
-	    ra = (unsigned long)p->hipe.ngra;
-	sdesc = hipe_find_sdesc(ra);
-	nsp = nstack_walk_next_frame(nsp, sdesc_size);
-    }
-    abort();
-}
-
-#else /* not INCREMENTAL */
-
-Eterm *ma_fullsweep_nstack(Process *p, Eterm *n_htop)
-{
-    /* known nstack walk state */
-    Eterm *nsp;
-    Eterm *nsp_end;
-    const struct sdesc *sdesc;
-    unsigned int sdesc_size;
-    unsigned long ra;
-    unsigned int i;
-    unsigned int mask;
-    /* arch-specific nstack walk state */
-    struct nstack_walk_state walk_state;
-
-    /* ma_fullsweep-specific state */
-    Eterm *gheap = global_heap;
-    Eterm *ghtop = global_htop;
-    Eterm *goheap = global_old_heap;
-    Eterm *gohtop = global_old_htop;
-
-    if (!nstack_walk_init_check(p))
-	return n_htop;
-
-    nsp = nstack_walk_nsp_begin(p);
-    nsp_end = nstack_walk_nsp_end(p);
-
-    sdesc = nstack_walk_init_sdesc(p, &walk_state);
-
-    for (;;) {
-	if (nstack_walk_nsp_reached_end(nsp, nsp_end)) {
-	    if (nsp == nsp_end)
-		return n_htop;
-	    fprintf(stderr, "%s: passed end of stack\r\n", __FUNCTION__);
-	    break;
-	}
-	sdesc_size = nstack_walk_frame_size(sdesc);
-	i = 0;
-	mask = sdesc->livebits[0];
-	for (;;) {
-	    if (mask & 1) {
-		Eterm *nsp_i = nstack_walk_frame_index(nsp, i);
-		Eterm gval = *nsp_i;
-		if (is_boxed(gval)) {
-		    Eterm *ptr = boxed_val(gval);
-		    Eterm val = *ptr;
-		    if (MY_IS_MOVED(val)) {
-			*nsp_i = val;
-		    } else if (ptr_within(ptr, gheap, ghtop)) {
-			MOVE_BOXED(ptr, val, n_htop, nsp_i);
-		    } else if (ptr_within(ptr, goheap, gohtop)) {
-			MOVE_BOXED(ptr, val, n_htop, nsp_i);
-		    }
-		} else if (is_list(gval)) {
-		    Eterm *ptr = list_val(gval);
-		    Eterm val = *ptr;
-		    if (is_non_value(val)) {
-			*nsp_i = ptr[1];
-		    } else if (ptr_within(ptr, gheap, ghtop)) {
-			MOVE_CONS(ptr, val, n_htop, nsp_i);
-		    } else if (ptr_within(ptr, gheap, ghtop)) {
-			MOVE_CONS(ptr, val, n_htop, nsp_i);
-		    }
-		}
-	    }
-	    if (++i >= sdesc_size)
-		break;
-	    if (i & 31)
-		mask >>= 1;
-	    else
-		mask = sdesc->livebits[i >> 5];
-	}
-	ra = nstack_walk_frame_ra(nsp, sdesc);
-	if (ra == (unsigned long)nbif_stack_trap_ra)
-	    ra = (unsigned long)p->hipe.ngra;
-	sdesc = hipe_find_sdesc(ra);
-	nsp = nstack_walk_next_frame(nsp, sdesc_size);
-    }
-    abort();
-}
-
-void ma_gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
-{
-    /* known nstack walk state */
-    Eterm *nsp;
-    Eterm *nsp_end;
-    const struct sdesc *sdesc;
-    unsigned int sdesc_size;
-    unsigned long ra;
-    unsigned int i;
-    unsigned int mask;
-    /* arch-specific nstack walk state */
-    struct nstack_walk_state walk_state;
-
-    /* ma_gensweep-specific state */
-    Eterm *low_water, *high_water, *surface;
-    Eterm *n_htop;
-    Eterm *old_htop;
-
-    if (!nstack_walk_init_check(p))
-	return;
-
-    nsp = nstack_walk_nsp_begin(p);
-    nsp_end = nstack_walk_nsp_end(p);
-
-    low_water = global_heap;
-    high_water = global_high_water;
-    surface = global_htop;
-
-    old_htop = *ptr_old_htop;
-    n_htop = *ptr_n_htop;
-
-    sdesc = nstack_walk_init_sdesc(p, &walk_state);
-
-    for (;;) {
-	if (nstack_walk_nsp_reached_end(nsp, nsp_end)) {
-	    if (nsp == nsp_end) {
-		*ptr_old_htop = old_htop;
-		*ptr_n_htop = n_htop;
-		return;
-	    }
-	    fprintf(stderr, "%s: passed end of stack\r\n", __FUNCTION__);
-	    break;
-	}
-	sdesc_size = nstack_walk_frame_size(sdesc);
-	i = 0;
-	mask = sdesc->livebits[0];
-	for (;;) {
-	    if (mask & 1) {
-		Eterm *nsp_i = nstack_walk_frame_index(nsp, i);
-		Eterm gval = *nsp_i;
-		if (is_boxed(gval)) {
-		    Eterm *ptr = boxed_val(gval);
-		    Eterm val = *ptr;
-		    if (MY_IS_MOVED(val)) {
-			*nsp_i = val;
-		    } else if (ptr_within(ptr, low_water, high_water)) {
-			MOVE_BOXED(ptr, val, old_htop, nsp_i);
-		    } else if (ptr_within(ptr, high_water, surface)) {
-			MOVE_BOXED(ptr, val, n_htop, nsp_i);
-		    }
-		} else if (is_list(gval)) {
-		    Eterm *ptr = list_val(gval);
-		    Eterm val = *ptr;
-		    if (is_non_value(val)) {
-			*nsp_i = ptr[1];
-		    } else if (ptr_within(ptr, low_water, high_water)) {
-			MOVE_CONS(ptr, val, old_htop, nsp_i);
-		    } else if (ptr_within(ptr, high_water, surface)) {
-			MOVE_CONS(ptr, val, n_htop, nsp_i);
-		    }
-		}
-	    }
-	    if (++i >= sdesc_size)
-		break;
-	    if (i & 31)
-		mask >>= 1;
-	    else
-		mask = sdesc->livebits[i >> 5];
-	}
-	ra = nstack_walk_frame_ra(nsp, sdesc);
-	if (ra == (unsigned long)nbif_stack_trap_ra)
-	    ra = (unsigned long)p->hipe.ngra;
-	sdesc = hipe_find_sdesc(ra);
-	nsp = nstack_walk_next_frame(nsp, sdesc_size);
-    }
-    abort();
-}
-#endif /* INCREMENTAL */
-
-#endif /* HYBRID */
diff --git a/erts/emulator/hipe/hipe_gc.h b/erts/emulator/hipe/hipe_gc.h
index 712d0ffa78..0d5614c9cf 100644
--- a/erts/emulator/hipe/hipe_gc.h
+++ b/erts/emulator/hipe/hipe_gc.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_GC_H
 #define HIPE_GC_H
 
diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c
index a77aec7919..6e9041c84a 100644
--- a/erts/emulator/hipe/hipe_mkliterals.c
+++ b/erts/emulator/hipe/hipe_mkliterals.c
@@ -1,24 +1,23 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
+
 #ifdef HAVE_CONFIG_H
 #include "config.h"
 #endif
@@ -212,6 +211,11 @@ static const unsigned int CRCTABLE[256] = {
     0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D,
 };
 
+/* For hipe cross compiler. Hard code all values.
+   No calls by hipe compiler to query the running emulator.
+*/
+static int is_xcomp = 0;
+
 /*
  *  The algorithm for calculating the 32 bit CRC checksum is based upon
  *  documentation and algorithms provided by Dr. Ross N. Williams in the
@@ -243,7 +247,7 @@ crc_update_buf(unsigned int crc_value,
 }
 
 static unsigned int
-crc_update_int(unsigned int crc_value, const unsigned int *p)
+crc_update_int(unsigned int crc_value, const int *p)
 {
     return crc_update_buf(crc_value, p, sizeof *p);
 }
@@ -256,12 +260,12 @@ crc_update_int(unsigned int crc_value, const unsigned int *p)
  */
 static const struct literal {
     const char *name;
-    unsigned int value;
+    int value;
 } literals[] = {
     /* Field offsets in a process struct */
     { "P_HP", offsetof(struct process, htop) },
     { "P_HP_LIMIT", offsetof(struct process, stop) },
-    { "P_OFF_HEAP_MSO", offsetof(struct process, off_heap.mso) },
+    { "P_OFF_HEAP_FIRST", offsetof(struct process, off_heap.first) },
     { "P_MBUF", offsetof(struct process, mbuf) },
     { "P_ID", offsetof(struct process, id) },
     { "P_FLAGS", offsetof(struct process, flags) },
@@ -289,6 +293,14 @@ static const struct literal {
     { "P_NRA", offsetof(struct process, hipe.nra) },
 #endif
     { "P_NARITY", offsetof(struct process, hipe.narity) },
+    { "P_FLOAT_RESULT",
+# ifdef NO_FPE_SIGNALS
+	offsetof(struct process, hipe.float_result)
+# endif
+    },
+# if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+    { "P_BIF_CALLEE", offsetof(struct process, hipe.bif_callee) },
+# endif
 #endif /* HIPE */
 
     /* process flags bits */
@@ -298,7 +310,7 @@ static const struct literal {
     { "FREASON_TRAP", TRAP },
 
     /* special Erlang constants */
-    { "THE_NON_VALUE", THE_NON_VALUE },
+    { "THE_NON_VALUE", (int)THE_NON_VALUE },
 
     /* funs */
 #ifdef HIPE
@@ -452,18 +464,14 @@ static const struct rts_param {
     unsigned int nr;
     const char *name;
     unsigned int is_defined;
-    unsigned int value;
+    int value;
 } rts_params[] = {
     { 1, "P_OFF_HEAP_FUNS",
-#if !defined(HYBRID)
-      1, offsetof(struct process, off_heap.funs)
-#endif
+      1, offsetof(struct process, off_heap.first)
     },
 
     { 4, "EFT_NEXT",
-#if !defined(HYBRID)
       1, offsetof(struct erl_fun_thing, next)
-#endif
     },
 
     /* These are always defined, but their values depend on the
@@ -484,7 +492,7 @@ static const struct rts_param {
 #endif
     },
     { 14, "P_FP_EXCEPTION",
-#if !defined(NO_FPE_SIGNALS)
+#if !defined(NO_FPE_SIGNALS) || defined(HIPE)
       1, offsetof(struct process, fp_exception)
 #endif
     },
@@ -497,6 +505,15 @@ static const struct rts_param {
       0
 #endif
     },
+    /* This flag is always defined, but its value is configuration-dependent. */
+    { 16, "ERTS_NO_FPE_SIGNALS",
+      1,
+#if defined(NO_FPE_SIGNALS)
+      1
+#else
+      0
+#endif
+    },
     /* This parameter is always defined, but its value depends on ERTS_SMP. */
     { 19, "MSG_MESSAGE",
       1, offsetof(struct erl_mesg, m[0])
@@ -528,12 +545,12 @@ static void compute_crc(void)
 
 static void c_define_literal(FILE *fp, const struct literal *literal)
 {
-    fprintf(fp, "#define %s %u\n", literal->name, literal->value);
+    fprintf(fp, "#define %s %d\n", literal->name, literal->value);
 }
 
 static void e_define_literal(FILE *fp, const struct literal *literal)
 {
-    fprintf(fp, "-define(%s, %u).\n", literal->name, literal->value);
+    fprintf(fp, "-define(%s, %d).\n", literal->name, literal->value);
 }
 
 static void print_literals(FILE *fp, void (*print_literal)(FILE*, const struct literal*))
@@ -560,7 +577,7 @@ static void print_atom_literals(FILE *fp, void (*print_atom_literal)(FILE*, cons
 static void c_define_param(FILE *fp, const struct rts_param *param)
 {
     if (param->is_defined)
-	fprintf(fp, "#define %s %u\n", param->name, param->value);
+	fprintf(fp, "#define %s %d\n", param->name, param->value);
 }
 
 static void c_case_param(FILE *fp, const struct rts_param *param)
@@ -568,7 +585,7 @@ static void c_case_param(FILE *fp, const struct rts_param *param)
     fprintf(fp, " \\\n");
     fprintf(fp, "\tcase %u: ", param->nr);
     if (param->is_defined)
-	fprintf(fp, "value = %u", param->value);
+	fprintf(fp, "value = %d", param->value);
     else
 	fprintf(fp, "is_defined = 0");
     fprintf(fp, "; break;");
@@ -576,7 +593,15 @@ static void c_case_param(FILE *fp, const struct rts_param *param)
 
 static void e_define_param(FILE *fp, const struct rts_param *param)
 {
-    fprintf(fp, "-define(%s, hipe_bifs:get_rts_param(%u)).\n", param->name, param->nr);
+    if (is_xcomp) {
+	if (param->is_defined)
+	    fprintf(fp, "-define(%s, %d).\n", param->name, param->value);
+	else
+	    fprintf(fp, "-define(%s, []).\n", param->name);	
+    }
+    else {
+	fprintf(fp, "-define(%s, hipe_bifs:get_rts_param(%u)).\n", param->name, param->nr);
+    }    
 }
 
 static void print_params(FILE *fp, void (*print_param)(FILE*,const struct rts_param*))
@@ -587,9 +612,9 @@ static void print_params(FILE *fp, void (*print_param)(FILE*,const struct rts_pa
 	(*print_param)(fp, &rts_params[i]);
 }
 
-static int do_c(FILE *fp)
+static int do_c(FILE *fp, const char* this_exe)
 {
-    fprintf(fp, "/* File: hipe_literals.h, generated by hipe_mkliterals */\n");
+    fprintf(fp, "/* File: hipe_literals.h, generated by %s */\n", this_exe);
     fprintf(fp, "#ifndef __HIPE_LITERALS_H__\n");
     fprintf(fp, "#define __HIPE_LITERALS_H__\n\n");
     print_literals(fp, c_define_literal);
@@ -603,9 +628,9 @@ static int do_c(FILE *fp)
     return 0;
 }
 
-static int do_e(FILE *fp)
+static int do_e(FILE *fp, const char* this_exe)
 {
-    fprintf(fp, "%%%% File: hipe_literals.hrl, generated by hipe_mkliterals");
+    fprintf(fp, "%%%% File: hipe_literals.hrl, generated by %s", this_exe);
     fprintf(fp, "\n\n");
     print_literals(fp, e_define_literal);
     fprintf(fp, "\n");
@@ -613,19 +638,40 @@ static int do_e(FILE *fp)
     fprintf(fp, "\n");
     print_params(fp, e_define_param);
     fprintf(fp, "\n");
-    fprintf(fp, "-define(HIPE_SYSTEM_CRC, hipe_bifs:system_crc(%u)).\n", literals_crc);
+    if (is_xcomp) {
+	fprintf(fp, "-define(HIPE_SYSTEM_CRC, %u).\n", system_crc);
+    }
+    else {
+	fprintf(fp, "-define(HIPE_SYSTEM_CRC, hipe_bifs:system_crc(%u)).\n",
+		literals_crc);
+    }
     return 0;
 }
 
 int main(int argc, const char **argv)
 {
+    int i;
+    int (*do_func_ptr)(FILE *, const char*) = NULL;
+
     compute_crc();
-    if (argc == 2) {
-	if (strcmp(argv[1], "-c") == 0)
-	    return do_c(stdout);
-	if (strcmp(argv[1], "-e") == 0)
-	    return do_e(stdout);
+    for (i = 1; i < argc; i++) {
+	if      (strcmp(argv[i], "-c") == 0)
+	    do_func_ptr = &do_c;
+	else if (strcmp(argv[i], "-e") == 0)
+	    do_func_ptr = &do_e;
+	else if (strcmp(argv[i], "-x") == 0)
+	    is_xcomp = 1;
+	else
+	    goto error;
+    }
+    if (do_func_ptr) {
+	return do_func_ptr(stdout, argv[0]);
     }
-    fprintf(stderr, "usage: %s [-c | -e] > output-file\n", argv[0]);
+error:
+    fprintf(stderr, "usage: %s [-x] [-c | -e] > output-file\n"
+	    "\t-c\tC header file\n"
+	    "\t-e\tErlang header file\n"
+	    "\t-x\tCross compile. No dependencies to compiling emulator\n",	    
+	    argv[0]);
     return 1;
 }
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index e5de244d25..6a3ce5608f 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -1,22 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_mode_switch.c
  */
 #ifdef HAVE_CONFIG_H
@@ -34,6 +35,17 @@
 #include "hipe_stack.h"
 #include "hipe_bif0.h"	/* hipe_mfa_info_table_init() */
 
+#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+#    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
+        if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN)
+#    define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \
+        if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN)
+#else
+#    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P)
+#    define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P)
+#endif
+
+
 /*
  * Internal debug support.
  * #define HIPE_DEBUG to the desired debug level:
@@ -208,6 +220,8 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
 #endif
 
     p->i = NULL;
+    /* Set current_function to undefined. stdlib hibernate tests rely on it. */
+    p->current = NULL;
 
     DPRINTF("cmd == %#x (%s)", cmd, code_str(cmd));
     HIPE_CHECK_PCB(p);
@@ -315,34 +329,46 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
 	   * Native code called a BIF, which "failed" with a TRAP to BEAM.
 	   * Prior to returning, the BIF stored (see BIF_TRAP<N>):
 
-	   * the callee's address in p->def_arg_reg[3]
-	   * the callee's parameters in p->def_arg_reg[0..2]
+	   * the callee's address in p->i
+	   * the callee's parameters in reg[0..2]
 	   * the callee's arity in p->arity (for BEAM gc purposes)
 	   *
 	   * We need to remove the BIF's parameters from the native
 	   * stack: to this end hipe_${ARCH}_glue.S stores the BIF's
 	   * arity in p->hipe.narity.
+	   *
+	   * If the BIF emptied the stack (typically hibernate), p->hipe.nstack
+	   * is NULL and there is no need to get rid of stacked parameters.
 	   */
-	  unsigned int i, is_recursive, callee_arity;
-
-	  /* Save p->arity, then update it with the original BIF's arity.
-	     Get rid of any stacked parameters in that call. */
-	  /* XXX: hipe_call_from_native_is_recursive() copies data to
-	     reg[], which is useless in the TRAP case. Maybe write a
-	     specialised hipe_trap_from_native_is_recursive() later. */
-	  callee_arity = p->arity;
-	  p->arity = p->hipe.narity; /* caller's arity */
-	  is_recursive = hipe_call_from_native_is_recursive(p, reg);
-
-	  p->i = (Eterm *)(p->def_arg_reg[3]);
-	  p->arity = callee_arity;
+	  unsigned int i, is_recursive = 0;
+
+          if (p->hipe.nstack != NULL) {
+	      ASSERT(p->hipe.nsp != NULL);
+	      is_recursive = hipe_trap_from_native_is_recursive(p);
+          }
+	  else {
+	      /* Some architectures (risc) need this re-reset of nsp as the
+	       * BIF wrapper do not detect stack change and causes an obsolete
+	       * stack pointer to be saved in p->hipe.nsp before return to us.
+	       */
+	      p->hipe.nsp = NULL;
+	  }
 
-	  for (i = 0; i < p->arity; ++i)
-	      reg[i] = p->def_arg_reg[i];
+	  /* Schedule next process if current process was hibernated or is waiting
+	     for messages */
+	  if (p->flags & F_HIBERNATE_SCHED) {
+	      p->flags &= ~F_HIBERNATE_SCHED;
+	      goto do_schedule;
+	  }
+	  if (p->status == P_WAITING) {
+	      for (i = 0; i < p->arity; ++i)
+		  p->arg_reg[i] = reg[i]; 	      
+	      goto do_schedule;
+	  }
 
 	  if (is_recursive)
 	      hipe_push_beam_trap_frame(p, reg, p->arity);
-
+	  
 	  result = HIPE_MODE_SWITCH_RES_CALL;
 	  break;
       }
@@ -451,10 +477,12 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
 #if !(NR_ARG_REGS > 5)
 	      int reds_in = p->def_arg_reg[5];
 #endif
+	      ERTS_SMP_UNREQ_PROC_MAIN_LOCK(p);
 	      p = schedule(p, reds_in - p->fcalls);
+	      ERTS_SMP_REQ_PROC_MAIN_LOCK(p);
 #ifdef ERTS_SMP
 	      p->hipe_smp.have_receive_locks = 0;
-	      reg = p->scheduler_data->save_reg;
+	      reg = p->scheduler_data->x_reg_array;
 #endif
 	  }
 	  {
@@ -592,6 +620,17 @@ void hipe_inc_nstack(Process *p)
 }
 #endif
 
+void hipe_empty_nstack(Process *p)
+{
+    if (p->hipe.nstack) {
+	erts_free(ERTS_ALC_T_HIPE, p->hipe.nstack);
+    }
+    p->hipe.nstgraylim = NULL;
+    p->hipe.nsp = NULL;
+    p->hipe.nstack = NULL;
+    p->hipe.nstend = NULL;
+}
+
 static void hipe_check_nstack(Process *p, unsigned nwords)
 {
     while (hipe_nstack_avail(p) < nwords)
@@ -618,7 +657,7 @@ Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s)
     if (depth < 1)
 	return NIL;
 
-    heap_size = 6 * depth;	/* each [{M,F,A}|_] is 2+4 == 6 words */
+    heap_size = 7 * depth;	/* each [{M,F,A,[]}|_] is 2+5 == 7 words */
     hp = HAlloc(p, heap_size);
     hp_end = hp + heap_size;
 
@@ -629,8 +668,8 @@ Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s)
 	ra = (const void*)s->trace[i];
 	if (!hipe_find_mfa_from_ra(ra, &m, &f, &a))
 	    continue;
-	mfa = TUPLE3(hp, m, f, make_small(a));
-	hp += 4;
+	mfa = TUPLE4(hp, m, f, make_small(a), NIL);
+	hp += 5;
 	next = CONS(hp, mfa, NIL);
 	*next_p = next;
 	next_p = &CDR(list_val(next));
diff --git a/erts/emulator/hipe/hipe_mode_switch.h b/erts/emulator/hipe/hipe_mode_switch.h
index 187b9145e2..a3e908a3b3 100644
--- a/erts/emulator/hipe/hipe_mode_switch.h
+++ b/erts/emulator/hipe/hipe_mode_switch.h
@@ -1,22 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_mode_switch.h
  */
 #ifndef HIPE_MODE_SWITCH_H
@@ -48,12 +49,13 @@
 
 #include "error.h"
 
-int hipe_modeswitch_debug;
+extern int hipe_modeswitch_debug;
 
 void hipe_mode_switch_init(void);
 void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure);
 Process *hipe_mode_switch(Process*, unsigned, Eterm*);
 void hipe_inc_nstack(Process *p);
+void hipe_empty_nstack(Process *p);
 void hipe_set_closure_stub(ErlFunEntry *fe, unsigned num_free);
 Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s);
 
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index f8c2502522..3be821f8f7 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_native_bif.c
  */
 #ifdef HAVE_CONFIG_H
@@ -41,9 +41,9 @@
  */
 
 /* for -Wmissing-prototypes :-( */
-extern Eterm hipe_check_process_code_2(Process*, Eterm, Eterm);
-extern Eterm hipe_garbage_collect_1(Process*, Eterm);
-extern Eterm hipe_show_nstack_1(Process*, Eterm);
+extern Eterm hipe_check_process_code_2(BIF_ALIST_2);
+extern Eterm hipe_garbage_collect_1(BIF_ALIST_1);
+extern Eterm hipe_show_nstack_1(BIF_ALIST_1);
 
 /* Used when a BIF can trigger a stack walk. */
 static __inline__ void hipe_set_narity(Process *p, unsigned int arity)
@@ -56,7 +56,7 @@ Eterm hipe_check_process_code_2(BIF_ALIST_2)
     Eterm ret;
 
     hipe_set_narity(BIF_P, 2);
-    ret = check_process_code_2(BIF_P, BIF_ARG_1, BIF_ARG_2);
+    ret = check_process_code_2(BIF_P, BIF__ARGS);
     hipe_set_narity(BIF_P, 0);
     return ret;
 }
@@ -66,7 +66,7 @@ Eterm hipe_garbage_collect_1(BIF_ALIST_1)
     Eterm ret;
 
     hipe_set_narity(BIF_P, 1);
-    ret = garbage_collect_1(BIF_P, BIF_ARG_1);
+    ret = garbage_collect_1(BIF_P, BIF__ARGS);
     hipe_set_narity(BIF_P, 0);
     return ret;
 }
@@ -76,7 +76,7 @@ Eterm hipe_show_nstack_1(BIF_ALIST_1)
     Eterm ret;
 
     hipe_set_narity(BIF_P, 1);
-    ret = hipe_bifs_show_nstack_1(BIF_P, BIF_ARG_1);
+    ret = hipe_bifs_show_nstack_1(BIF_P, BIF__ARGS);
     hipe_set_narity(BIF_P, 0);
     return ret;
 }
@@ -99,8 +99,10 @@ void hipe_gc(Process *p, Eterm need)
  *  has begun.
  * XXX: BUG: native code should check return status
  */
-Eterm hipe_set_timeout(Process *p, Eterm timeout_value)
+BIF_RETTYPE hipe_set_timeout(BIF_ALIST_1)
 {
+    Process* p = BIF_P;
+    Eterm timeout_value = BIF_ARG_1;
 #if !defined(ARCH_64)
     Uint time_val;
 #endif
@@ -187,6 +189,8 @@ void hipe_fclearerror_error(Process *p)
 {
 #if !defined(NO_FPE_SIGNALS)
     erts_fp_check_init_error(&p->fp_exception);
+#else
+    erl_exit(ERTS_ABORT_EXIT, "Emulated FPE not cleared by HiPE");
 #endif
 }
 
@@ -286,8 +290,13 @@ static struct StackTrace *get_trace_from_exc(Eterm exc)
  * This does what the (misnamed) Beam instruction 'raise_ss' does,
  * namely, a proper re-throw of an exception that was caught by 'try'.
  */
-Eterm hipe_rethrow(Process *c_p, Eterm exc, Eterm value)
+
+BIF_RETTYPE hipe_rethrow(BIF_ALIST_2)
 {
+    Process* c_p = BIF_P;
+    Eterm exc = BIF_ARG_1;
+    Eterm value = BIF_ARG_2;
+
     c_p->fvalue = value;
     if (c_p->freason == EXC_NULL) {
 	/* a safety check for the R10-0 case; should not happen */
@@ -334,7 +343,7 @@ char *hipe_bs_allocate(int len)
     bptr = erts_bin_nrml_alloc(len);
     bptr->flags = 0;
     bptr->orig_size = len;
-    erts_smp_atomic_init(&bptr->refc, 1);
+    erts_smp_atomic_init_nob(&bptr->refc, 1);
     return bptr->orig_bytes;
 }
 
@@ -411,8 +420,12 @@ Eterm hipe_bs_utf8_size(Eterm arg)
 	return make_small(4);
 }
 
-Eterm hipe_bs_put_utf8(Process *p, Eterm arg, byte *base, unsigned int offset)
+BIF_RETTYPE hipe_bs_put_utf8(BIF_ALIST_3)
 {
+    Process* p = BIF_P;
+    Eterm arg = BIF_ARG_1;
+    byte* base = (byte*) BIF_ARG_2;
+    Uint offset = (Uint) BIF_ARG_3;
     byte *save_bin_buf;
     Uint save_bin_offset;
     int res;
@@ -468,13 +481,21 @@ Eterm hipe_bs_put_utf16(Process *p, Eterm arg, byte *base, unsigned int offset,
     return new_offset;
 }
 
-Eterm hipe_bs_put_utf16be(Process *p, Eterm arg, byte *base, unsigned int offset)
+BIF_RETTYPE hipe_bs_put_utf16be(BIF_ALIST_3)
 {
+    Process *p = BIF_P;
+    Eterm arg = BIF_ARG_1;
+    byte *base = (byte*) BIF_ARG_2; 
+    Uint offset = (Uint) BIF_ARG_3;
     return hipe_bs_put_utf16(p, arg, base, offset, 0);
 }
 
-Eterm hipe_bs_put_utf16le(Process *p, Eterm arg, byte *base, unsigned int offset)
+BIF_RETTYPE hipe_bs_put_utf16le(BIF_ALIST_3)
 {
+    Process *p = BIF_P;
+    Eterm arg = BIF_ARG_1;
+    byte *base = (byte*) BIF_ARG_2; 
+    Uint offset = (Uint) BIF_ARG_3;
     return hipe_bs_put_utf16(p, arg, base, offset, BSF_LITTLE);
 }
 
@@ -489,8 +510,10 @@ static int validate_unicode(Eterm arg)
     return 1;
 }
 
-Eterm hipe_bs_validate_unicode(Process *p, Eterm arg)
+BIF_RETTYPE hipe_bs_validate_unicode(BIF_ALIST_1)
 {
+    Process *p = BIF_P;
+    Eterm arg = BIF_ARG_1;
     if (!validate_unicode(arg))
 	BIF_ERROR(p, BADARG);
     return NIL;
@@ -584,7 +607,7 @@ void hipe_clear_timeout(Process *c_p)
 
 void hipe_atomic_inc(int *counter)
 {
-    erts_smp_atomic_inc((erts_smp_atomic_t*)counter);
+    erts_smp_atomic_inc_nob((erts_smp_atomic_t*)counter);
 }
 
 #endif
diff --git a/erts/emulator/hipe/hipe_native_bif.h b/erts/emulator/hipe/hipe_native_bif.h
index 3b55b64a41..9e3a156fbc 100644
--- a/erts/emulator/hipe/hipe_native_bif.h
+++ b/erts/emulator/hipe/hipe_native_bif.h
@@ -1,28 +1,29 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_native_bif.h
  */
  
 #ifndef HIPE_NATIVE_BIF_H
 #define HIPE_NATIVE_BIF_H
 
+#include "bif.h"
 #include "hipe_arch.h"
 
 /*
@@ -71,27 +72,32 @@ AEXTERN(void,nbif_select_msg,(Process*));
 AEXTERN(Eterm,nbif_cmp_2,(void));
 AEXTERN(Eterm,nbif_eq_2,(void));
 
-Eterm hipe_nonclosure_address(Process*, Eterm, Uint);
-Eterm hipe_conv_big_to_float(Process*, Eterm);
+BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2);
+BIF_RETTYPE hipe_conv_big_to_float(BIF_ALIST_1);
 void hipe_fclearerror_error(Process*);
 void hipe_select_msg(Process*);
 void hipe_gc(Process*, Eterm);
-Eterm hipe_set_timeout(Process*, Eterm);
+BIF_RETTYPE hipe_set_timeout(BIF_ALIST_1);
 void hipe_handle_exception(Process*);
-Eterm hipe_rethrow(Process *c_p, Eterm exc, Eterm value);
+BIF_RETTYPE hipe_rethrow(BIF_ALIST_2);
 char *hipe_bs_allocate(int);
 Binary *hipe_bs_reallocate(Binary*, int);
 int hipe_bs_put_small_float(Process*, Eterm, Uint, byte*, unsigned, unsigned);
 void hipe_bs_put_bits(Eterm, Uint, byte*, unsigned, unsigned);
 Eterm hipe_bs_utf8_size(Eterm);
-Eterm hipe_bs_put_utf8(Process*, Eterm, byte*, unsigned int);
+BIF_RETTYPE hipe_bs_put_utf8(BIF_ALIST_3);
 Eterm hipe_bs_utf16_size(Eterm);
-Eterm hipe_bs_put_utf16be(Process*, Eterm, byte*, unsigned int);
-Eterm hipe_bs_put_utf16le(Process*, Eterm, byte*, unsigned int);
-Eterm hipe_bs_validate_unicode(Process*, Eterm);
+BIF_RETTYPE hipe_bs_put_utf16be(BIF_ALIST_3);
+BIF_RETTYPE hipe_bs_put_utf16le(BIF_ALIST_3);
+BIF_RETTYPE hipe_bs_validate_unicode(BIF_ALIST_1);
 struct erl_bin_match_buffer;
 int hipe_bs_validate_unicode_retract(struct erl_bin_match_buffer*, Eterm);
 
+#ifdef NO_FPE_SIGNALS
+AEXTERN(void,nbif_emulate_fpe,(Process*));
+void hipe_emulate_fpe(Process*);
+#endif
+
 /*
  * Stuff that is different in SMP and non-SMP.
  */
diff --git a/erts/emulator/hipe/hipe_ops.tab b/erts/emulator/hipe/hipe_ops.tab
index eb6f824d1c..50c3a4ae2f 100644
--- a/erts/emulator/hipe/hipe_ops.tab
+++ b/erts/emulator/hipe/hipe_ops.tab
@@ -1,19 +1,19 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2001-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2001-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
 
diff --git a/erts/emulator/hipe/hipe_perfctr.c b/erts/emulator/hipe/hipe_perfctr.c
index 69bb648854..371b3fb097 100644
--- a/erts/emulator/hipe/hipe_perfctr.c
+++ b/erts/emulator/hipe/hipe_perfctr.c
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifdef HAVE_CONFIG_H
 #include "config.h"
 #endif
diff --git a/erts/emulator/hipe/hipe_perfctr.h b/erts/emulator/hipe/hipe_perfctr.h
index 7b20c68cac..8fbf9ecf35 100644
--- a/erts/emulator/hipe/hipe_perfctr.h
+++ b/erts/emulator/hipe/hipe_perfctr.h
@@ -1,23 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
 
 extern int hipe_perfctr_hrvtime_open(void);
 extern void hipe_perfctr_hrvtime_close(void);
diff --git a/erts/emulator/hipe/hipe_perfctr.tab b/erts/emulator/hipe/hipe_perfctr.tab
index 663522f85e..eaecea4651 100644
--- a/erts/emulator/hipe/hipe_perfctr.tab
+++ b/erts/emulator/hipe/hipe_perfctr.tab
@@ -1,22 +1,21 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2004-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2004-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
-# $Id$
 
 bif hipe_bifs:vperfctr_open/0
 bif hipe_bifs:vperfctr_close/0
diff --git a/erts/emulator/hipe/hipe_ppc.c b/erts/emulator/hipe/hipe_ppc.c
index 3a0beedb68..2d8fd61e1e 100644
--- a/erts/emulator/hipe/hipe_ppc.c
+++ b/erts/emulator/hipe/hipe_ppc.c
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #include <stddef.h>	/* offsetof() */
 #ifdef HAVE_CONFIG_H
 #include "config.h"
@@ -80,55 +80,13 @@ static struct segment {
 } curseg;
 
 #define in_area(ptr,start,nbytes)	\
-	((unsigned long)((char*)(ptr) - (char*)(start)) < (nbytes))
+	((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
 
 /* Darwin breakage */
 #if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
 #define MAP_ANONYMOUS MAP_ANON
 #endif
 
-#if defined(__powerpc64__)
-static void *new_code_mapping(void)
-{
-    char *map_hint, *map_start;
-
-    /*
-     * Allocate a new 32MB code segment in the low 2GB of the address space.
-     *
-     * This is problematic for several reasons:
-     * - Linux/ppc64 lacks the MAP_32BIT flag that Linux/x86-64 has.
-     * - The address space hint to mmap is only respected if that
-     *   area is available. If it isn't, then mmap falls back to its
-     *   defaults, which (according to testing) results in very high
-     *   (and thus useless for us) addresses being returned.
-     * - Another mapping, presumably the brk, also occupies low addresses.
-     *
-     * As initial implementation, simply start allocating at the 0.5GB
-     * boundary. This leaves plenty of space for the brk before malloc
-     * needs to switch to mmap, while allowing for 1.5GB of code.
-     *
-     * A more robust implementation would be to parse /proc/self/maps,
-     * reserve all available space between (say) 0.5GB and 2GB with
-     * PROT_NONE MAP_NORESERVE mappings, and then allocate by releasing
-     * 32MB segments and re-mapping them properly. This would work on
-     * Linux/ppc64, I have no idea how things should be done on Darwin64.
-     */
-    if (curseg.base)
-	map_hint = (char*)curseg.base + SEGMENT_NRBYTES;
-    else
-	map_hint = (char*)(512*1024*1024); /* 0.5GB */
-    map_start = mmap(map_hint, SEGMENT_NRBYTES,
-		     PROT_EXEC|PROT_READ|PROT_WRITE,
-		     MAP_PRIVATE|MAP_ANONYMOUS,
-		     -1, 0);
-    if (map_start != MAP_FAILED &&
-	(((unsigned long)map_start + (SEGMENT_NRBYTES-1)) & ~0x7FFFFFFFUL)) {
-	fprintf(stderr, "mmap with hint %p returned code memory %p\r\n", map_hint, map_start);
-	abort();
-    }
-    return map_start;
-}
-#else
 static void *new_code_mapping(void)
 {
     return mmap(0, SEGMENT_NRBYTES,
@@ -136,7 +94,6 @@ static void *new_code_mapping(void)
 		MAP_PRIVATE|MAP_ANONYMOUS,
 		-1, 0);
 }
-#endif
 
 static int check_callees(Eterm callees)
 {
@@ -182,20 +139,30 @@ static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsig
 	unsigned int a = unsigned_val(tuple_val(mfa)[3]);
 	unsigned int *trampoline = hipe_mfa_get_trampoline(m, f, a);
 	if (!in_area(trampoline, base, SEGMENT_NRBYTES)) {
+#if defined(__powerpc64__)
+	    if (nrfreewords < 7)
+		return NULL;
+	    nrfreewords -= 7;
+	    tramp_pos = trampoline = tramp_pos - 7;
+	    trampoline[0] = 0x3D600000; /* addis r11,r0,0 */
+	    trampoline[1] = 0x616B0000; /* ori r11,r11,0 */
+	    trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */
+	    trampoline[3] = 0x656B0000; /* oris r11,r11,0 */
+	    trampoline[4] = 0x616B0000; /* ori r11,r11,0 */
+	    trampoline[5] = 0x7D6903A6; /* mtctr r11 */
+	    trampoline[6] = 0x4E800420; /* bctr */
+	    hipe_flush_icache_range(trampoline, 7*sizeof(int));
+#else
 	    if (nrfreewords < 4)
 		return NULL;
 	    nrfreewords -= 4;
 	    tramp_pos = trampoline = tramp_pos - 4;
-#if defined(__powerpc64__)
-	    trampoline[0] = 0x3D600000; /* addis r11,0,0 */
-	    trampoline[1] = 0x616B0000; /* ori r11,r11,0 */
-#else
 	    trampoline[0] = 0x39600000; /* addi r11,r0,0 */
 	    trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */
-#endif
 	    trampoline[2] = 0x7D6903A6; /* mtctr r11 */
 	    trampoline[3] = 0x4E800420; /* bctr */
 	    hipe_flush_icache_range(trampoline, 4*sizeof(int));
+#endif
 	    hipe_mfa_set_trampoline(m, f, a, trampoline);
 	}
 	trampvec[trampnr-1] = trampoline;
@@ -281,21 +248,22 @@ static void patch_imm16(Uint32 *address, unsigned int imm16)
 }
 
 #if defined(__powerpc64__)
+/*
+ * To load a 64-bit immediate value 'val' into Rd (Rd != R0):
+ *
+ * addis Rd, 0, val@highest // (val >> 48) & 0xFFFF
+ * ori Rd, Rd, val@higher   // (val >> 32) & 0xFFFF
+ * rldicr Rd, Rd, 32, 31
+ * oris Rd, Rd, val@h       // (val >> 16) & 0xFFFF
+ * ori Rd, Rd, val@l        // val & 0xFFFF
+ */
 static void patch_li64(Uint32 *address, Uint64 value)
 {
-    patch_imm16(address+0, value >> 48);/* addis r,0,value@highest */
-    patch_imm16(address+1, value >> 32);/* ori r,r,value@higher */
-    /* sldi r,r,32 */
-    patch_imm16(address+3, value >> 16);/* oris r,r,value@h */
-    patch_imm16(address+4, value);	/* ori r,r,value@l */
-}
-
-static int patch_li31(Uint32 *address, Uint32 value)
-{
-    if ((value >> 31) != 0)
-	return -1;
-    patch_imm16(address, value >> 16);	/* addis r,0,value@h */
-    patch_imm16(address+1, value);	/* ori r,r,value@l */
+    patch_imm16(address+0, value >> 48);
+    patch_imm16(address+1, value >> 32);
+    /* rldicr Rd, Rd, 32, 31 */
+    patch_imm16(address+3, value >> 16);
+    patch_imm16(address+4, value);
 }
 
 void hipe_patch_load_fe(Uint *address, Uint value)
@@ -308,11 +276,10 @@ int hipe_patch_insn(void *address, Uint64 value, Eterm type)
     switch (type) {
       case am_closure:
       case am_constant:
-	patch_li64((Uint32*)address, value);
-	return 0;
       case am_atom:
       case am_c_const:
-	return patch_li31((Uint32*)address, value);
+	patch_li64((Uint32*)address, value);
+	return 0;
       default:
 	return -1;
     }
@@ -442,34 +409,33 @@ static void patch_b(Uint32 *address, Sint32 offset, Uint32 AA)
 
 int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
 {
-    if ((Uint32)destAddress == ((Uint32)destAddress & 0x01FFFFFC)) {
+    if ((UWord)destAddress == ((UWord)destAddress & 0x01FFFFFC)) {
 	/* The destination is in the [0,32MB[ range.
 	   We can reach it with a ba/bla instruction.
 	   This is the typical case for BIFs and primops.
 	   It's also common for trap-to-BEAM stubs (on ppc32). */
-	patch_b((Uint32*)callAddress, (Uint32)destAddress >> 2, 2);
+	patch_b((Uint32*)callAddress, (Sint32)destAddress >> 2, 2);
     } else {
-	Sint32 destOffset = ((Sint32)destAddress - (Sint32)callAddress) >> 2;
+	SWord destOffset = ((SWord)destAddress - (SWord)callAddress) >> 2;
 	if (destOffset >= -0x800000 && destOffset <= 0x7FFFFF) {
 	    /* The destination is within a [-32MB,+32MB[ range from us.
 	       We can reach it with a b/bl instruction.
 	       This is typical for nearby Erlang code. */
-	    patch_b((Uint32*)callAddress, destOffset, 0);
+	    patch_b((Uint32*)callAddress, (Sint32)destOffset, 0);
 	} else {
 	    /* The destination is too distant for b/bl/ba/bla.
 	       Must do a b/bl to the trampoline. */
-	    Sint32 trampOffset = ((Sint32)trampoline - (Sint32)callAddress) >> 2;
+	    SWord trampOffset = ((SWord)trampoline - (SWord)callAddress) >> 2;
 	    if (trampOffset >= -0x800000 && trampOffset <= 0x7FFFFF) {
 		/* Update the trampoline's address computation.
 		   (May be redundant, but we can't tell.) */
 #if defined(__powerpc64__)
-		/* This relies on the fact that we allocate code below 2GB. */
-		patch_li31((Uint32*)trampoline, (Uint32)destAddress);
+		patch_li64((Uint32*)trampoline, (Uint64)destAddress);
 #else
 		patch_li((Uint32*)trampoline, (Uint32)destAddress);
 #endif
 		/* Update this call site. */
-		patch_b((Uint32*)callAddress, trampOffset, 0);
+		patch_b((Uint32*)callAddress, (Sint32)trampOffset, 0);
 	    } else
 		return -1;
 	}
diff --git a/erts/emulator/hipe/hipe_ppc.h b/erts/emulator/hipe/hipe_ppc.h
index e30ce30ed2..66000c1846 100644
--- a/erts/emulator/hipe/hipe_ppc.h
+++ b/erts/emulator/hipe/hipe_ppc.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_PPC_H
 #define HIPE_PPC_H
 
@@ -44,12 +44,20 @@ static __inline__ int hipe_word32_address_ok(void *address)
     return ((unsigned long)address & 0x3) == 0;
 }
 
+#if defined(__powerpc64__)
+/* for hipe_bifs_{read,write}_{s,u}64 */
+static __inline__ int hipe_word64_address_ok(void *address)
+{
+    return ((unsigned long)address & 0x7) == 0;
+}
+#endif
+
 /* Native stack growth direction. */
 #define HIPE_NSTACK_GROWS_DOWN
 
 #if defined(__powerpc64__)
 #define hipe_arch_name	am_ppc64
-#define AEXTERN(RET,NAME,PROTO)	extern const int NAME
+#define AEXTERN(RET,NAME,PROTO)	extern const int NAME[]
 AEXTERN(void,hipe_ppc_inc_stack,(void));
 #else
 #define hipe_arch_name	am_powerpc
diff --git a/erts/emulator/hipe/hipe_ppc.tab b/erts/emulator/hipe/hipe_ppc.tab
index a32dd820e7..38b7f46d3a 100644
--- a/erts/emulator/hipe/hipe_ppc.tab
+++ b/erts/emulator/hipe/hipe_ppc.tab
@@ -1,22 +1,22 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2004-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2004-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
-# $Id$
+
 # PowerPC-specific atoms
 
 atom fconv_constant
diff --git a/erts/emulator/hipe/hipe_ppc64.tab b/erts/emulator/hipe/hipe_ppc64.tab
index 513182721c..0a390a3bb8 100644
--- a/erts/emulator/hipe/hipe_ppc64.tab
+++ b/erts/emulator/hipe/hipe_ppc64.tab
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 2005-2009. All Rights Reserved.
+# Copyright Ericsson AB 2005-2011. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -16,7 +16,7 @@
 # 
 # %CopyrightEnd%
 #
-# $Id$
+
 # PPC64-specific atoms
 
 atom inc_stack_0
diff --git a/erts/emulator/hipe/hipe_ppc_abi.txt b/erts/emulator/hipe/hipe_ppc_abi.txt
index 4bf41e02b2..be0ef98b0d 100644
--- a/erts/emulator/hipe/hipe_ppc_abi.txt
+++ b/erts/emulator/hipe/hipe_ppc_abi.txt
@@ -2,7 +2,7 @@
  %CopyrightBegin%
  %CopyrightEnd%
 
-$Id$
+
 
 HiPE PowerPC ABI
 ================
diff --git a/erts/emulator/hipe/hipe_ppc_asm.m4 b/erts/emulator/hipe/hipe_ppc_asm.m4
index a0f8b78679..343402f9f0 100644
--- a/erts/emulator/hipe/hipe_ppc_asm.m4
+++ b/erts/emulator/hipe/hipe_ppc_asm.m4
@@ -1,25 +1,24 @@
 changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
+
 `#ifndef HIPE_PPC_ASM_H
 #define HIPE_PPC_ASM_H'
 
@@ -32,12 +31,23 @@ define(LOAD,ld)dnl
 define(STORE,std)dnl
 define(CMPI,cmpdi)dnl
 define(WSIZE,8)dnl
+`#define STORE_IA(ADDR, DST, TMP)  \
+	addis TMP, 0, ADDR@highest 	SEMI\
+	ori TMP, TMP, ADDR@higher  	SEMI\
+	rldicr TMP, TMP, 32, 31 	SEMI\
+	oris TMP, TMP, ADDR@h 		SEMI\
+	ori TMP, TMP, ADDR@l		SEMI\
+	std TMP, DST'
 ',`
 /* 32-bit PowerPC */
 define(LOAD,lwz)dnl
 define(STORE,stw)dnl
 define(CMPI,cmpwi)dnl
 define(WSIZE,4)dnl
+`#define STORE_IA(ADDR, DST, TMP)   \
+	lis TMP, ADDR@ha 	SEMI\
+	addi TMP, TMP, ADDR@l	SEMI\
+	stw TMP, DST'
 ')dnl
 `#define LOAD	'LOAD
 `#define STORE	'STORE
@@ -63,15 +73,31 @@ ifelse(OPSYS,darwin,``
 #define SEMI		@
 #define SET_SIZE(NAME)	/*empty*/
 #define TYPE_FUNCTION(NAME)	/*empty*/
+#define OPD(NAME)	/*empty*/
 '',``
 /* Not Darwin */''
 `ifelse(ARCH,ppc64,``
 /* 64-bit */
+/*
+ * The 64-bit PowerPC ABI requires us to setup Official Procedure Descriptors
+ * for functions called from C. These are exported as "func", while the entry
+ * point should is exported as ".func". A function pointer in C points to the
+ * function descriptor in the opd rather than to the function entry point.
+ */
 #define JOIN(X,Y)	X##Y
 #define CSYM(NAME)	JOIN(.,NAME)
+#define OPD(NAME)       			\
+	.pushsection .opd, "aw";		\
+	.align 3;				\
+	.global NAME;				\
+NAME:						\
+	.quad CSYM(NAME), .TOC.@tocbase, 0;	\
+	.type NAME, @function;			\
+	.popsection
 '',``
 /* 32-bit */
 #define CSYM(NAME)	NAME
+#define OPD(NAME)	/*empty*/
 '')'
 ``#define ASYM(NAME)	NAME
 #define GLOBAL(NAME)	.global NAME
diff --git a/erts/emulator/hipe/hipe_ppc_bifs.m4 b/erts/emulator/hipe/hipe_ppc_bifs.m4
index 3849d9113a..d09551d10d 100644
--- a/erts/emulator/hipe/hipe_ppc_bifs.m4
+++ b/erts/emulator/hipe/hipe_ppc_bifs.m4
@@ -1,42 +1,53 @@
 changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
 
 include(`hipe/hipe_ppc_asm.m4')
+#`include' "config.h"
 #`include' "hipe_literals.h"
 
+`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+#  define CALL_BIF(F)	STORE_IA(CSYM(F), P_BIF_CALLEE(P), r29); bl CSYM(hipe_debug_bif_wrapper) 
+#else
+#  define CALL_BIF(F)	bl	CSYM(F)
+#endif'
+
 	.text
 	.p2align 2
 
-`#define TEST_GOT_MBUF		LOAD r4, P_MBUF(P) SEMI CMPI r4, 0 SEMI bne- 3f SEMI 2:
-#define JOIN3(A,B,C)		A##B##C
-#define HANDLE_GOT_MBUF(ARITY)	3: bl CSYM(JOIN3(nbif_,ARITY,_gc_after_bif)) SEMI b 2b'
+define(TEST_GOT_MBUF,`LOAD r4, P_MBUF(P)	# `TEST_GOT_MBUF'
+	CMPI r4, 0
+	bne- 3f
+2:')
+define(HANDLE_GOT_MBUF,`
+3:	bl CSYM(nbif_$1_gc_after_bif)	# `HANDLE_GOT_MBUF'
+	b 2b')
+
 
 /*
  * standard_bif_interface_1(nbif_name, cbif_name)
  * standard_bif_interface_2(nbif_name, cbif_name)
  * standard_bif_interface_3(nbif_name, cbif_name)
+ * standard_bif_interface_0(nbif_name, cbif_name)
  *
- * Generate native interface for a BIF with 1-3 parameters and
+ * Generate native interface for a BIF with 0-3 parameters and
  * standard failure mode.
  */
 define(standard_bif_interface_1,
@@ -51,7 +62,9 @@ ASYM($1):
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_BIF
-	bl	CSYM($2)
+	STORE	r4, P_ARG0(r3)		# Store BIF__ARGS in def_arg_reg[]
+	addi	r4, r3, P_ARG0
+	CALL_BIF($2)
 	TEST_GOT_MBUF
 
 	/* Restore registers. Check for exception. */
@@ -79,7 +92,10 @@ ASYM($1):
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_BIF
-	bl	CSYM($2)
+	STORE	r4, P_ARG0(r3)		# Store BIF__ARGS in def_arg_reg[]
+	STORE	r5, P_ARG1(r3)
+	addi	r4, r3, P_ARG0
+	CALL_BIF($2)
 	TEST_GOT_MBUF
 
 	/* Restore registers. Check for exception. */
@@ -108,7 +124,11 @@ ASYM($1):
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_BIF
-	bl	CSYM($2)
+	STORE	r4, P_ARG0(r3)		# Store BIF__ARGS in def_arg_reg[]
+	STORE	r5, P_ARG1(r3)
+	STORE	r6, P_ARG2(r3)
+	addi	r4, r3, P_ARG0
+	CALL_BIF($2)
 	TEST_GOT_MBUF
 
 	/* Restore registers. Check for exception. */
@@ -123,13 +143,7 @@ ASYM($1):
 	TYPE_FUNCTION(ASYM($1))
 #endif')
 
-/*
- * fail_bif_interface_0(nbif_name, cbif_name)
- *
- * Generate native interface for a BIF with 0 parameters and
- * standard failure mode.
- */
-define(fail_bif_interface_0,
+define(standard_bif_interface_0,
 `
 #ifndef HAVE_$1
 #`define' HAVE_$1
@@ -140,7 +154,8 @@ ASYM($1):
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_BIF
-	bl	CSYM($2)
+	/* ignore empty BIF__ARGS */
+	CALL_BIF($2)
 	TEST_GOT_MBUF
 
 	/* Restore registers. Check for exception. */
@@ -175,7 +190,8 @@ ASYM($1):
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_GC
-	bl	CSYM($2)
+	/* ignore empty BIF__ARGS */
+	CALL_BIF($2)
 	TEST_GOT_MBUF
 
 	/* Restore registers. */
@@ -198,7 +214,9 @@ ASYM($1):
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_GC
-	bl	CSYM($2)
+	STORE	r4, P_ARG0(r3)		# Store BIF__ARGS in def_arg_reg[]
+	addi	r4, r3, P_ARG0
+	CALL_BIF($2)
 	TEST_GOT_MBUF
 
 	/* Restore registers. Check for exception. */
@@ -226,7 +244,10 @@ ASYM($1):
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_GC
-	bl	CSYM($2)
+	STORE	r4, P_ARG0(r3)		# Store BIF__ARGS in def_arg_reg[]
+	STORE	r5, P_ARG1(r3)
+	addi	r4, r3, P_ARG0
+	CALL_BIF($2)
 	TEST_GOT_MBUF
 
 	/* Restore registers. Check for exception. */
diff --git a/erts/emulator/hipe/hipe_ppc_gc.h b/erts/emulator/hipe/hipe_ppc_gc.h
index 796ebeb20a..823ba0ad06 100644
--- a/erts/emulator/hipe/hipe_ppc_gc.h
+++ b/erts/emulator/hipe/hipe_ppc_gc.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * Stack walking helpers for native stack GC procedures.
  * PowerPC version.
  */
diff --git a/erts/emulator/hipe/hipe_ppc_glue.S b/erts/emulator/hipe/hipe_ppc_glue.S
index 97b07353f9..6f0217c738 100644
--- a/erts/emulator/hipe/hipe_ppc_glue.S
+++ b/erts/emulator/hipe/hipe_ppc_glue.S
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #include "hipe_ppc_asm.h"
 #include "hipe_literals.h"
 #define ASM
@@ -198,6 +198,7 @@
  * int hipe_ppc_call_to_native(Process *p);
  * Emulated code recursively calls native code.
  */
+	OPD(hipe_ppc_call_to_native)
 	GLOBAL(CSYM(hipe_ppc_call_to_native))
 CSYM(hipe_ppc_call_to_native):
 	/* save C context */
@@ -229,6 +230,7 @@ ASYM(nbif_return):
  * int hipe_ppc_return_to_native(Process *p);
  * Emulated code returns to its native code caller.
  */
+	OPD(hipe_ppc_return_to_native)
 	GLOBAL(CSYM(hipe_ppc_return_to_native))
 CSYM(hipe_ppc_return_to_native):
 	/* save C context */
@@ -252,6 +254,7 @@ CSYM(hipe_ppc_return_to_native):
  * int hipe_ppc_tailcall_to_native(Process *p);
  * Emulated code tailcalls native code.
  */
+	OPD(hipe_ppc_tailcall_to_native)
 	GLOBAL(CSYM(hipe_ppc_tailcall_to_native))
 CSYM(hipe_ppc_tailcall_to_native):
 	/* save C context */
@@ -274,6 +277,7 @@ CSYM(hipe_ppc_tailcall_to_native):
  * int hipe_ppc_throw_to_native(Process *p);
  * Emulated code throws an exception to its native code caller.
  */
+	OPD(hipe_ppc_throw_to_native)
 	GLOBAL(CSYM(hipe_ppc_throw_to_native))
 CSYM(hipe_ppc_throw_to_native):
 	/* save C context */
@@ -455,6 +459,10 @@ ASYM(nbif_fail):
 	li	r3, HIPE_MODE_SWITCH_RES_THROW
 	b	.flush_exit	/* no need to save RA */
 
+	OPD(nbif_0_gc_after_bif)
+	OPD(nbif_1_gc_after_bif)
+	OPD(nbif_2_gc_after_bif)
+	OPD(nbif_3_gc_after_bif)
 	GLOBAL(CSYM(nbif_0_gc_after_bif))
 	GLOBAL(CSYM(nbif_1_gc_after_bif))
 	GLOBAL(CSYM(nbif_2_gc_after_bif))
@@ -476,6 +484,8 @@ CSYM(nbif_3_gc_after_bif):
 	STORE	TEMP_LR, P_NRA(P)
 	STORE	NSP, P_NSP(P)
 	mflr	TEMP_LR
+	li	r6, 0			/* Pass 0 in arity */
+	li	r5, 0			/* Pass NULL in regs */
 	mr	r4, r3
 	mr	r3, P
 	bl	CSYM(erts_gc_after_bif_call)
@@ -491,18 +501,22 @@ CSYM(nbif_3_gc_after_bif):
  * The heap pointer was just read from P.
  * TEMP_LR contains a copy of LR
  */
+	OPD(nbif_0_simple_exception)
 	GLOBAL(CSYM(nbif_0_simple_exception))
 CSYM(nbif_0_simple_exception):
 	li	r4, 0
 	b	.nbif_simple_exception
+	OPD(nbif_1_simple_exception)
 	GLOBAL(CSYM(nbif_1_simple_exception))
 CSYM(nbif_1_simple_exception):
 	li	r4, 1
 	b	.nbif_simple_exception
+	OPD(nbif_2_simple_exception)
 	GLOBAL(CSYM(nbif_2_simple_exception))
 CSYM(nbif_2_simple_exception):
 	li	r4, 2
 	b	.nbif_simple_exception
+	OPD(nbif_3_simple_exception)
 	GLOBAL(CSYM(nbif_3_simple_exception))
 CSYM(nbif_3_simple_exception):
 	li	r4, 3
@@ -539,7 +553,7 @@ CSYM(nbif_3_simple_exception):
 .handle_trap:
 	li	r3, HIPE_MODE_SWITCH_RES_TRAP
 	STORE	NSP, P_NSP(P)
-	STORE	r4, P_NARITY(P)
+	stw	r4, P_NARITY(P)	/* Note: narity is a 32-bit field */
 	STORE	TEMP_LR, P_NRA(P)
 	b	.nosave_exit
 
diff --git a/erts/emulator/hipe/hipe_ppc_glue.h b/erts/emulator/hipe/hipe_ppc_glue.h
index dcf5ec7644..f9c4460e60 100644
--- a/erts/emulator/hipe/hipe_ppc_glue.h
+++ b/erts/emulator/hipe/hipe_ppc_glue.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_PPC_GLUE_H
 #define HIPE_PPC_GLUE_H
 
diff --git a/erts/emulator/hipe/hipe_ppc_primops.h b/erts/emulator/hipe/hipe_ppc_primops.h
index 67205fe1d1..7dba0afc88 100644
--- a/erts/emulator/hipe/hipe_ppc_primops.h
+++ b/erts/emulator/hipe/hipe_ppc_primops.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #if !defined(__powerpc64__)
 PRIMOP_LIST(am_fconv_constant, &fconv_constant)
 #endif
diff --git a/erts/emulator/hipe/hipe_primops.h b/erts/emulator/hipe/hipe_primops.h
index cc2fc425d5..38509c105b 100644
--- a/erts/emulator/hipe/hipe_primops.h
+++ b/erts/emulator/hipe/hipe_primops.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_PRIMOPS_H
 #define HIPE_PRIMOPS_H
 
@@ -77,6 +77,10 @@ PRIMOP_LIST(am_nonclosure_address, &nbif_nonclosure_address)
 PRIMOP_LIST(am_conv_big_to_float, &nbif_conv_big_to_float)
 PRIMOP_LIST(am_fclearerror_error, &nbif_fclearerror_error)
 
+#ifdef NO_FPE_SIGNALS
+PRIMOP_LIST(am_emulate_fpe, &nbif_emulate_fpe)
+#endif
+
 #if defined(__sparc__)
 #include "hipe_sparc_primops.h"
 #endif
diff --git a/erts/emulator/hipe/hipe_process.h b/erts/emulator/hipe/hipe_process.h
index 5528e68826..4ee99d78a2 100644
--- a/erts/emulator/hipe/hipe_process.h
+++ b/erts/emulator/hipe/hipe_process.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * HiPE-specific process fields
  */
 #ifndef HIPE_PROCESS_H
@@ -42,6 +42,12 @@ struct hipe_process_state {
     void (*nra)(void);		/* Native code return address. */
 #endif
     unsigned int narity;	/* Arity of BIF call, for stack walks. */
+#ifdef NO_FPE_SIGNALS
+    double float_result;        /* to be checked for inf/NaN by hipe_emulate_fpe */ 
+#endif
+#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+    void (*bif_callee)(void);   /* When calling BIF's via debug wrapper */
+#endif
 };
 
 extern void hipe_arch_print_pcb(struct hipe_process_state *p);
diff --git a/erts/emulator/hipe/hipe_risc_gc.h b/erts/emulator/hipe/hipe_risc_gc.h
index 4a9a7878f0..947eb5956b 100644
--- a/erts/emulator/hipe/hipe_risc_gc.h
+++ b/erts/emulator/hipe/hipe_risc_gc.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2008-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * Stack walking helpers for native stack GC procedures.
  * Generic RISC version.
  */
diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h
index 3b2d6498d3..cc2671c016 100644
--- a/erts/emulator/hipe/hipe_risc_glue.h
+++ b/erts/emulator/hipe/hipe_risc_glue.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2008-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_RISC_GLUE_H
 #define HIPE_RISC_GLUE_H
 
@@ -199,6 +199,22 @@ hipe_call_from_native_is_recursive(Process *p, Eterm reg[])
     return 0;
 }
 
+/* BEAM called native, which called BIF that returned trap
+ * Discard bif parameters.
+ * If tailcall, also clean up native stub continuation. */
+static __inline__ int
+hipe_trap_from_native_is_recursive(Process *p)
+{
+    if (p->hipe.narity > NR_ARG_REGS) {
+	p->hipe.nsp += (p->hipe.narity - NR_ARG_REGS);
+    }
+    if (p->hipe.nra != (void(*)(void))&nbif_return)
+	return 1;
+    hipe_pop_risc_nra_frame(p);
+    return 0;
+}
+
+
 /* Native makes a call which needs to unload the parameters.
    This differs from hipe_call_from_native_is_recursive() in
    that it doesn't check for or pop the BEAM-calls-native frame.
diff --git a/erts/emulator/hipe/hipe_risc_stack.c b/erts/emulator/hipe/hipe_risc_stack.c
index 976ca0b85d..1183856c7e 100644
--- a/erts/emulator/hipe/hipe_risc_stack.c
+++ b/erts/emulator/hipe/hipe_risc_stack.c
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2008-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifdef HAVE_CONFIG_H
 #include "config.h"
 #endif
diff --git a/erts/emulator/hipe/hipe_signal.h b/erts/emulator/hipe/hipe_signal.h
index 3c3c844d52..4eacf52b5d 100644
--- a/erts/emulator/hipe/hipe_signal.h
+++ b/erts/emulator/hipe/hipe_signal.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2002-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_signal.h
  *
  * Architecture-specific initialisation of Unix signals.
diff --git a/erts/emulator/hipe/hipe_sparc.c b/erts/emulator/hipe/hipe_sparc.c
index 661b42130a..49d4da7bab 100644
--- a/erts/emulator/hipe/hipe_sparc.c
+++ b/erts/emulator/hipe/hipe_sparc.c
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #include <stddef.h>	/* offsetof() */
 #ifdef HAVE_CONFIG_H
 #include "config.h"
diff --git a/erts/emulator/hipe/hipe_sparc.h b/erts/emulator/hipe/hipe_sparc.h
index 53cb18ee45..1134b86004 100644
--- a/erts/emulator/hipe/hipe_sparc.h
+++ b/erts/emulator/hipe/hipe_sparc.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_SPARC_H
 #define HIPE_SPARC_H
 
diff --git a/erts/emulator/hipe/hipe_sparc.tab b/erts/emulator/hipe/hipe_sparc.tab
index f192e1f81c..c620c73c67 100644
--- a/erts/emulator/hipe/hipe_sparc.tab
+++ b/erts/emulator/hipe/hipe_sparc.tab
@@ -1,22 +1,22 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2004-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2004-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
-# $Id$
+
 # SPARC-specific atoms
 
 atom inc_stack_0
diff --git a/erts/emulator/hipe/hipe_sparc_abi.txt b/erts/emulator/hipe/hipe_sparc_abi.txt
index d016a96c1c..cb5cda310b 100644
--- a/erts/emulator/hipe/hipe_sparc_abi.txt
+++ b/erts/emulator/hipe/hipe_sparc_abi.txt
@@ -2,7 +2,7 @@
  %CopyrightBegin%
  %CopyrightEnd%
 
-$Id$
+
 
 HiPE SPARC ABI
 ==============
diff --git a/erts/emulator/hipe/hipe_sparc_asm.m4 b/erts/emulator/hipe/hipe_sparc_asm.m4
index 7a4403ac09..227d10ed80 100644
--- a/erts/emulator/hipe/hipe_sparc_asm.m4
+++ b/erts/emulator/hipe/hipe_sparc_asm.m4
@@ -1,25 +1,24 @@
 changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2007-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2007-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
+
 `#ifndef HIPE_SPARC_ASM_H
 #define HIPE_SPARC_ASM_H'
 
diff --git a/erts/emulator/hipe/hipe_sparc_bifs.m4 b/erts/emulator/hipe/hipe_sparc_bifs.m4
index f3753b3847..ca5af45d58 100644
--- a/erts/emulator/hipe/hipe_sparc_bifs.m4
+++ b/erts/emulator/hipe/hipe_sparc_bifs.m4
@@ -1,48 +1,61 @@
 changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
 
 include(`hipe/hipe_sparc_asm.m4')
+#`include' "config.h"
 #`include' "hipe_literals.h"
 
 	.section ".text"
 	.align	4
 
+`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+#  define CALL_BIF(F)	set F, %o7; st %o7, [%o0+P_BIF_CALLEE]; call hipe_debug_bif_wrapper 
+#else
+#  define CALL_BIF(F)	call	F
+#endif'
+
 /*
  * Test for exception. This macro executes its delay slot.
  */
-`#define __TEST_GOT_EXN(LABEL)	cmp %o0, THE_NON_VALUE; bz,pn %icc, LABEL
-#define TEST_GOT_EXN(ARITY)	__TEST_GOT_EXN(JOIN3(nbif_,ARITY,_simple_exception))'
+define(TEST_GOT_EXN,`cmp %o0, THE_NON_VALUE	! `TEST_GOT_EXN'
+	bz,pn %icc, nbif_$1_simple_exception')
 
-`#define TEST_GOT_MBUF		ld [P+P_MBUF], %o1; cmp %o1, 0; bne 3f; nop; 2:
-#define JOIN3(A,B,C)		A##B##C
-#define HANDLE_GOT_MBUF(ARITY)	3: call JOIN3(nbif_,ARITY,_gc_after_bif); nop; b 2b; nop'
+define(TEST_GOT_MBUF,`ld [P+P_MBUF], %o1	! `TEST_GOT_MBUF'
+	cmp %o1, 0
+	bne 3f
+	nop
+2:')
+define(HANDLE_GOT_MBUF,`
+3:	call nbif_$1_gc_after_bif	! `HANDLE_GOT_MBUF'
+	nop
+	b 2b
+	nop')
 
 /*
  * standard_bif_interface_1(nbif_name, cbif_name)
  * standard_bif_interface_2(nbif_name, cbif_name)
  * standard_bif_interface_3(nbif_name, cbif_name)
+ * standard_bif_interface_0(nbif_name, cbif_name)
  *
- * Generate native interface for a BIF with 1-3 parameters and
+ * Generate native interface for a BIF with 0-3 parameters and
  * standard failure mode.
  */
 define(standard_bif_interface_1,
@@ -57,7 +70,9 @@ $1:
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_BIF
-	call	$2
+	st 	%o1, [%o0+P_ARG0]	! Store BIF__ARGS in def_arg_reg
+	add	%o0, P_ARG0, %o1
+	CALL_BIF($2)
 	nop
 	TEST_GOT_MBUF
 
@@ -83,7 +98,10 @@ $1:
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_BIF
-	call	$2
+	st 	%o1, [%o0+P_ARG0]	! Store BIF__ARGS in def_arg_reg
+	st 	%o2, [%o0+P_ARG1]
+	add	%o0, P_ARG0, %o1
+	CALL_BIF($2)
 	nop
 	TEST_GOT_MBUF
 
@@ -110,7 +128,11 @@ $1:
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_BIF
-	call	$2
+	st 	%o1, [%o0+P_ARG0]	! Store BIF__ARGS in def_arg_reg
+	st 	%o2, [%o0+P_ARG1]
+	st 	%o3, [%o0+P_ARG2]
+	add	%o0, P_ARG0, %o1
+	CALL_BIF($2)
 	nop
 	TEST_GOT_MBUF
 
@@ -123,13 +145,7 @@ $1:
 	.type	$1, #function
 #endif')
 
-/*
- * fail_bif_interface_0(nbif_name, cbif_name)
- *
- * Generate native interface for a BIF with 0 parameters and
- * standard failure mode.
- */
-define(fail_bif_interface_0,
+define(standard_bif_interface_0,
 `
 #ifndef HAVE_$1
 #`define' HAVE_$1
@@ -140,7 +156,8 @@ $1:
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_BIF
-	call	$2
+	/* ignore empty BIF__ARGS */
+	CALL_BIF($2)
 	nop
 	TEST_GOT_MBUF
 
@@ -173,7 +190,8 @@ $1:
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_GC
-	call	$2
+	/* ignore empty BIF__ARGS */
+	CALL_BIF($2)
 	nop
 	TEST_GOT_MBUF
 
@@ -197,7 +215,9 @@ $1:
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_GC
-	call	$2
+	st 	%o1, [%o0+P_ARG0]	! Store BIF__ARGS in def_arg_reg
+	add	%o0, P_ARG0, %o1
+	CALL_BIF($2)
 	nop
 	TEST_GOT_MBUF
 
@@ -223,7 +243,10 @@ $1:
 
 	/* Save caller-save registers and call the C function. */
 	SAVE_CONTEXT_GC
-	call	$2
+	st 	%o1, [%o0+P_ARG0]	! Store BIF__ARGS in def_arg_reg
+	st 	%o2, [%o0+P_ARG1]
+	add	%o0, P_ARG0, %o1
+	CALL_BIF($2)
 	nop
 	TEST_GOT_MBUF
 
diff --git a/erts/emulator/hipe/hipe_sparc_gc.h b/erts/emulator/hipe/hipe_sparc_gc.h
index 9035f5baee..b870ddd59e 100644
--- a/erts/emulator/hipe/hipe_sparc_gc.h
+++ b/erts/emulator/hipe/hipe_sparc_gc.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * Stack walking helpers for native stack GC procedures.
  * SPARC version.
  */
diff --git a/erts/emulator/hipe/hipe_sparc_glue.S b/erts/emulator/hipe/hipe_sparc_glue.S
index d1af5c43f5..44bdf1bc7e 100644
--- a/erts/emulator/hipe/hipe_sparc_glue.S
+++ b/erts/emulator/hipe/hipe_sparc_glue.S
@@ -1,23 +1,24 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #include "hipe_sparc_asm.h"
 #include "hipe_literals.h"
 #define ASM
@@ -333,6 +334,8 @@ nbif_3_gc_after_bif:
 	st	TEMP_RA, [P+P_NRA]
 	st	NSP, [P+P_NSP]
 	mov	RA, TEMP_RA
+	mov	0, %o3			/* Pass 0 in arity */
+	mov	0, %o2			/* Pass NULL in regs */
 	mov	%o0, %o1
 	call	erts_gc_after_bif_call
 	mov	P, %o0			/* delay slot */
diff --git a/erts/emulator/hipe/hipe_sparc_glue.h b/erts/emulator/hipe/hipe_sparc_glue.h
index 3f881d2140..1404c0d4c0 100644
--- a/erts/emulator/hipe/hipe_sparc_glue.h
+++ b/erts/emulator/hipe/hipe_sparc_glue.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_SPARC_GLUE_H
 #define HIPE_SPARC_GLUE_H
 
diff --git a/erts/emulator/hipe/hipe_sparc_primops.h b/erts/emulator/hipe/hipe_sparc_primops.h
index 1fbb261c67..413371e5f0 100644
--- a/erts/emulator/hipe/hipe_sparc_primops.h
+++ b/erts/emulator/hipe/hipe_sparc_primops.h
@@ -1,21 +1,21 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 PRIMOP_LIST(am_inc_stack_0, &hipe_sparc_inc_stack)
diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c
index 82f7f022b6..da462a64e1 100644
--- a/erts/emulator/hipe/hipe_stack.c
+++ b/erts/emulator/hipe/hipe_stack.c
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifdef HAVE_CONFIG_H
 #include "config.h"
 #endif
diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h
index 354ac81b4c..4e3076caf5 100644
--- a/erts/emulator/hipe/hipe_stack.h
+++ b/erts/emulator/hipe/hipe_stack.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_STACK_H
 #define HIPE_STACK_H
 
@@ -116,13 +116,4 @@ extern int hipe_fill_stacktrace(Process*, int, Eterm**);
 extern Eterm *fullsweep_nstack(Process *p, Eterm *n_htop);
 extern void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop);
 
-#ifdef HYBRID
-#ifdef INCREMENTAL
-extern Eterm *ma_fullsweep_nstack(Process *p, Eterm *n_htop, Eterm *n_hend);
-#else
-extern Eterm *ma_fullsweep_nstack(Process *p, Eterm *n_htop);
-#endif
-extern void ma_gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop);
-#endif /* HYBRID */
-
 #endif /* HIPE_STACK_H */
diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c
index f79a2d53f4..24d232c968 100644
--- a/erts/emulator/hipe/hipe_x86.c
+++ b/erts/emulator/hipe/hipe_x86.c
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #include <stddef.h>	/* offsetof() */
 #ifdef HAVE_CONFIG_H
 #include "config.h"
diff --git a/erts/emulator/hipe/hipe_x86.h b/erts/emulator/hipe/hipe_x86.h
index 94ca39fc4f..97f09e38cd 100644
--- a/erts/emulator/hipe/hipe_x86.h
+++ b/erts/emulator/hipe/hipe_x86.h
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifndef HIPE_X86_H
 #define HIPE_X86_H
 
@@ -49,7 +49,9 @@ static __inline__ int hipe_word32_address_ok(void *address)
 #define hipe_arch_name	am_x86
 
 extern void nbif_inc_stack_0(void);
+#ifndef NO_FPE_SIGNALS
 extern void nbif_handle_fp_exception(void);
+#endif
 
 /* for hipe_bifs_enter_code_2 */
 extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p);
diff --git a/erts/emulator/hipe/hipe_x86.tab b/erts/emulator/hipe/hipe_x86.tab
index a38fe49156..fb33d0a6b9 100644
--- a/erts/emulator/hipe/hipe_x86.tab
+++ b/erts/emulator/hipe/hipe_x86.tab
@@ -1,22 +1,22 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2004-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2004-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
-# $Id$
+
 # x86-specific atoms
 
 atom handle_fp_exception
diff --git a/erts/emulator/hipe/hipe_x86_abi.txt b/erts/emulator/hipe/hipe_x86_abi.txt
index 62a704eef3..aa04a12611 100644
--- a/erts/emulator/hipe/hipe_x86_abi.txt
+++ b/erts/emulator/hipe/hipe_x86_abi.txt
@@ -1,7 +1,7 @@
 
  %CopyrightBegin%
  
- Copyright Ericsson AB 2001-2009. All Rights Reserved.
+ Copyright Ericsson AB 2001-2011. All Rights Reserved.
  
  The contents of this file are subject to the Erlang Public License,
  Version 1.1, (the "License"); you may not use this file except in
@@ -16,7 +16,7 @@
  
  %CopyrightEnd%
 
-$Id$
+
 
 HiPE x86 ABI
 ============
diff --git a/erts/emulator/hipe/hipe_x86_asm.m4 b/erts/emulator/hipe/hipe_x86_asm.m4
index 4c1d612ccd..020ccf8d4b 100644
--- a/erts/emulator/hipe/hipe_x86_asm.m4
+++ b/erts/emulator/hipe/hipe_x86_asm.m4
@@ -1,25 +1,24 @@
 changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2002-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
+
 `#ifndef HIPE_X86_ASM_H
 #define HIPE_X86_ASM_H'
 
diff --git a/erts/emulator/hipe/hipe_x86_bifs.m4 b/erts/emulator/hipe/hipe_x86_bifs.m4
index 80be74f7b2..3cb7d67be0 100644
--- a/erts/emulator/hipe/hipe_x86_bifs.m4
+++ b/erts/emulator/hipe/hipe_x86_bifs.m4
@@ -1,27 +1,26 @@
 changecom(`/*', `*/')dnl
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
 
 include(`hipe/hipe_x86_asm.m4')
+#`include' "config.h"
 #`include' "hipe_literals.h"
 
 `#if THE_NON_VALUE == 0
@@ -30,16 +29,27 @@ include(`hipe/hipe_x86_asm.m4')
 #define TEST_GOT_EXN	cmpl	$THE_NON_VALUE,%eax
 #endif'
 
-`#define TEST_GOT_MBUF		movl P_MBUF(P), %edx; testl %edx, %edx; jnz 3f; 2:
-#define JOIN3(A,B,C)		A##B##C
-#define HANDLE_GOT_MBUF(ARITY)	3: call JOIN3(nbif_,ARITY,_gc_after_bif); jmp 2b'
+`#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
+#  define CALL_BIF(F)	movl $CSYM(F), P_BIF_CALLEE(P); call CSYM(hipe_debug_bif_wrapper) 
+#else
+#  define CALL_BIF(F)	call	CSYM(F)
+#endif'
+
+define(TEST_GOT_MBUF,`movl P_MBUF(P), %edx	# `TEST_GOT_MBUF'
+	testl %edx, %edx
+	jnz 3f	
+2:')
+define(HANDLE_GOT_MBUF,`
+3:	call nbif_$1_gc_after_bif	# `HANDLE_GOT_MBUF'
+	jmp 2b')
 
 /*
  * standard_bif_interface_1(nbif_name, cbif_name)
  * standard_bif_interface_2(nbif_name, cbif_name)
  * standard_bif_interface_3(nbif_name, cbif_name)
+ * standard_bif_interface_0(nbif_name, cbif_name)
  *
- * Generate native interface for a BIF with 1-3 parameters and
+ * Generate native interface for a BIF with 0-3 parameters and
  * standard failure mode.
  */
 define(standard_bif_interface_1,
@@ -58,8 +68,10 @@ ASYM($1):
 
 	/* make the call on the C stack */
 	NBIF_ARG_REG(0,P)
-	NBIF_ARG(1,1,0)
-	call	CSYM($2)
+	NBIF_ARG(2,1,0)
+	lea 8(%esp), %eax
+	NBIF_ARG_REG(1,%eax)	# BIF__ARGS
+	CALL_BIF($2)
 	TEST_GOT_MBUF
 
 	/* switch to native stack */
@@ -90,9 +102,11 @@ ASYM($1):
 
 	/* make the call on the C stack */
 	NBIF_ARG_REG(0,P)
-	NBIF_ARG(1,2,0)
-	NBIF_ARG(2,2,1)
-	call	CSYM($2)
+	NBIF_ARG(2,2,0)
+	NBIF_ARG(3,2,1)
+	lea 8(%esp), %eax
+	NBIF_ARG_REG(1,%eax)	# BIF__ARGS
+	CALL_BIF($2)
 	TEST_GOT_MBUF
 
 	/* switch to native stack */
@@ -123,10 +137,12 @@ ASYM($1):
 
 	/* make the call on the C stack */
 	NBIF_ARG_REG(0,P)
-	NBIF_ARG(1,3,0)
-	NBIF_ARG(2,3,1)
-	NBIF_ARG(3,3,2)
-	call	CSYM($2)
+	NBIF_ARG(2,3,0)
+	NBIF_ARG(3,3,1)
+	NBIF_ARG(4,3,2)
+	lea 8(%esp), %eax
+	NBIF_ARG_REG(1,%eax)	# BIF__ARGS
+	CALL_BIF($2)
 	TEST_GOT_MBUF
 
 	/* switch to native stack */
@@ -141,13 +157,7 @@ ASYM($1):
 	TYPE_FUNCTION(ASYM($1))
 #endif')
 
-/*
- * fail_bif_interface_0(nbif_name, cbif_name)
- *
- * Generate native interface for a BIF with 0 parameters and
- * standard failure mode.
- */
-define(fail_bif_interface_0,
+define(standard_bif_interface_0,
 `
 #ifndef HAVE_$1
 #`define' HAVE_$1
@@ -160,7 +170,8 @@ ASYM($1):
 
 	/* make the call on the C stack */
 	NBIF_ARG_REG(0,P)
-	call	CSYM($2)
+	/* skip BIF__ARGS */	
+	CALL_BIF($2)
 	TEST_GOT_MBUF
 
 	/* switch to native stack */
@@ -610,7 +621,9 @@ ASYM($1):
 /*
  * x86-specific primops.
  */
+#ifndef NO_FPE_SIGNALS
 noproc_primop_interface_0(nbif_handle_fp_exception, erts_restore_fpu)
+#endif /* NO_FPE_SIGNALS */
 
 /*
  * Implement gc_bif_interface_0 as nofail_primop_interface_0.
diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h
index 4f17f767df..e4607ad27d 100644
--- a/erts/emulator/hipe/hipe_x86_gc.h
+++ b/erts/emulator/hipe/hipe_x86_gc.h
@@ -1,22 +1,22 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * Stack walking helpers for native stack GC procedures.
  */
 #ifndef HIPE_X86_GC_H
diff --git a/erts/emulator/hipe/hipe_x86_glue.S b/erts/emulator/hipe/hipe_x86_glue.S
index 2f7dff39f5..88b86f4de7 100644
--- a/erts/emulator/hipe/hipe_x86_glue.S
+++ b/erts/emulator/hipe/hipe_x86_glue.S
@@ -1,24 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
 
 #include "hipe_x86_asm.h"
 #include "hipe_literals.h"
@@ -320,11 +319,13 @@ nbif_3_gc_after_bif:
 	.align	4
 .gc_after_bif:
 	movl	%edx, P_NARITY(P)
-	subl	$(16-4), %esp
+	subl	$(32-4), %esp
 	movl	P, (%esp)
 	movl	%eax, 4(%esp)
+	movl	$0, 8(%esp)		# Pass NULL in regs
+	movl	$0, 12(%esp)		# Pass 0 in arity
 	call	CSYM(erts_gc_after_bif_call)
-	addl	$(16-4), %esp
+	addl	$(32-4), %esp
 	movl	$0, P_NARITY(P)
 	ret
 
diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h
index 4c9c92c52f..b0db93267c 100644
--- a/erts/emulator/hipe/hipe_x86_glue.h
+++ b/erts/emulator/hipe/hipe_x86_glue.h
@@ -1,24 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/*
- * $Id$
- */
+
+
 #ifndef HIPE_X86_GLUE_H
 #define HIPE_X86_GLUE_H
 
@@ -187,6 +186,25 @@ hipe_call_from_native_is_recursive(Process *p, Eterm reg[])
     return 0;
 }
 
+/* BEAM called native, which called BIF that returned trap
+ * Discard bif parameters.
+ * If tailcall, also clean up native stub continuation. */
+static __inline__ int
+hipe_trap_from_native_is_recursive(Process *p)
+{
+    Eterm nra = *(p->hipe.nsp++);
+
+    if (p->hipe.narity > NR_ARG_REGS) {
+	p->hipe.nsp += (p->hipe.narity - NR_ARG_REGS);
+    }
+    if (nra != (Eterm)nbif_return) {
+	*--(p->hipe.nsp) = nra;
+	return 1;
+    }
+    return 0;
+}
+
+
 /* Native makes a call which needs to unload the parameters.
    This differs from hipe_call_from_native_is_recursive() in
    that it doesn't check for or pop the BEAM-calls-native frame.
diff --git a/erts/emulator/hipe/hipe_x86_primops.h b/erts/emulator/hipe/hipe_x86_primops.h
index 757da484ad..111b1fa8bd 100644
--- a/erts/emulator/hipe/hipe_x86_primops.h
+++ b/erts/emulator/hipe/hipe_x86_primops.h
@@ -1,22 +1,25 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 PRIMOP_LIST(am_inc_stack_0, &nbif_inc_stack_0)
+#ifndef NO_FPE_SIGNALS
 PRIMOP_LIST(am_handle_fp_exception, &nbif_handle_fp_exception)
+#endif
+
diff --git a/erts/emulator/hipe/hipe_x86_signal.c b/erts/emulator/hipe/hipe_x86_signal.c
index a4fff4ce31..64c0e0da3e 100644
--- a/erts/emulator/hipe/hipe_x86_signal.c
+++ b/erts/emulator/hipe/hipe_x86_signal.c
@@ -1,22 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
+/*
  * hipe_x86_signal.c
  *
  * Erlang code compiled to x86 native code uses the x86 %esp as its
@@ -195,7 +196,7 @@ static void do_init(void)
 #define INIT()	do { if (!init_done()) do_init(); } while (0)
 #endif /* __DARWIN__ */
 
-#if !defined(__GLIBC__) && !defined(__DARWIN__)
+#if !defined(__GLIBC__) && !defined(__DARWIN__) && !defined(__NetBSD__)
 /*
  * Assume Solaris/x86 2.8.
  * There is a number of sigaction() procedures in libc:
@@ -231,6 +232,7 @@ static void do_init(void)
 #define INIT()	do { if (!init_done()) do_init(); } while (0)
 #endif	/* not glibc or darwin */
 
+#if !defined(__NetBSD__)
 /*
  * This is our wrapper for sigaction(). sigaction() can be called before
  * hipe_signal_init() has been executed, especially when threads support
@@ -253,7 +255,7 @@ static int my_sigaction(int signum, const struct sigaction *act, struct sigactio
     }
     return __next_sigaction(signum, act, oldact);
 }
-
+#endif
 /*
  * This overrides the C library's core sigaction() procedure, catching
  * all its internal calls.
@@ -268,7 +270,7 @@ int __SIGACTION(int signum, const struct sigaction *act, struct sigaction *oldac
 /*
  * This catches the application's own sigaction() calls.
  */
-#if !defined(__DARWIN__)
+#if !defined(__DARWIN__) && !defined(__NetBSD__)
 int sigaction(int signum, const struct sigaction *act, struct sigaction *oldact)
 {
     return my_sigaction(signum, act, oldact);
@@ -326,7 +328,9 @@ void hipe_signal_init(void)
     struct sigaction sa;
     int i;
 
+#ifndef __NetBSD__
     INIT();
+#endif
 
     hipe_sigaltstack_init();
 
diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c
index b459593883..9ad3fa9d31 100644
--- a/erts/emulator/hipe/hipe_x86_stack.c
+++ b/erts/emulator/hipe/hipe_x86_stack.c
@@ -1,23 +1,23 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2001-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
-/* $Id$
- */
+
+
 #ifdef HAVE_CONFIG_H
 #include "config.h"
 #endif
diff --git a/erts/emulator/internal_doc/dec.dat b/erts/emulator/internal_doc/dec.dat
new file mode 100644
index 0000000000..771ef51baa
--- /dev/null
+++ b/erts/emulator/internal_doc/dec.dat
@@ -0,0 +1,942 @@
+{[59],894}.
+{[96],8175}.
+{[180],8189}.
+{[183],903}.
+{[198],1236}.
+{[230],1237}.
+{[399],1240}.
+{[415],1256}.
+{[439],1248}.
+{[601],1241}.
+{[629],1257}.
+{[658],1249}.
+{[697],884}.
+{[768],832}.
+{[768,65],192}.
+{[768,69],200}.
+{[768,73],204}.
+{[768,79],210}.
+{[768,85],217}.
+{[768,87],7808}.
+{[768,89],7922}.
+{[768,97],224}.
+{[768,101],232}.
+{[768,105],236}.
+{[768,111],242}.
+{[768,117],249}.
+{[768,119],7809}.
+{[768,121],7923}.
+{[768,168],8173}.
+{[768,770,65],7846}.
+{[768,770,69],7872}.
+{[768,770,79],7890}.
+{[768,770,97],7847}.
+{[768,770,101],7873}.
+{[768,770,111],7891}.
+{[768,772,69],7700}.
+{[768,772,79],7760}.
+{[768,772,101],7701}.
+{[768,772,111],7761}.
+{[768,774,65],7856}.
+{[768,774,97],7857}.
+{[768,776,85],475}.
+{[768,776,117],476}.
+{[768,776,953],8146}.
+{[768,776,965],8162}.
+{[768,787,837,913],8074}.
+{[768,787,837,919],8090}.
+{[768,787,837,937],8106}.
+{[768,787,837,945],8066}.
+{[768,787,837,951],8082}.
+{[768,787,837,969],8098}.
+{[768,787,913],7946}.
+{[768,787,917],7962}.
+{[768,787,919],7978}.
+{[768,787,921],7994}.
+{[768,787,927],8010}.
+{[768,787,937],8042}.
+{[768,787,945],7938}.
+{[768,787,949],7954}.
+{[768,787,951],7970}.
+{[768,787,953],7986}.
+{[768,787,959],8002}.
+{[768,787,965],8018}.
+{[768,787,969],8034}.
+{[768,788,837,913],8075}.
+{[768,788,837,919],8091}.
+{[768,788,837,937],8107}.
+{[768,788,837,945],8067}.
+{[768,788,837,951],8083}.
+{[768,788,837,969],8099}.
+{[768,788,913],7947}.
+{[768,788,917],7963}.
+{[768,788,919],7979}.
+{[768,788,921],7995}.
+{[768,788,927],8011}.
+{[768,788,933],8027}.
+{[768,788,937],8043}.
+{[768,788,945],7939}.
+{[768,788,949],7955}.
+{[768,788,951],7971}.
+{[768,788,953],7987}.
+{[768,788,959],8003}.
+{[768,788,965],8019}.
+{[768,788,969],8035}.
+{[768,795,79],7900}.
+{[768,795,85],7914}.
+{[768,795,111],7901}.
+{[768,795,117],7915}.
+{[768,837,945],8114}.
+{[768,837,951],8130}.
+{[768,837,969],8178}.
+{[768,913],8122}.
+{[768,917],8136}.
+{[768,919],8138}.
+{[768,921],8154}.
+{[768,927],8184}.
+{[768,933],8170}.
+{[768,937],8186}.
+{[768,945],8048}.
+{[768,949],8050}.
+{[768,951],8052}.
+{[768,953],8054}.
+{[768,959],8056}.
+{[768,965],8058}.
+{[768,969],8060}.
+{[768,8127],8141}.
+{[768,8190],8157}.
+{[769],833}.
+{[769,65],193}.
+{[769,67],262}.
+{[769,69],201}.
+{[769,71],500}.
+{[769,73],205}.
+{[769,75],7728}.
+{[769,76],313}.
+{[769,77],7742}.
+{[769,78],323}.
+{[769,79],211}.
+{[769,80],7764}.
+{[769,82],340}.
+{[769,83],346}.
+{[769,85],218}.
+{[769,87],7810}.
+{[769,89],221}.
+{[769,90],377}.
+{[769,97],225}.
+{[769,99],263}.
+{[769,101],233}.
+{[769,103],501}.
+{[769,105],237}.
+{[769,107],7729}.
+{[769,108],314}.
+{[769,109],7743}.
+{[769,110],324}.
+{[769,111],243}.
+{[769,112],7765}.
+{[769,114],341}.
+{[769,115],347}.
+{[769,117],250}.
+{[769,119],7811}.
+{[769,121],253}.
+{[769,122],378}.
+{[769,168],8174}.
+{[769,198],508}.
+{[769,216],510}.
+{[769,230],509}.
+{[769,248],511}.
+{[769,770,65],7844}.
+{[769,770,69],7870}.
+{[769,770,79],7888}.
+{[769,770,97],7845}.
+{[769,770,101],7871}.
+{[769,770,111],7889}.
+{[769,771,79],7756}.
+{[769,771,85],7800}.
+{[769,771,111],7757}.
+{[769,771,117],7801}.
+{[769,772,69],7702}.
+{[769,772,79],7762}.
+{[769,772,101],7703}.
+{[769,772,111],7763}.
+{[769,774,65],7854}.
+{[769,774,97],7855}.
+{[769,776,73],7726}.
+{[769,776,85],471}.
+{[769,776,105],7727}.
+{[769,776,117],472}.
+{[769,776,953],8147}.
+{[769,776,965],8163}.
+{[769,778,65],506}.
+{[769,778,97],507}.
+{[769,787,837,913],8076}.
+{[769,787,837,919],8092}.
+{[769,787,837,937],8108}.
+{[769,787,837,945],8068}.
+{[769,787,837,951],8084}.
+{[769,787,837,969],8100}.
+{[769,787,913],7948}.
+{[769,787,917],7964}.
+{[769,787,919],7980}.
+{[769,787,921],7996}.
+{[769,787,927],8012}.
+{[769,787,937],8044}.
+{[769,787,945],7940}.
+{[769,787,949],7956}.
+{[769,787,951],7972}.
+{[769,787,953],7988}.
+{[769,787,959],8004}.
+{[769,787,965],8020}.
+{[769,787,969],8036}.
+{[769,788,837,913],8077}.
+{[769,788,837,919],8093}.
+{[769,788,837,937],8109}.
+{[769,788,837,945],8069}.
+{[769,788,837,951],8085}.
+{[769,788,837,969],8101}.
+{[769,788,913],7949}.
+{[769,788,917],7965}.
+{[769,788,919],7981}.
+{[769,788,921],7997}.
+{[769,788,927],8013}.
+{[769,788,933],8029}.
+{[769,788,937],8045}.
+{[769,788,945],7941}.
+{[769,788,949],7957}.
+{[769,788,951],7973}.
+{[769,788,953],7989}.
+{[769,788,959],8005}.
+{[769,788,965],8021}.
+{[769,788,969],8037}.
+{[769,795,79],7898}.
+{[769,795,85],7912}.
+{[769,795,111],7899}.
+{[769,795,117],7913}.
+{[769,807,67],7688}.
+{[769,807,99],7689}.
+{[769,837,945],8116}.
+{[769,837,951],8132}.
+{[769,837,959],8180}.
+{[769,913],8123}.
+{[769,917],8137}.
+{[769,919],8139}.
+{[769,921],8155}.
+{[769,927],8185}.
+{[769,933],8171}.
+{[769,937],8187}.
+{[769,945],8049}.
+{[769,949],8051}.
+{[769,951],8053}.
+{[769,953],8055}.
+{[769,959],8057}.
+{[769,965],8059}.
+{[769,969],8061}.
+{[769,1043],1027}.
+{[769,1050],1036}.
+{[769,1075],1107}.
+{[769,1082],1116}.
+{[769,8127],8142}.
+{[769,8190],8158}.
+{[770,65],194}.
+{[770,67],264}.
+{[770,69],202}.
+{[770,71],284}.
+{[770,72],292}.
+{[770,73],206}.
+{[770,74],308}.
+{[770,79],212}.
+{[770,83],348}.
+{[770,85],219}.
+{[770,87],372}.
+{[770,89],374}.
+{[770,90],7824}.
+{[770,97],226}.
+{[770,99],265}.
+{[770,101],234}.
+{[770,103],285}.
+{[770,104],293}.
+{[770,105],238}.
+{[770,106],309}.
+{[770,111],244}.
+{[770,115],349}.
+{[770,117],251}.
+{[770,119],373}.
+{[770,121],375}.
+{[770,122],7825}.
+{[770,803,65],7852}.
+{[770,803,69],7878}.
+{[770,803,79],7896}.
+{[770,803,97],7853}.
+{[770,803,101],7879}.
+{[770,803,111],7897}.
+{[771,65],195}.
+{[771,69],7868}.
+{[771,73],296}.
+{[771,78],209}.
+{[771,79],213}.
+{[771,85],360}.
+{[771,86],7804}.
+{[771,89],7928}.
+{[771,97],227}.
+{[771,101],7869}.
+{[771,105],297}.
+{[771,110],241}.
+{[771,111],245}.
+{[771,117],361}.
+{[771,118],7805}.
+{[771,121],7929}.
+{[771,770,65],7850}.
+{[771,770,69],7876}.
+{[771,770,79],7894}.
+{[771,770,97],7851}.
+{[771,770,101],7877}.
+{[771,770,111],7895}.
+{[771,774,65],7860}.
+{[771,774,97],7861}.
+{[771,795,79],7904}.
+{[771,795,85],7918}.
+{[771,795,111],7905}.
+{[771,795,117],7919}.
+{[772,65],256}.
+{[772,69],274}.
+{[772,71],7712}.
+{[772,73],298}.
+{[772,79],332}.
+{[772,85],362}.
+{[772,97],257}.
+{[772,101],275}.
+{[772,103],7713}.
+{[772,105],299}.
+{[772,111],333}.
+{[772,117],363}.
+{[772,198],482}.
+{[772,230],483}.
+{[772,775,65],480}.
+{[772,775,97],481}.
+{[772,776,65],478}.
+{[772,776,85],469}.
+{[772,776,97],479}.
+{[772,776,117],470}.
+{[772,803,76],7736}.
+{[772,803,82],7772}.
+{[772,803,108],7737}.
+{[772,803,114],7773}.
+{[772,808,79],492}.
+{[772,808,111],493}.
+{[772,913],8121}.
+{[772,921],8153}.
+{[772,933],8169}.
+{[772,945],8113}.
+{[772,953],8145}.
+{[772,965],8161}.
+{[772,1048],1250}.
+{[772,1059],1262}.
+{[772,1080],1251}.
+{[772,1091],1263}.
+{[774,65],258}.
+{[774,69],276}.
+{[774,71],286}.
+{[774,73],300}.
+{[774,79],334}.
+{[774,85],364}.
+{[774,97],259}.
+{[774,101],277}.
+{[774,103],287}.
+{[774,105],301}.
+{[774,111],335}.
+{[774,117],365}.
+{[774,803,65],7862}.
+{[774,803,97],7863}.
+{[774,807,69],7708}.
+{[774,807,101],7709}.
+{[774,913],8120}.
+{[774,921],8152}.
+{[774,933],8168}.
+{[774,945],8112}.
+{[774,953],8144}.
+{[774,965],8160}.
+{[774,1040],1232}.
+{[774,1045],1238}.
+{[774,1046],1217}.
+{[774,1048],1049}.
+{[774,1059],1038}.
+{[774,1072],1233}.
+{[774,1077],1239}.
+{[774,1078],1218}.
+{[774,1080],1081}.
+{[774,1091],1118}.
+{[775,66],7682}.
+{[775,67],266}.
+{[775,68],7690}.
+{[775,69],278}.
+{[775,70],7710}.
+{[775,71],288}.
+{[775,72],7714}.
+{[775,73],304}.
+{[775,77],7744}.
+{[775,78],7748}.
+{[775,80],7766}.
+{[775,82],7768}.
+{[775,83],7776}.
+{[775,84],7786}.
+{[775,87],7814}.
+{[775,88],7818}.
+{[775,89],7822}.
+{[775,90],379}.
+{[775,98],7683}.
+{[775,99],267}.
+{[775,100],7691}.
+{[775,101],279}.
+{[775,102],7711}.
+{[775,103],289}.
+{[775,104],7715}.
+{[775,109],7745}.
+{[775,110],7749}.
+{[775,112],7767}.
+{[775,114],7769}.
+{[775,115],7777}.
+{[775,116],7787}.
+{[775,119],7815}.
+{[775,120],7819}.
+{[775,121],7823}.
+{[775,122],380}.
+{[775,383],7835}.
+{[775,769,83],7780}.
+{[775,769,115],7781}.
+{[775,774],784}.
+{[775,780,83],7782}.
+{[775,780,115],7783}.
+{[775,803,83],7784}.
+{[775,803,115],7785}.
+{[776,65],196}.
+{[776,69],203}.
+{[776,72],7718}.
+{[776,73],207}.
+{[776,79],214}.
+{[776,85],220}.
+{[776,87],7812}.
+{[776,88],7820}.
+{[776,89],376}.
+{[776,97],228}.
+{[776,101],235}.
+{[776,104],7719}.
+{[776,105],239}.
+{[776,111],246}.
+{[776,116],7831}.
+{[776,117],252}.
+{[776,119],7813}.
+{[776,120],7821}.
+{[776,121],255}.
+{[776,399],1242}.
+{[776,415],1258}.
+{[776,601],1243}.
+{[776,629],1259}.
+{[776,771,79],7758}.
+{[776,771,111],7759}.
+{[776,772,85],7802}.
+{[776,772,117],7803}.
+{[776,921],938}.
+{[776,933],939}.
+{[776,953],970}.
+{[776,965],971}.
+{[776,978],980}.
+{[776,1030],1031}.
+{[776,1040],1234}.
+{[776,1045],1025}.
+{[776,1046],1244}.
+{[776,1047],1246}.
+{[776,1048],1252}.
+{[776,1054],1254}.
+{[776,1059],1264}.
+{[776,1063],1268}.
+{[776,1067],1272}.
+{[776,1072],1235}.
+{[776,1077],1105}.
+{[776,1078],1245}.
+{[776,1079],1247}.
+{[776,1080],1253}.
+{[776,1086],1255}.
+{[776,1091],1265}.
+{[776,1095],1269}.
+{[776,1099],1273}.
+{[776,1110],1111}.
+{[777,65],7842}.
+{[777,69],7866}.
+{[777,73],7880}.
+{[777,79],7886}.
+{[777,85],7910}.
+{[777,89],7926}.
+{[777,97],7843}.
+{[777,101],7867}.
+{[777,105],7881}.
+{[777,111],7887}.
+{[777,117],7911}.
+{[777,121],7927}.
+{[777,770,65],7848}.
+{[777,770,69],7874}.
+{[777,770,79],7892}.
+{[777,770,97],7849}.
+{[777,770,101],7875}.
+{[777,770,111],7893}.
+{[777,774,65],7858}.
+{[777,774,97],7859}.
+{[777,795,79],7902}.
+{[777,795,85],7916}.
+{[777,795,111],7903}.
+{[777,795,117],7917}.
+{[778,65],197}.
+{[778,85],366}.
+{[778,97],229}.
+{[778,117],367}.
+{[778,119],7832}.
+{[778,121],7833}.
+{[779,79],336}.
+{[779,85],368}.
+{[779,111],337}.
+{[779,117],369}.
+{[779,1059],1266}.
+{[779,1091],1267}.
+{[780,65],461}.
+{[780,67],268}.
+{[780,68],270}.
+{[780,69],282}.
+{[780,71],486}.
+{[780,73],463}.
+{[780,75],488}.
+{[780,76],317}.
+{[780,78],327}.
+{[780,79],465}.
+{[780,82],344}.
+{[780,83],352}.
+{[780,84],356}.
+{[780,85],467}.
+{[780,90],381}.
+{[780,97],462}.
+{[780,99],269}.
+{[780,100],271}.
+{[780,101],283}.
+{[780,103],487}.
+{[780,105],464}.
+{[780,106],496}.
+{[780,107],489}.
+{[780,108],318}.
+{[780,110],328}.
+{[780,111],466}.
+{[780,114],345}.
+{[780,115],353}.
+{[780,116],357}.
+{[780,117],468}.
+{[780,122],382}.
+{[780,439],494}.
+{[780,658],495}.
+{[780,776,85],473}.
+{[780,776,117],474}.
+{[781,168],901}.
+{[781,776],836}.
+{[781,776,953],912}.
+{[781,776,965],944}.
+{[781,913],902}.
+{[781,917],904}.
+{[781,919],905}.
+{[781,921],906}.
+{[781,927],908}.
+{[781,933],910}.
+{[781,937],911}.
+{[781,945],940}.
+{[781,949],941}.
+{[781,951],942}.
+{[781,953],943}.
+{[781,959],972}.
+{[781,965],973}.
+{[781,969],974}.
+{[781,978],979}.
+{[783,65],512}.
+{[783,69],516}.
+{[783,73],520}.
+{[783,79],524}.
+{[783,82],528}.
+{[783,85],532}.
+{[783,97],513}.
+{[783,101],517}.
+{[783,105],521}.
+{[783,111],525}.
+{[783,114],529}.
+{[783,117],533}.
+{[783,1140],1142}.
+{[783,1141],1143}.
+{[785,65],514}.
+{[785,69],518}.
+{[785,73],522}.
+{[785,79],526}.
+{[785,82],530}.
+{[785,85],534}.
+{[785,97],515}.
+{[785,101],519}.
+{[785,105],523}.
+{[785,111],527}.
+{[785,114],531}.
+{[785,117],535}.
+{[787],835}.
+{[787,837,913],8072}.
+{[787,837,919],8088}.
+{[787,837,937],8104}.
+{[787,837,945],8064}.
+{[787,837,951],8080}.
+{[787,837,969],8096}.
+{[787,913],7944}.
+{[787,917],7960}.
+{[787,919],7976}.
+{[787,921],7992}.
+{[787,927],8008}.
+{[787,937],8040}.
+{[787,945],7936}.
+{[787,949],7952}.
+{[787,951],7968}.
+{[787,953],7984}.
+{[787,959],8000}.
+{[787,961],8164}.
+{[787,965],8016}.
+{[787,969],8032}.
+{[788,837,913],8073}.
+{[788,837,919],8089}.
+{[788,837,937],8105}.
+{[788,837,945],8065}.
+{[788,837,951],8081}.
+{[788,837,969],8097}.
+{[788,913],7945}.
+{[788,917],7961}.
+{[788,919],7977}.
+{[788,921],7993}.
+{[788,927],8009}.
+{[788,929],8172}.
+{[788,933],8025}.
+{[788,937],8041}.
+{[788,945],7937}.
+{[788,949],7953}.
+{[788,951],7969}.
+{[788,953],7985}.
+{[788,959],8001}.
+{[788,961],8165}.
+{[788,965],8017}.
+{[788,969],8033}.
+{[795,79],416}.
+{[795,85],431}.
+{[795,111],417}.
+{[795,117],432}.
+{[803,65],7840}.
+{[803,66],7684}.
+{[803,68],7692}.
+{[803,69],7864}.
+{[803,72],7716}.
+{[803,73],7882}.
+{[803,75],7730}.
+{[803,76],7734}.
+{[803,77],7746}.
+{[803,78],7750}.
+{[803,79],7884}.
+{[803,82],7770}.
+{[803,83],7778}.
+{[803,84],7788}.
+{[803,85],7908}.
+{[803,86],7806}.
+{[803,87],7816}.
+{[803,89],7924}.
+{[803,90],7826}.
+{[803,97],7841}.
+{[803,98],7685}.
+{[803,100],7693}.
+{[803,101],7865}.
+{[803,104],7717}.
+{[803,105],7883}.
+{[803,107],7731}.
+{[803,108],7735}.
+{[803,109],7747}.
+{[803,110],7751}.
+{[803,111],7885}.
+{[803,114],7771}.
+{[803,115],7779}.
+{[803,116],7789}.
+{[803,117],7909}.
+{[803,118],7807}.
+{[803,119],7817}.
+{[803,121],7925}.
+{[803,122],7827}.
+{[803,795,79],7906}.
+{[803,795,85],7920}.
+{[803,795,111],7907}.
+{[803,795,117],7921}.
+{[804,85],7794}.
+{[804,117],7795}.
+{[805,65],7680}.
+{[805,97],7681}.
+{[807,67],199}.
+{[807,68],7696}.
+{[807,71],290}.
+{[807,72],7720}.
+{[807,75],310}.
+{[807,76],315}.
+{[807,78],325}.
+{[807,82],342}.
+{[807,83],350}.
+{[807,84],354}.
+{[807,99],231}.
+{[807,100],7697}.
+{[807,103],291}.
+{[807,104],7721}.
+{[807,107],311}.
+{[807,108],316}.
+{[807,110],326}.
+{[807,114],343}.
+{[807,115],351}.
+{[807,116],355}.
+{[808,65],260}.
+{[808,69],280}.
+{[808,73],302}.
+{[808,79],490}.
+{[808,85],370}.
+{[808,97],261}.
+{[808,101],281}.
+{[808,105],303}.
+{[808,111],491}.
+{[808,117],371}.
+{[813,68],7698}.
+{[813,69],7704}.
+{[813,76],7740}.
+{[813,78],7754}.
+{[813,84],7792}.
+{[813,85],7798}.
+{[813,100],7699}.
+{[813,101],7705}.
+{[813,108],7741}.
+{[813,110],7755}.
+{[813,116],7793}.
+{[813,117],7799}.
+{[814,72],7722}.
+{[814,104],7723}.
+{[816,69],7706}.
+{[816,73],7724}.
+{[816,85],7796}.
+{[816,101],7707}.
+{[816,105],7725}.
+{[816,117],7797}.
+{[817,66],7686}.
+{[817,68],7694}.
+{[817,75],7732}.
+{[817,76],7738}.
+{[817,78],7752}.
+{[817,82],7774}.
+{[817,84],7790}.
+{[817,90],7828}.
+{[817,98],7687}.
+{[817,100],7695}.
+{[817,104],7830}.
+{[817,107],7733}.
+{[817,108],7739}.
+{[817,110],7753}.
+{[817,114],7775}.
+{[817,116],7791}.
+{[817,122],7829}.
+{[834,168],8129}.
+{[834,776,953],8151}.
+{[834,776,965],8167}.
+{[834,787,837,913],8078}.
+{[834,787,837,919],8094}.
+{[834,787,837,937],8110}.
+{[834,787,837,945],8070}.
+{[834,787,837,951],8086}.
+{[834,787,837,969],8102}.
+{[834,787,913],7950}.
+{[834,787,919],7982}.
+{[834,787,921],7998}.
+{[834,787,937],8046}.
+{[834,787,945],7942}.
+{[834,787,951],7974}.
+{[834,787,953],7990}.
+{[834,787,965],8022}.
+{[834,787,969],8038}.
+{[834,788,837,913],8079}.
+{[834,788,837,919],8095}.
+{[834,788,837,937],8111}.
+{[834,788,837,945],8071}.
+{[834,788,837,951],8087}.
+{[834,788,837,969],8103}.
+{[834,788,913],7951}.
+{[834,788,919],7983}.
+{[834,788,921],7999}.
+{[834,788,933],8031}.
+{[834,788,937],8047}.
+{[834,788,945],7943}.
+{[834,788,951],7975}.
+{[834,788,953],7991}.
+{[834,788,965],8023}.
+{[834,788,969],8039}.
+{[834,837,945],8119}.
+{[834,837,951],8135}.
+{[834,837,969],8183}.
+{[834,945],8118}.
+{[834,951],8134}.
+{[834,953],8150}.
+{[834,965],8166}.
+{[834,969],8182}.
+{[834,8127],8143}.
+{[834,8190],8159}.
+{[837,913],8124}.
+{[837,919],8140}.
+{[837,937],8188}.
+{[837,945],8115}.
+{[837,951],8131}.
+{[837,969],8179}.
+{[953],8126}.
+{[1463,1488],64302}.
+{[1463,1522],64287}.
+{[1464,1488],64303}.
+{[1465,1493],64331}.
+{[1468,1488],64304}.
+{[1468,1489],64305}.
+{[1468,1490],64306}.
+{[1468,1491],64307}.
+{[1468,1492],64308}.
+{[1468,1493],64309}.
+{[1468,1494],64310}.
+{[1468,1496],64312}.
+{[1468,1497],64313}.
+{[1468,1498],64314}.
+{[1468,1499],64315}.
+{[1468,1500],64316}.
+{[1468,1502],64318}.
+{[1468,1504],64320}.
+{[1468,1505],64321}.
+{[1468,1507],64323}.
+{[1468,1508],64324}.
+{[1468,1510],64326}.
+{[1468,1511],64327}.
+{[1468,1512],64328}.
+{[1468,1513],64329}.
+{[1468,1514],64330}.
+{[1471,1489],64332}.
+{[1471,1499],64333}.
+{[1471,1508],64334}.
+{[1473,1468,1513],64300}.
+{[1473,1513],64298}.
+{[1474,1468,1513],64301}.
+{[1474,1513],64299}.
+{[2364,2325],2392}.
+{[2364,2326],2393}.
+{[2364,2327],2394}.
+{[2364,2332],2395}.
+{[2364,2337],2396}.
+{[2364,2338],2397}.
+{[2364,2344],2345}.
+{[2364,2347],2398}.
+{[2364,2351],2399}.
+{[2364,2352],2353}.
+{[2364,2355],2356}.
+{[2492,2465],2524}.
+{[2492,2466],2525}.
+{[2492,2476],2480}.
+{[2492,2479],2527}.
+{[2494,2503],2507}.
+{[2519,2503],2508}.
+{[2620,2582],2649}.
+{[2620,2583],2650}.
+{[2620,2588],2651}.
+{[2620,2593],2652}.
+{[2620,2603],2654}.
+{[2876,2849],2908}.
+{[2876,2850],2909}.
+{[2876,2863],2911}.
+{[2878,2887],2891}.
+{[2902,2887],2888}.
+{[2903,2887],2892}.
+{[3006,3014],3018}.
+{[3006,3015],3019}.
+{[3031,2962],2964}.
+{[3031,3014],3020}.
+{[3158,3142],3144}.
+{[3266,3270],3274}.
+{[3285,3263],3264}.
+{[3285,3266,3270],3275}.
+{[3285,3270],3271}.
+{[3286,3270],3272}.
+{[3390,3398],3402}.
+{[3390,3399],3403}.
+{[3415,3398],3404}.
+{[3634,3661],3635}.
+{[3762,3789],3763}.
+{[3953,3954],3955}.
+{[3953,3956],3957}.
+{[3953,3968],3969}.
+{[3953,3968,4018],3959}.
+{[3953,3968,4019],3961}.
+{[3968,4018],3958}.
+{[3968,4019],3960}.
+{[4021,3904],3945}.
+{[4021,3984],4025}.
+{[4023,3906],3907}.
+{[4023,3916],3917}.
+{[4023,3921],3922}.
+{[4023,3926],3927}.
+{[4023,3931],3932}.
+{[4023,3986],3987}.
+{[4023,3996],3997}.
+{[4023,4001],4002}.
+{[4023,4006],4007}.
+{[4023,4011],4012}.
+{[12441,12358],12436}.
+{[12441,12363],12364}.
+{[12441,12365],12366}.
+{[12441,12367],12368}.
+{[12441,12369],12370}.
+{[12441,12371],12372}.
+{[12441,12373],12374}.
+{[12441,12375],12376}.
+{[12441,12377],12378}.
+{[12441,12379],12380}.
+{[12441,12381],12382}.
+{[12441,12383],12384}.
+{[12441,12385],12386}.
+{[12441,12388],12389}.
+{[12441,12390],12391}.
+{[12441,12392],12393}.
+{[12441,12399],12400}.
+{[12441,12402],12403}.
+{[12441,12405],12406}.
+{[12441,12408],12409}.
+{[12441,12411],12412}.
+{[12441,12445],12446}.
+{[12441,12454],12532}.
+{[12441,12459],12460}.
+{[12441,12461],12462}.
+{[12441,12463],12464}.
+{[12441,12465],12466}.
+{[12441,12467],12468}.
+{[12441,12469],12470}.
+{[12441,12471],12472}.
+{[12441,12473],12474}.
+{[12441,12475],12476}.
+{[12441,12477],12478}.
+{[12441,12479],12480}.
+{[12441,12481],12482}.
+{[12441,12484],12485}.
+{[12441,12486],12487}.
+{[12441,12488],12489}.
+{[12441,12495],12496}.
+{[12441,12498],12499}.
+{[12441,12501],12502}.
+{[12441,12504],12505}.
+{[12441,12507],12508}.
+{[12441,12527],12535}.
+{[12441,12528],12536}.
+{[12441,12529],12537}.
+{[12441,12530],12538}.
+{[12441,12541],12542}.
+{[12442,12399],12401}.
+{[12442,12402],12404}.
+{[12442,12405],12407}.
+{[12442,12408],12410}.
+{[12442,12411],12413}.
+{[12442,12495],12497}.
+{[12442,12498],12500}.
+{[12442,12501],12503}.
+{[12442,12504],12506}.
+{[12442,12507],12509}.
diff --git a/erts/emulator/internal_doc/dec.erl b/erts/emulator/internal_doc/dec.erl
new file mode 100644
index 0000000000..0315f2a52d
--- /dev/null
+++ b/erts/emulator/internal_doc/dec.erl
@@ -0,0 +1,237 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2000-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%% This program is used to generate a header file with data for
+%% normalizing denormalized unicode.
+
+%% The C header is generated from a text file containing tuples in the 
+%% following format:
+%% {RevList,Translation}
+%% Where 'RevList' is a reversed list of the denormalized repressentation of
+%% the character 'Translation'. An example would be the swedish character 
+%% '�', which would be represented in the file as:
+%% {[776,111],246}, as the denormalized representation of codepoint 246
+%% is [111,776] (i.e an 'o' followed by the "double dot accent character 776),
+%% while '�' instead is represented as {[776,97],228}, as the denormalized 
+%% form would be [97,776] (same accent but an 'a' instead).
+%% The datafile is generated from the table on Apple's developer connection
+%% http://developer.apple.com/library/mac/#technotes/tn/tn1150table.html
+%% The generating is done whenever new data is present (i.e. dec.dat has 
+%% to be changed) and not for every build. The product (the C header) is copied
+%% to $ERL_TOP/erts/beam after generation and checked in.
+%% The program and the data file is included for reference.
+
+-module(dec).
+
+-compile(export_all).
+
+-define(HASH_SIZE_FACTOR,2).
+-define(BIG_PREFIX_SIZE,392).
+
+-define(INPUT_FILE_NAME,"dec.dat").
+-define(OUTPUT_FILE_NAME,"erl_unicode_normalize.h").
+
+read(FName) ->
+    {ok,L} = file:consult(FName),
+    [{A,B} || {A,B} <- L,
+	      length(A) > 1% , hd(A) < 769
+		 ].
+
+dec() ->
+    L = read(?INPUT_FILE_NAME),
+    G = group(L),
+    {ok,Out} = file:open(?OUTPUT_FILE_NAME,[write]),
+    io:format
+      (Out,
+       "/*~n"
+       "* %CopyrightBegin%~n"
+       "*~n"
+       "* Copyright Ericsson AB 1999-2010. All Rights Reserved.~n"
+       "*~n"
+       "* The contents of this file are subject to the Erlang Public License,~n"
+       "* Version 1.1, (the \"License\"); you may not use this file except in~n"
+       "* compliance with the License. You should have received a copy of the~n"
+       "* Erlang Public License along with this software. If not, it can be~n"
+       "* retrieved online at http://www.erlang.org/.~n"
+       "*~n"
+       "* Software distributed under the License is distributed on an "
+       "\"AS IS\"~n"
+       "* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See~n"
+       "* the License for the specific language governing rights and "
+       "limitations~n"
+       "* under the License.~n"
+       "*~n"
+       "* %CopyrightEnd%~n"
+       "*/~n"
+       "/*~n"
+       "* This file is automatically generated by ~p.erl, "
+       "do not edit manually~n"
+       "*/~n",
+       [?MODULE]),
+
+    io:format(Out,
+	      "#define HASH_SIZE_FACTOR ~w~n"
+	      "typedef struct _compose_entry {~n"
+	      "    Uint16 c;~n"
+	      "    Uint16 res;~n"
+	      "    Uint16 num_subs;~n"
+	      "    struct _compose_entry *subs;~n"
+	      "    int *hash;~n"
+	      "} CompEntry;~n~n"
+	      "static int compose_tab_size = ~p;~n", 
+	      [?HASH_SIZE_FACTOR,length(G)]),
+    d(Out,G,[],0),
+    PreTab = tuple_to_list(make_prefix_table(G,erlang:make_tuple(102,0))),
+    dump_prefixes(Out,PreTab),
+%% Using this cuts down on the searching in the
+%% actual implementation, but wastes memory with little real gain..
+%%    LL = lists:flatten([PartList || {PartList,_} <- L]),
+%%    BigPreTab = tuple_to_list(
+%%		  make_big_prefixes(LL,
+%%				    erlang:make_tuple(?BIG_PREFIX_SIZE,0))),
+%%    dump_big_prefixes(Out,BigPreTab),
+    file:close(Out),
+    ok.
+    
+   
+
+d(Out,List,D,C) ->
+    d_sub(Out,List,D,C),
+    d_top_hash(Out,List,D,C),
+    d_top(Out,List,D,C).
+d_sub(_Out,[],_D,_C) ->
+    ok;
+d_sub(Out,[{_CP,[],_Res}|T],D,C) ->
+    d_sub(Out,T,D,C+1);
+d_sub(Out,[{_CP,Subs,_Res0}|T],D,C) ->
+    d(Out,Subs,[C|D],0),
+    d_sub(Out,T,D,C+1).
+d_top(Out,L,D,C) ->
+    io:format(Out,"static CompEntry ~s[] = {~n",[format_depth(D)]),
+    d_top_1(Out,L,D,C),
+    io:format(Out,"}; /* ~s */ ~n",[format_depth(D)]).
+    
+d_top_1(_Out,[],_D,_C) ->
+    ok;
+d_top_1(Out,[{CP,[],Res}|T],D,C) ->
+    io:format(Out,
+	          "{~w, ~w, 0, NULL, NULL}",[CP,Res]),
+    if 
+	T =:= [] ->
+	    io:format(Out,"~n",[]);
+	true ->
+	    io:format(Out,",~n",[])
+    end,
+    d_top_1(Out,T,D,C+1);
+d_top_1(Out,[{CP,Subs,_Res}|T],D,C) ->
+    io:format(Out,
+	          "{~w, 0, ~w, ~s, ~s}",[CP,length(Subs),
+					  format_depth([C|D]),
+					 "hash_"++format_depth([C|D])]),
+    if 
+	T =:= [] ->
+	    io:format(Out,"~n",[]);
+	true ->
+	    io:format(Out,",~n",[])
+    end,
+    d_top_1(Out,T,D,C+1).
+
+
+d_top_hash(Out,List,D,_C) ->
+     HSize = length(List)*?HASH_SIZE_FACTOR,
+     io:format(Out,"static int ~s[~p] = ~n",["hash_"++format_depth(D),HSize]),
+     Tup = d_top_hash_1(List,0,erlang:make_tuple(HSize,-1),HSize),
+     io:format(Out,"~p; /* ~s */ ~n",[Tup,"hash_"++format_depth(D)]).
+
+d_top_hash_1([],_,Hash,_HSize) -> 
+    Hash;
+d_top_hash_1([{CP,_,_}|T],Index,Hash,HSize) ->
+    Bucket = hash_search(Hash,HSize,CP rem HSize),
+    d_top_hash_1(T,Index+1,erlang:setelement(Bucket+1,Hash,Index),HSize).
+
+hash_search(Hash,_HSize,Bucket) when element(Bucket+1,Hash) =:= -1 ->
+    Bucket;
+hash_search(Hash,HSize,Bucket) ->
+    hash_search(Hash,HSize,(Bucket + 1) rem HSize). 
+
+format_depth(D) ->
+    lists:reverse(tl(lists:reverse(lists:flatten(["compose_tab_",[ integer_to_list(X) ++ "_" || X <- lists:reverse(D) ]])))).
+
+
+
+
+make_prefix_table([],Table) ->
+    Table;
+make_prefix_table([{C,_,_}|T],Table) when C =< 4023 ->
+    Index = (C div 32) + 1 - 24,
+    Pos = C rem 32,
+    X = element(Index,Table),
+    Y = X bor (1 bsl Pos),
+    NewTab = setelement(Index,Table,Y),
+    make_prefix_table(T,NewTab);
+make_prefix_table([_|T],Tab) ->
+    make_prefix_table(T,Tab).
+
+dump_prefixes(Out,L) ->
+    io:format(Out,"#define COMP_CANDIDATE_MAP_OFFSET 24~n",[]),
+    io:format(Out,"static Uint32 comp_candidate_map[] = {~n",[]),
+    dump_prefixes_1(Out,L).
+dump_prefixes_1(Out,[H]) ->
+    io:format(Out,"    0x~8.16.0BU~n",[H]),
+    io:format(Out,"};~n",[]);
+dump_prefixes_1(Out,[H|T]) ->
+    io:format(Out,"    0x~8.16.0BU,~n",[H]),
+    dump_prefixes_1(Out,T).
+
+%% make_big_prefixes([],Table) ->
+%%     Table;
+%% make_big_prefixes([C|T],Table) ->
+%%     Index = (C div 32) + 1,
+%%     Pos = C rem 32,
+%%     X = element(Index,Table),
+%%     Y = X bor (1 bsl Pos),
+%%     NewTab = setelement(Index,Table,Y),
+%%     make_big_prefixes(T,NewTab).
+
+%% dump_big_prefixes(Out,L) ->
+%%     io:format(Out,"#define BIG_COMP_CANDIDATE_SIZE ~w~n", [?BIG_PREFIX_SIZE]),
+%%     io:format(Out,"static Uint32 big_comp_candidate_map[] = {~n",[]),
+%%     dump_prefixes_1(Out,L).
+   
+pick([],_,Acc) ->
+    {lists:reverse(Acc),[]};
+pick([{[H|TT],N}|T],H,Acc) ->
+    pick(T,H,[{TT,N}|Acc]);
+pick([{[H|_],_}|_]=L,M,Acc) when H =/= M ->
+    {lists:reverse(Acc),L}.
+    
+
+group([]) ->
+    [];
+group([{[H],N}|T]) ->
+    {Part,Rest} = pick(T,H,[]),
+    [{H,group(Part),N}| group(Rest)];
+group([{[H|_],_}|_]=L) ->
+    {Part,Rest} = pick(L,H,[]),
+    [{H,group(Part),0}| group(Rest)].
+    
+    
+
+
+    
diff --git a/erts/emulator/obsolete/driver.h b/erts/emulator/obsolete/driver.h
deleted file mode 100644
index 708fe68e1a..0000000000
--- a/erts/emulator/obsolete/driver.h
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-/*
- * OLD, OBSOLETE include file for erlang driver writers.
- * New drivers should use erl_driver.h instead.
- */
-
-#ifndef __DRIVER_H__
-#define __DRIVER_H__
-
-#include <stdlib.h>
-#include "driver_int.h"
-
-#undef _ANSI_ARGS_
-#undef CONST
-
-#if ((defined(__STDC__) || defined(SABER)) && !defined(NO_PROTOTYPE)) || defined(__cplusplus) || defined(USE_PROTOTYPE)
-#   define _USING_PROTOTYPES_ 1
-#   define _ANSI_ARGS_(x)	x
-#   define CONST const
-#else
-#   define _ANSI_ARGS_(x)	()
-#   define CONST
-#endif
-
-#ifdef __cplusplus
-#   define EXTERN extern "C"
-#else
-#   define EXTERN extern
-#endif
-
-/* Values for mode arg to driver_select() */
-
-#define DO_READ	 (1 << 0)
-#define DO_WRITE (1 << 1)
-
-/* Flags for set_port_control_flags() */
-#define PORT_CONTROL_FLAG_BINARY	1
-#define PORT_CONTROL_FLAG_HEAVY		2
-
-/* This macro is used to name a dynamic driver's init function in */
-/* a way that doesn't lead to conflicts. This is crucial when using */
-/* operating systems that has one namespace for all symbols */
-/* (e.g. VxWorks). Example: if you have an dynamic driver C source */
-/* file named echo_drv.c, you use the macro like this: */
-/* int DRIVER_INIT(echo_drv)(void *handle) */
-#if defined(VXWORKS)
-#  define DRIVER_INIT(DRIVER_NAME) DRIVER_NAME  ## _init
-#elif defined(__WIN32__)
-#  define DRIVER_INIT(DRIVER_NAME) __declspec(dllexport) driver_init
-#else 
-#  define DRIVER_INIT(DRIVER_NAME)  driver_init
-#endif
-
-typedef int (*F_PTR)();    /* a function pointer */
-typedef long (*L_PTR)();   /* pointer to a function returning long */
-
-extern int null_func();
-
-/* This structure MUST match Binary in global.h exactly!!! */
-typedef struct driver_binary {
-    int orig_size;        /* total length of binary */
-    char orig_bytes[1];   /* the data (char instead of byte!) */
-} DriverBinary;
-
-typedef struct {
-    int vsize;     /* length of vectors */
-    int size;      /* total size in bytes */
-    SysIOVec* iov;
-    DriverBinary**  binv;
-} ErlIOVec;
-
-/*
- * OLD, OBSOLETE driver entry structure.
- */
-
-typedef struct driver_entry {
-    F_PTR init;          /* called at system start up (no args) */
-    L_PTR start;         /* called when some one does an open_port
-			 args: port, command (nul-terminated),
-			 additional/alternate args for fd/vanilla/spawn driver.
-			 return value -1 means failure, other
-			 is saved and passed to the other funcs */
-    F_PTR stop;          /* called when port is closed, and when the
-			    emulator is halted - arg: start_return */
-    F_PTR output;	 /* called when we have output from erlang to the port
-		         args: start_return, buf, buflen */
-    F_PTR ready_input;   /* called when we have input from one of the driver's
-			 file descriptors - args: start_return, fd */
-    F_PTR ready_output;  /* called when output is possible to one of the driver's
-			 file descriptors - args: start_return, fd */
-    char *driver_name;   /* name supplied as {driver,Name,Args} to open_port */
-
-    F_PTR finish;        /* called before unloading (DYNAMIC DRIVERS ONLY) */
-    void *handle;        /* file handle             (DYNAMIC DRIVERS ONLY) */
-    F_PTR control;	 /* "ioctl" for drivers (invoked by port_command/3) */
-    F_PTR timeout;       /* Reserved */
-    F_PTR outputv;       /* Reserved */
-    F_PTR ready_async;   /* Completion routine for driver_async */
-    F_PTR padding1[3];   /* pad to match size of modern driver struct */
-    int padding2[4];     /* more pad */
-    F_PTR padding3[3];   /* even more padding */
-} DriverEntry;
-
-
-/* These are the kernel functions available for driver writers */
-
-EXTERN int driver_select _ANSI_ARGS_((int,int,int,int));
-
-EXTERN int driver_output _ANSI_ARGS_((int, char*, int));
-EXTERN int driver_output2 _ANSI_ARGS_((int, char*, int, char*, int));
-EXTERN int driver_output_binary _ANSI_ARGS_((int, char*, int,
-					     DriverBinary*, int, int));
-EXTERN int driver_outputv _ANSI_ARGS_((int, char*,int,ErlIOVec*,int));
-
-EXTERN int driver_vec_to_buf _ANSI_ARGS_((ErlIOVec*, char*, int));
-
-EXTERN int driver_set_timer _ANSI_ARGS_((int, unsigned long));
-EXTERN int driver_cancel_timer _ANSI_ARGS_((int));
-
-/*
- * The following functions are used to initiate a close of a port
- * from a driver.
- */
-EXTERN int driver_failure_eof _ANSI_ARGS_((int));
-EXTERN int driver_failure_atom _ANSI_ARGS_((int, char *));
-EXTERN int driver_failure_posix _ANSI_ARGS_((int, int));
-EXTERN int driver_failure _ANSI_ARGS_((int, int));
-EXTERN int driver_exit _ANSI_ARGS_ ((int, int));
-
-EXTERN char* erl_errno_id _ANSI_ARGS_((int error));
-EXTERN void set_busy_port _ANSI_ARGS_((int, int));
-EXTERN void add_driver_entry _ANSI_ARGS_((DriverEntry *));
-EXTERN int remove_driver_entry _ANSI_ARGS_((DriverEntry *));
-EXTERN void set_port_control_flags _ANSI_ARGS_((int, int));
-
-/* Binary interface */
-/* NOTE: DO NOT overwrite a binary with new data (if the data is delivered);
-** since the binary is a shared object it MUST be written once.
-*/
-
-EXTERN DriverBinary* driver_alloc_binary _ANSI_ARGS_((int));
-EXTERN DriverBinary* driver_realloc_binary _ANSI_ARGS_((DriverBinary*, int));
-EXTERN void driver_free_binary _ANSI_ARGS_((DriverBinary*));
-
-
-/* Queue interface */
-EXTERN int driver_enqv _ANSI_ARGS_((int, ErlIOVec*, int));
-EXTERN int driver_pushqv _ANSI_ARGS_((int, ErlIOVec*, int));
-EXTERN int driver_deq _ANSI_ARGS_((int, int));
-EXTERN SysIOVec* driver_peekq _ANSI_ARGS_((int, int*));
-EXTERN int driver_sizeq _ANSI_ARGS_((int));
-EXTERN int driver_enq_bin _ANSI_ARGS_((int, DriverBinary*, int, int));
-EXTERN int driver_enq _ANSI_ARGS_((int, char*, int));
-EXTERN int driver_pushq_bin _ANSI_ARGS_((int, DriverBinary*, int, int));
-EXTERN int driver_pushq _ANSI_ARGS_((int, char*, int));
-
-/* Memory management */
-EXTERN void *driver_alloc _ANSI_ARGS_((size_t));
-EXTERN void *driver_realloc _ANSI_ARGS_((void*, size_t));
-EXTERN void driver_free _ANSI_ARGS_((void*));
-
-/* Shared / dynamic link libraries */
-EXTERN void *driver_dl_open _ANSI_ARGS_((char *));
-EXTERN void *driver_dl_sym _ANSI_ARGS_((void *, char *));
-EXTERN int driver_dl_close _ANSI_ARGS_((void *));
-EXTERN char *driver_dl_error _ANSI_ARGS_((void));
-
-/* Async IO functions */
-EXTERN long driver_async _ANSI_ARGS_((int,
-				      unsigned int*,
-				      void (*)(void*), 
-				      void *,
-				      void (*)(void*)));
-EXTERN int driver_async_cancel _ANSI_ARGS_((long));
-
-EXTERN int driver_lock_driver _ANSI_ARGS_((int));
-
-/* Threads */
-typedef void* erl_mutex_t;
-typedef void* erl_cond_t;
-typedef void* erl_thread_t;
-
-EXTERN erl_mutex_t erts_mutex_create _ANSI_ARGS_((void));
-EXTERN int erts_mutex_destroy _ANSI_ARGS_((erl_mutex_t));
-EXTERN int erts_mutex_lock _ANSI_ARGS_((erl_mutex_t));
-EXTERN int erts_mutex_unlock _ANSI_ARGS_((erl_mutex_t));
-
-EXTERN erl_cond_t erts_cond_create _ANSI_ARGS_((void));
-EXTERN int erts_cond_destroy _ANSI_ARGS_((erl_cond_t));
-EXTERN int erts_cond_signal _ANSI_ARGS_((erl_cond_t));
-EXTERN int erts_cond_broadcast _ANSI_ARGS_((erl_cond_t));
-EXTERN int erts_cond_wait _ANSI_ARGS_((erl_cond_t, erl_mutex_t));
-EXTERN int erts_cond_timedwait _ANSI_ARGS_((erl_cond_t, erl_mutex_t, long));
-
-EXTERN int erts_thread_create _ANSI_ARGS_((erl_thread_t*,
-					 void* (*func)(void*),
-					 void* arg,
-					 int detached));
-EXTERN erl_thread_t erts_thread_self _ANSI_ARGS_((void));
-EXTERN void erts_thread_exit _ANSI_ARGS_((void*));
-EXTERN int  erts_thread_join _ANSI_ARGS_((erl_thread_t, void**));
-EXTERN int  erts_thread_kill _ANSI_ARGS_((erl_thread_t));
-
-
-typedef unsigned long DriverTermData;
-
-#define TERM_DATA(x) ((DriverTermData) (x))
-
-/* Possible types to send from driver          Argument type */
-#define ERL_DRV_NIL         ((DriverTermData) 1)  /* None */
-#define ERL_DRV_ATOM        ((DriverTermData) 2)  /* driver_mk_atom(string) */
-#define ERL_DRV_INT         ((DriverTermData) 3)  /* int */
-#define ERL_DRV_PORT        ((DriverTermData) 4)  /* driver_mk_port(ix) */
-#define ERL_DRV_BINARY      ((DriverTermData) 5)  /* ErlDriverBinary*, int */
-#define ERL_DRV_STRING      ((DriverTermData) 6)  /* char*, int */
-#define ERL_DRV_TUPLE       ((DriverTermData) 7)  /* int */
-#define ERL_DRV_LIST        ((DriverTermData) 8)  /* int */
-#define ERL_DRV_STRING_CONS ((DriverTermData) 9)  /* char*, int */
-#define ERL_DRV_PID         ((DriverTermData) 10) /* driver_connected,... */
-
-/* DriverTermData is the type to use for casts when building 
- * terms that should be sent to connected process,
- * for instance a tuple on the form {tcp, Port, [Tag|Binary]}
- *
- * DriverTermData spec[] = {
- *    ERL_DRV_ATOM, driver_mk_atom("tcp"),
- *    ERL_DRV_PORT, driver_mk_port(drv->ix),
- *             ERL_DRV_INT, REPLY_TAG,
- *             ERL_DRV_BIN, 50, TERM_DATA(buffer),
- *             ERL_DRV_LIST, 2,
- *    ERL_DRV_TUPLE, 3,
- *  }       
- *             
- */
-
-EXTERN DriverTermData driver_mk_atom _ANSI_ARGS_ ((char*));
-EXTERN DriverTermData driver_mk_port _ANSI_ARGS_ ((int));
-EXTERN DriverTermData driver_connected _ANSI_ARGS_((int));
-EXTERN DriverTermData driver_caller _ANSI_ARGS_((int));
-
-EXTERN int driver_output_term _ANSI_ARGS_((int, DriverTermData *, int));
-EXTERN int driver_send_term _ANSI_ARGS_((int, DriverTermData, DriverTermData *, int));
-
-#endif
-
-
diff --git a/erts/emulator/pcre/Makefile.in b/erts/emulator/pcre/Makefile.in
deleted file mode 100644
index f62700ec4e..0000000000
--- a/erts/emulator/pcre/Makefile.in
+++ /dev/null
@@ -1,165 +0,0 @@
-# Makefile for zlib
-# Copyright (C) 1995-1996 Jean-loup Gailly.
-# For conditions of distribution and use, see copyright notice in zlib.h 
-
-# To compile and test, type:
-#   ./configure; make test
-# The call of configure is optional if you don't have special requirements
-
-# To install /usr/local/lib/libz.* and /usr/local/include/zlib.h, type:
-#    make install
-# To install in $HOME instead of /usr/local, use:
-#    make install prefix=$HOME
-
-# %ExternalCopyright%
-
-ARFLAGS = rc
-
-O = \
-pcre_latin_1_table.o \
-pcre_compile.o \
-pcre_config.o \
-pcre_dfa_exec.o \
-pcre_exec.o \
-pcre_fullinfo.o \
-pcre_get.o \
-pcre_globals.o \
-pcre_info.o \
-pcre_maketables.o \
-pcre_newline.o \
-pcre_ord2utf8.o \
-pcre_refcount.o \
-pcre_study.o \
-pcre_tables.o \
-pcre_try_flipped.o \
-pcre_ucp_searchfuncs.o \
-pcre_valid_utf8.o \
-pcre_version.o \
-pcre_xclass.o
-
-OBJS = $(O:%=$(OBJDIR)/%)
-
-GENINC = pcre_exec_loop_break_cases.inc
-
-#### Begin OTP targets
-
-include $(ERL_TOP)/make/target.mk
-
-# On windows we need a separate zlib during debug build
-ifeq ($(TARGET),win32)
-
-ifeq ($(TYPE),debug)
-CFLAGS = $(subst -O2, -g, @CFLAGS@ @DEFS@ @DEBUG_FLAGS@ @EMU_THR_DEFS@ -DERLANG_INTEGRATION)
-else  # debug
-CFLAGS = @CFLAGS@ @DEFS@ @EMU_THR_DEFS@ -DERLANG_INTEGRATION
-endif # debug
-
-else  # win32
-
-ifeq ($(TYPE),debug)
-TYPE_FLAGS = @DEBUG_CFLAGS@ 
-else  # debug
-ifeq ($(TYPE),gcov)
-TYPE_FLAGS = -O0 -fprofile-arcs -ftest-coverage
-else  # gcov 
-TYPE_FLAGS = -O3
-endif # gcov
-endif # debug
-
-CFLAGS =  $(TYPE_FLAGS) $(subst -O2,, @CFLAGS@) @DEFS@ @EMU_THR_DEFS@ -DERLANG_INTEGRATION 
-
-endif # win32
-
-OBJDIR = $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)
-
-include $(ERL_TOP)/make/$(TARGET)/otp.mk
-
-ifeq ($(TARGET), win32)
-LIBRARY=$(OBJDIR)/epcre.lib
-else
-LIBRARY=$(OBJDIR)/libepcre.a
-endif
-
-all: $(LIBRARY)
-
-# ----------------------------------------------------
-# Release Target
-# ---------------------------------------------------- 
-include $(ERL_TOP)/make/otp_release_targets.mk
-
-release_spec: opt
-
-tests release_tests:
-
-docs release_docs release_docs_spec:
-
-clean:
-	rm -f $(OBJS) $(OBJDIR)/libepcre.a
-
-#### end OTP targets
-
-ifeq ($(TARGET), win32)
-$(LIBRARY): $(OBJS)
-	$(AR) -out:$@ $(OBJS)
-else
-$(LIBRARY): $(OBJS)
-	$(AR) $(ARFLAGS) $@ $(OBJS)
-	-@ ($(RANLIB) $@ || true) 2>/dev/null
-endif
-
-$(OBJDIR)/%.o: %.c
-	$(CC) -c $(CFLAGS) -o $@ $<
-
-$(GENINC): pcre_exec.c
-	for x in `grep -n COST_CHK pcre_exec.c | grep -v 'COST_CHK(N)' | awk -F: '{print $$1}'`; \
-	do \
-		N=`expr $$x + 100`; \
-		echo "case $$N: goto L_LOOP_COUNT_$${x};"; \
-	done > $(GENINC)     
-
-table: ./gen_table
-	./gen_table pcre_latin_1_table.c
-
-./gen_table: pcre_make_latin1_default.c make_latin1_table.c
-	$(CC) $(CFLAGS) -o gen_table  pcre_make_latin1_default.c make_latin1_table.c
-
-# DO NOT DELETE THIS LINE -- make depend depends on it.
-
-$(OBJDIR)/pcre_chartables.o: pcre_chartables.c pcre_internal.h local_config.h \
-  pcre.h ucp.h
-$(OBJDIR)/pcre_compile.o: pcre_compile.c pcre_internal.h local_config.h \
-  pcre.h ucp.h
-$(OBJDIR)/pcre_config.o: pcre_config.c pcre_internal.h local_config.h pcre.h \
-  ucp.h
-$(OBJDIR)/pcre_dfa_exec.o: pcre_dfa_exec.c pcre_internal.h local_config.h \
-  pcre.h ucp.h
-$(OBJDIR)/pcre_exec.o: pcre_exec.c pcre_internal.h local_config.h pcre.h ucp.h \
-  $(GENINC)
-$(OBJDIR)/pcre_fullinfo.o: pcre_fullinfo.c pcre_internal.h local_config.h \
-  pcre.h ucp.h
-$(OBJDIR)/pcre_get.o: pcre_get.c pcre_internal.h local_config.h pcre.h ucp.h
-$(OBJDIR)/pcre_globals.o: pcre_globals.c pcre_internal.h local_config.h \
-  pcre.h ucp.h
-$(OBJDIR)/pcre_info.o: pcre_info.c pcre_internal.h local_config.h pcre.h ucp.h
-$(OBJDIR)/pcre_maketables.o: pcre_maketables.c pcre_internal.h local_config.h \
-  pcre.h ucp.h
-$(OBJDIR)/pcre_newline.o: pcre_newline.c pcre_internal.h local_config.h \
-  pcre.h ucp.h
-$(OBJDIR)/pcre_ord2utf8.o: pcre_ord2utf8.c pcre_internal.h local_config.h \
-  pcre.h ucp.h
-$(OBJDIR)/pcre_refcount.o: pcre_refcount.c pcre_internal.h local_config.h \
-  pcre.h ucp.h
-$(OBJDIR)/pcre_study.o: pcre_study.c pcre_internal.h local_config.h pcre.h \
-  ucp.h
-$(OBJDIR)/pcre_tables.o: pcre_tables.c pcre_internal.h local_config.h pcre.h \
- ucp.h
-$(OBJDIR)/pcre_try_flipped.o: pcre_try_flipped.c pcre_internal.h \
-  local_config.h pcre.h ucp.h
-$(OBJDIR)/pcre_ucp_searchfuncs.o: pcre_ucp_searchfuncs.c pcre_internal.h \
-  local_config.h pcre.h ucp.h ucpinternal.h ucptable.h
-$(OBJDIR)/pcre_valid_utf8.o: pcre_valid_utf8.c pcre_internal.h local_config.h \
-  pcre.h ucp.h
-pcre_version.o: pcre_version.c pcre_internal.h local_config.h pcre.h \
-  ucp.h
-$(OBJDIR)/pcre_xclass.o: pcre_xclass.c pcre_internal.h local_config.h pcre.h \
-  ucp.h
diff --git a/erts/emulator/pcre/pcre.mk b/erts/emulator/pcre/pcre.mk
new file mode 100644
index 0000000000..352137b341
--- /dev/null
+++ b/erts/emulator/pcre/pcre.mk
@@ -0,0 +1,111 @@
+#
+# %CopyrightBegin%
+#
+# Copyright Ericsson AB 2012. All Rights Reserved.
+#
+# The contents of this file are subject to the Erlang Public License,
+# Version 1.1, (the "License"); you may not use this file except in
+# compliance with the License. You should have received a copy of the
+# Erlang Public License along with this software. If not, it can be
+# retrieved online at http://www.erlang.org/.
+#
+# Software distributed under the License is distributed on an "AS IS"
+# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+# the License for the specific language governing rights and limitations
+# under the License.
+#
+# %CopyrightEnd%
+#
+
+PCRE_O = \
+pcre_latin_1_table.o \
+pcre_compile.o \
+pcre_config.o \
+pcre_dfa_exec.o \
+pcre_exec.o \
+pcre_fullinfo.o \
+pcre_get.o \
+pcre_globals.o \
+pcre_info.o \
+pcre_maketables.o \
+pcre_newline.o \
+pcre_ord2utf8.o \
+pcre_refcount.o \
+pcre_study.o \
+pcre_tables.o \
+pcre_try_flipped.o \
+pcre_ucp_searchfuncs.o \
+pcre_valid_utf8.o \
+pcre_version.o \
+pcre_xclass.o
+
+PCRE_OBJS = $(PCRE_O:%=$(PCRE_OBJDIR)/%)
+
+PCRE_GENINC = $(ERL_TOP)/erts/emulator/pcre/pcre_exec_loop_break_cases.inc
+
+PCRE_OBJDIR = $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)
+
+PCRE_CFLAGS = $(filter-out -DDEBUG,$(CFLAGS)) -DERLANG_INTEGRATION
+
+ifeq ($(TARGET), win32)
+$(EPCRE_LIB): $(PCRE_OBJS)
+	$(AR) -out:$@ $(PCRE_OBJS)
+else
+$(EPCRE_LIB): $(PCRE_OBJS)
+	$(AR) $(ARFLAGS) $@ $(PCRE_OBJS)
+	-@ ($(RANLIB) $@ || true) 2>/dev/null
+endif
+
+$(PCRE_OBJDIR)/%.o: pcre/%.c
+	$(CC) -c $(PCRE_CFLAGS) -o $@ $<
+
+$(PCRE_GENINC): pcre/pcre_exec.c
+	for x in `grep -n COST_CHK pcre/pcre_exec.c | grep -v 'COST_CHK(N)' | awk -F: '{print $$1}'`; \
+	do \
+		N=`expr $$x + 100`; \
+		echo "case $$N: goto L_LOOP_COUNT_$${x};"; \
+	done > $(PCRE_GENINC)
+
+# Dependencies.
+
+$(PCRE_OBJDIR)/pcre_chartables.o: pcre/pcre_chartables.c pcre/pcre_internal.h \
+  pcre/local_config.h pcre/pcre.h pcre/ucp.h
+$(PCRE_OBJDIR)/pcre_compile.o: pcre/pcre_compile.c pcre/pcre_internal.h \
+  pcre/local_config.h pcre/pcre.h pcre/ucp.h
+$(PCRE_OBJDIR)/pcre_config.o: pcre/pcre_config.c pcre/pcre_internal.h \
+  pcre/local_config.h pcre/pcre.h pcre/ucp.h
+$(PCRE_OBJDIR)/pcre_dfa_exec.o: pcre/pcre_dfa_exec.c pcre/pcre_internal.h \
+  pcre/local_config.h pcre/pcre.h pcre/ucp.h
+$(PCRE_OBJDIR)/pcre_exec.o: pcre/pcre_exec.c pcre/pcre_internal.h \
+  pcre/local_config.h pcre/pcre.h pcre/ucp.h $(PCRE_GENINC)
+$(PCRE_OBJDIR)/pcre_fullinfo.o: pcre/pcre_fullinfo.c pcre/pcre_internal.h \
+  pcre/local_config.h pcre/pcre.h pcre/ucp.h
+$(PCRE_OBJDIR)/pcre_get.o: pcre/pcre_get.c pcre/pcre_internal.h \
+  pcre/local_config.h pcre/pcre.h pcre/ucp.h
+$(PCRE_OBJDIR)/pcre_globals.o: pcre/pcre_globals.c pcre/pcre_internal.h \
+  pcre/local_config.h pcre/pcre.h pcre/ucp.h
+$(PCRE_OBJDIR)/pcre_info.o: pcre/pcre_info.c pcre/pcre_internal.h \
+  pcre/local_config.h pcre/pcre.h pcre/ucp.h
+$(PCRE_OBJDIR)/pcre_maketables.o: pcre/pcre_maketables.c pcre/pcre_internal.h \
+  pcre/local_config.h pcre/pcre.h pcre/ucp.h
+$(PCRE_OBJDIR)/pcre_newline.o: pcre/pcre_newline.c pcre/pcre_internal.h \
+  pcre/local_config.h pcre/pcre.h pcre/ucp.h
+$(PCRE_OBJDIR)/pcre_ord2utf8.o: pcre/pcre_ord2utf8.c pcre/pcre_internal.h \
+  pcre/local_config.h pcre/pcre.h pcre/ucp.h
+$(PCRE_OBJDIR)/pcre/pcre_refcount.o: pcre/pcre_refcount.c pcre/pcre_internal.h \
+  pcre/local_config.h pcre/pcre.h pcre/ucp.h
+$(PCRE_OBJDIR)/pcre_study.o: pcre/pcre_study.c pcre/pcre_internal.h \
+  pcre/local_config.h pcre/pcre.h pcre/ucp.h
+$(PCRE_OBJDIR)/pcre_tables.o: pcre/pcre_tables.c pcre/pcre_internal.h \
+  pcre/local_config.h pcre/pcre.h pcre/ucp.h
+$(PCRE_OBJDIR)/pcre_try_flipped.o: pcre/pcre_try_flipped.c pcre/pcre_internal.h \
+  pcre/local_config.h pcre/pcre.h pcre/ucp.h
+$(PCRE_OBJDIR)/pcre_ucp_searchfuncs.o: pcre/pcre_ucp_searchfuncs.c \
+  pcre/pcre_internal.h pcre/local_config.h pcre/pcre.h pcre/ucp.h \
+  pcre/ucpinternal.h pcre/ucptable.h
+$(PCRE_OBJDIR)/pcre_valid_utf8.o: pcre/pcre_valid_utf8.c pcre/pcre_internal.h \
+  pcre/local_config.h pcre/pcre.h pcre/ucp.h
+pcre_version.o: pcre/pcre_version.c pcre/pcre_internal.h pcre/local_config.h \
+  pcre/pcre.h pcre/ucp.h
+$(PCRE_OBJDIR)/pcre/pcre_xclass.o: pcre/pcre_xclass.c pcre/pcre_internal.h \
+  pcre/local_config.h pcre/pcre.h pcre/ucp.h
diff --git a/erts/emulator/pcre/pcre_compile.c b/erts/emulator/pcre/pcre_compile.c
index 29743362d4..9508c5a697 100644
--- a/erts/emulator/pcre/pcre_compile.c
+++ b/erts/emulator/pcre/pcre_compile.c
@@ -92,6 +92,11 @@ is 4 there is plenty of room. */
 
 #define COMPILE_WORK_SIZE (4096)
 
+/* The overrun tests check for a slightly smaller size so that they detect the
+overrun before it actually does run off the end of the data block. */
+
+#define WORK_SIZE_CHECK (COMPILE_WORK_SIZE - 100)
+
 
 /* Table for handling escaped characters in the range '0'-'z'. Positive returns
 are simple data values; negative values are for special things like \d and so
@@ -2445,7 +2450,7 @@ for (;; ptr++)
 #ifdef DEBUG
     if (code > cd->hwm) cd->hwm = code;                 /* High water info */
 #endif
-    if (code > cd->start_workspace + COMPILE_WORK_SIZE) /* Check for overrun */
+    if (code > cd->start_workspace + WORK_SIZE_CHECK)   /* Check for overrun */
       {
       *errorcodeptr = ERR52;
       goto FAILED;
@@ -2494,7 +2499,7 @@ for (;; ptr++)
   /* In the real compile phase, just check the workspace used by the forward
   reference list. */
 
-  else if (cd->hwm > cd->start_workspace + COMPILE_WORK_SIZE)
+  else if (cd->hwm > cd->start_workspace + WORK_SIZE_CHECK)
     {
     *errorcodeptr = ERR52;
     goto FAILED;
diff --git a/erts/emulator/pcre/pcre_exec.c b/erts/emulator/pcre/pcre_exec.c
index 51625130c3..3fe13ca32e 100644
--- a/erts/emulator/pcre/pcre_exec.c
+++ b/erts/emulator/pcre/pcre_exec.c
@@ -5191,7 +5191,6 @@ for(;;)
       EDEBUGF(("Loop limit break detected"));
       return PCRE_ERROR_LOOP_LIMIT;
   RESTART_INTERRUPTED:
-      md->match_call_count = 0; 
       md->loop_limit = extra_data->loop_limit;
       rc = match(NULL,NULL,NULL,0,md,0,NULL,0,0);
       *extra_data->loop_counter_return = 
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 218bd79584..c1336c60d9 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2012. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -35,6 +35,8 @@
 #include "sys.h"
 #include "global.h"
 #include "erl_check_io.h"
+#include "erl_thr_progress.h"
+#include "dtrace-wrapper.h"
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
 #  define ERTS_DRV_EV_STATE_EXTRA_SIZE 128
@@ -66,6 +68,9 @@ typedef char EventStateFlags;
 
 #define ERTS_CIO_POLL_CTL	ERTS_POLL_EXPORT(erts_poll_control)
 #define ERTS_CIO_POLL_WAIT	ERTS_POLL_EXPORT(erts_poll_wait)
+#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
+#define ERTS_CIO_POLL_AS_INTR 	ERTS_POLL_EXPORT(erts_poll_async_sig_interrupt)
+#endif
 #define ERTS_CIO_POLL_INTR 	ERTS_POLL_EXPORT(erts_poll_interrupt)
 #define ERTS_CIO_POLL_INTR_TMD	ERTS_POLL_EXPORT(erts_poll_interrupt_timed)
 #define ERTS_CIO_NEW_POLLSET 	ERTS_POLL_EXPORT(erts_poll_create_pollset)
@@ -218,7 +223,7 @@ remember_removed(ErtsDrvEventState *state, struct pollset_info* psi)
 #ifdef ERTS_SMP
     struct removed_fd *fdlp;
     ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(fd_mtx(state->fd)));
-    if (erts_smp_atomic_read(&psi->in_poll_wait)) {
+    if (erts_smp_atomic_read_nob(&psi->in_poll_wait)) {
 	state->remove_cnt++;
 	ASSERT(state->remove_cnt > 0);
 	fdlp = removed_fd_alloc();
@@ -310,7 +315,8 @@ forget_removed(struct pollset_info* psi)
 	erts_smp_mtx_unlock(mtx);
 	if (drv_ptr) {
 	    int was_unmasked = erts_block_fpe();
-	    (*drv_ptr->stop_select) (fd, NULL);
+	    DTRACE1(driver_stop_select, drv_ptr->name);
+	    (*drv_ptr->stop_select) ((ErlDrvEvent) fd, NULL);
 	    erts_unblock_fpe(was_unmasked);
 	    if (drv_ptr->handle) {
 		erts_ddll_dereference_driver(drv_ptr->handle);
@@ -333,7 +339,7 @@ grow_drv_ev_state(int min_ix)
 	new_len = max_fds;
 
     erts_smp_mtx_lock(&drv_ev_state_grow_lock);
-    if (erts_smp_atomic_read(&drv_ev_state_len) <= min_ix) {
+    if (erts_smp_atomic_read_nob(&drv_ev_state_len) <= min_ix) {
 	for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { /* lock all fd's */
 	    erts_smp_mtx_lock(&drv_ev_state_locks[i].lck);
 	}
@@ -343,7 +349,7 @@ grow_drv_ev_state(int min_ix)
 				       sizeof(ErtsDrvEventState)*new_len)
 			: erts_alloc(ERTS_ALC_T_DRV_EV_STATE,
 				     sizeof(ErtsDrvEventState)*new_len));
-	for (i = erts_smp_atomic_read(&drv_ev_state_len); i < new_len; i++) {
+	for (i = erts_smp_atomic_read_nob(&drv_ev_state_len); i < new_len; i++) {
 	    drv_ev_state[i].fd = (ErtsSysFdType) i;
 	    drv_ev_state[i].driver.select = NULL;
 	    drv_ev_state[i].events = 0;
@@ -351,7 +357,7 @@ grow_drv_ev_state(int min_ix)
 	    drv_ev_state[i].type = ERTS_EV_TYPE_NONE;
 	    drv_ev_state[i].flags = 0;
 	}
-	erts_smp_atomic_set(&drv_ev_state_len, new_len);
+	erts_smp_atomic_set_nob(&drv_ev_state_len, new_len);
 	for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) {
 	    erts_smp_mtx_unlock(&drv_ev_state_locks[i].lck);
 	}
@@ -492,12 +498,15 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
     ErtsDrvEventState *state;
     int wake_poller;
     int ret;
+#ifdef USE_VM_PROBES
+    DTRACE_CHARBUF(name, 64);
+#endif
     
     ERTS_SMP_LC_ASSERT(erts_drvport2port(ix)
 		       && erts_lc_is_port_locked(erts_drvport2port(ix)));
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    if ((unsigned)fd >= (unsigned)erts_smp_atomic_read(&drv_ev_state_len)) {
+    if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) {
 	if (fd < 0) {
 	    return -1;    
 	}
@@ -521,6 +530,10 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
 	if (IS_FD_UNKNOWN(state)) {
 	    /* fast track to stop_select callback */
 	    stop_select_fn = erts_drvport2port(ix)->drv_ptr->stop_select;
+#ifdef USE_VM_PROBES
+	    strncpy(name, erts_drvport2port(ix)->drv_ptr->name, sizeof(name)-1);
+	    name[sizeof(name)-1] = '\0';
+#endif
 	    ret = 0;
 	    goto done_unknown;
 	}
@@ -657,6 +670,10 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix,
 		    /* Safe to close fd now as it is not in pollset
 		       or there was no need to eject fd (kernel poll) */
 		    stop_select_fn = drv_ptr->stop_select;
+#ifdef USE_VM_PROBES
+		    strncpy(name, erts_drvport2port(ix)->drv_ptr->name, sizeof(name)-1);
+		    name[sizeof(name)-1] = '\0';
+#endif
 		}
 		else {
 		    /* Not safe to close fd, postpone stop_select callback. */
@@ -682,6 +699,7 @@ done_unknown:
     erts_smp_mtx_unlock(fd_mtx(fd));
     if (stop_select_fn) {
 	int was_unmasked = erts_block_fpe();
+	DTRACE1(driver_stop_select, name);
 	(*stop_select_fn)(e, NULL);
 	erts_unblock_fpe(was_unmasked);
     }
@@ -709,7 +727,7 @@ ERTS_CIO_EXPORT(driver_event)(ErlDrvPort ix,
 		       && erts_lc_is_port_locked(erts_drvport2port(ix)));
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    if ((unsigned)fd >= (unsigned)erts_smp_atomic_read(&drv_ev_state_len)) {
+    if ((unsigned)fd >= (unsigned)erts_smp_atomic_read_nob(&drv_ev_state_len)) {
 	if (fd < 0)
 	    return -1;    
 	if (fd >= max_fds) {
@@ -1115,6 +1133,14 @@ eready(Eterm id, ErtsDrvEventState *state, ErlDrvEventData event_data)
 
 static void bad_fd_in_pollset( ErtsDrvEventState *, Eterm, Eterm, ErtsPollEvents);
 
+#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
+void
+ERTS_CIO_EXPORT(erts_check_io_async_sig_interrupt)(void)
+{
+    ERTS_CIO_POLL_AS_INTR(pollset.ps);
+}
+#endif
+
 void
 ERTS_CIO_EXPORT(erts_check_io_interrupt)(int set)
 {
@@ -1122,7 +1148,8 @@ ERTS_CIO_EXPORT(erts_check_io_interrupt)(int set)
 }
 
 void
-ERTS_CIO_EXPORT(erts_check_io_interrupt_timed)(int set, long msec)
+ERTS_CIO_EXPORT(erts_check_io_interrupt_timed)(int set,
+					       erts_short_time_t msec)
 {
     ERTS_CIO_POLL_INTR_TMD(pollset.ps, set, msec);
 }
@@ -1137,6 +1164,11 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 
  restart:
 
+#ifdef ERTS_BREAK_REQUESTED
+    if (ERTS_BREAK_REQUESTED)
+	erts_do_break_handling();
+#endif
+
     /* Figure out timeout value */
     if (do_wait) {
 	erts_time_remaining(&wait_time);
@@ -1148,17 +1180,15 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_check_exact(NULL, 0); /* No locks should be locked */
 #endif
-    erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
     pollres_len = sizeof(pollres)/sizeof(ErtsPollResFd);
 
-    erts_smp_atomic_set(&pollset.in_poll_wait, 1);
+    erts_smp_atomic_set_nob(&pollset.in_poll_wait, 1);
 
     poll_ret = ERTS_CIO_POLL_WAIT(pollset.ps, pollres, &pollres_len, &wait_time);
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_check_exact(NULL, 0); /* No locks should be locked */
 #endif
-    erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL);
 
     erts_deliver_time(); /* sync the machine's idea of time */
 
@@ -1168,7 +1198,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 #endif
 
     if (poll_ret != 0) {
-	erts_smp_atomic_set(&pollset.in_poll_wait, 0);
+	erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0);
 	forget_removed(&pollset);
 	if (poll_ret == EAGAIN) {
 	    goto restart;
@@ -1299,7 +1329,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 #endif
     }
 
-    erts_smp_atomic_set(&pollset.in_poll_wait, 0);
+    erts_smp_atomic_set_nob(&pollset.in_poll_wait, 0);
     forget_removed(&pollset);
 }
 
@@ -1414,7 +1444,7 @@ static void drv_ev_state_free(void *des)
 void
 ERTS_CIO_EXPORT(erts_init_check_io)(void)
 {
-    erts_smp_atomic_init(&pollset.in_poll_wait, 0);
+    erts_smp_atomic_init_nob(&pollset.in_poll_wait, 0);
     ERTS_CIO_POLL_INIT();
     pollset.ps = ERTS_CIO_NEW_POLLSET();
 
@@ -1436,7 +1466,7 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
 #endif
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     max_fds = ERTS_CIO_POLL_MAX_FDS();
-    erts_smp_atomic_init(&drv_ev_state_len, 0);
+    erts_smp_atomic_init_nob(&drv_ev_state_len, 0);
     drv_ev_state = NULL;
     erts_smp_mtx_init(&drv_ev_state_grow_lock, "drv_ev_state_grow");
 #else
@@ -1474,7 +1504,7 @@ ERTS_CIO_EXPORT(erts_check_io_size)(void)
     ERTS_CIO_POLL_INFO(pollset.ps, &pi);
     res = pi.memory_size;
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    res += sizeof(ErtsDrvEventState) * erts_smp_atomic_read(&drv_ev_state_len);
+    res += sizeof(ErtsDrvEventState) * erts_smp_atomic_read_nob(&drv_ev_state_len);
 #else
     res += safe_hash_table_sz(&drv_ev_state_tab);
     {
@@ -1501,7 +1531,7 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc)
     ERTS_CIO_POLL_INFO(pollset.ps, &pi);
     memory_size = pi.memory_size;
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    memory_size += sizeof(ErtsDrvEventState) * erts_smp_atomic_read(&drv_ev_state_len);
+    memory_size += sizeof(ErtsDrvEventState) * erts_smp_atomic_read_nob(&drv_ev_state_len);
 #else
     memory_size += safe_hash_table_sz(&drv_ev_state_tab);
     {
@@ -1865,13 +1895,12 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(void)
 
     erts_printf("--- fds in pollset --------------------------------------\n");
 
-#ifdef ERTS_SMP
-# ifdef ERTS_ENABLE_LOCK_CHECK
+#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
     erts_lc_check_exact(NULL, 0); /* No locks should be locked */
-# endif
-    erts_block_system(0); /* stop the world to avoid messy locking */
 #endif
 
+    erts_smp_thr_progress_block(); /* stop the world to avoid messy locking */
+
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     counters.epep = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollEvents)*max_fds);
     ERTS_POLL_EXPORT(erts_poll_get_selected_events)(pollset.ps, counters.epep, max_fds);
@@ -1881,7 +1910,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(void)
     counters.num_errors = 0;
 
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
-    len = erts_smp_atomic_read(&drv_ev_state_len);
+    len = erts_smp_atomic_read_nob(&drv_ev_state_len);
     for (fd = 0; fd < len; fd++) {
 	doit_erts_check_io_debug((void *) &drv_ev_state[fd], (void *) &counters);
     }
@@ -1893,9 +1922,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(void)
     safe_hash_for_each(&drv_ev_state_tab, &doit_erts_check_io_debug, (void *) &counters);
 #endif
 
-#ifdef ERTS_SMP
-    erts_release_system();
-#endif
+    erts_smp_thr_progress_unblock();
 
     erts_printf("\n");
     erts_printf("used fds=%d\n", counters.used_fds);
diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h
index 9b45a63913..edab7947ba 100644
--- a/erts/emulator/sys/common/erl_check_io.h
+++ b/erts/emulator/sys/common/erl_check_io.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -40,10 +40,14 @@ Eterm erts_check_io_info_kp(void *);
 Eterm erts_check_io_info_nkp(void *);
 int erts_check_io_max_files_kp(void);
 int erts_check_io_max_files_nkp(void);
+#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
+void erts_check_io_async_sig_interrupt_kp(void);
+void erts_check_io_async_sig_interrupt_nkp(void);
+#endif
 void erts_check_io_interrupt_kp(int);
 void erts_check_io_interrupt_nkp(int);
-void erts_check_io_interrupt_timed_kp(int, long);
-void erts_check_io_interrupt_timed_nkp(int, long);
+void erts_check_io_interrupt_timed_kp(int, erts_short_time_t);
+void erts_check_io_interrupt_timed_nkp(int, erts_short_time_t);
 void erts_check_io_kp(int);
 void erts_check_io_nkp(int);
 void erts_init_check_io_kp(void);
@@ -56,8 +60,11 @@ int erts_check_io_debug_nkp(void);
 Uint erts_check_io_size(void);
 Eterm erts_check_io_info(void *);
 int erts_check_io_max_files(void);
+#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
+void erts_check_io_async_sig_interrupt(void);
+#endif
 void erts_check_io_interrupt(int);
-void erts_check_io_interrupt_timed(int, long);
+void erts_check_io_interrupt_timed(int, erts_short_time_t);
 void erts_check_io(int);
 void erts_init_check_io(void);
 
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index 5dfd66bd7c..db2854fa40 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -35,14 +35,13 @@
 #include "global.h"
 #include "erl_threads.h"
 #include "erl_mtrace.h"
+#include "erl_time.h"
+#include "erl_alloc.h"
 #include "big.h"
+#include "erl_thr_progress.h"
 
 #if HAVE_ERTS_MSEG
 
-#if defined(USE_THREADS) && !defined(ERTS_SMP)
-#  define ERTS_THREADS_NO_SMP
-#endif
-
 #define SEGTYPE ERTS_MTRACE_SEGMENT_ID
 
 #ifndef HAVE_GETPAGESIZE
@@ -74,19 +73,24 @@
 
 static int atoms_initialized;
 
-static Uint cache_check_interval;
+typedef struct mem_kind_t MemKind;
+
+static void mseg_clear_cache(MemKind*);
 
-static void check_cache(void *unused);
-static void mseg_clear_cache(void);
-static int is_cache_check_scheduled;
-#ifdef ERTS_THREADS_NO_SMP
-static int is_cache_check_requested;
+#if HALFWORD_HEAP
+static int initialize_pmmap(void);
+static void *pmmap(size_t size);
+static int pmunmap(void *p, size_t size);
+static void *pmremap(void *old_address, size_t old_size,
+		     size_t new_size);
 #endif
 
 #if HAVE_MMAP
 /* Mmap ... */
 
 #define MMAP_PROT		(PROT_READ|PROT_WRITE)
+
+
 #ifdef MAP_ANON
 #  define MMAP_FLAGS		(MAP_ANON|MAP_PRIVATE)
 #  define MMAP_FD		(-1)
@@ -102,19 +106,36 @@ static int mmap_fd;
 #  define HAVE_MSEG_RECREATE 0
 #endif
 
+#if HALFWORD_HEAP
+#define CAN_PARTLY_DESTROY 0
+#else
 #define CAN_PARTLY_DESTROY 1
+#endif
 #else  /* #if HAVE_MMAP */
 #define CAN_PARTLY_DESTROY 0
 #error "Not supported"
 #endif /* #if HAVE_MMAP */
 
+#if defined(ERTS_MSEG_FAKE_SEGMENTS) && HALFWORD_HEAP
+#  warning "ERTS_MSEG_FAKE_SEGMENTS will only be used for high memory segments" 
+#endif 
 
 #if defined(ERTS_MSEG_FAKE_SEGMENTS)
 #undef CAN_PARTLY_DESTROY
 #define CAN_PARTLY_DESTROY 0
 #endif
 
-static const ErtsMsegOpt_t default_opt = ERTS_MSEG_DEFAULT_OPT_INITIALIZER;
+const ErtsMsegOpt_t erts_mseg_default_opt = {
+    1,			/* Use cache		     */
+    1,			/* Preserv data		     */
+    0,			/* Absolute shrink threshold */
+    0,			/* Relative shrink threshold */
+    0			/* Scheduler specific        */
+#if HALFWORD_HEAP
+    ,0                  /* need low memory */
+#endif
+};
+
 
 typedef struct cache_desc_t_ {
     void *seg;
@@ -128,11 +149,10 @@ typedef struct {
     Uint32 no;
 } CallCounter;
 
-static int is_init_done;
 static Uint page_size;
 static Uint page_shift;
 
-static struct {
+typedef struct {
     CallCounter alloc;
     CallCounter dealloc;
     CallCounter realloc;
@@ -143,407 +163,526 @@ static struct {
 #endif
     CallCounter clear_cache;
     CallCounter check_cache;
-} calls;
+} ErtsMsegCalls;
+
+typedef struct ErtsMsegAllctr_t_ ErtsMsegAllctr_t;
+
+struct mem_kind_t {
+    cache_desc_t cache_descs[MAX_CACHE_SIZE];
+    cache_desc_t *free_cache_descs;
+    cache_desc_t *cache;
+    cache_desc_t *cache_end;
+
+    Uint cache_size;
+    Uint min_cached_seg_size;
+    Uint max_cached_seg_size;
+    Uint cache_hits;
 
-static cache_desc_t cache_descs[MAX_CACHE_SIZE];
-static cache_desc_t *free_cache_descs;
-static cache_desc_t *cache;
-static cache_desc_t *cache_end;
-static Uint cache_hits;
-static Uint cache_size;
-static Uint min_cached_seg_size;
-static Uint max_cached_seg_size;
+    struct {
+	struct {
+	    Uint watermark;
+	    Uint no;
+	    Uint sz;
+	} current;
+	struct {
+	    Uint no;
+	    Uint sz;
+	} max;
+	struct {
+	    Uint no;
+	    Uint sz;
+	} max_ever;
+    } segments;
+
+    ErtsMsegAllctr_t *ma;
+    const char* name;
+    MemKind* next;
+};/*MemKind*/
+
+struct ErtsMsegAllctr_t_ {
+    int ix;
+
+    int is_init_done;
+    int is_thread_safe;
+    erts_mtx_t mtx;
+
+    int is_cache_check_scheduled;
+
+    MemKind* mk_list;
+
+#if HALFWORD_HEAP
+    MemKind low_mem;
+    MemKind hi_mem;
+#else
+    MemKind the_mem;
+#endif
 
-static Uint max_cache_size;
-static Uint abs_max_cache_bad_fit;
-static Uint rel_max_cache_bad_fit;
+    Uint max_cache_size;
+    Uint abs_max_cache_bad_fit;
+    Uint rel_max_cache_bad_fit;
+
+    ErtsMsegCalls calls;
 
 #if CAN_PARTLY_DESTROY
-static Uint min_seg_size;
+    Uint min_seg_size;
 #endif
 
-struct {
-    struct {
-	Uint watermark;
-	Uint no;
-	Uint sz;
-    } current;
-    struct {
-	Uint no;
-	Uint sz;
-    } max;
-    struct {
-	Uint no;
-	Uint sz;
-    } max_ever;
-} segments;
+};
+
+typedef union {
+    ErtsMsegAllctr_t mseg_alloc;
+    char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsMsegAllctr_t))];
+} ErtsAlgndMsegAllctr_t;
+
+static int no_mseg_allocators;
+static ErtsAlgndMsegAllctr_t *aligned_mseg_allctr;
+
+#ifdef ERTS_SMP
+
+#define ERTS_MSEG_ALLCTR_IX(IX) \
+  (&aligned_mseg_allctr[(IX)].mseg_alloc)
+
+#define ERTS_MSEG_ALLCTR_SS() \
+  ERTS_MSEG_ALLCTR_IX((int) erts_get_scheduler_id())
+
+#define ERTS_MSEG_ALLCTR_OPT(OPT) \
+  ((OPT)->sched_spec ? ERTS_MSEG_ALLCTR_SS() : ERTS_MSEG_ALLCTR_IX(0))
+
+#else
 
-#define ERTS_MSEG_ALLOC_STAT(SZ)					\
+#define ERTS_MSEG_ALLCTR_IX(IX) \
+  (&aligned_mseg_allctr[0].mseg_alloc)
+
+#define ERTS_MSEG_ALLCTR_SS() \
+  (&aligned_mseg_allctr[0].mseg_alloc)
+
+#define ERTS_MSEG_ALLCTR_OPT(OPT) \
+  (&aligned_mseg_allctr[0].mseg_alloc)
+
+#endif
+
+#define ERTS_MSEG_LOCK(MA)		\
+do {					\
+    if ((MA)->is_thread_safe)		\
+	erts_mtx_lock(&(MA)->mtx);	\
+} while (0)
+
+#define ERTS_MSEG_UNLOCK(MA)		\
+do {					\
+    if ((MA)->is_thread_safe)		\
+	erts_mtx_unlock(&(MA)->mtx);	\
+} while (0)
+
+#define ERTS_MSEG_ALLOC_STAT(C,SZ)					\
 do {									\
-    segments.current.no++;						\
-    if (segments.max.no < segments.current.no)				\
-	segments.max.no = segments.current.no;				\
-    if (segments.current.watermark < segments.current.no)		\
-	segments.current.watermark = segments.current.no;		\
-    segments.current.sz += (SZ);					\
-    if (segments.max.sz < segments.current.sz)				\
-	segments.max.sz = segments.current.sz;				\
+    C->segments.current.no++;						\
+    if (C->segments.max.no < C->segments.current.no)			\
+	C->segments.max.no = C->segments.current.no;			\
+    if (C->segments.current.watermark < C->segments.current.no)		\
+	C->segments.current.watermark = C->segments.current.no;		\
+    C->segments.current.sz += (SZ);					\
+    if (C->segments.max.sz < C->segments.current.sz)			\
+	C->segments.max.sz = C->segments.current.sz;			\
 } while (0)
 
-#define ERTS_MSEG_DEALLOC_STAT(SZ)					\
+#define ERTS_MSEG_DEALLOC_STAT(C,SZ)					\
 do {									\
-    ASSERT(segments.current.no > 0);					\
-    segments.current.no--;						\
-    ASSERT(segments.current.sz >= (SZ));				\
-    segments.current.sz -= (SZ);					\
+    ASSERT(C->segments.current.no > 0);					\
+    C->segments.current.no--;						\
+    ASSERT(C->segments.current.sz >= (SZ));				\
+    C->segments.current.sz -= (SZ);					\
 } while (0) 
 
-#define ERTS_MSEG_REALLOC_STAT(OSZ, NSZ)				\
+#define ERTS_MSEG_REALLOC_STAT(C,OSZ, NSZ)				\
 do {									\
-    ASSERT(segments.current.sz >= (OSZ));				\
-    segments.current.sz -= (OSZ);					\
-    segments.current.sz += (NSZ);					\
+    ASSERT(C->segments.current.sz >= (OSZ));				\
+    C->segments.current.sz -= (OSZ);					\
+    C->segments.current.sz += (NSZ);					\
 } while (0)
 
 #define ONE_GIGA (1000000000)
 
-#define ZERO_CC(CC) (calls.CC.no = 0, calls.CC.giga_no = 0)
+#define ZERO_CC(MA, CC) ((MA)->calls.CC.no = 0,				\
+			 (MA)->calls.CC.giga_no = 0)
 
-#define INC_CC(CC) (calls.CC.no == ONE_GIGA - 1				\
-		    ? (calls.CC.giga_no++, calls.CC.no = 0)		\
-		    : calls.CC.no++)
+#define INC_CC(MA, CC) ((MA)->calls.CC.no == ONE_GIGA - 1		\
+			? ((MA)->calls.CC.giga_no++,			\
+			   (MA)->calls.CC.no = 0)			\
+			: (MA)->calls.CC.no++)
 
-#define DEC_CC(CC) (calls.CC.no == 0					\
-		    ? (calls.CC.giga_no--,				\
-		       calls.CC.no = ONE_GIGA - 1)			\
-		    : calls.CC.no--)
+#define DEC_CC(MA, CC) ((MA)->calls.CC.no == 0				\
+			? ((MA)->calls.CC.giga_no--,			\
+			   (MA)->calls.CC.no = ONE_GIGA - 1)		\
+			: (MA)->calls.CC.no--)
 
 
-static erts_mtx_t mseg_mutex; /* Also needed when !USE_THREADS */
 static erts_mtx_t init_atoms_mutex; /* Also needed when !USE_THREADS */
 
-#ifdef USE_THREADS
-#ifdef ERTS_THREADS_NO_SMP
-static erts_tid_t main_tid;
-static int async_handle = -1;
-#endif
-
-static void thread_safe_init(void)
-{
-    erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms");
-    erts_mtx_init(&mseg_mutex, "mseg");
-#ifdef ERTS_THREADS_NO_SMP
-    main_tid = erts_thr_self();
-#endif
-}
-
-#endif
-
-static ErlTimer cache_check_timer;
 
 static ERTS_INLINE void
-schedule_cache_check(void)
+schedule_cache_check(ErtsMsegAllctr_t *ma)
 {
-    if (!is_cache_check_scheduled && is_init_done) {
-#ifdef ERTS_THREADS_NO_SMP
-	if (!erts_equal_tids(erts_thr_self(), main_tid)) {
-	    if (!is_cache_check_requested) {
-		is_cache_check_requested = 1;
-		sys_async_ready(async_handle);
-	    }
-	}
-	else
-#endif
-	{
-	    cache_check_timer.active = 0;
-	    erl_set_timer(&cache_check_timer,
-			  check_cache,
-			  NULL,
-			  NULL,
-			  cache_check_interval);
-	    is_cache_check_scheduled = 1;
-#ifdef ERTS_THREADS_NO_SMP
-	    is_cache_check_requested = 0;
-#endif
-	}
-    }
-}
-
-#ifdef ERTS_THREADS_NO_SMP
 
-static void
-check_schedule_cache_check(void)
-{
-    erts_mtx_lock(&mseg_mutex);
-    if (is_cache_check_requested
-	&& !is_cache_check_scheduled) {
-	schedule_cache_check();
+    if (!ma->is_cache_check_scheduled && ma->is_init_done) {
+	erts_set_aux_work_timeout(ma->ix,
+				  ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK,
+				  1);
+	ma->is_cache_check_scheduled = 1;
     }
-    erts_mtx_unlock(&mseg_mutex);    
-}
-
-#endif
-
-static void
-mseg_shutdown(void)
-{
-    erts_mtx_lock(&mseg_mutex);
-    mseg_clear_cache();
-    erts_mtx_unlock(&mseg_mutex);
 }
 
 static ERTS_INLINE void *
-mseg_create(Uint size)
+mseg_create(ErtsMsegAllctr_t *ma, MemKind* mk, Uint size)
 {
     void *seg;
 
     ASSERT(size % page_size == 0);
 
+#if HALFWORD_HEAP
+    if (mk == &ma->low_mem) {
+	seg = pmmap(size);
+	if ((unsigned long) seg & CHECK_POINTER_MASK) {
+	    erts_fprintf(stderr,"Pointer mask failure (0x%08lx)\n",(unsigned long) seg);
+	    return NULL;
+	}
+    }
+    else
+#endif
+    {
 #if defined(ERTS_MSEG_FAKE_SEGMENTS)
-    seg = erts_sys_alloc(ERTS_ALC_N_INVALID, NULL, size);
+	seg = erts_sys_alloc(ERTS_ALC_N_INVALID, NULL, size);
 #elif HAVE_MMAP
-    seg = (void *) mmap((void *) 0, (size_t) size,
-			MMAP_PROT, MMAP_FLAGS, MMAP_FD, 0);
-    if (seg == (void *) MAP_FAILED)
-	seg = NULL;
+	{
+	    seg = (void *) mmap((void *) 0, (size_t) size,
+				MMAP_PROT, MMAP_FLAGS, MMAP_FD, 0);
+	    if (seg == (void *) MAP_FAILED)
+		seg = NULL;
+	}
 #else
-#error "Missing mseg_create() implementation"
+# error "Missing mseg_create() implementation"
 #endif
+    }
 
-    INC_CC(create);
+    INC_CC(ma, create);
 
     return seg;
 }
 
 static ERTS_INLINE void
-mseg_destroy(void *seg, Uint size)
+mseg_destroy(ErtsMsegAllctr_t *ma, MemKind* mk, void *seg, Uint size)
 {
-#if defined(ERTS_MSEG_FAKE_SEGMENTS)
-    erts_sys_free(ERTS_ALC_N_INVALID, NULL, seg);
-#elif HAVE_MMAP
-
 #ifdef DEBUG
-    int res =
+    int res;
 #endif
 
-	munmap((void *) seg, size);
+#if HALFWORD_HEAP
+    if (mk == &ma->low_mem) {
+#ifdef DEBUG
+	res = 
+#endif
+	    pmunmap((void *) seg, size);
+    }
+    else
+#endif
+    {
+#ifdef ERTS_MSEG_FAKE_SEGMENTS
+	erts_sys_free(ERTS_ALC_N_INVALID, NULL, seg);
+#ifdef DEBUG
+	res = 0;
+#endif
+#elif HAVE_MMAP
+#ifdef DEBUG
+	res = 
+#endif
+	    munmap((void *) seg, size);
+#else
+# error "Missing mseg_destroy() implementation"
+#endif
+    }
 
     ASSERT(size % page_size == 0);
     ASSERT(res == 0);
-#else
-#error "Missing mseg_destroy() implementation"
-#endif
 
-    INC_CC(destroy);
+    INC_CC(ma, destroy);
 
 }
 
 #if HAVE_MSEG_RECREATE
 
 static ERTS_INLINE void *
-mseg_recreate(void *old_seg, Uint old_size, Uint new_size)
+mseg_recreate(ErtsMsegAllctr_t *ma, MemKind* mk, void *old_seg, Uint old_size, Uint new_size)
 {
     void *new_seg;
 
     ASSERT(old_size % page_size == 0);
     ASSERT(new_size % page_size == 0);
 
+#if HALFWORD_HEAP
+    if (mk == &ma->low_mem) {
+	new_seg = (void *) pmremap((void *) old_seg,
+				   (size_t) old_size,
+				   (size_t) new_size);
+    }
+    else
+#endif
+    {
 #if defined(ERTS_MSEG_FAKE_SEGMENTS)
-    new_seg = erts_sys_realloc(ERTS_ALC_N_INVALID, NULL, old_seg, new_size);
+	new_seg = erts_sys_realloc(ERTS_ALC_N_INVALID, NULL, old_seg, new_size);
 #elif HAVE_MREMAP
-    new_seg = (void *) mremap((void *) old_seg,
-			      (size_t) old_size,
-			      (size_t) new_size,
-			      MREMAP_MAYMOVE);
-    if (new_seg == (void *) MAP_FAILED)
-	new_seg = NULL;
+
+    #if defined(__NetBSD__)
+	new_seg = (void *) mremap((void *) old_seg,
+				  (size_t) old_size,
+				  NULL,
+				  (size_t) new_size,
+				  0);
+    #else
+	new_seg = (void *) mremap((void *) old_seg,
+				  (size_t) old_size,
+				  (size_t) new_size,
+				  MREMAP_MAYMOVE);
+    #endif
+	if (new_seg == (void *) MAP_FAILED)
+	    new_seg = NULL;
 #else
 #error "Missing mseg_recreate() implementation"
 #endif
+    }
 
-    INC_CC(recreate);
+    INC_CC(ma, recreate);
 
     return new_seg;
 }
 
 #endif /* #if HAVE_MSEG_RECREATE */
 
+#ifdef DEBUG
+#define ERTS_DBG_MA_CHK_THR_ACCESS(MA)					\
+do {									\
+    if ((MA)->is_thread_safe)						\
+	ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&(MA)->mtx)		\
+		       || erts_smp_thr_progress_is_blocking()		\
+		       || ERTS_IS_CRASH_DUMPING);			\
+    else								\
+	ERTS_LC_ASSERT((MA)->ix == (int) erts_get_scheduler_id()	\
+		       || erts_smp_thr_progress_is_blocking()		\
+		       || ERTS_IS_CRASH_DUMPING);			\
+} while (0)
+#define ERTS_DBG_MK_CHK_THR_ACCESS(MK) \
+  ERTS_DBG_MA_CHK_THR_ACCESS((MK)->ma)
+#else
+#define ERTS_DBG_MA_CHK_THR_ACCESS(MA)
+#define ERTS_DBG_MK_CHK_THR_ACCESS(MK)
+#endif
 
 static ERTS_INLINE cache_desc_t * 
-alloc_cd(void)
+alloc_cd(MemKind* mk)
 {    
-    cache_desc_t *cd = free_cache_descs;
-    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex));
+    cache_desc_t *cd = mk->free_cache_descs;
+    ERTS_DBG_MK_CHK_THR_ACCESS(mk);
     if (cd)
-	free_cache_descs = cd->next;
+	mk->free_cache_descs = cd->next;
     return cd;
 }
 
 static ERTS_INLINE void
-free_cd(cache_desc_t *cd)
+free_cd(MemKind* mk, cache_desc_t *cd)
 {
-    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex));
-    cd->next = free_cache_descs;
-    free_cache_descs = cd;
+    ERTS_DBG_MK_CHK_THR_ACCESS(mk);
+    cd->next = mk->free_cache_descs;
+    mk->free_cache_descs = cd;
 }
 
 
 static ERTS_INLINE void
-link_cd(cache_desc_t *cd)
+link_cd(MemKind* mk, cache_desc_t *cd)
 {
-    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex));
-    if (cache)
-	cache->prev = cd;
-    cd->next = cache;
+    ERTS_DBG_MK_CHK_THR_ACCESS(mk);
+    if (mk->cache)
+	mk->cache->prev = cd;
+    cd->next = mk->cache;
     cd->prev = NULL;
-    cache = cd;
+    mk->cache = cd;
 
-    if (!cache_end) {
+    if (!mk->cache_end) {
 	ASSERT(!cd->next);
-	cache_end = cd;
+	mk->cache_end = cd;
     }
 
-    cache_size++;
+    mk->cache_size++;
 }
 
+#if CAN_PARTLY_DESTROY
 static ERTS_INLINE void
-end_link_cd(cache_desc_t *cd)
+end_link_cd(MemKind* mk, cache_desc_t *cd)
 {
-    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex));
-    if (cache_end)
-	cache_end->next = cd;
+    ERTS_DBG_MK_CHK_THR_ACCESS(mk);
+    if (mk->cache_end)
+	mk->cache_end->next = cd;
     cd->next = NULL;
-    cd->prev = cache_end;
-    cache_end = cd;
+    cd->prev = mk->cache_end;
+    mk->cache_end = cd;
 
-    if (!cache) {
+    if (!mk->cache) {
 	ASSERT(!cd->prev);
-	cache = cd;
+	mk->cache = cd;
     }
 
-    cache_size++;
+    mk->cache_size++;
 }
+#endif
 
 static ERTS_INLINE void
-unlink_cd(cache_desc_t *cd)
+unlink_cd(MemKind* mk, cache_desc_t *cd)
 {
-    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex));
+    ERTS_DBG_MK_CHK_THR_ACCESS(mk);
     if (cd->next)
 	cd->next->prev = cd->prev;
     else
-	cache_end = cd->prev;
+	mk->cache_end = cd->prev;
     
     if (cd->prev)
 	cd->prev->next = cd->next;
     else
-	cache = cd->next;
-    ASSERT(cache_size > 0);
-    cache_size--;
+	mk->cache = cd->next;
+    ASSERT(mk->cache_size > 0);
+    mk->cache_size--;
 }
 
 static ERTS_INLINE void
-check_cache_limits(void)
+check_cache_limits(MemKind* mk)
 {
     cache_desc_t *cd;
-    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex));
-    max_cached_seg_size = 0;
-    min_cached_seg_size = ~((Uint) 0);
-    for (cd = cache; cd; cd = cd->next) {
-	if (cd->size < min_cached_seg_size)
-	    min_cached_seg_size = cd->size;
-	if (cd->size > max_cached_seg_size)
-	    max_cached_seg_size = cd->size;
+    ERTS_DBG_MK_CHK_THR_ACCESS(mk);
+    mk->max_cached_seg_size = 0;
+    mk->min_cached_seg_size = ~((Uint) 0);
+    for (cd = mk->cache; cd; cd = cd->next) {
+	if (cd->size < mk->min_cached_seg_size)
+	    mk->min_cached_seg_size = cd->size;
+	if (cd->size > mk->max_cached_seg_size)
+	    mk->max_cached_seg_size = cd->size;
     }
-
 }
 
 static ERTS_INLINE void
-adjust_cache_size(int force_check_limits)
+adjust_cache_size(MemKind* mk, int force_check_limits)
 {
     cache_desc_t *cd;
     int check_limits = force_check_limits;
-    Sint max_cached = ((Sint) segments.current.watermark
-		       - (Sint) segments.current.no);
-    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex));
-    while (((Sint) cache_size) > max_cached && ((Sint) cache_size) > 0) {
-	ASSERT(cache_end);
-	cd = cache_end;
+    Sint max_cached = ((Sint) mk->segments.current.watermark
+		       - (Sint) mk->segments.current.no);
+    ERTS_DBG_MK_CHK_THR_ACCESS(mk);
+    while (((Sint) mk->cache_size) > max_cached && ((Sint) mk->cache_size) > 0) {
+	ASSERT(mk->cache_end);
+	cd = mk->cache_end;
 	if (!check_limits &&
-	    !(min_cached_seg_size < cd->size
-	      && cd->size < max_cached_seg_size)) {
+	    !(mk->min_cached_seg_size < cd->size
+	      && cd->size < mk->max_cached_seg_size)) {
 	    check_limits = 1;
 	}
 	if (erts_mtrace_enabled)
 	    erts_mtrace_crr_free(SEGTYPE, SEGTYPE, cd->seg);
-	mseg_destroy(cd->seg, cd->size);
-	unlink_cd(cd);
-	free_cd(cd);
+	mseg_destroy(mk->ma, mk, cd->seg, cd->size);
+	unlink_cd(mk,cd);
+	free_cd(mk,cd);
     }
 
     if (check_limits)
-	check_cache_limits();
+	check_cache_limits(mk);
+}
 
+static Uint
+check_one_cache(MemKind* mk)
+{
+    if (mk->segments.current.watermark > mk->segments.current.no)
+	mk->segments.current.watermark--;
+    adjust_cache_size(mk, 0);
+
+    if (mk->cache_size)
+	schedule_cache_check(mk->ma);
+    return mk->cache_size;
 }
 
-static void
-check_cache(void *unused)
+static void do_cache_check(ErtsMsegAllctr_t *ma)
 {
-    erts_mtx_lock(&mseg_mutex);
+    int empty_cache = 1;
+    MemKind* mk;
 
-    is_cache_check_scheduled = 0;
+    ERTS_MSEG_LOCK(ma);
+
+    for (mk=ma->mk_list; mk; mk=mk->next) {
+	if (check_one_cache(mk))
+	    empty_cache = 0;
+    }
 
-    if (segments.current.watermark > segments.current.no)
-	segments.current.watermark--;
-    adjust_cache_size(0);
+    if (empty_cache) {
+	ma->is_cache_check_scheduled = 0;
+	erts_set_aux_work_timeout(ma->ix,
+				  ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK,
+				  0);
+    }
 
-    if (cache_size)
-	schedule_cache_check();
+    INC_CC(ma, check_cache);
 
-    INC_CC(check_cache);
+    ERTS_MSEG_UNLOCK(ma);
+}
 
-    erts_mtx_unlock(&mseg_mutex);
+void erts_mseg_cache_check(void)
+{
+    do_cache_check(ERTS_MSEG_ALLCTR_SS());
 }
 
 static void
-mseg_clear_cache(void)
+mseg_clear_cache(MemKind* mk)
 {
-    segments.current.watermark = 0;
+    mk->segments.current.watermark = 0;
 
-    adjust_cache_size(1);
+    adjust_cache_size(mk, 1);
 
-    ASSERT(!cache);
-    ASSERT(!cache_end);
-    ASSERT(!cache_size);
+    ASSERT(!mk->cache);
+    ASSERT(!mk->cache_end);
+    ASSERT(!mk->cache_size);
 
-    segments.current.watermark = segments.current.no;
+    mk->segments.current.watermark = mk->segments.current.no;
 
-    INC_CC(clear_cache);
+    INC_CC(mk->ma, clear_cache);
 }
 
-static void *
-mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt)
+static ERTS_INLINE MemKind* memkind(ErtsMsegAllctr_t *ma,
+				    const ErtsMsegOpt_t *opt)
 {
+#if HALFWORD_HEAP
+    return opt->low_mem ? &ma->low_mem : &ma->hi_mem;
+#else
+    return &ma->the_mem;
+#endif
+}
 
+static void *
+mseg_alloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, Uint *size_p,
+	   const ErtsMsegOpt_t *opt)
+{
     Uint max, min, diff_size, size;
     cache_desc_t *cd, *cand_cd;
     void *seg;
+    MemKind* mk = memkind(ma, opt);
 
-    INC_CC(alloc);
+    INC_CC(ma, alloc);
 
     size = PAGE_CEILING(*size_p);
 
 #if CAN_PARTLY_DESTROY
-    if (size < min_seg_size)	
-	min_seg_size = size;
+    if (size < ma->min_seg_size)	
+	ma->min_seg_size = size;
 #endif
 
     if (!opt->cache) {
     create_seg:
-	adjust_cache_size(0);
-	seg = mseg_create(size);
+	adjust_cache_size(mk,0);
+	seg = mseg_create(ma, mk, size);
 	if (!seg) {
-	    mseg_clear_cache();
-	    seg = mseg_create(size);
+	    mseg_clear_cache(mk);
+	    seg = mseg_create(ma, mk, size);
 	    if (!seg)
 		size = 0;
 	}
@@ -552,22 +691,22 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt)
 	if (seg) {
 	    if (erts_mtrace_enabled)
 		erts_mtrace_crr_alloc(seg, atype, ERTS_MTRACE_SEGMENT_ID, size);
-	    ERTS_MSEG_ALLOC_STAT(size);
+	    ERTS_MSEG_ALLOC_STAT(mk,size);
 	}
 	return seg;
     }
 
-    if (size > max_cached_seg_size)
+    if (size > mk->max_cached_seg_size)
 	goto create_seg;
 
-    if (size < min_cached_seg_size) {
+    if (size < mk->min_cached_seg_size) {
 
-	diff_size = min_cached_seg_size - size;
+	diff_size = mk->min_cached_seg_size - size;
 
-	if (diff_size > abs_max_cache_bad_fit)
+	if (diff_size > ma->abs_max_cache_bad_fit)
 	    goto create_seg;
 
-	if (100*PAGES(diff_size) > rel_max_cache_bad_fit*PAGES(size))
+	if (100*PAGES(diff_size) > ma->rel_max_cache_bad_fit*PAGES(size))
 	    goto create_seg;
 
     }
@@ -576,7 +715,7 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt)
     min = ~((Uint) 0);
     cand_cd = NULL;
 
-    for (cd = cache; cd; cd = cd->next) {
+    for (cd = mk->cache; cd; cd = cd->next) {
 	if (cd->size >= size) {
 	    if (!cand_cd) {
 		cand_cd = cd;
@@ -597,30 +736,30 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt)
 	    min = cd->size;
     }
 
-    min_cached_seg_size = min;
-    max_cached_seg_size = max;
+    mk->min_cached_seg_size = min;
+    mk->max_cached_seg_size = max;
 
     if (!cand_cd)
 	goto create_seg;
 
     diff_size = cand_cd->size - size;
 
-    if (diff_size > abs_max_cache_bad_fit
-	|| 100*PAGES(diff_size) > rel_max_cache_bad_fit*PAGES(size)) {
-	if (max_cached_seg_size < cand_cd->size)	
-	    max_cached_seg_size = cand_cd->size;
-	if (min_cached_seg_size > cand_cd->size)
-	    min_cached_seg_size = cand_cd->size;
+    if (diff_size > ma->abs_max_cache_bad_fit
+	|| 100*PAGES(diff_size) > ma->rel_max_cache_bad_fit*PAGES(size)) {
+	if (mk->max_cached_seg_size < cand_cd->size)
+	    mk->max_cached_seg_size = cand_cd->size;
+	if (mk->min_cached_seg_size > cand_cd->size)
+	    mk->min_cached_seg_size = cand_cd->size;
 	goto create_seg;
     }
 
-    cache_hits++;
+    mk->cache_hits++;
 
     size = cand_cd->size;
     seg = cand_cd->seg;
 
-    unlink_cd(cand_cd);
-    free_cd(cand_cd);
+    unlink_cd(mk,cand_cd);
+    free_cd(mk,cand_cd);
 
     *size_p = size;
 
@@ -630,50 +769,52 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt)
     }
 
     if (seg)
-	ERTS_MSEG_ALLOC_STAT(size);
+	ERTS_MSEG_ALLOC_STAT(mk,size);
+
     return seg;
 }
 
 
 static void
-mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size,
+mseg_dealloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, Uint size,
 	     const ErtsMsegOpt_t *opt)
 {
+    MemKind* mk = memkind(ma, opt);
     cache_desc_t *cd;
 
-    ERTS_MSEG_DEALLOC_STAT(size);
+    ERTS_MSEG_DEALLOC_STAT(mk,size);
 
-    if (!opt->cache || max_cache_size == 0) {
+    if (!opt->cache || ma->max_cache_size == 0) {
 	if (erts_mtrace_enabled)
 	    erts_mtrace_crr_free(atype, SEGTYPE, seg);
-	mseg_destroy(seg, size);
+	mseg_destroy(ma, mk, seg, size);
     }
     else {
 	int check_limits = 0;
 	
-	if (size < min_cached_seg_size)
-	    min_cached_seg_size = size;
-	if (size > max_cached_seg_size)
-	    max_cached_seg_size = size;
-
-	if (!free_cache_descs) {
-	    cd = cache_end;
-	    if (!(min_cached_seg_size < cd->size
-		  && cd->size < max_cached_seg_size)) {
+	if (size < mk->min_cached_seg_size)
+	    mk->min_cached_seg_size = size;
+	if (size > mk->max_cached_seg_size)
+	    mk->max_cached_seg_size = size;
+
+	if (!mk->free_cache_descs) {
+	    cd = mk->cache_end;
+	    if (!(mk->min_cached_seg_size < cd->size
+		  && cd->size < mk->max_cached_seg_size)) {
 		check_limits = 1;
 	    }
 	    if (erts_mtrace_enabled)
 		erts_mtrace_crr_free(SEGTYPE, SEGTYPE, cd->seg);
-	    mseg_destroy(cd->seg, cd->size);
-	    unlink_cd(cd);
-	    free_cd(cd);
+	    mseg_destroy(ma, mk, cd->seg, cd->size);
+	    unlink_cd(mk,cd);
+	    free_cd(mk,cd);
 	}
 
-	cd = alloc_cd();
+	cd = alloc_cd(mk);
 	ASSERT(cd);
 	cd->seg = seg;
 	cd->size = size;
-	link_cd(cd);
+	link_cd(mk,cd);
 
 	if (erts_mtrace_enabled) {
 	    erts_mtrace_crr_free(atype, SEGTYPE, seg);
@@ -683,34 +824,36 @@ mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size,
 	/* ASSERT(segments.current.watermark >= segments.current.no + cache_size); */
 
 	if (check_limits)
-	    check_cache_limits();
+	    check_cache_limits(mk);
 
-	schedule_cache_check();
+	schedule_cache_check(ma);
 
     }
 
-    INC_CC(dealloc);
+    INC_CC(ma, dealloc);
 }
 
 static void *
-mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p,
-	     const ErtsMsegOpt_t *opt)
+mseg_realloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg,
+	     Uint old_size, Uint *new_size_p, const ErtsMsegOpt_t *opt)
 {
+    MemKind* mk;
     void *new_seg;
     Uint new_size;
 
     if (!seg || !old_size) {
-	new_seg = mseg_alloc(atype, new_size_p, opt);
-	DEC_CC(alloc);
+	new_seg = mseg_alloc(ma, atype, new_size_p, opt);
+	DEC_CC(ma, alloc);
 	return new_seg;
     }
 
     if (!(*new_size_p)) {
-	mseg_dealloc(atype, seg, old_size, opt);
-	DEC_CC(dealloc);
+	mseg_dealloc(ma, atype, seg, old_size, opt);
+	DEC_CC(ma, dealloc);
 	return NULL;
     }
 
+    mk = memkind(ma, opt);
     new_seg = seg;
     new_size = PAGE_CEILING(*new_size_p);
 
@@ -720,8 +863,8 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p,
 	Uint shrink_sz = old_size - new_size;
 
 #if CAN_PARTLY_DESTROY
-	if (new_size < min_seg_size)	
-	    min_seg_size = new_size;
+	if (new_size < ma->min_seg_size)	
+	    ma->min_seg_size = new_size;
 #endif
 
 	if (shrink_sz < opt->abs_shrink_th
@@ -732,16 +875,16 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p,
 
 #if CAN_PARTLY_DESTROY
 
-	    if (shrink_sz > min_seg_size
-		&& free_cache_descs
+	    if (shrink_sz > ma->min_seg_size
+		&& mk->free_cache_descs
 		&& opt->cache) {
 		cache_desc_t *cd;
 
-		cd = alloc_cd();
+		cd = alloc_cd(mk);
 		ASSERT(cd);
 		cd->seg = ((char *) seg) + new_size;
 		cd->size = shrink_sz;
-		end_link_cd(cd);
+		end_link_cd(mk,cd);
 
 		if (erts_mtrace_enabled) {
 		    erts_mtrace_crr_realloc(new_seg,
@@ -751,7 +894,7 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p,
 					    new_size);
 		    erts_mtrace_crr_alloc(cd->seg, SEGTYPE, SEGTYPE, cd->size);
 		}
-		schedule_cache_check();
+		schedule_cache_check(ma);
 	    }
 	    else {
 		if (erts_mtrace_enabled)
@@ -760,7 +903,7 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p,
 					    SEGTYPE,
 					    seg,
 					    new_size);
-		mseg_destroy(((char *) seg) + new_size, shrink_sz);
+		mseg_destroy(ma, mk, ((char *) seg) + new_size, shrink_sz);
 	    }
 
 #elif HAVE_MSEG_RECREATE
@@ -769,14 +912,14 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p,
 
 #else
 
-	    new_seg = mseg_alloc(atype, &new_size, opt);
+	    new_seg = mseg_alloc(ma, atype, &new_size, opt);
 	    if (!new_seg)
 		new_size = old_size;
 	    else {
 		sys_memcpy(((char *) new_seg),
 			   ((char *) seg),
 			   MIN(new_size, old_size));
-		mseg_dealloc(atype, seg, old_size, opt);
+		mseg_dealloc(ma, atype, seg, old_size, opt);
 	    }
 
 #endif
@@ -786,38 +929,38 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p,
     else {
 
 	if (!opt->preserv) {
-	    mseg_dealloc(atype, seg, old_size, opt);
-	    new_seg = mseg_alloc(atype, &new_size, opt);
+	    mseg_dealloc(ma, atype, seg, old_size, opt);
+	    new_seg = mseg_alloc(ma, atype, &new_size, opt);
 	}
 	else {
 #if HAVE_MSEG_RECREATE
 #if !CAN_PARTLY_DESTROY
 	do_recreate:
 #endif
-	    new_seg = mseg_recreate((void *) seg, old_size, new_size);
+	    new_seg = mseg_recreate(ma, mk, (void *) seg, old_size, new_size);
 	    if (erts_mtrace_enabled)
 		erts_mtrace_crr_realloc(new_seg, atype, SEGTYPE, seg, new_size);
 	    if (!new_seg)
 		new_size = old_size;
 #else
-	    new_seg = mseg_alloc(atype, &new_size, opt);
+	    new_seg = mseg_alloc(ma, atype, &new_size, opt);
 	    if (!new_seg)
 		new_size = old_size;
 	    else {
 		sys_memcpy(((char *) new_seg),
 			   ((char *) seg),
 			   MIN(new_size, old_size));
-		mseg_dealloc(atype, seg, old_size, opt);
+		mseg_dealloc(ma, atype, seg, old_size, opt);
 	    }
 #endif
 	}
     }
 
-    INC_CC(realloc);
+    INC_CC(ma, realloc);
 
     *new_size_p = new_size;
 
-    ERTS_MSEG_REALLOC_STAT(old_size, new_size);
+    ERTS_MSEG_REALLOC_STAT(mk, old_size, new_size);
 
     return new_seg;
 }
@@ -831,8 +974,9 @@ static struct {
     Eterm amcbf;
     Eterm rmcbf;
     Eterm mcs;
-    Eterm cci;
 
+    Eterm memkind;
+    Eterm name;
     Eterm status;
     Eterm cached_segments;
     Eterm cache_hits;
@@ -865,13 +1009,13 @@ static void ERTS_INLINE atom_init(Eterm *atom, char *name)
 #define AM_INIT(AM) atom_init(&am.AM, #AM)
 
 static void
-init_atoms(void)
+init_atoms(ErtsMsegAllctr_t *ma)
 {
 #ifdef DEBUG
     Eterm *atom;
 #endif
 
-    erts_mtx_unlock(&mseg_mutex);
+    ERTS_MSEG_UNLOCK(ma);
     erts_mtx_lock(&init_atoms_mutex);
 
     if (!atoms_initialized) {
@@ -882,12 +1026,13 @@ init_atoms(void)
 #endif
 
 	AM_INIT(version);
+	AM_INIT(memkind);
+	AM_INIT(name);
 
 	AM_INIT(options);
 	AM_INIT(amcbf);
 	AM_INIT(rmcbf);
 	AM_INIT(mcs);
-	AM_INIT(cci);
 
 	AM_INIT(status);
 	AM_INIT(cached_segments);
@@ -915,7 +1060,7 @@ init_atoms(void)
 #endif
     }
 
-    erts_mtx_lock(&mseg_mutex);
+    ERTS_MSEG_LOCK(ma);
     atoms_initialized = 1;
     erts_mtx_unlock(&init_atoms_mutex);
 }
@@ -972,7 +1117,8 @@ add_4tup(Uint **hpp, Uint *szp, Eterm *lp,
 }
 
 static Eterm
-info_options(char *prefix,
+info_options(ErtsMsegAllctr_t *ma,
+	     char *prefix,
 	     int *print_to_p,
 	     void *print_to_arg,
 	     Uint **hpp,
@@ -983,30 +1129,26 @@ info_options(char *prefix,
     if (print_to_p) {
 	int to = *print_to_p;
 	void *arg = print_to_arg;
-	erts_print(to, arg, "%samcbf: %bpu\n", prefix, abs_max_cache_bad_fit);
-	erts_print(to, arg, "%srmcbf: %bpu\n", prefix, rel_max_cache_bad_fit);
-	erts_print(to, arg, "%smcs: %bpu\n", prefix, max_cache_size);
-	erts_print(to, arg, "%scci: %bpu\n", prefix, cache_check_interval);
+	erts_print(to, arg, "%samcbf: %beu\n", prefix, ma->abs_max_cache_bad_fit);
+	erts_print(to, arg, "%srmcbf: %beu\n", prefix, ma->rel_max_cache_bad_fit);
+	erts_print(to, arg, "%smcs: %beu\n", prefix, ma->max_cache_size);
     }
 
     if (hpp || szp) {
 
 	if (!atoms_initialized)
-	    init_atoms();
+	    init_atoms(ma);
 
 	res = NIL;
 	add_2tup(hpp, szp, &res,
-		 am.cci,
-		 bld_uint(hpp, szp, cache_check_interval));
-	add_2tup(hpp, szp, &res,
 		 am.mcs,
-		 bld_uint(hpp, szp, max_cache_size));
+		 bld_uint(hpp, szp, ma->max_cache_size));
 	add_2tup(hpp, szp, &res,
 		 am.rmcbf,
-		 bld_uint(hpp, szp, rel_max_cache_bad_fit));
+		 bld_uint(hpp, szp, ma->rel_max_cache_bad_fit));
 	add_2tup(hpp, szp, &res,
 		 am.amcbf,
-		 bld_uint(hpp, szp, abs_max_cache_bad_fit));
+		 bld_uint(hpp, szp, ma->abs_max_cache_bad_fit));
 
     }
 
@@ -1014,18 +1156,18 @@ info_options(char *prefix,
 }
 
 static Eterm
-info_calls(int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp)
+info_calls(ErtsMsegAllctr_t *ma, int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp)
 {
     Eterm res = THE_NON_VALUE;
 
     if (print_to_p) {
 
-#define PRINT_CC(TO, TOA, CC)						\
-    if (calls.CC.giga_no == 0)						\
-	erts_print(TO, TOA, "mseg_%s calls: %bpu\n", #CC, calls.CC.no);	\
-    else								\
-	erts_print(TO, TOA, "mseg_%s calls: %bpu%09bpu\n", #CC,		\
-		   calls.CC.giga_no, calls.CC.no)
+#define PRINT_CC(TO, TOA, CC)							\
+    if (ma->calls.CC.giga_no == 0)						\
+	erts_print(TO, TOA, "mseg_%s calls: %b32u\n", #CC, ma->calls.CC.no);	\
+    else									\
+	erts_print(TO, TOA, "mseg_%s calls: %b32u%09b32u\n", #CC,		\
+		   ma->calls.CC.giga_no, ma->calls.CC.no)
 
 	int to = *print_to_p;
 	void *arg = print_to_arg;
@@ -1051,108 +1193,131 @@ info_calls(int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp)
 
 	add_3tup(hpp, szp, &res,
 		 am.mseg_check_cache,
-		 bld_unstable_uint(hpp, szp, calls.check_cache.giga_no),
-		 bld_unstable_uint(hpp, szp, calls.check_cache.no));
+		 bld_unstable_uint(hpp, szp, ma->calls.check_cache.giga_no),
+		 bld_unstable_uint(hpp, szp, ma->calls.check_cache.no));
 	add_3tup(hpp, szp, &res,
 		 am.mseg_clear_cache,
-		 bld_unstable_uint(hpp, szp, calls.clear_cache.giga_no),
-		 bld_unstable_uint(hpp, szp, calls.clear_cache.no));
+		 bld_unstable_uint(hpp, szp, ma->calls.clear_cache.giga_no),
+		 bld_unstable_uint(hpp, szp, ma->calls.clear_cache.no));
 
 #if HAVE_MSEG_RECREATE
 	add_3tup(hpp, szp, &res,
 		 am.mseg_recreate,
-		 bld_unstable_uint(hpp, szp, calls.recreate.giga_no),
-		 bld_unstable_uint(hpp, szp, calls.recreate.no));
+		 bld_unstable_uint(hpp, szp, ma->calls.recreate.giga_no),
+		 bld_unstable_uint(hpp, szp, ma->calls.recreate.no));
 #endif
 	add_3tup(hpp, szp, &res,
 		 am.mseg_destroy,
-		 bld_unstable_uint(hpp, szp, calls.destroy.giga_no),
-		 bld_unstable_uint(hpp, szp, calls.destroy.no));
+		 bld_unstable_uint(hpp, szp, ma->calls.destroy.giga_no),
+		 bld_unstable_uint(hpp, szp, ma->calls.destroy.no));
 	add_3tup(hpp, szp, &res,
 		 am.mseg_create,
-		 bld_unstable_uint(hpp, szp, calls.create.giga_no),
-		 bld_unstable_uint(hpp, szp, calls.create.no));
+		 bld_unstable_uint(hpp, szp, ma->calls.create.giga_no),
+		 bld_unstable_uint(hpp, szp, ma->calls.create.no));
 
 
 	add_3tup(hpp, szp, &res,
 		 am.mseg_realloc,
-		 bld_unstable_uint(hpp, szp, calls.realloc.giga_no),
-		 bld_unstable_uint(hpp, szp, calls.realloc.no));
+		 bld_unstable_uint(hpp, szp, ma->calls.realloc.giga_no),
+		 bld_unstable_uint(hpp, szp, ma->calls.realloc.no));
 	add_3tup(hpp, szp, &res,
 		 am.mseg_dealloc,
-		 bld_unstable_uint(hpp, szp, calls.dealloc.giga_no),
-		 bld_unstable_uint(hpp, szp, calls.dealloc.no));
+		 bld_unstable_uint(hpp, szp, ma->calls.dealloc.giga_no),
+		 bld_unstable_uint(hpp, szp, ma->calls.dealloc.no));
 	add_3tup(hpp, szp, &res,
 		 am.mseg_alloc,
-		 bld_unstable_uint(hpp, szp, calls.alloc.giga_no),
-		 bld_unstable_uint(hpp, szp, calls.alloc.no));
+		 bld_unstable_uint(hpp, szp, ma->calls.alloc.giga_no),
+		 bld_unstable_uint(hpp, szp, ma->calls.alloc.no));
     }
 
     return res;
 }
 
 static Eterm
-info_status(int *print_to_p,
-	    void *print_to_arg,
-	    int begin_new_max_period,
-	    Uint **hpp,
-	    Uint *szp)
+info_status(ErtsMsegAllctr_t *ma, MemKind* mk, int *print_to_p, void *print_to_arg,
+	    int begin_new_max_period, Uint **hpp, Uint *szp)
 {
     Eterm res = THE_NON_VALUE;
     
-    if (segments.max_ever.no < segments.max.no)
-	segments.max_ever.no = segments.max.no;
-    if (segments.max_ever.sz < segments.max.sz)
-	segments.max_ever.sz = segments.max.sz;
+    if (mk->segments.max_ever.no < mk->segments.max.no)
+	mk->segments.max_ever.no = mk->segments.max.no;
+    if (mk->segments.max_ever.sz < mk->segments.max.sz)
+	mk->segments.max_ever.sz = mk->segments.max.sz;
 
     if (print_to_p) {
 	int to = *print_to_p;
 	void *arg = print_to_arg;
 
-	erts_print(to, arg, "cached_segments: %bpu\n", cache_size);
-	erts_print(to, arg, "cache_hits: %bpu\n", cache_hits);
-	erts_print(to, arg, "segments: %bpu %bpu %bpu\n",
-		   segments.current.no, segments.max.no, segments.max_ever.no);
-	erts_print(to, arg, "segments_size: %bpu %bpu %bpu\n",
-		   segments.current.sz, segments.max.sz, segments.max_ever.sz);
-	erts_print(to, arg, "segments_watermark: %bpu\n",
-		   segments.current.watermark);
+	erts_print(to, arg, "cached_segments: %beu\n", mk->cache_size);
+	erts_print(to, arg, "cache_hits: %beu\n", mk->cache_hits);
+	erts_print(to, arg, "segments: %beu %beu %beu\n",
+		   mk->segments.current.no, mk->segments.max.no, mk->segments.max_ever.no);
+	erts_print(to, arg, "segments_size: %beu %beu %beu\n",
+		   mk->segments.current.sz, mk->segments.max.sz, mk->segments.max_ever.sz);
+	erts_print(to, arg, "segments_watermark: %beu\n",
+		   mk->segments.current.watermark);
     }
 
     if (hpp || szp) {
 	res = NIL;
 	add_2tup(hpp, szp, &res,
 		 am.segments_watermark,
-		 bld_unstable_uint(hpp, szp, segments.current.watermark));
+		 bld_unstable_uint(hpp, szp, mk->segments.current.watermark));
 	add_4tup(hpp, szp, &res,
 		 am.segments_size,
-		 bld_unstable_uint(hpp, szp, segments.current.sz),
-		 bld_unstable_uint(hpp, szp, segments.max.sz),
-		 bld_unstable_uint(hpp, szp, segments.max_ever.sz));
+		 bld_unstable_uint(hpp, szp, mk->segments.current.sz),
+		 bld_unstable_uint(hpp, szp, mk->segments.max.sz),
+		 bld_unstable_uint(hpp, szp, mk->segments.max_ever.sz));
 	add_4tup(hpp, szp, &res,
 		 am.segments,
-		 bld_unstable_uint(hpp, szp, segments.current.no),
-		 bld_unstable_uint(hpp, szp, segments.max.no),
-		 bld_unstable_uint(hpp, szp, segments.max_ever.no));
+		 bld_unstable_uint(hpp, szp, mk->segments.current.no),
+		 bld_unstable_uint(hpp, szp, mk->segments.max.no),
+		 bld_unstable_uint(hpp, szp, mk->segments.max_ever.no));
 	add_2tup(hpp, szp, &res,
 		 am.cache_hits,
-		 bld_unstable_uint(hpp, szp, cache_hits));
+		 bld_unstable_uint(hpp, szp, mk->cache_hits));
 	add_2tup(hpp, szp, &res,
 		 am.cached_segments,
-		 bld_unstable_uint(hpp, szp, cache_size));
+		 bld_unstable_uint(hpp, szp, mk->cache_size));
 
     }
 
     if (begin_new_max_period) {
-	segments.max.no = segments.current.no;
-	segments.max.sz = segments.current.sz;
+	mk->segments.max.no = mk->segments.current.no;
+	mk->segments.max.sz = mk->segments.current.sz;
+    }
+
+    return res;
+}
+
+static Eterm info_memkind(ErtsMsegAllctr_t *ma, MemKind* mk, int *print_to_p, void *print_to_arg,
+			  int begin_max_per, Uint **hpp, Uint *szp)
+{
+    Eterm res = THE_NON_VALUE;
+    Eterm atoms[3];
+    Eterm values[3];
+
+    if (print_to_p) {
+	erts_print(*print_to_p, print_to_arg, "memory kind: %s\n", mk->name);
     }
+    if (hpp || szp) {
+	atoms[0] = am.name;
+	atoms[1] = am.status;
+	atoms[2] = am.calls;
+	values[0] = erts_bld_string(hpp, szp, mk->name);
+    }
+    values[1] = info_status(ma, mk, print_to_p, print_to_arg, begin_max_per, hpp, szp);
+    values[2] = info_calls(ma, print_to_p, print_to_arg, hpp, szp);
+
+    if (hpp || szp)
+	res = bld_2tup_list(hpp, szp, 3, atoms, values);
 
     return res;
 }
 
+
 static Eterm
-info_version(int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp)
+info_version(ErtsMsegAllctr_t *ma, int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp)
 {
     Eterm res = THE_NON_VALUE;
 
@@ -1173,53 +1338,64 @@ info_version(int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp)
 \*                                                                           */
 
 Eterm
-erts_mseg_info_options(int *print_to_p, void *print_to_arg,
+erts_mseg_info_options(int ix,
+		       int *print_to_p, void *print_to_arg,
 		       Uint **hpp, Uint *szp)
 {
+    ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_IX(ix);
     Eterm res;
 
-    erts_mtx_lock(&mseg_mutex);
+    ERTS_MSEG_LOCK(ma);
+
+    ERTS_DBG_MA_CHK_THR_ACCESS(ma);
 
-    res = info_options("option ", print_to_p, print_to_arg, hpp, szp);
+    res = info_options(ma, "option ", print_to_p, print_to_arg, hpp, szp);
 
-    erts_mtx_unlock(&mseg_mutex);
+    ERTS_MSEG_UNLOCK(ma);
 
     return res;
 }
 
 Eterm
-erts_mseg_info(int *print_to_p,
+erts_mseg_info(int ix,
+	       int *print_to_p,
 	       void *print_to_arg,
 	       int begin_max_per,
 	       Uint **hpp,
 	       Uint *szp)
 {
+    ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_IX(ix);
     Eterm res = THE_NON_VALUE;
     Eterm atoms[4];
     Eterm values[4];
+    Uint n = 0;
 
-    erts_mtx_lock(&mseg_mutex);
+    ERTS_MSEG_LOCK(ma);
+
+    ERTS_DBG_MA_CHK_THR_ACCESS(ma);
 
     if (hpp || szp) {
 	
 	if (!atoms_initialized)
-	    init_atoms();
+	    init_atoms(ma);
 
 	atoms[0] = am.version;
 	atoms[1] = am.options;
-	atoms[2] = am.status;
-	atoms[3] = am.calls;
+	atoms[2] = am.memkind;
+	atoms[3] = am.memkind;
     }
-
-    values[0] = info_version(print_to_p, print_to_arg, hpp, szp);
-    values[1] = info_options("option ", print_to_p, print_to_arg, hpp, szp);
-    values[2] = info_status(print_to_p, print_to_arg, begin_max_per, hpp, szp);
-    values[3] = info_calls(print_to_p, print_to_arg, hpp, szp);
-
+    values[n++] = info_version(ma, print_to_p, print_to_arg, hpp, szp);
+    values[n++] = info_options(ma, "option ", print_to_p, print_to_arg, hpp, szp);
+#if HALFWORD_HEAP
+    values[n++] = info_memkind(ma, &ma->low_mem, print_to_p, print_to_arg, begin_max_per, hpp, szp);
+    values[n++] = info_memkind(ma, &ma->hi_mem, print_to_p, print_to_arg, begin_max_per, hpp, szp);
+#else
+    values[n++] = info_memkind(ma, &ma->the_mem, print_to_p, print_to_arg, begin_max_per, hpp, szp);
+#endif
     if (hpp || szp)
-	res = bld_2tup_list(hpp, szp, 4, atoms, values);
+	res = bld_2tup_list(hpp, szp, n, atoms, values);
 
-    erts_mtx_unlock(&mseg_mutex);
+    ERTS_MSEG_UNLOCK(ma);
 
     return res;
 }
@@ -1227,67 +1403,93 @@ erts_mseg_info(int *print_to_p,
 void *
 erts_mseg_alloc_opt(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt)
 {
+    ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_OPT(opt);
     void *seg;
-    erts_mtx_lock(&mseg_mutex);
-    seg = mseg_alloc(atype, size_p, opt);
-    erts_mtx_unlock(&mseg_mutex);
+    ERTS_MSEG_LOCK(ma);
+    ERTS_DBG_MA_CHK_THR_ACCESS(ma);
+    seg = mseg_alloc(ma, atype, size_p, opt);
+    ERTS_MSEG_UNLOCK(ma);
     return seg;
 }
 
 void *
 erts_mseg_alloc(ErtsAlcType_t atype, Uint *size_p)
 {
-    return erts_mseg_alloc_opt(atype, size_p, &default_opt);
+    return erts_mseg_alloc_opt(atype, size_p, &erts_mseg_default_opt);
 }
 
 void
-erts_mseg_dealloc_opt(ErtsAlcType_t atype, void *seg, Uint size,
-		      const ErtsMsegOpt_t *opt)
+erts_mseg_dealloc_opt(ErtsAlcType_t atype, void *seg,
+		      Uint size, const ErtsMsegOpt_t *opt)
 {
-    erts_mtx_lock(&mseg_mutex);
-    mseg_dealloc(atype, seg, size, opt);
-    erts_mtx_unlock(&mseg_mutex);
+    ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_OPT(opt);
+    ERTS_MSEG_LOCK(ma);
+    ERTS_DBG_MA_CHK_THR_ACCESS(ma);
+    mseg_dealloc(ma, atype, seg, size, opt);
+    ERTS_MSEG_UNLOCK(ma);
 }
 
 void
 erts_mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size)
 {
-    erts_mseg_dealloc_opt(atype, seg, size, &default_opt);
+    erts_mseg_dealloc_opt(atype, seg, size, &erts_mseg_default_opt);
 }
 
 void *
-erts_mseg_realloc_opt(ErtsAlcType_t atype, void *seg, Uint old_size,
-		      Uint *new_size_p, const ErtsMsegOpt_t *opt)
+erts_mseg_realloc_opt(ErtsAlcType_t atype, void *seg,
+		      Uint old_size, Uint *new_size_p,
+		      const ErtsMsegOpt_t *opt)
 {
+    ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_OPT(opt);
     void *new_seg;
-    erts_mtx_lock(&mseg_mutex);
-    new_seg = mseg_realloc(atype, seg, old_size, new_size_p, opt);
-    erts_mtx_unlock(&mseg_mutex);
+    ERTS_MSEG_LOCK(ma);
+    ERTS_DBG_MA_CHK_THR_ACCESS(ma);
+    new_seg = mseg_realloc(ma, atype, seg, old_size, new_size_p, opt);
+    ERTS_MSEG_UNLOCK(ma);
     return new_seg;
 }
 
 void *
-erts_mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size,
-		  Uint *new_size_p)
+erts_mseg_realloc(ErtsAlcType_t atype, void *seg,
+		  Uint old_size, Uint *new_size_p)
 {
-    return erts_mseg_realloc_opt(atype, seg, old_size, new_size_p, &default_opt);
+    return erts_mseg_realloc_opt(atype, seg, old_size, new_size_p,
+				 &erts_mseg_default_opt);
 }
 
 void
 erts_mseg_clear_cache(void)
 {
-    erts_mtx_lock(&mseg_mutex);
-    mseg_clear_cache();
-    erts_mtx_unlock(&mseg_mutex);
+    ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_SS();
+    MemKind* mk;
+
+start:
+
+    ERTS_MSEG_LOCK(ma);
+    ERTS_DBG_MA_CHK_THR_ACCESS(ma);
+    for (mk=ma->mk_list; mk; mk=mk->next) {
+	mseg_clear_cache(mk);
+    }
+    ERTS_MSEG_UNLOCK(ma);
+
+    if (ma->ix != 0) {
+	ma = ERTS_MSEG_ALLCTR_IX(0);
+	goto start;
+    }
 }
 
 Uint
-erts_mseg_no(void)
+erts_mseg_no(const ErtsMsegOpt_t *opt)
 {
-    Uint n;
-    erts_mtx_lock(&mseg_mutex);
-    n = segments.current.no;
-    erts_mtx_unlock(&mseg_mutex);
+    ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_OPT(opt);
+    MemKind* mk;
+    Uint n = 0;
+    ERTS_MSEG_LOCK(ma);
+    ERTS_DBG_MA_CHK_THR_ACCESS(ma);
+    for (mk=ma->mk_list; mk; mk=mk->next) {
+	n += mk->segments.current.no;
+    }
+    ERTS_MSEG_UNLOCK(ma);
     return n;
 }
 
@@ -1297,83 +1499,141 @@ erts_mseg_unit_size(void)
     return page_size;
 }
 
-void
-erts_mseg_init(ErtsMsegInit_t *init)
+static void mem_kind_init(ErtsMsegAllctr_t *ma, MemKind* mk, const char* name)
 {
     unsigned i;
 
-    atoms_initialized = 0;
-    is_init_done = 0;
+    mk->cache = NULL;
+    mk->cache_end = NULL;
+    mk->max_cached_seg_size = 0;
+    mk->min_cached_seg_size = ~((Uint) 0);
+    mk->cache_size = 0;
+    mk->cache_hits = 0;
+
+    if (ma->max_cache_size > 0) {
+	for (i = 0; i < ma->max_cache_size - 1; i++)
+	    mk->cache_descs[i].next = &mk->cache_descs[i + 1];
+	mk->cache_descs[ma->max_cache_size - 1].next = NULL;
+	mk->free_cache_descs = &mk->cache_descs[0];
+    }
+    else
+	mk->free_cache_descs = NULL;
+
+    mk->segments.current.watermark = 0;
+    mk->segments.current.no = 0;
+    mk->segments.current.sz = 0;
+    mk->segments.max.no = 0;
+    mk->segments.max.sz = 0;
+    mk->segments.max_ever.no = 0;
+    mk->segments.max_ever.sz = 0;
+
+    mk->ma = ma;
+    mk->name = name;
+    mk->next = ma->mk_list;
+    ma->mk_list = mk;
+}
 
-    /* Options ... */
 
-    abs_max_cache_bad_fit	= init->amcbf;
-    rel_max_cache_bad_fit	= init->rmcbf;
-    max_cache_size		= init->mcs;
-    cache_check_interval	= init->cci;
 
-    /* */
 
-#ifdef USE_THREADS
-    thread_safe_init();
+void
+erts_mseg_init(ErtsMsegInit_t *init)
+{
+    int i;
+    UWord x;
+
+#ifdef ERTS_SMP
+    no_mseg_allocators = init->nos + 1;
+#else
+    no_mseg_allocators = 1;
 #endif
 
+    x = (UWord) malloc(sizeof(ErtsAlgndMsegAllctr_t)
+		       *no_mseg_allocators
+		       + (ERTS_CACHE_LINE_SIZE-1));
+    if (x & ERTS_CACHE_LINE_MASK)
+	x = (x & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE;
+    ASSERT((x & ERTS_CACHE_LINE_MASK) == 0);
+    aligned_mseg_allctr = (ErtsAlgndMsegAllctr_t *) x;
+
+    atoms_initialized = 0;
+
+    erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms");
+
 #if HAVE_MMAP && !defined(MAP_ANON)
     mmap_fd = open("/dev/zero", O_RDWR);
     if (mmap_fd < 0)
 	erl_exit(ERTS_ABORT_EXIT, "erts_mseg: unable to open /dev/zero\n");
 #endif
 
+#if HAVE_MMAP && HALFWORD_HEAP
+    initialize_pmmap();
+#endif
+
     page_size = GET_PAGE_SIZE;
 
     page_shift = 1;
     while ((page_size >> page_shift) != 1) {
 	if ((page_size & (1 << (page_shift - 1))) != 0)
 	    erl_exit(ERTS_ABORT_EXIT,
-		     "erts_mseg: Unexpected page_size %bpu\n", page_size);
+		     "erts_mseg: Unexpected page_size %beu\n", page_size);
 	page_shift++;
     }
 
-    sys_memzero((void *) &calls, sizeof(calls));
+    for (i = 0; i < no_mseg_allocators; i++) {
+	ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_IX(i);
 
-#if CAN_PARTLY_DESTROY
-    min_seg_size = ~((Uint) 0);
-#endif
+	ma->ix = i;
+
+	ma->is_init_done = 0;
+
+	if (i != 0)
+	    ma->is_thread_safe = 0;
+	else {
+	    ma->is_thread_safe = 1;
+	    erts_mtx_init(&ma->mtx, "mseg");
+	}
 
-    cache = NULL;
-    cache_end = NULL;
-    cache_hits = 0;
-    max_cached_seg_size = 0;
-    min_cached_seg_size = ~((Uint) 0);
-    cache_size = 0;
+	ma->is_cache_check_scheduled = 0;
 
-    is_cache_check_scheduled = 0;
-#ifdef ERTS_THREADS_NO_SMP
-    is_cache_check_requested = 0;
+	/* Options ... */
+
+	ma->abs_max_cache_bad_fit = init->amcbf;
+	ma->rel_max_cache_bad_fit = init->rmcbf;
+	ma->max_cache_size = init->mcs;
+
+	if (ma->max_cache_size > MAX_CACHE_SIZE)
+	    ma->max_cache_size = MAX_CACHE_SIZE;
+
+	ma->mk_list = NULL;
+
+#if HALFWORD_HEAP
+	mem_kind_init(ma, &ma->low_mem, "low memory");
+	mem_kind_init(ma, &ma->hi_mem, "high memory");
+#else
+	mem_kind_init(ma, &ma->the_mem, "all memory");
 #endif
 
-    if (max_cache_size > MAX_CACHE_SIZE)
-	max_cache_size = MAX_CACHE_SIZE;
+	sys_memzero((void *) &ma->calls, sizeof(ErtsMsegCalls));
 
-    if (max_cache_size > 0) {
-	for (i = 0; i < max_cache_size - 1; i++)
-	    cache_descs[i].next = &cache_descs[i + 1];
-	cache_descs[max_cache_size - 1].next = NULL;
-	free_cache_descs = &cache_descs[0];
+#if CAN_PARTLY_DESTROY
+	ma->min_seg_size = ~((Uint) 0);
+#endif
     }
-    else
-	free_cache_descs = NULL;
-
-    segments.current.watermark = 0;
-    segments.current.no = 0;
-    segments.current.sz = 0;
-    segments.max.no = 0;
-    segments.max.sz = 0;
-    segments.max_ever.no = 0;
-    segments.max_ever.sz = 0;
 }
 
 
+static ERTS_INLINE Uint tot_cache_size(ErtsMsegAllctr_t *ma)
+{
+    MemKind* mk;
+    Uint sz = 0;
+    ERTS_DBG_MA_CHK_THR_ACCESS(ma);
+    for (mk=ma->mk_list; mk; mk=mk->next) {
+	sz += mk->cache_size;
+    }
+    return sz;
+}
+
 /*
  * erts_mseg_late_init() have to be called after all allocators,
  * threads and timers have been initialized.
@@ -1381,25 +1641,13 @@ erts_mseg_init(ErtsMsegInit_t *init)
 void
 erts_mseg_late_init(void)
 {
-#ifdef ERTS_THREADS_NO_SMP
-    int handle =
-	erts_register_async_ready_callback(
-	    check_schedule_cache_check);
-#endif
-    erts_mtx_lock(&mseg_mutex);
-    is_init_done = 1;
-#ifdef ERTS_THREADS_NO_SMP
-    async_handle = handle;
-#endif
-    if (cache_size)
-	schedule_cache_check();
-    erts_mtx_unlock(&mseg_mutex);
-}
-
-void
-erts_mseg_exit(void)
-{
-    mseg_shutdown();
+    ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_SS();
+    ERTS_MSEG_LOCK(ma);
+    ERTS_DBG_MA_CHK_THR_ACCESS(ma);
+    ma->is_init_done = 1;
+    if (tot_cache_size(ma))
+	schedule_cache_check(ma);
+    ERTS_MSEG_UNLOCK(ma);
 }
 
 #endif /* #if HAVE_ERTS_MSEG */
@@ -1428,12 +1676,13 @@ erts_mseg_test(unsigned long op,
 	erts_mseg_clear_cache();
 	return (unsigned long) 0;
     case 0x405:
-	return (unsigned long) erts_mseg_no();
+	return (unsigned long) erts_mseg_no(&erts_mseg_default_opt);
     case 0x406: {
+	ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_IX(0);
 	unsigned long res;
-	erts_mtx_lock(&mseg_mutex);
-	res = (unsigned long) cache_size;
-	erts_mtx_unlock(&mseg_mutex);
+	ERTS_MSEG_LOCK(ma);
+	res = (unsigned long) tot_cache_size(ma);
+	ERTS_MSEG_UNLOCK(ma);
 	return res;
     }
 #else /* #if HAVE_ERTS_MSEG */
@@ -1446,3 +1695,432 @@ erts_mseg_test(unsigned long op,
 }
 
 
+#if HALFWORD_HEAP
+/*
+ * Very simple page oriented mmap replacer. Works in the lower
+ * 32 bit address range of a 64bit program.
+ * Implements anonymous mmap mremap and munmap with address order first fit.
+ * The free list is expected to be very short...
+ * To be used for compressed pointers in Erlang halfword emulator
+ * implementation. The MacOS X version is more of a toy, it's not really
+ * for production as the halfword erlang VM relies on Linux specific memory
+ * mapping tricks.
+ */
+
+/*#define HARDDEBUG 1*/
+
+#ifdef __APPLE__
+#define MAP_ANONYMOUS MAP_ANON
+#endif
+
+#define INIT_LOCK() do {erts_mtx_init(&pmmap_mutex, "pmmap");} while(0)
+
+#define TAKE_LOCK()  do {erts_mtx_lock(&pmmap_mutex);} while(0)
+
+#define RELEASE_LOCK() do {erts_mtx_unlock(&pmmap_mutex);} while(0)
+
+static erts_mtx_t pmmap_mutex; /* Also needed when !USE_THREADS */
+
+typedef struct _free_block {
+    unsigned long num; /*pages*/
+    struct _free_block *next;
+} FreeBlock;
+
+/* Assigned once and for all */
+static size_t pagsz;
+
+/* Protect with lock */
+static FreeBlock *first;
+
+static size_t round_up_to_pagesize(size_t size)
+{
+    size_t x  = size / pagsz;
+
+    if ((size % pagsz)) {
+	++x;
+    }
+
+    return pagsz * x;
+}
+
+static size_t round_down_to_pagesize(size_t size)
+{
+    size_t x  = size / pagsz;
+
+    return pagsz * x;
+}
+
+static void *do_map(void *ptr, size_t sz)
+{
+    void *res;
+
+    if (round_up_to_pagesize(sz) != sz) {
+#ifdef HARDDEBUG
+	fprintf(stderr,"Mapping of address %p with size %ld "
+		"does not map complete pages\r\n",
+		(void *) ptr, (unsigned long) sz);
+#endif
+	return NULL;
+    }
+
+    if (((unsigned long) ptr) % pagsz) {
+#ifdef HARDDEBUG
+	fprintf(stderr,"Mapping of address %p with size %ld "
+		"is not page aligned\r\n",
+		(void *) ptr, (unsigned long) sz);
+#endif
+	return NULL;
+    }
+
+#if HAVE_MMAP
+    res = mmap(ptr, sz,
+	       PROT_READ | PROT_WRITE, MAP_PRIVATE |
+	       MAP_ANONYMOUS | MAP_FIXED,
+	       -1 , 0);
+#else
+#  error "Missing mmap support"
+#endif
+
+    if (res == MAP_FAILED) {
+#ifdef HARDDEBUG
+	fprintf(stderr,"Mapping of address %p with size %ld failed!\r\n",
+		(void *) ptr, (unsigned long) sz);
+#endif
+	return NULL;
+    }
+
+    return res;
+}
+
+static int do_unmap(void *ptr, size_t sz)
+{
+    void *res;
+
+    if (round_up_to_pagesize(sz) != sz) {
+#ifdef HARDDEBUG
+	fprintf(stderr,"Mapping of address %p with size %ld "
+		"does not map complete pages\r\n",
+		(void *) ptr, (unsigned long) sz);
+#endif
+	return 1;
+    }
+
+    if (((unsigned long) ptr) % pagsz) {
+#ifdef HARDDEBUG
+	fprintf(stderr,"Mapping of address %p with size %ld "
+		"is not page aligned\r\n",
+		(void *) ptr, (unsigned long) sz);
+#endif
+	return 1;
+    }
+
+
+    res = mmap(ptr, sz,
+	       PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE
+	       | MAP_FIXED,
+	       -1 , 0);
+
+    if (res == MAP_FAILED) {
+#ifdef HARDDEBUG
+	fprintf(stderr,"Mapping of address %p with size %ld failed!\r\n",
+		(void *) ptr, (unsigned long) sz);
+#endif
+	return 1;
+    }
+
+    return 0;
+}
+
+#ifdef __APPLE__
+/*
+ * The first 4 gig's are protected on Macos X for 64bit processes :(
+ * The range 0x1000000000 - 0x10FFFFFFFF is selected as an arbitrary
+ * value of a normally unused range... Real MMAP's will avoid
+ * it and all 32bit compressed pointers can be in that range...
+ * More expensive than on Linux where expansion of compressed
+ * poiters involves no masking (as they are in the first 4 gig's).
+ * It's also very uncertain if the MAP_NORESERVE flag really has
+ * any effect in MacOS X. Swap space may always be allocated...
+ */
+#define SET_RANGE_MIN() /* nothing */
+#define RANGE_MIN 0x1000000000UL
+#define RANGE_MAX 0x1100000000UL
+#define RANGE_MASK (RANGE_MIN)
+#define EXTRA_MAP_FLAGS (MAP_FIXED)
+#else
+static size_t range_min;
+#define SET_RANGE_MIN() do { range_min = (size_t) sbrk(0); } while (0)
+#define RANGE_MIN range_min
+#define RANGE_MAX 0x100000000UL
+#define RANGE_MASK 0UL
+#define EXTRA_MAP_FLAGS (0)
+#endif
+
+static int initialize_pmmap(void)
+{
+    char *p,*q,*rptr;
+    size_t rsz;
+    FreeBlock *initial;
+
+
+    pagsz = getpagesize();
+    SET_RANGE_MIN();
+    if (sizeof(void *) != 8) {
+	erl_exit(1,"Halfword emulator cannot be run in 32bit mode");
+    }
+
+    p = (char *) RANGE_MIN;
+    q = (char *) RANGE_MAX;
+
+    rsz = round_down_to_pagesize(q - p);
+
+    rptr = mmap((void *) p, rsz,
+		PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS |
+		MAP_NORESERVE | EXTRA_MAP_FLAGS,
+		-1 , 0);
+#ifdef HARDDEBUG
+    printf("p=%p, rsz = %ld, pages = %ld, got range = %p -> %p\r\n",
+	   p, (unsigned long) rsz, (unsigned long) (rsz / pagsz),
+	   (void *) rptr, (void*)(rptr + rsz));
+#endif
+    if ((UWord)(rptr + rsz) > RANGE_MAX) {
+	size_t rsz_trunc = RANGE_MAX - (UWord)rptr;
+#ifdef HARDDEBUG
+	printf("Reducing mmap'ed memory from %lu to %lu Mb, reduced range = %p -> %p\r\n",
+		rsz/(1024*1024), rsz_trunc/(1024*1024), rptr, rptr+rsz_trunc);
+#endif
+	munmap((void*)RANGE_MAX, rsz - rsz_trunc);
+	rsz = rsz_trunc;
+    }
+    if (!do_map(rptr,pagsz)) {
+	erl_exit(1,"Could not actually mmap first page for halfword emulator...\n");
+    }
+    initial = (FreeBlock *) rptr;
+    initial->num = (rsz / pagsz);
+    initial->next = NULL;
+    first = initial;
+    INIT_LOCK();
+    return 0;
+}
+
+#ifdef HARDDEBUG
+static void dump_freelist(void)
+{
+    FreeBlock *p = first;
+
+    while (p) {
+	printf("p = %p\r\np->num = %ld\r\np->next = %p\r\n\r\n",
+	       (void *) p, (unsigned long) p->num, (void *) p->next);
+	p = p->next;
+    }
+}
+#endif
+
+
+static void *pmmap(size_t size)
+{
+    size_t real_size = round_up_to_pagesize(size);
+    size_t num_pages = real_size / pagsz;
+    FreeBlock **block;
+    FreeBlock *tail;
+    FreeBlock *res;
+    TAKE_LOCK();
+    for (block = &first;
+	 *block != NULL && (*block)->num < num_pages;
+	 block = &((*block)->next))
+	;
+    if (!(*block)) {
+	RELEASE_LOCK();
+	return NULL;
+    }
+    if ((*block)->num == num_pages) {
+	/* nice, perfect fit */
+	res = *block;
+	*block = (*block)->next;
+    } else {
+	tail = (FreeBlock *) (((char *) ((void *) (*block))) + real_size);
+	if (!do_map(tail,pagsz)) {
+#ifdef HARDDEBUG
+	    fprintf(stderr, "Could not actually allocate page at %p...\r\n",
+		    (void *) tail);
+#endif
+	    RELEASE_LOCK();
+	    return NULL;
+	}
+	tail->num = (*block)->num - num_pages;
+	tail->next = (*block)->next;
+	res = *block;
+	*block = tail;
+    }
+    RELEASE_LOCK();
+    if (!do_map(res,real_size)) {
+#ifdef HARDDEBUG
+	fprintf(stderr, "Could not actually allocate %ld at %p...\r\n",
+		(unsigned long) real_size, (void *) res);
+#endif
+	return NULL;
+    }
+
+    return (void *) res;
+}
+
+static int pmunmap(void *p, size_t size)
+{
+    size_t real_size = round_up_to_pagesize(size);
+    size_t num_pages = real_size / pagsz;
+    FreeBlock *block;
+    FreeBlock *last;
+    FreeBlock *nb = (FreeBlock *) p;
+
+    ASSERT(((unsigned long)p & CHECK_POINTER_MASK)==0);
+    if (real_size > pagsz) {
+	if (do_unmap(((char *) p) + pagsz,real_size - pagsz)) {
+	    return 1;
+	}
+    }
+
+    TAKE_LOCK();
+
+    last = NULL;
+    block = first;
+    while(block != NULL && ((void *) block) < p) {
+	last = block;
+	block = block->next;
+    }
+
+    if (block != NULL &&
+	((void *) block) == ((void *) (((char *) p) + real_size))) {
+	/* Merge new free block with following */
+	nb->num = block->num + num_pages;
+	nb->next = block->next;
+	if (do_unmap(block,pagsz)) {
+	    RELEASE_LOCK();
+	    return 1;
+	}
+    } else {
+	/* just link in */
+	nb->num = num_pages;
+	nb->next = block;
+    }
+    if (last != NULL) {
+	if (p == ((void *) (((char *) last) + (last->num * pagsz)))) {
+	    /* Merge with previous */
+	    last->num += nb->num;
+	    last->next = nb->next;
+	    if (do_unmap(nb,pagsz)) {
+		RELEASE_LOCK();
+		return 1;
+	    }
+	} else {
+	    last->next = nb;
+	}
+    } else {
+	first = nb;
+    }
+    RELEASE_LOCK();
+    return 0;
+}
+
+static void *pmremap(void *old_address, size_t old_size,
+	      size_t new_size)
+{
+    size_t new_real_size = round_up_to_pagesize(new_size);
+    size_t new_num_pages = new_real_size / pagsz;
+    size_t old_real_size = round_up_to_pagesize(old_size);
+    size_t old_num_pages = old_real_size / pagsz;
+    if (new_num_pages == old_num_pages) {
+	return old_address;
+    } else if (new_num_pages < old_num_pages) { /* Shrink */
+	size_t nfb_pages = old_num_pages - new_num_pages;
+	size_t nfb_real_size = old_real_size - new_real_size;
+	void *vnfb = (void *) (((char *)old_address) + new_real_size);
+	FreeBlock *nfb = (FreeBlock *) vnfb;
+	FreeBlock **block;
+	TAKE_LOCK();
+	for (block = &first;
+	     *block != NULL && (*block) < nfb;
+	     block = &((*block)->next))
+	;
+	if (!(*block) ||
+	    (*block) > ((FreeBlock *)(((char *) vnfb) + nfb_real_size))) {
+	    /* Normal link in */
+	    if (nfb_pages > 1) {
+		if (do_unmap((void *)(((char *) vnfb) + pagsz),
+			     (nfb_pages - 1)*pagsz)) {
+		    return NULL;
+		}
+	    }
+	    nfb->next = (*block);
+	    nfb->num = nfb_pages;
+	    (*block) = nfb;
+	} else { /* block merge */
+	    nfb->next = (*block)->next;
+	    nfb->num = nfb_pages + (*block)->num;
+	    /* unmap also the first page of the next freeblock */
+	    (*block) = nfb;
+	    if (do_unmap((void *)(((char *) vnfb) + pagsz),
+			 nfb_pages*pagsz)) {
+		return NULL;
+	    }
+	}
+	RELEASE_LOCK();
+	return old_address;
+    } else { /* Enlarge */
+	FreeBlock **block;
+	void *old_end = (void *) (((char *)old_address) + old_real_size);
+	TAKE_LOCK();
+	for (block = &first;
+	     *block != NULL && (*block) < (FreeBlock *) old_address;
+	     block = &((*block)->next))
+	    ;
+	if ((*block) == NULL || old_end > ((void *) RANGE_MAX) ||
+	    (*block) != old_end ||
+	    (*block)->num < (new_num_pages - old_num_pages)) {
+	    /* cannot extend */
+	    void *result;
+	    RELEASE_LOCK();
+	    result = pmmap(new_size);
+	    if (result == NULL) {
+		return NULL;
+	    }
+	    memcpy(result,old_address,old_size);
+	    if (pmunmap(old_address,old_size)) {
+		/* Oups... */
+		pmunmap(result,new_size);
+		return NULL;
+	    }
+	    return result;
+	} else { /* extend */
+	    size_t remaining_pages = (*block)->num -
+		(new_num_pages - old_num_pages);
+	    if (!remaining_pages) {
+		void *p = (void *) (((char *) (*block)) + pagsz);
+		void *n = (*block)->next;
+		size_t x = ((*block)->num - 1) * pagsz;
+		if (x > 0) {
+		    if (do_map(p,x) == NULL) {
+			RELEASE_LOCK();
+			return NULL;
+		    }
+		}
+		(*block) = n;
+	    } else {
+		FreeBlock *nfb = (FreeBlock *) ((void *)
+						(((char *) old_address) +
+						 new_real_size));
+		void *p = (void *) (((char *) (*block)) + pagsz);
+		if (do_map(p,new_real_size - old_real_size) == NULL) {
+		    RELEASE_LOCK();
+		    return NULL;
+		}
+		nfb->num = remaining_pages;
+		nfb->next = (*block)->next;
+		(*block) = nfb;
+	    }
+	    RELEASE_LOCK();
+	    return old_address;
+	}
+    }
+}
+
+#endif /* HALFWORD_HEAP */
diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h
index 1c5aa63e90..741080fb78 100644
--- a/erts/emulator/sys/common/erl_mseg.h
+++ b/erts/emulator/sys/common/erl_mseg.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2002-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -44,7 +44,7 @@ typedef struct {
     Uint amcbf;
     Uint rmcbf;
     Uint mcs;
-    Uint cci;
+    Uint nos;
 } ErtsMsegInit_t;
 
 #define ERTS_MSEG_INIT_DEFAULT_INITIALIZER				\
@@ -58,17 +58,15 @@ typedef struct {
 typedef struct {
     int  cache;
     int  preserv;
-    Uint abs_shrink_th;
-    Uint rel_shrink_th;
+    UWord abs_shrink_th;
+    UWord rel_shrink_th;
+    int sched_spec;
+#if HALFWORD_HEAP
+    int low_mem;
+#endif
 } ErtsMsegOpt_t;
 
-#define ERTS_MSEG_DEFAULT_OPT_INITIALIZER				\
-{									\
-    1,			/* Use cache				*/	\
-    1,			/* Preserv data				*/	\
-    0,			/* Absolute shrink threshold		*/	\
-    0			/* Relative shrink threshold		*/	\
-}
+extern const ErtsMsegOpt_t erts_mseg_default_opt;
 
 void *erts_mseg_alloc(ErtsAlcType_t, Uint *);
 void *erts_mseg_alloc_opt(ErtsAlcType_t, Uint *, const ErtsMsegOpt_t *);
@@ -78,14 +76,14 @@ void *erts_mseg_realloc(ErtsAlcType_t, void *, Uint, Uint *);
 void *erts_mseg_realloc_opt(ErtsAlcType_t, void *, Uint, Uint *,
 			    const ErtsMsegOpt_t *);
 void  erts_mseg_clear_cache(void);
-Uint  erts_mseg_no(void);
+void  erts_mseg_cache_check(void);
+Uint  erts_mseg_no( const ErtsMsegOpt_t *);
 Uint  erts_mseg_unit_size(void);
 void  erts_mseg_init(ErtsMsegInit_t *init);
 void  erts_mseg_late_init(void); /* Have to be called after all allocators,
 				   threads and timers have been initialized. */
-void  erts_mseg_exit(void);
-Eterm erts_mseg_info_options(int *, void*, Uint **, Uint *);
-Eterm erts_mseg_info(int *, void*, int, Uint **, Uint *);
+Eterm erts_mseg_info_options(int, int *, void*, Uint **, Uint *);
+Eterm erts_mseg_info(int, int *, void*, int, Uint **, Uint *);
 
 #endif /* #if HAVE_ERTS_MSEG */
 
diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index 5cca33d7eb..26858052c4 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2006-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -68,6 +68,7 @@
 #    endif
 #  endif
 #endif
+#include "erl_thr_progress.h"
 #include "erl_driver.h"
 #include "erl_alloc.h"
 
@@ -114,7 +115,7 @@
 #endif
 
 #define ERTS_POLL_USE_WAKEUP_PIPE \
-   (ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(ERTS_SMP))
+   (ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(USE_THREADS))
 
 #ifdef ERTS_SMP
 
@@ -124,20 +125,11 @@
   erts_smp_mtx_unlock(&(PS)->mtx)
 
 #define ERTS_POLLSET_SET_POLLED_CHK(PS) \
-  ((int) erts_smp_atomic_xchg(&(PS)->polled, (long) 1))
+  ((int) erts_atomic32_xchg_nob(&(PS)->polled, (erts_aint32_t) 1))
 #define ERTS_POLLSET_UNSET_POLLED(PS) \
-  erts_smp_atomic_set(&(PS)->polled, (long) 0)
+  erts_atomic32_set_nob(&(PS)->polled, (erts_aint32_t) 0)
 #define ERTS_POLLSET_IS_POLLED(PS) \
-  ((int) erts_smp_atomic_read(&(PS)->polled))
-
-#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) \
-  ((int) erts_smp_atomic_xchg(&(PS)->woken, (long) 1))
-#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) \
-  erts_smp_atomic_set(&(PS)->woken, (long) 1)
-#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) \
-  erts_smp_atomic_set(&(PS)->woken, (long) 0)
-#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) \
-  ((int) erts_smp_atomic_read(&(PS)->woken))
+  ((int) erts_atomic32_read_nob(&(PS)->polled))
 
 #else
 
@@ -147,64 +139,21 @@
 #define ERTS_POLLSET_UNSET_POLLED(PS)
 #define ERTS_POLLSET_IS_POLLED(PS) 0
 
-#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
-
-/*
- * Ideally, the ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) operation would
- * be atomic. This operation isn't, but we will do okay anyway. The
- * "woken check" is only an optimization. The only requirement we have:
- * If (PS)->woken is set to a value != 0 when interrupting, we have to
- * write on the the wakeup pipe at least once. Multiple writes are okay.
- */
-#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) ((PS)->woken++)
-#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) ((PS)->woken = 1, (void) 0)
-#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) ((PS)->woken = 0, (void) 0)
-#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) ((PS)->woken)
-
-#else
-
-#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) 1
-#define ERTS_POLLSET_SET_POLLER_WOKEN(PS)
-#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS)
-#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) 1
-
-#endif
-
 #endif
 
 #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
 #define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) \
-  erts_smp_atomic_set(&(PS)->have_update_requests, (long) 1)
+  erts_smp_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 1)
 #define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) \
-  erts_smp_atomic_set(&(PS)->have_update_requests, (long) 0)
+  erts_smp_atomic32_set_nob(&(PS)->have_update_requests, (erts_aint32_t) 0)
 #define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) \
-  ((int) erts_smp_atomic_read(&(PS)->have_update_requests))
+  ((int) erts_smp_atomic32_read_nob(&(PS)->have_update_requests))
 #else
 #define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS)
 #define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS)
 #define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) 0
 #endif
 
-#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP)
-
-#define ERTS_POLLSET_UNSET_INTERRUPTED_CHK(PS) unset_interrupted_chk((PS))
-#define ERTS_POLLSET_UNSET_INTERRUPTED(PS) ((PS)->interrupt = 0, (void) 0)
-#define ERTS_POLLSET_SET_INTERRUPTED(PS) ((PS)->interrupt = 1, (void) 0)
-#define ERTS_POLLSET_IS_INTERRUPTED(PS) ((PS)->interrupt)
-
-#else
-
-#define ERTS_POLLSET_UNSET_INTERRUPTED_CHK(PS) \
-  ((int) erts_smp_atomic_xchg(&(PS)->interrupt, (long) 0))
-#define ERTS_POLLSET_UNSET_INTERRUPTED(PS) \
-  erts_smp_atomic_set(&(PS)->interrupt, (long) 0)
-#define ERTS_POLLSET_SET_INTERRUPTED(PS) \
-  erts_smp_atomic_set(&(PS)->interrupt, (long) 1)
-#define ERTS_POLLSET_IS_INTERRUPTED(PS) \
-  ((int) erts_smp_atomic_read(&(PS)->interrupt))
-
-#endif
-
 #if ERTS_POLL_USE_FALLBACK
 #  if ERTS_POLL_USE_POLL
 #    define ERTS_POLL_NEED_FALLBACK(PS) ((PS)->no_poll_fds > 1)
@@ -276,7 +225,7 @@ struct ErtsPollSet_ {
     ErtsPollSet next;
     int internal_fd_limit;
     ErtsFdStatus *fds_status;
-    int no_of_user_fds;
+    erts_smp_atomic_t no_of_user_fds;
     int fds_status_len;
 #if ERTS_POLL_USE_KERNEL_POLL
     int kp_fd;
@@ -308,14 +257,11 @@ struct ErtsPollSet_ {
 #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
     ErtsPollSetUpdateRequestsBlock update_requests;
     ErtsPollSetUpdateRequestsBlock *curr_upd_req_block;
-    erts_smp_atomic_t have_update_requests;
+    erts_smp_atomic32_t have_update_requests;
 #endif
 #ifdef ERTS_SMP
-    erts_smp_atomic_t polled;
-    erts_smp_atomic_t woken;
+    erts_atomic32_t polled;
     erts_smp_mtx_t mtx;
-#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
-    volatile int woken;
 #endif
 #if ERTS_POLL_USE_WAKEUP_PIPE
     int wake_fds[2];
@@ -323,12 +269,10 @@ struct ErtsPollSet_ {
 #if ERTS_POLL_USE_FALLBACK
     int fallback_used;
 #endif
-#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP)
-    volatile int interrupt;
-#else
-    erts_smp_atomic_t interrupt;
+#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
+    erts_atomic32_t wakeup_state;
 #endif
-    erts_smp_atomic_t timeout;
+    erts_smp_atomic32_t timeout;
 #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
     erts_smp_atomic_t no_avoided_wakeups;
     erts_smp_atomic_t no_avoided_interrupts;
@@ -336,20 +280,6 @@ struct ErtsPollSet_ {
 #endif
 };
 
-#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP)
-
-static ERTS_INLINE int
-unset_interrupted_chk(ErtsPollSet ps)
-{
-    /* This operation isn't atomic, but we have no need at all for an
-       atomic operation here... */
-    int res = ps->interrupt;
-    ps->interrupt = 0;
-    return res;
-}
-
-#endif
-
 void erts_silence_warn_unused_result(long unused);
 static void fatal_error(char *format, ...);
 static void fatal_error_async_signal_safe(char *error_str);
@@ -406,6 +336,52 @@ static void check_poll_status(ErtsPollSet ps);
 static void print_misc_debug_info(void);
 #endif
 
+#define ERTS_POLL_NOT_WOKEN	0
+#define ERTS_POLL_WOKEN		-1
+#define ERTS_POLL_WOKEN_INTR	1
+
+static ERTS_INLINE void
+reset_wakeup_state(ErtsPollSet ps)
+{
+#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
+    erts_atomic32_set_mb(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
+#endif
+}
+
+static ERTS_INLINE int
+is_woken(ErtsPollSet ps)
+{
+#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
+    return erts_atomic32_read_acqb(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN;
+#else
+    return 0;
+#endif
+}
+
+static ERTS_INLINE int
+is_interrupted_reset(ErtsPollSet ps)
+{
+#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
+    return (erts_atomic32_xchg_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN)
+	    == ERTS_POLL_WOKEN_INTR);
+#else
+    return 0;
+#endif
+}
+
+static ERTS_INLINE void
+woke_up(ErtsPollSet ps)
+{
+#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
+    erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
+    if (wakeup_state == ERTS_POLL_NOT_WOKEN)
+	(void) erts_atomic32_cmpxchg_nob(&ps->wakeup_state,
+					 ERTS_POLL_WOKEN,
+					 ERTS_POLL_NOT_WOKEN);
+    ASSERT(erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN);
+#endif
+}
+
 /*
  * --- Wakeup pipe -----------------------------------------------------------
  */
@@ -413,14 +389,33 @@ static void print_misc_debug_info(void);
 #if ERTS_POLL_USE_WAKEUP_PIPE
 
 static ERTS_INLINE void
-wake_poller(ErtsPollSet ps)
+wake_poller(ErtsPollSet ps, int interrupted, int async_signal_safe)
 {
+    int wake;
+    if (async_signal_safe)
+	wake = 1;
+    else {
+	erts_aint32_t wakeup_state;
+	if (!interrupted)
+	    wakeup_state = erts_atomic32_cmpxchg_relb(&ps->wakeup_state,
+						      ERTS_POLL_WOKEN,
+						      ERTS_POLL_NOT_WOKEN);
+	else {
+	    /*
+	     * We might unnecessarily write to the pipe, however,
+	     * that isn't problematic.
+	     */
+	    wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
+	    erts_atomic32_set_relb(&ps->wakeup_state, ERTS_POLL_WOKEN_INTR);
+	}
+	wake = wakeup_state == ERTS_POLL_NOT_WOKEN;
+    }
     /*
      * NOTE: This function might be called from signal handlers in the
      *       non-smp case; therefore, it has to be async-signal safe in
      *       the non-smp case.
      */
-    if (!ERTS_POLLSET_SET_POLLER_WOKEN_CHK(ps)) {
+    if (wake) {
 	ssize_t res;
 	if (ps->wake_fds[1] < 0)
 	    return; /* Not initialized yet */
@@ -429,9 +424,17 @@ wake_poller(ErtsPollSet ps)
 	    res = write(ps->wake_fds[1], "!", 1);
 	} while (res < 0 && errno == EINTR);
 	if (res <= 0 && errno != ERRNO_BLOCK) {
-	    fatal_error_async_signal_safe(__FILE__
-					  ":XXX:wake_poller(): "
-					  "Failed to write on wakeup pipe\n");
+	    if (async_signal_safe)
+		fatal_error_async_signal_safe(__FILE__
+					      ":XXX:wake_poller(): "
+					      "Failed to write on wakeup pipe\n");
+	    else
+		fatal_error("%s:%d:wake_poller(): "
+			    "Failed to write to wakeup pipe fd=%d: "
+			    "%s (%d)\n",
+			    __FILE__, __LINE__,
+			    ps->wake_fds[1],
+			    erl_errno_id(errno), errno);
 	}
     }
 }
@@ -439,11 +442,18 @@ wake_poller(ErtsPollSet ps)
 static ERTS_INLINE void
 cleanup_wakeup_pipe(ErtsPollSet ps)
 {
+#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
+    int intr = 0;
+#endif
     int fd = ps->wake_fds[0];
     int res;
     do {
 	char buf[32];
 	res = read(fd, buf, sizeof(buf));
+#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
+	if (res > 0)
+	    intr = 1;
+#endif
     } while (res > 0 || (res < 0 && errno == EINTR));
     if (res < 0 && errno != ERRNO_BLOCK) {
 	fatal_error("%s:%d:cleanup_wakeup_pipe(): "
@@ -453,6 +463,10 @@ cleanup_wakeup_pipe(ErtsPollSet ps)
 		    fd,
 		    erl_errno_id(errno), errno);
     }
+#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
+    if (intr)
+	erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_WOKEN_INTR);
+#endif
 }
 
 static void
@@ -756,7 +770,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp)
 		short filter;
 		int fd = (int) ebuf[i].ident;
 
-		switch ((int) ebuf[i].udata) {
+		switch ((int) (long) ebuf[i].udata) {
 
 		    /*
 		     * Since we use a lazy update approach EV_DELETE will
@@ -795,7 +809,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp)
 			if (fd == (int) ebuf[j].ident) {
 			    ebuf[j].udata = (void *) ERTS_POLL_KQ_OP_HANDLED;
 			    if (!(ebuf[j].flags & EV_ERROR)) {
-				switch ((int) ebuf[j].udata) {
+				switch ((int) (long) ebuf[j].udata) {
 				case ERTS_POLL_KQ_OP_ADD2_W:
 				    filter = EVFILT_WRITE;
 				    goto rm_add_fb;
@@ -813,7 +827,8 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp)
 			}
 		    }
 		    /* The other add succeded... */
-		    filter = (((int) ebuf[i].udata == ERTS_POLL_KQ_OP_ADD2_W)
+		    filter = ((((int) (long) ebuf[i].udata)
+			       == ERTS_POLL_KQ_OP_ADD2_W)
 			      ? EVFILT_READ
 			      : EVFILT_WRITE);
 		rm_add_fb:
@@ -828,7 +843,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp)
 		    ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK;
 		    ASSERT(ps->fds_status[fd].used_events);
 		    ps->fds_status[fd].used_events = 0;
-		    ps->no_of_user_fds--;
+		    erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
 		    update_fallback_pollset(ps, fd);
 		    ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK);
 		    break;
@@ -878,11 +893,11 @@ batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp)
     events = ERTS_POLL_EV_E2N(ps->fds_status[fd].events);
     if (!events) {
 	buf[buf_len].events = POLLREMOVE;
-	ps->no_of_user_fds--;
+	erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
     }
     else if (!ps->fds_status[fd].used_events) {
 	buf[buf_len].events = events;
-	ps->no_of_user_fds++;
+	erts_smp_atomic_inc_nob(&ps->no_of_user_fds);
     }
     else {
 	if ((ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST)
@@ -972,12 +987,12 @@ batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp)
 	}
 	if (used_events) {
 	    if (!events) {
-		ps->no_of_user_fds--;
+		erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
 	    }
 	}
 	else {
 	    if (events)
-		ps->no_of_user_fds++;
+		erts_smp_atomic_inc_nob(&ps->no_of_user_fds);
 	}
 	ASSERT((events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0);
 	ASSERT((used_events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0);
@@ -1051,7 +1066,7 @@ update_pollset(ErtsPollSet ps, int fd)
 	    epe.data.fd = epe_templ.data.fd;
 	    res = epoll_ctl(ps->kp_fd, EPOLL_CTL_DEL, fd, &epe);
 	} while (res != 0 && errno == EINTR);
-	ps->no_of_user_fds--;
+	erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
 	ps->fds_status[fd].used_events = 0;
     }
 
@@ -1059,11 +1074,11 @@ update_pollset(ErtsPollSet ps, int fd)
 	/* A note on EPOLL_CTL_DEL: linux kernel versions before 2.6.9
 	   need a non-NULL event pointer even though it is ignored... */
 	op = EPOLL_CTL_DEL;
-	ps->no_of_user_fds--;
+	erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
     }
     else if (!ps->fds_status[fd].used_events) {
 	op = EPOLL_CTL_ADD;
-	ps->no_of_user_fds++;
+	erts_smp_atomic_inc_nob(&ps->no_of_user_fds);
     }
     else {
 	op = EPOLL_CTL_MOD;
@@ -1113,7 +1128,7 @@ update_pollset(ErtsPollSet ps, int fd)
 	/* Fall through ... */
 	case EPOLL_CTL_ADD: {
 	    ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK;
-	    ps->no_of_user_fds--;
+	    erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
 #if ERTS_POLL_USE_CONCURRENT_UPDATE
 	    if (!*update_fallback) {
 		*update_fallback = 1;
@@ -1201,7 +1216,7 @@ static int update_pollset(ErtsPollSet ps, int fd)
 #if ERTS_POLL_USE_FALLBACK
 	ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK);
 #endif
-	ps->no_of_user_fds--;
+	erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
 	last_pix = --ps->no_poll_fds;
 	if (pix != last_pix) {
 	/* Move last pix to this pix */
@@ -1228,7 +1243,7 @@ static int update_pollset(ErtsPollSet ps, int fd)
 	    ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK)
 		    || fd == ps->kp_fd);
 #endif
-	    ps->no_of_user_fds++;
+	    erts_smp_atomic_inc_nob(&ps->no_of_user_fds);
 	    ps->fds_status[fd].pix = pix = ps->no_poll_fds++;
 	    if (pix >= ps->poll_fds_len)
 		grow_poll_fds(ps, pix);
@@ -1279,7 +1294,7 @@ static int update_pollset(ErtsPollSet ps, int fd)
 
 	if (!ps->fds_status[fd].used_events) {
 	    ASSERT(events);
-	    ps->no_of_user_fds++;
+	    erts_smp_atomic_inc_nob(&ps->no_of_user_fds);
 #if ERTS_POLL_USE_FALLBACK
 	    ps->no_select_fds++;
 	    ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_INFLBCK;
@@ -1287,7 +1302,7 @@ static int update_pollset(ErtsPollSet ps, int fd)
 	}
 	else if (!events) {
 	    ASSERT(ps->fds_status[fd].used_events);
-	    ps->no_of_user_fds--;
+	    erts_smp_atomic_dec_nob(&ps->no_of_user_fds);
 	    ps->fds_status[fd].events = events;
 #if ERTS_POLL_USE_FALLBACK
 	    ps->no_select_fds--;
@@ -1363,9 +1378,7 @@ handle_update_requests(ErtsPollSet ps)
 #endif /* ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE */
 
 static ERTS_INLINE ErtsPollEvents
-poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on,
-	     int *have_set_have_update_requests,
-	     int *do_wake)
+poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake)
 {
     ErtsPollEvents new_events;
 
@@ -1469,7 +1482,6 @@ ERTS_POLL_EXPORT(erts_poll_controlv)(ErtsPollSet ps,
 				     int len)
 {
     int i;
-    int hshur = 0;
     int do_wake;
     int final_do_wake = 0;
 
@@ -1481,17 +1493,17 @@ ERTS_POLL_EXPORT(erts_poll_controlv)(ErtsPollSet ps,
 				      pcev[i].fd,
 				      pcev[i].events,
 				      pcev[i].on,
-				      &hshur,
 				      &do_wake);
 	final_do_wake |= do_wake;
     }
 
+    ERTS_POLLSET_UNLOCK(ps);
+
 #ifdef ERTS_SMP
     if (final_do_wake)
-	wake_poller(ps);
+	wake_poller(ps, 0, 0);
 #endif /* ERTS_SMP */
 
-    ERTS_POLLSET_UNLOCK(ps);
 }
 
 ErtsPollEvents
@@ -1502,20 +1514,20 @@ ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet ps,
 				    int* do_wake) /* In: Wake up polling thread */
 				                  /* Out: Poller is woken */
 {
-    int hshur = 0;
     ErtsPollEvents res;
 
     ERTS_POLLSET_LOCK(ps);
 
-    res = poll_control(ps, fd, events, on, &hshur, do_wake);
+    res = poll_control(ps, fd, events, on, do_wake);
+
+    ERTS_POLLSET_UNLOCK(ps);
 
 #ifdef ERTS_SMP
     if (*do_wake) {
-	wake_poller(ps);
+	wake_poller(ps, 0, 0);
     }
 #endif /* ERTS_SMP */
 
-    ERTS_POLLSET_UNLOCK(ps);
     return res;
 }
 
@@ -1885,17 +1897,20 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res,
 }
 
 static ERTS_INLINE int
-check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked)
+check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res)
 {
-    ASSERT(!*ps_locked);
-    if (ps->no_of_user_fds == 0 && tv->tv_usec == 0 && tv->tv_sec == 0) {
+    int res;
+    if (erts_smp_atomic_read_nob(&ps->no_of_user_fds) == 0
+	&& tv->tv_usec == 0 && tv->tv_sec == 0) {
 	/* Nothing to poll and zero timeout; done... */
 	return 0;
     }
     else {
 	long timeout = tv->tv_sec*1000 + tv->tv_usec/1000;
+	if (timeout > ERTS_AINT32_T_MAX)
+	    timeout = ERTS_AINT32_T_MAX;
 	ASSERT(timeout >= 0);
-	erts_smp_atomic_set(&ps->timeout, timeout);
+	erts_smp_atomic32_set_relb(&ps->timeout, (erts_aint32_t) timeout);
 #if ERTS_POLL_USE_FALLBACK
 	if (!(ps->fallback_used = ERTS_POLL_NEED_FALLBACK(ps))) {
 
@@ -1904,16 +1919,23 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked)
 		timeout = INT_MAX;
 	    if (max_res > ps->res_events_len)
 		grow_res_events(ps, max_res);
-	    return epoll_wait(ps->kp_fd, ps->res_events, max_res, (int)timeout);
+#ifdef ERTS_SMP
+	    if (timeout)
+		erts_thr_progress_prepare_wait(NULL);
+#endif
+	    res = epoll_wait(ps->kp_fd, ps->res_events, max_res, (int)timeout);
 #elif ERTS_POLL_USE_KQUEUE	/* --- kqueue ------------------------------ */
 	    struct timespec ts;
-	    ts.tv_sec = tv->tv_sec;
-	    ts.tv_nsec = tv->tv_usec*1000;
 	    if (max_res > ps->res_events_len)
 		grow_res_events(ps, max_res);
-	    return kevent(ps->kp_fd, NULL, 0, ps->res_events, max_res, &ts);
+#ifdef ERTS_SMP
+	    if (timeout)
+		erts_thr_progress_prepare_wait(NULL);
+#endif
+	    ts.tv_sec = tv->tv_sec;
+	    ts.tv_nsec = tv->tv_usec*1000;
+	    res = kevent(ps->kp_fd, NULL, 0, ps->res_events, max_res, &ts);
 #endif				/* ----------------------------------------- */
-
 	}
 	else /* use fallback (i.e. poll() or select()) */
 #endif /* ERTS_POLL_USE_FALLBACK */
@@ -1926,8 +1948,8 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked)
 	     * the maximum number of file descriptors in the poll set.
 	     */
 	    struct dvpoll poll_res;
-	    int nfds = ps->no_of_user_fds;
-#ifdef ERTS_SMP
+	    int nfds = (int) erts_smp_atomic_read_nob(&ps->no_of_user_fds);
+#if ERTS_POLL_USE_WAKEUP_PIPE
 	    nfds++; /* Wakeup pipe */
 #endif
 	    if (timeout > INT_MAX)
@@ -1936,22 +1958,38 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked)
 	    if (poll_res.dp_nfds > ps->res_events_len)
 		grow_res_events(ps, poll_res.dp_nfds);
 	    poll_res.dp_fds = ps->res_events;
+#ifdef ERTS_SMP
+	    if (timeout)
+		erts_thr_progress_prepare_wait(NULL);
+#endif
 	    poll_res.dp_timeout = (int) timeout;
-	    return ioctl(ps->kp_fd, DP_POLL, &poll_res);
+	    res = ioctl(ps->kp_fd, DP_POLL, &poll_res);
 #elif ERTS_POLL_USE_POLL	/* --- poll -------------------------------- */
 	    if (timeout > INT_MAX)
 		timeout = INT_MAX;
-	    return poll(ps->poll_fds, ps->no_poll_fds, (int) timeout);
+#ifdef ERTS_SMP
+	    if (timeout)
+		erts_thr_progress_prepare_wait(NULL);
+#endif
+	    res = poll(ps->poll_fds, ps->no_poll_fds, (int) timeout);
 #elif ERTS_POLL_USE_SELECT	/* --- select ------------------------------ */
-	    int res;
+	    SysTimeval to = *tv;
+
 	    ps->res_input_fds = ps->input_fds;
 	    ps->res_output_fds = ps->output_fds;
+
+#ifdef ERTS_SMP
+	    if (to.tv_sec || to.tv_usec)
+		erts_thr_progress_prepare_wait(NULL);
+#endif
 	    res = select(ps->max_fd + 1,
 			 &ps->res_input_fds,
 			 &ps->res_output_fds,
 			 NULL,
-			 tv);
+			 &to);
 #ifdef ERTS_SMP
+	    if (to.tv_sec || to.tv_usec)
+		erts_thr_progress_finalize_wait(NULL);
 	    if (res < 0
 		&& errno == EBADF
 		&& ERTS_POLLSET_HAVE_UPDATE_REQUESTS(ps)) {
@@ -1967,15 +2005,16 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked)
 		 * have triggered, we fake an EAGAIN error and let the caller
 		 * restart us.
 		 */
-		SysTimeval zero_tv = {0, 0};
-		*ps_locked = 1;
+		to.tv_sec = 0;
+		to.tv_usec = 0;
 		ERTS_POLLSET_LOCK(ps);
 		handle_update_requests(ps);
+		ERTS_POLLSET_UNLOCK(ps);
 		res = select(ps->max_fd + 1,
 			     &ps->res_input_fds,
 			     &ps->res_output_fds,
 			     NULL,
-			     &zero_tv);
+			     &to);
 		if (res == 0) {
 		    errno = EAGAIN;
 		    res = -1;
@@ -1985,6 +2024,11 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked)
 	    return res;
 #endif				/* ----------------------------------------- */
 	}
+#ifdef ERTS_SMP
+	if (timeout)
+	    erts_thr_progress_finalize_wait(NULL);
+#endif
+	return res;
     }
 }
 
@@ -1996,7 +2040,9 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
 {
     int res, no_fds;
     int ebadf = 0;
-    int ps_locked;
+#ifdef ERTS_SMP
+    int ps_locked = 0;
+#endif
     SysTimeval *tvp;
     SysTimeval itv;
 
@@ -2017,15 +2063,14 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
 		 (int) tv->tv_sec*1000 + tv->tv_usec/1000);
 #endif
 
-    ERTS_POLLSET_UNSET_POLLER_WOKEN(ps);
     if (ERTS_POLLSET_SET_POLLED_CHK(ps)) {
 	res = EINVAL; /* Another thread is in erts_poll_wait()
 			 on this pollset... */
 	goto done;
     }
 
-    if (ERTS_POLLSET_IS_INTERRUPTED(ps)) {
-	/* Interrupt use zero timeout */
+    if (is_woken(ps)) {
+	/* Use zero timeout */
 	itv.tv_sec = 0;
 	itv.tv_usec = 0;
 	tvp = &itv;
@@ -2039,10 +2084,9 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
     }
 #endif
 
-    ps_locked = 0;
-    res = check_fd_events(ps, tvp, no_fds, &ps_locked);
+    res = check_fd_events(ps, tvp, no_fds);
 
-    ERTS_POLLSET_SET_POLLER_WOKEN(ps);
+    woke_up(ps);
 
     if (res == 0) {
 	res = ETIMEDOUT;
@@ -2062,10 +2106,8 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
 #endif
 
 #ifdef ERTS_SMP
-	if (!ps_locked) {
-	    ps_locked = 1;
-	    ERTS_POLLSET_LOCK(ps);
-	}
+	ps_locked = 1;
+	ERTS_POLLSET_LOCK(ps);
 #endif
 
 	no_fds = save_poll_result(ps, pr, no_fds, res, ebadf);
@@ -2074,9 +2116,7 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
 	check_poll_result(pr, no_fds);
 #endif
 
-	res = (no_fds == 0
-	       ? (ERTS_POLLSET_UNSET_INTERRUPTED_CHK(ps) ? EINTR : EAGAIN)
-	       : 0);
+	res = (no_fds == 0 ? (is_interrupted_reset(ps) ? EINTR : EAGAIN) : 0);
 	*len = no_fds;
     }
 
@@ -2087,7 +2127,7 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
 #endif
 
  done:
-    erts_smp_atomic_set(&ps->timeout, LONG_MAX);
+    erts_smp_atomic32_set_relb(&ps->timeout, ERTS_AINT32_T_MAX);
 #ifdef ERTS_POLL_DEBUG_PRINT
     erts_printf("Leaving %s = erts_poll_wait()\n",
 		 res == 0 ? "0" : erl_errno_id(res));
@@ -2103,21 +2143,25 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps,
 void
 ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet ps, int set)
 {
+#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
+    if (!set)
+	reset_wakeup_state(ps);
+    else
+	wake_poller(ps, 1, 0);
+#endif
+}
+
+#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
+void
+ERTS_POLL_EXPORT(erts_poll_async_sig_interrupt)(ErtsPollSet ps)
+{
     /*
-     * NOTE: This function might be called from signal handlers in the
-     *       non-smp case; therefore, it has to be async-signal safe in
-     *       the non-smp case.
+     * NOTE: This function is called from signal handlers, it,
+     *       therefore, it has to be async-signal safe.
      */
-    if (set) {
-	ERTS_POLLSET_SET_INTERRUPTED(ps);
-#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(ERTS_SMP)
-	wake_poller(ps);
-#endif
-    }
-    else {
-	ERTS_POLLSET_UNSET_INTERRUPTED(ps);
-    }
+    wake_poller(ps, 1, 1);
 }
+#endif
 
 /*
  * erts_poll_interrupt_timed():
@@ -2125,27 +2169,26 @@ ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet ps, int set)
  * is not guaranteed that it will timeout before 'msec' milli seconds.
  */
 void
-ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps, int set, long msec)
+ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps,
+					    int set,
+					    erts_short_time_t msec)
 {
-    if (set) {
-	if (erts_smp_atomic_read(&ps->timeout) > msec) {
-	    ERTS_POLLSET_SET_INTERRUPTED(ps);
 #if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(ERTS_SMP)
-	    wake_poller(ps);
-#endif
-	}
+    if (!set)
+	reset_wakeup_state(ps);
+    else {
+	if (erts_smp_atomic32_read_acqb(&ps->timeout) > (erts_aint32_t) msec)
+	    wake_poller(ps, 1, 0);
 #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
 	else {
 	    if (ERTS_POLLSET_IS_POLLED(ps))
-		erts_smp_atomic_inc(&ps->no_avoided_wakeups);
-	    erts_smp_atomic_inc(&ps->no_avoided_interrupts);
+		erts_smp_atomic_inc_nob(&ps->no_avoided_wakeups);
+	    erts_smp_atomic_inc_nob(&ps->no_avoided_interrupts);
 	}
-	erts_smp_atomic_inc(&ps->no_interrupt_timed);
+	erts_smp_atomic_inc_nob(&ps->no_interrupt_timed);
 #endif
     }
-    else {
-	ERTS_POLLSET_UNSET_INTERRUPTED(ps);
-    }
+#endif
 }
 
 int
@@ -2204,7 +2247,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
     ps->internal_fd_limit = 0;
     ps->fds_status = NULL;
     ps->fds_status_len = 0;
-    ps->no_of_user_fds = 0;
+    erts_smp_atomic_init_nob(&ps->no_of_user_fds, 0);
 #if ERTS_POLL_USE_KERNEL_POLL
     ps->kp_fd = -1;
 #if ERTS_POLL_USE_EPOLL
@@ -2256,14 +2299,14 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
     ps->update_requests.next = NULL;
     ps->update_requests.len = 0;
     ps->curr_upd_req_block = &ps->update_requests;
-    erts_smp_atomic_init(&ps->have_update_requests, 0);
+    erts_smp_atomic32_init_nob(&ps->have_update_requests, 0);
 #endif
 #ifdef ERTS_SMP
-    erts_smp_atomic_init(&ps->polled, 0);
-    erts_smp_atomic_init(&ps->woken, 0);
+    erts_atomic32_init_nob(&ps->polled, 0);
     erts_smp_mtx_init(&ps->mtx, "pollset");
-#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
-    ps->woken = 0;
+#endif
+#if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
+    erts_atomic32_init_nob(&ps->wakeup_state, (erts_aint32_t) 0);
 #endif
 #if ERTS_POLL_USE_WAKEUP_PIPE
     create_wakeup_pipe(ps);
@@ -2285,16 +2328,11 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
 	ps->internal_fd_limit = kp_fd + 1;
     ps->kp_fd = kp_fd;
 #endif
-#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP)
-    ps->interrupt = 0;
-#else
-    erts_smp_atomic_init(&ps->interrupt, 0);
-#endif
-    erts_smp_atomic_init(&ps->timeout, LONG_MAX);
+    erts_smp_atomic32_init_nob(&ps->timeout, ERTS_AINT32_T_MAX);
 #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
-    erts_smp_atomic_init(&ps->no_avoided_wakeups, 0);
-    erts_smp_atomic_init(&ps->no_avoided_interrupts, 0);
-    erts_smp_atomic_init(&ps->no_interrupt_timed, 0);
+    erts_smp_atomic_init_nob(&ps->no_avoided_wakeups, 0);
+    erts_smp_atomic_init_nob(&ps->no_avoided_interrupts, 0);
+    erts_smp_atomic_init_nob(&ps->no_interrupt_timed, 0);
 #endif
 #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE
     handle_update_requests(ps);
@@ -2302,7 +2340,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
 #if ERTS_POLL_USE_FALLBACK
     ps->fallback_used = 0;
 #endif
-    ps->no_of_user_fds = 0; /* Don't count wakeup pipe and fallback fd */
+    erts_smp_atomic_set_nob(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */
 
     erts_smp_spin_lock(&pollsets_lock);
     ps->next = pollsets;
@@ -2405,6 +2443,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
 	while (urqbp) {
 	    size += sizeof(ErtsPollSetUpdateRequestsBlock);
 	    pending_updates += urqbp->len;
+	    urqbp = urqbp->next;
 	}
     }
 #endif
@@ -2447,8 +2486,8 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
 
     pip->memory_size = size;
 
-    pip->poll_set_size = ps->no_of_user_fds;
-#ifdef ERTS_SMP
+    pip->poll_set_size = (int) erts_smp_atomic_read_nob(&ps->no_of_user_fds);
+#if ERTS_POLL_USE_WAKEUP_PIPE
     pip->poll_set_size++; /* Wakeup pipe */
 #endif
 
@@ -2505,9 +2544,9 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip)
     pip->max_fds = max_fds;
 
 #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS
-    pip->no_avoided_wakeups = erts_smp_atomic_read(&ps->no_avoided_wakeups);
-    pip->no_avoided_interrupts = erts_smp_atomic_read(&ps->no_avoided_interrupts);
-    pip->no_interrupt_timed = erts_smp_atomic_read(&ps->no_interrupt_timed);
+    pip->no_avoided_wakeups = erts_smp_atomic_read_nob(&ps->no_avoided_wakeups);
+    pip->no_avoided_interrupts = erts_smp_atomic_read_nob(&ps->no_avoided_interrupts);
+    pip->no_interrupt_timed = erts_smp_atomic_read_nob(&ps->no_interrupt_timed);
 #endif
 
     ERTS_POLLSET_UNLOCK(ps);
@@ -2527,7 +2566,7 @@ fatal_error(char *format, ...)
 {
     va_list ap;
 
-    if (ERTS_IS_CRASH_DUMPING || ERTS_GOT_SIGUSR1) {
+    if (ERTS_SOMEONE_IS_CRASH_DUMPING || ERTS_GOT_SIGUSR1) {
 	/*
 	 * Crash dump writing and reception of sigusr1 (which will
 	 * result in a crash dump) closes all file descriptors. This
@@ -2547,7 +2586,7 @@ fatal_error(char *format, ...)
 static void
 fatal_error_async_signal_safe(char *error_str)
 {
-    if (ERTS_IS_CRASH_DUMPING || ERTS_GOT_SIGUSR1) {
+    if (ERTS_SOMEONE_IS_CRASH_DUMPING || ERTS_GOT_SIGUSR1) {
 	/* See comment above in fatal_error() */
 	return;
     }
diff --git a/erts/emulator/sys/common/erl_poll.h b/erts/emulator/sys/common/erl_poll.h
index 725a77a152..8dde619105 100644
--- a/erts/emulator/sys/common/erl_poll.h
+++ b/erts/emulator/sys/common/erl_poll.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -216,11 +216,14 @@ typedef struct {
 #endif
 } ErtsPollInfo;
 
+#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
+void		ERTS_POLL_EXPORT(erts_poll_async_sig_interrupt)(ErtsPollSet);
+#endif
 void		ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet,
 						      int);
 void		ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet,
 							    int,
-							    long);
+							    erts_short_time_t);
 ErtsPollEvents	ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet,
 						    ErtsSysFdType,
 						    ErtsPollEvents,
diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c
new file mode 100644
index 0000000000..461e763f03
--- /dev/null
+++ b/erts/emulator/sys/common/erl_sys_common_misc.c
@@ -0,0 +1,107 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2006-2010. All Rights Reserved.
+ * 
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ * 
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+
+
+/*
+ * Darwin needs conversion!
+ * http://developer.apple.com/library/mac/#qa/qa2001/qa1235.html
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "sys.h"
+#include "global.h"
+
+#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__)
+#define __DARWIN__ 1
+#endif
+
+#if !defined(__WIN32__)
+#include <locale.h>
+#if !defined(HAVE_SETLOCALE) || !defined(HAVE_NL_LANGINFO) || !defined(HAVE_LANGINFO_H)
+#define PRIMITIVE_UTF8_CHECK 1
+#else
+#include <langinfo.h>
+#endif
+#endif
+
+/* Written once and only once */
+
+static int filename_encoding = ERL_FILENAME_UNKNOWN;
+#if defined(__WIN32__) || defined(__DARWIN__)
+static int user_filename_encoding = ERL_FILENAME_UTF8; /* Default unicode on windows */
+#else
+static int user_filename_encoding = ERL_FILENAME_LATIN1;
+#endif
+void erts_set_user_requested_filename_encoding(int encoding)
+{
+    user_filename_encoding = encoding;
+}
+
+int erts_get_user_requested_filename_encoding(void)
+{
+    return user_filename_encoding;
+}
+
+void erts_init_sys_common_misc(void)
+{
+#if defined(__WIN32__)
+    /* win_efile will totally fail if this is not set. */
+    filename_encoding = ERL_FILENAME_WIN_WCHAR;
+#else
+    if (user_filename_encoding != ERL_FILENAME_UNKNOWN) {
+	filename_encoding = user_filename_encoding;
+    } else {
+	char *l;
+	filename_encoding = ERL_FILENAME_LATIN1;
+#  ifdef PRIMITIVE_UTF8_CHECK
+	setlocale(LC_CTYPE, "");  /* Set international environment, 
+				     ignore result */
+	if (((l = getenv("LC_ALL"))   && *l) ||
+	    ((l = getenv("LC_CTYPE")) && *l) ||
+	    ((l = getenv("LANG"))     && *l)) {
+	    if (strstr(l, "UTF-8")) {
+		filename_encoding = ERL_FILENAME_UTF8;
+	    } 
+	}
+	
+#  else
+	l = setlocale(LC_CTYPE, "");  /* Set international environment */
+	if (l != NULL) {
+	    if (strcmp(nl_langinfo(CODESET), "UTF-8") == 0) {
+		filename_encoding = ERL_FILENAME_UTF8;
+	    }
+	}
+#  endif
+    }
+#  if defined(__DARWIN__)
+    if (filename_encoding == ERL_FILENAME_UTF8) {
+	filename_encoding = ERL_FILENAME_UTF8_MAC;
+    }
+#  endif
+#endif
+}
+
+int erts_get_native_filename_encoding(void)
+{
+    return filename_encoding;
+}
diff --git a/erts/emulator/sys/unix/erl9_start.c b/erts/emulator/sys/unix/erl9_start.c
deleted file mode 100644
index 578062d7e2..0000000000
--- a/erts/emulator/sys/unix/erl9_start.c
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2002-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-#ifdef HAVE_CONFIG_H
-#  include "config.h"
-#endif
-#include "sys.h"
-#include "erl_vm.h"
-#include "global.h"
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-/*
- * XXX This is a temporary dummy to make sys.c happy until we'll rewrite it.
- */
-unsigned preloaded_size_ring0 = 1;
-unsigned char preloaded_ring0[1] = {0};
-
-Preload pre_loaded[] = {
-    {"ring0", 1, preloaded_ring0},
-    {0, 0, 0}
-};
-
-int
-main(int argc, char** argv)
-{
-    char sbuf[1024];
-    struct {
-	void* p;
-	int sz;
-    } bins[2];
-    int bin_num = 0;
-    FILE* fp;
-    char* progname = argv[0];
-    char* eq;
-
-    argv++, argc--;
-
-    if (argc > 0 && argv[0][0] == '-') {
-	argv++, argc--;
-    }
-    if (argc < 1) {
-	abort();
-    }
-    if ((fp = fopen(argv[0], "r")) == NULL) {
-	abort();
-    }
-    
-    /* Needs to be called before any memory allocation */
-    erts_short_init();
-
-    while (fgets(sbuf, sizeof sbuf, fp)) {
-	if (sbuf[0] == '#') {
-	    continue;		/* Comment */
-	} else if (sbuf[0] == 'e' && strncmp("exec", sbuf, 4) == 0) {
-	    continue;		/* Comment ;-) */
-	} else if ((eq = strchr(sbuf, '=')) != NULL) {
-	    char* val;
-	    char* p = strchr(sbuf, '\n');
-	    if (p) {
-		*p = '\0';
-	    }
-	    *eq = '\0';
-	    val = erts_read_env(sbuf);
-	    if (val == NULL) {
-		*eq = '=';
-		erts_sys_putenv(sbuf, eq - &sbuf[0]);
-	    }
-	    erts_free_read_env(val);
-	} else if (sbuf[0] == ':' && '0' <= sbuf[1] && sbuf[1] <= '9') {
-	    int load_size = atoi(sbuf+1);
-	    void* bin;
-	    
-	    bin = malloc(load_size);
-	    if (fread(bin, 1, load_size, fp) != load_size) {
-		abort();
-	    }
-	    bins[bin_num].p = bin;
-	    bins[bin_num].sz = load_size;
-	    bin_num++;
-	} else if (strcmp(sbuf, "--end--\n") == 0) {
-	    int rval;
-	    Eterm mod = NIL;
-	    char *val;
-
-	    fclose(fp);
-
-	    if (bin_num != 2) {
-		abort();
-	    }
-
-	    val = erts_read_env("ERLBREAKHANDLER");
- 	    if (val) {
-		init_break_handler();
-	    }
-	    erts_free_read_env(val);
-
-	    if ((rval = erts_load_module(NULL, 0, NIL, &mod, bins[0].p, bins[0].sz)) < 0) {
-		fprintf(stderr, "%s: Load of initial module failed: %d\n",
-			progname, rval);
-		abort();
-	    }
-	    erts_first_process(mod, bins[1].p, bins[1].sz, argc, argv);
-	    free(bins[0].p);
-	    free(bins[1].p);
-	    process_main();
-	    abort();
-	} else {
-	    fprintf(stderr, "%s: bad line: %s\n", progname, sbuf);
-	    abort();
-	}
-    }
-    abort();
-}
diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h
index 2d5ef882f6..c8fcec8547 100644
--- a/erts/emulator/sys/unix/erl_unix_sys.h
+++ b/erts/emulator/sys/unix/erl_unix_sys.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -129,10 +129,12 @@
 #define HAVE_ERTS_CHECK_IO_DEBUG
 int erts_check_io_debug(void);
 
-
-#ifndef ENABLE_CHILD_WAITER_THREAD
+#ifndef ERTS_SMP
 #  undef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
 #  define ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
+#endif
+
+#ifndef ENABLE_CHILD_WAITER_THREAD
 #  ifdef ERTS_SMP
 #    define ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
 void erts_check_children(void);
@@ -144,6 +146,7 @@ typedef void *GETENV_STATE;
 /*
 ** For the erl_timer_sup module.
 */
+typedef time_t erts_time_t;
 
 typedef struct timeval SysTimeval;
 
@@ -329,11 +332,4 @@ extern int exit_async(void);
 
 #define ERTS_EXIT_AFTER_DUMP _exit
 
-#ifdef ERTS_TIMER_THREAD
-struct erts_iwait; /* opaque for clients */
-extern struct erts_iwait *erts_iwait_init(void);
-extern void erts_iwait_wait(struct erts_iwait *iwait, struct timeval *delay);
-extern void erts_iwait_interrupt(struct erts_iwait *iwait);
-#endif /* ERTS_TIMER_THREAD */
-
 #endif /* #ifndef _ERL_UNIX_SYS_H */
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index 31ab5d03de..bf69f3bf90 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -52,6 +52,12 @@
 #define ERTS_WANT_GOT_SIGUSR1
 #define WANT_NONBLOCKING    /* must define this to pull in defs from sys.h */
 #include "sys.h"
+#include "erl_thr_progress.h"
+
+#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__)
+#define __DARWIN__ 1
+#endif
+
 
 #ifdef USE_THREADS
 #include "erl_threads.h"
@@ -75,6 +81,7 @@ static erts_smp_rwmtx_t environ_rwmtx;
 
 #include "erl_sys_driver.h"
 #include "erl_check_io.h"
+#include "erl_cpu_topology.h"
 
 #ifndef DISABLE_VFORK
 #define DISABLE_VFORK 0
@@ -121,10 +128,7 @@ static ErtsSysReportExit *report_exit_list;
 static ErtsSysReportExit *report_exit_transit_list;
 #endif
 
-extern int  check_async_ready(void);
 extern int  driver_interrupt(int, int);
-/*EXTERN_FUNCTION(void, increment_time, (int));*/
-/*EXTERN_FUNCTION(int, next_time, (_VOID_));*/
 extern void do_break(void);
 
 extern void erl_sys_args(int*, char**);
@@ -161,14 +165,14 @@ static int debug_log = 0;
 #endif
 
 #ifdef ERTS_SMP
-erts_smp_atomic_t erts_got_sigusr1;
+erts_smp_atomic32_t erts_got_sigusr1;
 #define ERTS_SET_GOT_SIGUSR1 \
-  erts_smp_atomic_set(&erts_got_sigusr1, 1)
+  erts_smp_atomic32_set_mb(&erts_got_sigusr1, 1)
 #define ERTS_UNSET_GOT_SIGUSR1 \
-  erts_smp_atomic_set(&erts_got_sigusr1, 0)
-static erts_smp_atomic_t have_prepared_crash_dump;
+  erts_smp_atomic32_set_mb(&erts_got_sigusr1, 0)
+static erts_smp_atomic32_t have_prepared_crash_dump;
 #define ERTS_PREPARED_CRASH_DUMP \
-  ((int) erts_smp_atomic_xchg(&have_prepared_crash_dump, 1))
+  ((int) erts_smp_atomic32_xchg_nob(&have_prepared_crash_dump, 1))
 #else
 volatile int erts_got_sigusr1;
 #define ERTS_SET_GOT_SIGUSR1 (erts_got_sigusr1 = 1)
@@ -221,10 +225,10 @@ static struct fd_data {
 } *fd_data;			/* indexed by fd */
 
 /* static FUNCTION(int, write_fill, (int, char*, int)); unused? */
-static FUNCTION(void, note_child_death, (int, int));
+static void note_child_death(int, int);
 
 #if CHLDWTHR
-static FUNCTION(void *, child_waiter, (void *));
+static void* child_waiter(void *);
 #endif
 
 /********************* General functions ****************************/
@@ -236,11 +240,11 @@ static int max_files = -1;
  * a few variables used by the break handler 
  */
 #ifdef ERTS_SMP
-erts_smp_atomic_t erts_break_requested;
+erts_smp_atomic32_t erts_break_requested;
 #define ERTS_SET_BREAK_REQUESTED \
-  erts_smp_atomic_set(&erts_break_requested, (long) 1)
+  erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1)
 #define ERTS_UNSET_BREAK_REQUESTED \
-  erts_smp_atomic_set(&erts_break_requested, (long) 0)
+  erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0)
 #else
 volatile int erts_break_requested = 0;
 #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1)
@@ -259,8 +263,9 @@ int erts_use_kernel_poll = 0;
 struct {
     int (*select)(ErlDrvPort, ErlDrvEvent, int, int);
     int (*event)(ErlDrvPort, ErlDrvEvent, ErlDrvEventData);
+    void (*check_io_as_interrupt)(void);
     void (*check_io_interrupt)(int);
-    void (*check_io_interrupt_tmd)(int, long);
+    void (*check_io_interrupt_tmd)(int, erts_short_time_t);
     void (*check_io)(int);
     Uint (*size)(void);
     Eterm (*info)(void *);
@@ -298,6 +303,9 @@ init_check_io(void)
     if (erts_use_kernel_poll) {
 	io_func.select			= driver_select_kp;
 	io_func.event			= driver_event_kp;
+#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
+	io_func.check_io_as_interrupt	= erts_check_io_async_sig_interrupt_kp;
+#endif
 	io_func.check_io_interrupt	= erts_check_io_interrupt_kp;
 	io_func.check_io_interrupt_tmd	= erts_check_io_interrupt_timed_kp;
 	io_func.check_io		= erts_check_io_kp;
@@ -310,6 +318,9 @@ init_check_io(void)
     else {
 	io_func.select			= driver_select_nkp;
 	io_func.event			= driver_event_nkp;
+#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
+	io_func.check_io_as_interrupt	= erts_check_io_async_sig_interrupt_nkp;
+#endif
 	io_func.check_io_interrupt	= erts_check_io_interrupt_nkp;
 	io_func.check_io_interrupt_tmd	= erts_check_io_interrupt_timed_nkp;
 	io_func.check_io		= erts_check_io_nkp;
@@ -321,6 +332,11 @@ init_check_io(void)
     }
 }
 
+#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
+#define ERTS_CHK_IO_AS_INTR()	(*io_func.check_io_as_interrupt)()
+#else
+#define ERTS_CHK_IO_AS_INTR()	(*io_func.check_io_interrupt)(1)
+#endif
 #define ERTS_CHK_IO_INTR	(*io_func.check_io_interrupt)
 #define ERTS_CHK_IO_INTR_TMD	(*io_func.check_io_interrupt_tmd)
 #define ERTS_CHK_IO		(*io_func.check_io)
@@ -335,6 +351,11 @@ init_check_io(void)
     max_files = erts_check_io_max_files();
 }
 
+#ifdef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
+#define ERTS_CHK_IO_AS_INTR()	erts_check_io_async_sig_interrupt()
+#else
+#define ERTS_CHK_IO_AS_INTR()	erts_check_io_interrupt(1)
+#endif
 #define ERTS_CHK_IO_INTR	erts_check_io_interrupt
 #define ERTS_CHK_IO_INTR_TMD	erts_check_io_interrupt_timed
 #define ERTS_CHK_IO		erts_check_io
@@ -342,15 +363,15 @@ init_check_io(void)
 
 #endif
 
-#ifdef ERTS_SMP
 void
 erts_sys_schedule_interrupt(int set)
 {
     ERTS_CHK_IO_INTR(set);
 }
 
+#ifdef ERTS_SMP
 void
-erts_sys_schedule_interrupt_timed(int set, long msec)
+erts_sys_schedule_interrupt_timed(int set, erts_short_time_t msec)
 {
     ERTS_CHK_IO_INTR_TMD(set, msec);
 }
@@ -360,14 +381,14 @@ Uint
 erts_sys_misc_mem_sz(void)
 {
     Uint res = ERTS_CHK_IO_SZ();
-    res += erts_smp_atomic_read(&sys_misc_mem_sz);
+    res += erts_smp_atomic_read_mb(&sys_misc_mem_sz);
     return res;
 }
 
 /*
  * reset the terminal to the original settings on exit
  */
-void sys_tty_reset(void)
+void sys_tty_reset(int exit_code)
 {
   if (using_oldshell && !replace_intr) {
     SET_BLOCKING(0);
@@ -384,18 +405,6 @@ MALLOC_USE_HASH(1);
 #endif
 
 #ifdef USE_THREADS
-static void *ethr_internal_alloc(size_t size)
-{
-    return erts_alloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, (Uint) size);
-}
-static void *ethr_internal_realloc(void *ptr, size_t size)
-{
-    return erts_realloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, ptr, (Uint) size);
-}
-static void ethr_internal_free(void *ptr)
-{
-    erts_free(ERTS_ALC_T_ETHR_INTERNAL, ptr);
-}
 
 #ifdef ERTS_THR_HAVE_SIG_FUNCS
 /*
@@ -413,7 +422,7 @@ typedef struct {
 #ifdef ERTS_THR_HAVE_SIG_FUNCS
     sigset_t saved_sigmask;
 #endif
-    int unbind_child;
+    int sched_bind_data;
 } erts_thr_create_data_t;
 
 /*
@@ -424,15 +433,13 @@ static void *
 thr_create_prepare(void)
 {
     erts_thr_create_data_t *tcdp;
-    ErtsSchedulerData *esdp;
 
     tcdp = erts_alloc(ERTS_ALC_T_TMP, sizeof(erts_thr_create_data_t));
 
 #ifdef ERTS_THR_HAVE_SIG_FUNCS
     erts_thr_sigmask(SIG_BLOCK, &thr_create_sigmask, &tcdp->saved_sigmask);
 #endif
-    esdp = erts_get_scheduler_data();
-    tcdp->unbind_child = esdp && erts_is_scheduler_bound(esdp);
+    tcdp->sched_bind_data = erts_sched_bind_atthrcreate_prepare();
 
     return (void *) tcdp;
 }
@@ -444,6 +451,8 @@ thr_create_cleanup(void *vtcdp)
 {
     erts_thr_create_data_t *tcdp = (erts_thr_create_data_t *) vtcdp;
 
+    erts_sched_bind_atthrcreate_parent(tcdp->sched_bind_data);
+
 #ifdef ERTS_THR_HAVE_SIG_FUNCS
     /* Restore signalmask... */
     erts_thr_sigmask(SIG_SETMASK, &tcdp->saved_sigmask, NULL);
@@ -457,6 +466,10 @@ thr_create_prepare_child(void *vtcdp)
 {
     erts_thr_create_data_t *tcdp = (erts_thr_create_data_t *) vtcdp;
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_thread_setup();
+#endif
+
 #ifndef NO_FPE_SIGNALS
     /*
      * We do not want fp exeptions in other threads than the
@@ -466,12 +479,7 @@ thr_create_prepare_child(void *vtcdp)
     erts_thread_disable_fpe();
 #endif
 
-    if (tcdp->unbind_child) {
-	erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx);
-	erts_unbind_from_cpu(erts_cpuinfo);
-	erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx);
-    }
-    
+    erts_sched_bind_atthrcreate_child(tcdp->sched_bind_data);
 }
 
 #endif /* #ifdef USE_THREADS */
@@ -484,9 +492,6 @@ erts_sys_pre_init(void)
 #ifdef USE_THREADS
     {
     erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER;
-    eid.alloc = ethr_internal_alloc;
-    eid.realloc = ethr_internal_realloc;
-    eid.free = ethr_internal_free;
 
     eid.thread_create_child_func = thr_create_prepare_child;
     /* Before creation in parent */
@@ -521,9 +526,9 @@ erts_sys_pre_init(void)
 #endif
     }
 #ifdef ERTS_SMP
-    erts_smp_atomic_init(&erts_break_requested, 0);
-    erts_smp_atomic_init(&erts_got_sigusr1, 0);
-    erts_smp_atomic_init(&have_prepared_crash_dump, 0);
+    erts_smp_atomic32_init_nob(&erts_break_requested, 0);
+    erts_smp_atomic32_init_nob(&erts_got_sigusr1, 0);
+    erts_smp_atomic32_init_nob(&have_prepared_crash_dump, 0);
 #else
     erts_break_requested = 0;
     erts_got_sigusr1 = 0;
@@ -533,14 +538,14 @@ erts_sys_pre_init(void)
     children_died = 0;
 #endif
 #endif /* USE_THREADS */
-    erts_smp_atomic_init(&sys_misc_mem_sz, 0);
-    erts_smp_rwmtx_init(&environ_rwmtx, "environ");
+    erts_smp_atomic_init_nob(&sys_misc_mem_sz, 0);
 }
 
 void
 erl_sys_init(void)
 {
 #if !DISABLE_VFORK
+ {
     int res;
     char bindir[MAXPATHLEN];
     size_t bindirsz = sizeof(bindir);
@@ -564,12 +569,13 @@ erl_sys_init(void)
 		   + sizeof(CHILD_SETUP_PROG_NAME)
 		   + 1);
     child_setup_prog = erts_alloc(ERTS_ALC_T_CS_PROG_PATH, csp_path_sz);
-    erts_smp_atomic_add(&sys_misc_mem_sz, csp_path_sz);
+    erts_smp_atomic_add_nob(&sys_misc_mem_sz, csp_path_sz);
     sprintf(child_setup_prog,
             "%s%c%s",
             bindir,
             DIR_SEPARATOR_CHAR,
             CHILD_SETUP_PROG_NAME);
+ }
 #endif
 
 #ifdef USE_SETLINEBUF
@@ -742,7 +748,7 @@ break_requested(void)
       erl_exit(ERTS_INTR_EXIT, "");
 
   ERTS_SET_BREAK_REQUESTED;
-  ERTS_CHK_IO_INTR(1); /* Make sure we don't sleep in poll */
+  ERTS_CHK_IO_AS_INTR(); /* Make sure we don't sleep in poll */
 }
 
 /* set up signal handlers for break and quit */
@@ -942,18 +948,13 @@ void
 os_flavor(char* namebuf, 	/* Where to return the name. */
 	  unsigned size) 	/* Size of name buffer. */
 {
-    static int called = 0;
-    static struct utsname uts;	/* Information about the system. */
-
-    if (!called) {
-	char* s;
+    struct utsname uts;		/* Information about the system. */
+    char* s;
 
-	(void) uname(&uts);
-	called = 1;
-	for (s = uts.sysname; *s; s++) {
-	    if (isupper((int) *s)) {
-		*s = tolower((int) *s);
-	    }
+    (void) uname(&uts);
+    for (s = uts.sysname; *s; s++) {
+	if (isupper((int) *s)) {
+	    *s = tolower((int) *s);
 	}
     }
     strcpy(namebuf, uts.sysname);
@@ -1032,14 +1033,15 @@ static struct driver_data {
 /* Driver interfaces */
 static ErlDrvData spawn_start(ErlDrvPort, char*, SysDriverOpts*);
 static ErlDrvData fd_start(ErlDrvPort, char*, SysDriverOpts*);
-static int fd_control(ErlDrvData, unsigned int, char *, int, char **, int);
+static ErlDrvSSizeT fd_control(ErlDrvData, unsigned int, char *, ErlDrvSizeT,
+			       char **, ErlDrvSizeT);
 static ErlDrvData vanilla_start(ErlDrvPort, char*, SysDriverOpts*);
 static int spawn_init(void);
 static void fd_stop(ErlDrvData);
 static void stop(ErlDrvData);
 static void ready_input(ErlDrvData, ErlDrvEvent);
 static void ready_output(ErlDrvData, ErlDrvEvent);
-static void output(ErlDrvData, char*, int);
+static void output(ErlDrvData, char*, ErlDrvSizeT);
 static void outputv(ErlDrvData, ErlIOVec*);
 static void stop_select(ErlDrvEvent, void*);
 
@@ -1118,31 +1120,6 @@ struct erl_drv_entry vanilla_driver_entry = {
     stop_select
 };
 
-#if defined(USE_THREADS) && !defined(ERTS_SMP)
-static int  async_drv_init(void);
-static ErlDrvData async_drv_start(ErlDrvPort, char*, SysDriverOpts*);
-static void async_drv_stop(ErlDrvData);
-static void async_drv_input(ErlDrvData, ErlDrvEvent);
-
-/* INTERNAL use only */
-
-struct erl_drv_entry async_driver_entry = {
-    async_drv_init,
-    async_drv_start,
-    async_drv_stop,
-    NULL,
-    async_drv_input,
-    NULL,
-    "async",
-    NULL,
-    NULL,
-    NULL,
-    NULL,
-    NULL,
-    NULL
-};
-#endif
-
 /* Handle SIGCHLD signals. */
 #if (defined(SIG_SIGSET) || defined(SIG_SIGNAL))
 static RETSIGTYPE onchld(void)
@@ -1156,7 +1133,7 @@ static RETSIGTYPE onchld(int signum)
     smp_sig_notify('C');
 #else
     children_died = 1;
-    ERTS_CHK_IO_INTR(1); /* Make sure we don't sleep in poll */
+    ERTS_CHK_IO_AS_INTR(); /* Make sure we don't sleep in poll */
 #endif
 }
 
@@ -1186,6 +1163,8 @@ static int set_driver_data(int port_num,
 	report_exit_list = report_exit;
     }
 
+    erts_port[port_num].os_pid = pid;
+
     if (read_write & DO_READ) {
 	driver_data[ifd].packet_bytes = packet_bytes;
 	driver_data[ifd].port_num = port_num;
@@ -1226,8 +1205,8 @@ static int spawn_init()
    sys_sigset(SIGPIPE, SIG_IGN); /* Ignore - we'll handle the write failure */
    driver_data = (struct driver_data *)
        erts_alloc(ERTS_ALC_T_DRV_TAB, max_files * sizeof(struct driver_data));
-   erts_smp_atomic_add(&sys_misc_mem_sz,
-		       max_files * sizeof(struct driver_data));
+   erts_smp_atomic_add_nob(&sys_misc_mem_sz,
+			   max_files * sizeof(struct driver_data));
 
    for (i = 0; i < max_files; i++)
       driver_data[i].pid = -1;
@@ -1324,9 +1303,18 @@ static char **build_unix_environment(char *block)
 	}
     }
 
-    for (j = 0; j < i; j++) {
-	if (cpp[j][strlen(cpp[j])-1] == '=') {
+    for (j = 0; j < i; ) {
+        size_t last = strlen(cpp[j])-1;
+	if (cpp[j][last] == '=' && strchr(cpp[j], '=') == cpp[j]+last) {
 	    cpp[j] = cpp[--len];
+	    if (len < i) {
+		i--;
+	    } else {
+		j++;
+	    }
+	}
+	else {
+	    j++;
 	}
     }
 
@@ -1463,9 +1451,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op
 
     CHLD_STAT_LOCK;
 
-    unbind = erts_is_scheduler_bound(NULL);
-    if (unbind)
-	erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx);
+    unbind = erts_sched_bind_atfork_prepare();
 
 #if !DISABLE_VFORK
     /* See fork/vfork discussion before this function. */
@@ -1478,7 +1464,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op
 	if (pid == 0) {
 	    /* The child! Setup child... */
 
-	    if (unbind && erts_unbind_from_cpu(erts_cpuinfo) != 0)
+	    if (erts_sched_bind_atfork_child(unbind) != 0)
 		goto child_error;
 
 	    /* OBSERVE!
@@ -1579,8 +1565,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op
 
 	cs_argv[CS_ARGV_PROGNAME_IX] = child_setup_prog;
 	cs_argv[CS_ARGV_WD_IX] = opts->wd ? opts->wd : ".";
-	cs_argv[CS_ARGV_UNBIND_IX]
-	    = (unbind ? erts_get_unbind_from_cpu_str(erts_cpuinfo) : "false");
+	cs_argv[CS_ARGV_UNBIND_IX] = erts_sched_bind_atvfork_child(unbind);
 	cs_argv[CS_ARGV_FD_CR_IX] = fd_close_range;
 	for (i = 0; i < CS_ARGV_NO_OF_DUP2_OPS; i++)
 	    cs_argv[CS_ARGV_DUP2_OP_IX(i)] = &dup2_op[i][0];
@@ -1629,8 +1614,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op
     }
 #endif
 
-    if (unbind)
-	erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx);
+    erts_sched_bind_atfork_parent(unbind);
 
     if (pid == -1) {
         saved_errno = errno;
@@ -1745,10 +1729,10 @@ static int fd_get_window_size(int fd, Uint32 *width, Uint32 *height)
     return -1;
 }
 
-static int fd_control(ErlDrvData drv_data,
-		      unsigned int command,
-		      char *buf, int len,
-		      char **rbuf, int rlen)
+static ErlDrvSSizeT fd_control(ErlDrvData drv_data,
+			       unsigned int command,
+			       char *buf, ErlDrvSizeT len,
+			       char **rbuf, ErlDrvSizeT rlen)
 {
     int fd = (int)(long)drv_data;
     char resbuff[2*sizeof(Uint32)];
@@ -1930,8 +1914,8 @@ static void clear_fd_data(int fd)
 {
     if (fd_data[fd].sz > 0) {
 	erts_free(ERTS_ALC_T_FD_ENTRY_BUF, (void *) fd_data[fd].buf);
-	ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= fd_data[fd].sz);
-	erts_smp_atomic_add(&sys_misc_mem_sz, -1*fd_data[fd].sz);
+	ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= fd_data[fd].sz);
+	erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*fd_data[fd].sz);
     }
     fd_data[fd].buf = NULL;
     fd_data[fd].sz = 0;
@@ -2020,18 +2004,20 @@ static void outputv(ErlDrvData e, ErlIOVec* ev)
     int ix = driver_data[fd].port_num;
     int pb = driver_data[fd].packet_bytes;
     int ofd = driver_data[fd].ofd;
-    int n;
-    int sz;
+    ssize_t n;
+    ErlDrvSizeT sz;
     char lb[4];
     char* lbp;
-    int len = ev->size;
+    ErlDrvSizeT len = ev->size;
 
     /* (len > ((unsigned long)-1 >> (4-pb)*8)) */
+    /*    if (pb >= 0 && (len & (((ErlDrvSizeT)1 << (pb*8))) - 1) != len) {*/
     if (((pb == 2) && (len > 0xffff)) || (pb == 1 && len > 0xff)) {
 	driver_failure_posix(ix, EINVAL);
 	return; /* -1; */
     }
-    put_int32(len, lb);
+    /* Handles 0 <= pb <= 4 only */
+    put_int32((Uint32) len, lb);
     lbp = lb + (4-pb);
 
     ev->iov[0].iov_base = lbp;
@@ -2062,14 +2048,14 @@ static void outputv(ErlDrvData e, ErlIOVec* ev)
 }
 
 
-static void output(ErlDrvData e, char* buf, int len)
+static void output(ErlDrvData e, char* buf, ErlDrvSizeT len)
 {
     int fd = (int)(long)e;
     int ix = driver_data[fd].port_num;
     int pb = driver_data[fd].packet_bytes;
     int ofd = driver_data[fd].ofd;
-    int n;
-    int sz;
+    ssize_t n;
+    ErlDrvSizeT sz;
     char lb[4];
     char* lbp;
     struct iovec iv[2];
@@ -2266,7 +2252,7 @@ static void ready_input(ErlDrvData e, ErlDrvEvent ready_fd)
 			port_inp_failure(port_num, ready_fd, -1);
 		    }
 		    else {
-			erts_smp_atomic_add(&sys_misc_mem_sz, h);
+			erts_smp_atomic_add_nob(&sys_misc_mem_sz, h);
 			sys_memcpy(buf, cpos, bytes_left);
 			fd_data[ready_fd].buf = buf;
 			fd_data[ready_fd].sz = h;
@@ -2322,87 +2308,6 @@ static void stop_select(ErlDrvEvent fd, void* _)
     close((int)fd);
 }
 
-/*
-** Async opertation support
-*/
-#if defined(USE_THREADS) && !defined(ERTS_SMP)
-static void
-sys_async_ready_failed(int fd, int r, int err)
-{
-    char buf[120];
-    sprintf(buf, "sys_async_ready(): Fatal error: fd=%d, r=%d, errno=%d\n",
-	     fd, r, err);
-    erts_silence_warn_unused_result(write(2, buf, strlen(buf)));
-    abort();
-}
-
-/* called from threads !! */
-void sys_async_ready(int fd)
-{
-    int r;
-    while (1) {
-	r = write(fd, "0", 1);  /* signal main thread fd MUST be async_fd[1] */
-	if (r == 1) {
-	    DEBUGF(("sys_async_ready(): r = 1\r\n"));
-	    break;
-	}
-	if (r < 0 && errno == EINTR) {
-	    DEBUGF(("sys_async_ready(): r = %d\r\n", r));
-	    continue;
-	}
-	sys_async_ready_failed(fd, r, errno);
-    }
-}
-
-static int async_drv_init(void)
-{
-    async_fd[0] = -1;
-    async_fd[1] = -1;
-    return 0;
-}
-
-static ErlDrvData async_drv_start(ErlDrvPort port_num,
-				  char* name, SysDriverOpts* opts)
-{
-    if (async_fd[0] != -1)
-	return ERL_DRV_ERROR_GENERAL;
-    if (pipe(async_fd) < 0)
-	return ERL_DRV_ERROR_GENERAL;
-
-    DEBUGF(("async_drv_start: %d\r\n", port_num));
-
-    SET_NONBLOCKING(async_fd[0]);
-    driver_select(port_num, async_fd[0], ERL_DRV_READ, 1);
-
-    if (init_async(async_fd[1]) < 0)
-	return ERL_DRV_ERROR_GENERAL;
-    return (ErlDrvData)port_num;
-}
-
-static void async_drv_stop(ErlDrvData e)
-{
-    int port_num = (int)(long)e;
-
-    DEBUGF(("async_drv_stop: %d\r\n", port_num));
-
-    exit_async();
-
-    driver_select(port_num, async_fd[0], ERL_DRV_READ, 0);
-
-    close(async_fd[0]);
-    close(async_fd[1]);
-    async_fd[0] = async_fd[1] = -1;
-}
-
-
-static void async_drv_input(ErlDrvData e, ErlDrvEvent fd)
-{
-    char *buf[32];
-    DEBUGF(("async_drv_input\r\n"));
-    while (read((int) fd, (void *) buf, 32) > 0); /* fd MUST be async_fd[0] */
-    check_async_ready();  /* invoke all async_ready */
-}
-#endif
 
 void erts_do_break_handling(void)
 {
@@ -2414,11 +2319,7 @@ void erts_do_break_handling(void)
      * therefore, make sure that all threads but this one are blocked before
      * proceeding!
      */
-    erts_smp_block_system(0);
-    /*
-     * NOTE: since we allow gc we are not allowed to lock
-     *       (any) process main locks while blocking system...
-     */
+    erts_smp_thr_progress_block();
 
     /* during break we revert to initial settings */
     /* this is done differently for oldshell */
@@ -2446,7 +2347,7 @@ void erts_do_break_handling(void)
       tcsetattr(0,TCSANOW,&temp_mode);
     }
 
-    erts_smp_release_system();
+    erts_smp_thr_progress_unblock();
 }
 
 /* Fills in the systems representation of the jam/beam process identifier.
@@ -2470,7 +2371,7 @@ erts_sys_putenv(char *buffer, int sep_ix)
 #else
     Uint sz = strlen(buffer)+1;
     env = erts_alloc(ERTS_ALC_T_PUTENV_STR, sz);
-    erts_smp_atomic_add(&sys_misc_mem_sz, sz);
+    erts_smp_atomic_add_nob(&sys_misc_mem_sz, sz);
     strcpy(env,buffer);
 #endif
     erts_smp_rwmtx_rwlock(&environ_rwmtx);
@@ -2480,12 +2381,10 @@ erts_sys_putenv(char *buffer, int sep_ix)
 }
 
 int
-erts_sys_getenv(char *key, char *value, size_t *size)
+erts_sys_getenv__(char *key, char *value, size_t *size)
 {
-    char *orig_value;
     int res;
-    erts_smp_rwmtx_rlock(&environ_rwmtx);
-    orig_value = getenv(key);
+    char *orig_value = getenv(key);
     if (!orig_value)
 	res = -1;
     else {
@@ -2500,6 +2399,15 @@ erts_sys_getenv(char *key, char *value, size_t *size)
 	    res = 0;
 	}
     }
+    return res;
+}
+
+int
+erts_sys_getenv(char *key, char *value, size_t *size)
+{
+    int res;
+    erts_smp_rwmtx_rlock(&environ_rwmtx);
+    res = erts_sys_getenv__(key, value, size);
     erts_smp_rwmtx_runlock(&environ_rwmtx);
     return res;
 }
@@ -2509,33 +2417,8 @@ sys_init_io(void)
 {
     fd_data = (struct fd_data *)
 	erts_alloc(ERTS_ALC_T_FD_TAB, max_files * sizeof(struct fd_data));
-    erts_smp_atomic_add(&sys_misc_mem_sz,
-			max_files * sizeof(struct fd_data));
-
-#ifdef USE_THREADS
-#ifdef ERTS_SMP
-    if (init_async(-1) < 0)
-	erl_exit(1, "Failed to initialize async-threads\n");
-#else
-    {
-	/* This is speical stuff, starting a driver from the 
-	 * system routines, but is a nice way of handling stuff
-	 * the erlang way
-	 */
-	SysDriverOpts dopts;
-	int ret;
-
-	sys_memset((void*)&dopts, 0, sizeof(SysDriverOpts));
-	add_driver_entry(&async_driver_entry);
-	ret = erts_open_driver(NULL, NIL, "async", &dopts, NULL);
-	DEBUGF(("open_driver = %d\n", ret));
-	if (ret < 0)
-	    erl_exit(1, "Failed to open async driver\n");
-	erts_port[ret].status |= ERTS_PORT_SFLG_IMMORTAL;
-    }
-#endif
-#endif
-
+    erts_smp_atomic_add_nob(&sys_misc_mem_sz,
+			    max_files * sizeof(struct fd_data));
 }
 
 #if (0) /* unused? */
@@ -2562,7 +2445,6 @@ extern Preload pre_loaded[];
 
 void erts_sys_alloc_init(void)
 {
-    elib_ensure_initialized();
 }
 
 void *erts_sys_alloc(ErtsAlcType_t t, void *x, Uint sz)
@@ -2763,15 +2645,7 @@ initiate_report_exit_status(ErtsSysReportExit *rep, int status)
     rep->next = report_exit_transit_list;
     rep->status = status;
     report_exit_transit_list = rep;
-    /*
-     * We need the scheduler thread to call check_children().
-     * If the scheduler thread is sleeping in a poll with a
-     * timeout, we need to wake the scheduler thread. We use the
-     * functionality of the async driver to do this, instead of
-     * implementing yet another driver doing the same thing. A
-     * little bit ugly, but it works...
-     */
-    sys_async_ready(async_fd[1]);
+    erts_sys_schedule_interrupt(1);
 }
 
 static int check_children(void)
@@ -2858,20 +2732,11 @@ erl_sys_schedule(int runnable)
 {
 #ifdef ERTS_SMP
     ERTS_CHK_IO(!runnable);
-    ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING);
 #else
-    ERTS_CHK_IO_INTR(0);
-    if (runnable) {
-	ERTS_CHK_IO(0);		/* Poll for I/O */
-	check_async_ready();	/* Check async completions */
-    } else {
-	int wait_for_io = !check_async_ready();
-	if (wait_for_io)
-	    wait_for_io = !check_children();
-	ERTS_CHK_IO(wait_for_io);
-    }
-    (void) check_children();
+    ERTS_CHK_IO(runnable ? 0 : !check_children());
 #endif
+    ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
+    (void) check_children();
 }
 
 
@@ -2899,8 +2764,8 @@ smp_sig_notify(char c)
 static void *
 signal_dispatcher_thread_func(void *unused)
 {
-    int initialized = 0;
 #if !CHLDWTHR
+    int initialized = 0;
     int notify_check_children = 0;
 #endif
 #ifdef ERTS_ENABLE_LOCK_CHECK
@@ -2928,20 +2793,20 @@ signal_dispatcher_thread_func(void *unused)
 	     *         to other threads.
 	     *
 	     * NOTE 2: The signal dispatcher thread is not a blockable
-	     *         thread (i.e., it hasn't called 
-	     *         erts_register_blockable_thread()). This is
-	     *         intentional. We want to be able to interrupt
-	     *         writing of a crash dump by hitting C-c twice.
-	     *         Since it isn't a blockable thread it is important
-	     *         that it doesn't change the state of any data that
-	     *         a blocking thread expects to have exclusive access
-	     *         to (unless the signal dispatcher itself explicitly
-	     *         is blocking all blockable threads).
+	     *         thread (i.e., not a thread managed by the
+	     *         erl_thr_progress module). This is intentional.
+	     *         We want to be able to interrupt writing of a crash
+	     *         dump by hitting C-c twice. Since it isn't a
+	     *         blockable thread it is important that it doesn't
+	     *         change the state of any data that a blocking thread
+	     *         expects to have exclusive access to (unless the
+	     *         signal dispatcher itself explicitly is blocking all
+	     *         blockable threads).
 	     */
 	    switch (buf[i]) {
 	    case 0: /* Emulator initialized */
-		initialized = 1;
 #if !CHLDWTHR
+		initialized = 1;
 		if (!notify_check_children)
 #endif
 		    break;
@@ -2976,7 +2841,7 @@ signal_dispatcher_thread_func(void *unused)
 			 buf[i]);
 	    }
 	}
-	ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING);
+	ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
     }
     return NULL;
 }
@@ -3000,11 +2865,27 @@ init_smp_sig_notify(void)
 			NULL,
 			&thr_opts);
 }
+#ifdef __DARWIN__
+
+int erts_darwin_main_thread_pipe[2];
+int erts_darwin_main_thread_result_pipe[2];
+
+static void initialize_darwin_main_thread_pipes(void) 
+{
+    if (pipe(erts_darwin_main_thread_pipe) < 0 || 
+	pipe(erts_darwin_main_thread_result_pipe) < 0) {
+	erl_exit(1,"Fatal error initializing Darwin main thread stealing");
+    }
+}
 
+#endif
 void
 erts_sys_main_thread(void)
 {
     erts_thread_disable_fpe();
+#ifdef __DARWIN__
+    initialize_darwin_main_thread_pipes();
+#endif
     /* Become signal receiver thread... */
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_set_thread_name("signal_receiver");
@@ -3013,6 +2894,27 @@ erts_sys_main_thread(void)
     smp_sig_notify(0); /* Notify initialized */
     while (1) {
 	/* Wait for a signal to arrive... */
+#ifdef __DARWIN__
+	/*
+	 * The wx driver needs to be able to steal the main thread for Cocoa to
+	 * work properly.
+	 */
+	fd_set readfds;
+	int res;
+
+	FD_ZERO(&readfds);
+	FD_SET(erts_darwin_main_thread_pipe[0], &readfds);
+	res = select(erts_darwin_main_thread_pipe[0] + 1, &readfds, NULL, NULL, NULL);
+	if (res > 0 && FD_ISSET(erts_darwin_main_thread_pipe[0],&readfds)) {
+	    void* (*func)(void*);
+	    void* arg;
+	    void *resp;
+	    read(erts_darwin_main_thread_pipe[0],&func,sizeof(void* (*)(void*)));
+	    read(erts_darwin_main_thread_pipe[0],&arg, sizeof(void*));
+	    resp = (*func)(arg);
+	    write(erts_darwin_main_thread_result_pipe[1],&resp,sizeof(void *));
+	}
+#else
 #ifdef DEBUG
 	int res =
 #else
@@ -3021,6 +2923,7 @@ erts_sys_main_thread(void)
 	    select(0, NULL, NULL, NULL, NULL);
 	ASSERT(res < 0);
 	ASSERT(errno == EINTR);
+#endif
     }
 }
 
@@ -3058,6 +2961,8 @@ erl_sys_args(int* argc, char** argv)
 {
     int i, j;
 
+    erts_smp_rwmtx_init(&environ_rwmtx, "environ");
+
     i = 1;
 
     ASSERT(argc && argv);
@@ -3119,227 +3024,5 @@ erl_sys_args(int* argc, char** argv)
 	    argv[j++] = argv[i];
     }
     *argc = j;
-}
-
-#ifdef ERTS_TIMER_THREAD
-
-/*
- * Interruptible-wait facility: low-level synchronisation state
- * and methods that are implementation dependent.
- *
- * Constraint: Every implementation must define 'struct erts_iwait'
- * with a field 'erts_smp_atomic_t state;'.
- */
-
-/* values for struct erts_iwait's state field */
-#define IWAIT_WAITING	0
-#define IWAIT_AWAKE	1
-#define IWAIT_INTERRUPT	2
-
-#if 0	/* XXX: needs feature test in erts/configure.in */
-
-/*
- * This is an implementation of the interruptible wait facility on
- * top of Linux-specific futexes.
- */
-#include <asm/unistd.h>
-#define FUTEX_WAIT		0
-#define FUTEX_WAKE		1
-static int sys_futex(void *futex, int op, int val, const struct timespec *timeout)
-{
-    return syscall(__NR_futex, futex, op, val, timeout);
-}
-
-struct erts_iwait {
-    erts_smp_atomic_t state; /* &state.counter is our futex */
-};
-
-static void iwait_lowlevel_init(struct erts_iwait *iwait) { /* empty */ }
-
-static void iwait_lowlevel_wait(struct erts_iwait *iwait, struct timeval *delay)
-{
-    struct timespec timeout;
-    int res;
-
-    timeout.tv_sec = delay->tv_sec;
-    timeout.tv_nsec = delay->tv_usec * 1000;
-    res = sys_futex((void*)&iwait->state.counter, FUTEX_WAIT, IWAIT_WAITING, &timeout);
-    if (res < 0 && errno != ETIMEDOUT && errno != EWOULDBLOCK && errno != EINTR)
-	perror("FUTEX_WAIT");
-}
-
-static void iwait_lowlevel_interrupt(struct erts_iwait *iwait)
-{
-    int res = sys_futex((void*)&iwait->state.counter, FUTEX_WAKE, 1, NULL);
-    if (res < 0)
-	perror("FUTEX_WAKE");
-}
-
-#else	/* using poll() or select() */
-
-/*
- * This is an implementation of the interruptible wait facility on
- * top of pipe(), poll() or select(), read(), and write().
- */
-struct erts_iwait {
-    erts_smp_atomic_t state;
-    int read_fd;	/* wait polls and reads this fd */
-    int write_fd;	/* interrupt writes this fd */
-};
-
-static void iwait_lowlevel_init(struct erts_iwait *iwait)
-{
-    int fds[2];
 
-    if (pipe(fds) < 0) {
-	perror("pipe()");
-	exit(1);
-    }
-    iwait->read_fd = fds[0];
-    iwait->write_fd = fds[1];
-}
-
-#if defined(ERTS_USE_POLL)
-
-#include <sys/poll.h>
-#define PERROR_POLL "poll()"
-
-static int iwait_lowlevel_poll(int read_fd, struct timeval *delay)
-{
-    struct pollfd pollfd;
-    int timeout;
-
-    pollfd.fd = read_fd;
-    pollfd.events = POLLIN;
-    pollfd.revents = 0;
-    timeout = delay->tv_sec * 1000 + delay->tv_usec / 1000;
-    return poll(&pollfd, 1, timeout);
-}
-
-#else	/* !ERTS_USE_POLL */
-
-#include <sys/select.h>
-#define PERROR_POLL "select()"
-
-static int iwait_lowlevel_poll(int read_fd, struct timeval *delay)
-{
-    fd_set readfds;
-
-    FD_ZERO(&readfds);
-    FD_SET(read_fd, &readfds);
-    return select(read_fd + 1, &readfds, NULL, NULL, delay);
-}
-
-#endif	/* !ERTS_USE_POLL */
-
-static void iwait_lowlevel_wait(struct erts_iwait *iwait, struct timeval *delay)
-{
-    int res;
-    char buf[64];
-
-    res = iwait_lowlevel_poll(iwait->read_fd, delay);
-    if (res > 0)
-	(void)read(iwait->read_fd, buf, sizeof buf);
-    else if (res < 0 && errno != EINTR)
-	perror(PERROR_POLL);
-}
-
-static void iwait_lowlevel_interrupt(struct erts_iwait *iwait)
-{
-    int res = write(iwait->write_fd, "!", 1);
-    if (res < 0)
-	perror("write()");
 }
-
-#endif	/* using poll() or select() */
-
-#if 0	/* not using poll() or select() */
-/*
- * This is an implementation of the interruptible wait facility on
- * top of pthread_cond_timedwait(). This has two problems:
- * 1. pthread_cond_timedwait() requires an absolute time point,
- *    so the relative delay must be converted to absolute time.
- *    Worse, this breaks if the machine's time is adjusted while
- *    we're preparing to wait.
- * 2. Each cond operation requires additional mutex lock/unlock operations.
- *
- * Problem 2 is probably not too bad on Linux (they'll just become
- * relatively cheap futex operations), but problem 1 is the real killer.
- * Only use this implementation if no better alternatives are available!
- */
-struct erts_iwait {
-    erts_smp_atomic_t state;
-    pthread_cond_t cond;
-    pthread_mutex_t mutex;
-};
-
-static void iwait_lowlevel_init(struct erts_iwait *iwait)
-{
-    iwait->cond = (pthread_cond_t) PTHREAD_COND_INITIALIZER;
-    iwait->mutex = (pthread_mutex_t) PTHREAD_MUTEX_INITIALIZER;
-}
-
-static void iwait_lowlevel_wait(struct erts_iwait *iwait, struct timeval *delay)
-{
-    struct timeval tmp;
-    struct timespec timeout;
-
-    /* Due to pthread_cond_timedwait()'s use of absolute
-       time, this must be the real gettimeofday(), _not_
-       the "smoothed" one beam/erl_time_sup.c implements. */
-    gettimeofday(&tmp, NULL);
-
-    tmp.tv_sec += delay->tv_sec;
-    tmp.tv_usec += delay->tv_usec;
-    if (tmp.tv_usec >= 1000*1000) {
-	tmp.tv_usec -= 1000*1000;
-	tmp.tv_sec += 1;
-    }
-    timeout.tv_sec = tmp.tv_sec;
-    timeout.tv_nsec = tmp.tv_usec * 1000;
-    pthread_mutex_lock(&iwait->mutex);
-    pthread_cond_timedwait(&iwait->cond, &iwait->mutex, &timeout);
-    pthread_mutex_unlock(&iwait->mutex);
-}
-
-static void iwait_lowlevel_interrupt(struct erts_iwait *iwait)
-{
-    pthread_mutex_lock(&iwait->mutex);
-    pthread_cond_signal(&iwait->cond);
-    pthread_mutex_unlock(&iwait->mutex);
-}
-
-#endif /* not using POLL */
-
-/*
- * Interruptible-wait facility. This is just a wrapper around the
- * low-level synchronisation code, where we maintain our logical
- * state in order to suppress some state transitions.
- */
-
-struct erts_iwait *erts_iwait_init(void)
-{
-    struct erts_iwait *iwait = malloc(sizeof *iwait);
-    if (!iwait) {
-	perror("malloc");
-	exit(1);
-    }
-    iwait_lowlevel_init(iwait);
-    erts_smp_atomic_init(&iwait->state, IWAIT_AWAKE);
-    return iwait;
-}
-
-void erts_iwait_wait(struct erts_iwait *iwait, struct timeval *delay)
-{
-    if (erts_smp_atomic_xchg(&iwait->state, IWAIT_WAITING) != IWAIT_INTERRUPT)
-	iwait_lowlevel_wait(iwait, delay);
-    erts_smp_atomic_set(&iwait->state, IWAIT_AWAKE);
-}
-
-void erts_iwait_interrupt(struct erts_iwait *iwait)
-{
-    if (erts_smp_atomic_xchg(&iwait->state, IWAIT_INTERRUPT) == IWAIT_WAITING)
-	iwait_lowlevel_interrupt(iwait);
-}
-
-#endif /* ERTS_TIMER_THREAD */
diff --git a/erts/emulator/sys/unix/sys_float.c b/erts/emulator/sys/unix/sys_float.c
index c59c99f65e..8ec7b31ce0 100644
--- a/erts/emulator/sys/unix/sys_float.c
+++ b/erts/emulator/sys/unix/sys_float.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -36,11 +36,6 @@ erts_sys_init_float(void)
 # endif
 }
 
-static ERTS_INLINE void set_current_fp_exception(unsigned long pc)
-{
-    /* nothing to do */
-}
-
 #else  /* !NO_FPE_SIGNALS */
 
 #ifdef ERTS_SMP
@@ -476,7 +471,7 @@ static int mask_fpe(void)
 
 #endif
 
-#if (defined(__linux__) && (defined(__i386__) || defined(__x86_64__) || defined(__sparc__) || defined(__powerpc__))) || (defined(__DARWIN__) && (defined(__i386__) || defined(__x86_64__) || defined(__ppc__))) || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__))) || (defined(__OpenBSD__) && defined(__x86_64__)) || (defined(__sun__) && defined(__x86_64__))
+#if (defined(__linux__) && (defined(__i386__) || defined(__x86_64__) || defined(__sparc__) || defined(__powerpc__))) || (defined(__DARWIN__) && (defined(__i386__) || defined(__x86_64__) || defined(__ppc__))) || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__))) || ((defined(__NetBSD__) || defined(__OpenBSD__)) && defined(__x86_64__)) || (defined(__sun__) && defined(__x86_64__))
 
 #if defined(__linux__) && defined(__i386__)
 #if !defined(X86_FXSR_MAGIC)
@@ -519,6 +514,10 @@ static int mask_fpe(void)
 #define mc_pc(mc)	((mc)->mc_rip)
 #elif defined(__FreeBSD__) && defined(__i386__)
 #define mc_pc(mc)	((mc)->mc_eip)
+#elif defined(__NetBSD__) && defined(__x86_64__)
+#define mc_pc(mc)	((mc)->__gregs[_REG_RIP])
+#elif defined(__NetBSD__) && defined(__i386__)
+#define mc_pc(mc)	((mc)->__gregs[_REG_EIP])
 #elif defined(__OpenBSD__) && defined(__x86_64__)
 #define mc_pc(mc)	((mc)->sc_rip)
 #elif defined(__sun__) && defined(__x86_64__)
@@ -610,6 +609,23 @@ static void fpe_sig_action(int sig, siginfo_t *si, void *puc)
 	struct env87 *env87 = &savefpu->sv_87.sv_env;
 	env87->en_sw &= ~0xFF;
     }
+#elif defined(__NetBSD__) && defined(__x86_64__)
+    mcontext_t *mc = &uc->uc_mcontext;
+    struct fxsave64 *fxsave = (struct fxsave64 *)&mc->__fpregs;
+    pc = mc_pc(mc);
+    fxsave->fx_mxcsr = 0x1F80;
+    fxsave->fx_fsw &= ~0xFF;
+#elif defined(__NetBSD__) && defined(__i386__)
+    mcontext_t *mc = &uc->uc_mcontext;
+    pc = mc_pc(mc);
+    if (uc->uc_flags & _UC_FXSAVE) {
+	struct envxmm *envxmm = (struct envxmm *)&mc->__fpregs;
+	envxmm->en_mxcsr = 0x1F80;
+	envxmm->en_sw &= ~0xFF;
+    } else {
+	struct env87 *env87 = (struct env87 *)&mc->__fpregs;
+	env87->en_sw &= ~0xFF;
+    }
 #elif defined(__OpenBSD__) && defined(__x86_64__)
     struct fxsave64 *fxsave = uc->sc_fpstate;
     pc = mc_pc(uc);
@@ -799,8 +815,17 @@ sys_chars_to_double(char* buf, double* fp)
     }
 
 #ifdef NO_FPE_SIGNALS
-    if (errno == ERANGE && (*fp == 0.0 || *fp == HUGE_VAL || *fp == -HUGE_VAL)) {
-	return -1;
+    if (errno == ERANGE) {
+	if (*fp == HUGE_VAL || *fp == -HUGE_VAL) {
+	    /* overflow, should give error */
+	    return -1;
+	} else if (t == s && *fp == 0.0) {
+	    /* This should give 0.0 - OTP-7178 */
+	    errno = 0;
+
+	} else if (*fp == 0.0) {
+	    return -1;
+	}
     }
 #endif
     return 0;
@@ -810,7 +835,9 @@ int
 matherr(struct exception *exc)
 {
 #if !defined(NO_FPE_SIGNALS)
-    set_current_fp_exception((unsigned long)__builtin_return_address(0));
+    volatile unsigned long *fpexnp = erts_get_current_fp_exception();
+    if (fpexnp != NULL)
+	*fpexnp = (unsigned long)__builtin_return_address(0);
 #endif
     return 1;
 }
diff --git a/erts/emulator/sys/vxworks/erl_vxworks_sys.h b/erts/emulator/sys/vxworks/erl_vxworks_sys.h
index ae46403600..3d53238ea6 100644
--- a/erts/emulator/sys/vxworks/erl_vxworks_sys.h
+++ b/erts/emulator/sys/vxworks/erl_vxworks_sys.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -158,6 +158,7 @@ typedef struct _vxworks_tms {
 
 typedef long long SysHrTime;
 
+typedef time_t erts_time_t;
 typedef struct timeval SysTimeval;
 
 extern int sys_init_hrtime(void);
diff --git a/erts/emulator/sys/vxworks/sys.c b/erts/emulator/sys/vxworks/sys.c
index abddc7e107..739b026fb1 100644
--- a/erts/emulator/sys/vxworks/sys.c
+++ b/erts/emulator/sys/vxworks/sys.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /*
@@ -85,7 +85,7 @@ EXTERN_FUNCTION(void, erl_exit, (int n, char*, _DOTS_));
 EXTERN_FUNCTION(void, erl_error, (char*, va_list));
 EXTERN_FUNCTION(int, driver_interrupt, (int, int));
 EXTERN_FUNCTION(void, increment_time, (int));
-EXTERN_FUNCTION(int, next_time, (_VOID_));
+EXTERN_FUNCTION(int, erts_next_time, (_VOID_));
 EXTERN_FUNCTION(void, set_reclaim_free_function, (FreeFunction));
 EXTERN_FUNCTION(int, erl_mem_info_get, (MEM_PART_STATS *));
 EXTERN_FUNCTION(void, erl_crash_dump, (char* file, int line, char* fmt, ...));
@@ -143,6 +143,14 @@ volatile int erts_break_requested;
 
 /********************* General functions ****************************/
 
+/*
+ * Reset the terminal to the original settings on exit
+ * (nothing to do for WxWorks).
+ */
+void sys_tty_reset(int exit_code)
+{
+}
+
 Uint
 erts_sys_misc_mem_sz(void)
 {
@@ -230,6 +238,12 @@ erl_sys_args(int* argc, char** argv)
     ASSERT(max_files <= erts_vxworks_max_files);
 }
 
+void
+erts_sys_schedule_interrupt(int set)
+{
+    erts_check_io_interrupt(set);
+}
+
 /*
  * Called from schedule() when it runs out of runnable processes,
  * or when Erlang code has performed INPUT_REDUCTIONS reduction
@@ -238,7 +252,6 @@ erl_sys_args(int* argc, char** argv)
 void
 erl_sys_schedule(int runnable)
 {	
-    erts_check_io_interrupt(0);
     erts_check_io(!runnable);
 }
 
@@ -301,7 +314,7 @@ static void request_break(void)
   fprintf(stderr,"break!\n");
 #endif
   erts_break_requested = 1;
-  erts_check_io_interrupt(1); /* Make sure we don't sleep in erts_poll_wait */
+  erts_check_io_async_sig_interrupt(1); /* Make sure we don't sleep in erts_poll_wait */
 }
 
 static void do_quit(void)
@@ -552,7 +565,7 @@ static void fd_stop(ErlDrvData);
 static void stop(ErlDrvData);
 static void ready_input(ErlDrvData fd, ErlDrvEvent ready_fd);
 static void ready_output(ErlDrvData fd, ErlDrvEvent ready_fd);
-static void output(ErlDrvData fd, char *buf, int len);
+static void output(ErlDrvData fd, char *buf, ErlDrvSizeT len);
 static void stop_select(ErlDrvEvent, void*);
 
 struct erl_drv_entry spawn_driver_entry = {
@@ -1174,7 +1187,7 @@ static int sched_write(int port_num,int fd, char *buf, int len, int pb)
 }
 
 /* Fd is the value returned as drv_data by the start func */
-static void output(ErlDrvData drv_data, char *buf, int len)
+static void output(ErlDrvData drv_data, char *buf, ErlDrvSizeT len)
 {
     int buf_done, port_num, wval, pb, ofd;
     byte lb[4];
@@ -1507,6 +1520,12 @@ erts_sys_getenv(char *key, char *value, size_t *size)
     return res;
 }
 
+int
+erts_sys_getenv__(char *key, char *value, size_t *size)
+{
+    return erts_sys_getenv(key, value, size);
+}
+
 void
 sys_init_io(void)
 {
@@ -2017,9 +2036,6 @@ int erl_memory_show(int p0, int p1, int p2, int p3, int p4, int p5,
 	erts_printf("The memory block used by elib is save_malloc'ed "
 		   "at 0x%08x.\n", (unsigned int) alloc_pool_ptr);
     }
-#ifdef NO_FIX_ALLOC
-    erts_printf("Fix_alloc is disabled in this build\n");
-#endif
     erts_printf("Statistics from elib_malloc:\n");
     ELIB_LOCK;
 
diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c
index d816cc2c07..7a1d129cd5 100644
--- a/erts/emulator/sys/win32/erl_poll.c
+++ b/erts/emulator/sys/win32/erl_poll.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2007-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2007-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 #ifdef HAVE_CONFIG_H
@@ -274,7 +274,6 @@ struct ErtsPollSet_ {
     Waiter** waiter;
     int allocated_waiters;  /* Size ow waiter array */ 
     int num_waiters;	    /* Number of waiter threads. */
-    erts_atomic_t sys_io_ready; /* Tells us there is I/O ready (already). */
     int restore_events;        /* Tells us to restore waiters events 
 				  next time around */
     HANDLE event_io_ready;     /* To be used when waiting for io */
@@ -282,12 +281,11 @@ struct ErtsPollSet_ {
     volatile int standby_wait_counter; /* Number of threads to wait for */
     CRITICAL_SECTION standby_crit;     /* CS to guard the counter */
     HANDLE standby_wait_event;         /* Event signalled when counte == 0 */
+    erts_atomic32_t wakeup_state;
 #ifdef ERTS_SMP
-    erts_smp_atomic_t woken;
     erts_smp_mtx_t mtx;
-    erts_smp_atomic_t interrupt;
 #endif
-    erts_smp_atomic_t timeout;
+    erts_smp_atomic32_t timeout;
 };
 
 #ifdef ERTS_SMP
@@ -296,99 +294,24 @@ struct ErtsPollSet_ {
   erts_smp_mtx_lock(&(PS)->mtx)
 #define ERTS_POLLSET_UNLOCK(PS) \
   erts_smp_mtx_unlock(&(PS)->mtx)
-#define ERTS_POLLSET_SET_POLLED_CHK(PS) \
-  ((int) erts_smp_atomic_xchg(&(PS)->polled, (long) 1))
-#define ERTS_POLLSET_SET_POLLED(PS) \
-  erts_smp_atomic_set(&(PS)->polled, (long) 1)
-#define ERTS_POLLSET_UNSET_POLLED(PS) \
-  erts_smp_atomic_set(&(PS)->polled, (long) 0)
-#define ERTS_POLLSET_IS_POLLED(PS) \
-  ((int) erts_smp_atomic_read(&(PS)->polled))
-#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) \
-  ((int) erts_smp_atomic_xchg(&(PS)->woken, (long) 1))
-#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) \
-  erts_smp_atomic_set(&(PS)->woken, (long) 1)
-#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) \
-  erts_smp_atomic_set(&(PS)->woken, (long) 0)
-#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) \
-  ((int) erts_smp_atomic_read(&(PS)->woken))
-
-#define ERTS_POLLSET_UNSET_INTERRUPTED_CHK(PS) \
-  ((int) erts_smp_atomic_xchg(&(PS)->interrupt, (long) 0))
-#define ERTS_POLLSET_UNSET_INTERRUPTED(PS) \
-  erts_smp_atomic_set(&(PS)->interrupt, (long) 0)
-#define ERTS_POLLSET_SET_INTERRUPTED(PS) \
-  erts_smp_atomic_set(&(PS)->interrupt, (long) 1)
-#define ERTS_POLLSET_IS_INTERRUPTED(PS) \
-  ((int) erts_smp_atomic_read(&(PS)->interrupt))
 
 #else
 
 #define ERTS_POLLSET_LOCK(PS)
 #define ERTS_POLLSET_UNLOCK(PS)
-#define ERTS_POLLSET_SET_POLLED_CHK(PS) 0
-#define ERTS_POLLSET_UNSET_POLLED(PS)
-#define ERTS_POLLSET_IS_POLLED(PS) 0
-#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) 1
-#define ERTS_POLLSET_SET_POLLER_WOKEN(PS)
-#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS)
-#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) 1
-
-
-#endif
 
-/*
- * While atomics are not yet implemented for windows in the common library...
- *
- * MSDN doc states that SMP machines and old compilers require
- * InterLockedExchange to properly read and write interlocked
- * variables, otherwise the processors might reschedule
- * the access and order of atomics access is destroyed... 
- * While they only mention it in white-papers, the problem
- * in VS2003 is due to the IA64 arch, so we can still count
- * on the CPU not rescheduling the access to volatile in X86 arch using 
- * even the slightly older compiler...
- *
- * So here's (hopefully) a subset of the generally working atomic 
- * variable access...
- */
-
-#if defined(__GNUC__)
-#  if defined(__i386__) || defined(__x86_64__)
-#    define VOLATILE_IN_SEQUENCE 1
-#  else
-#    define VOLATILE_IN_SEQUENCE 0
-#  endif 
-#elif defined(_MSC_VER)
-#  if _MSC_VER < 1300
-#    define VOLATILE_IN_SEQUENCE 0 /* Dont trust really old compilers */
-#  else
-#    if defined(_M_IX86)
-#      define VOLATILE_IN_SEQUENCE 1
-#    else /* I.e. IA64 */
-#      if _MSC_VER >= 1400 
-#        define VOLATILE_IN_SEQUENCE 1
-#      else
-#        define VOLATILE_IN_SEQUENCE 0
-#      endif
-#    endif
-#  endif
-#else
-# define VOLATILE_IN_SEQUENCE 0
 #endif
- 
-
 
 /*
  * Communication with sys_interrupt
  */
 
 #ifdef ERTS_SMP
-extern erts_smp_atomic_t erts_break_requested;
+extern erts_smp_atomic32_t erts_break_requested;
 #define ERTS_SET_BREAK_REQUESTED \
-  erts_smp_atomic_set(&erts_break_requested, (long) 1)
+  erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1)
 #define ERTS_UNSET_BREAK_REQUESTED \
-  erts_smp_atomic_set(&erts_break_requested, (long) 0)
+  erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0)
 #else
 extern volatile int erts_break_requested;
 #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1)
@@ -397,7 +320,7 @@ extern volatile int erts_break_requested;
 
 static erts_mtx_t break_waiter_lock;
 static HANDLE break_happened_event;
-static erts_atomic_t break_waiter_state;
+static erts_atomic32_t break_waiter_state;
 #define BREAK_WAITER_GOT_BREAK 1
 #define BREAK_WAITER_GOT_HALT 2
 
@@ -440,29 +363,172 @@ do { \
     wait_standby(PS); \
  } while(0)
 
-#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP)
+#define ERTS_POLL_NOT_WOKEN		((erts_aint32_t) 0)
+#define ERTS_POLL_WOKEN_IO_READY	((erts_aint32_t) 1)
+#define ERTS_POLL_WOKEN_INTR		((erts_aint32_t) 2)
+#define ERTS_POLL_WOKEN_TIMEDOUT	((erts_aint32_t) 3)
 
 static ERTS_INLINE int
-unset_interrupted_chk(ErtsPollSet ps)
+is_io_ready(ErtsPollSet ps)
 {
-    /* This operation isn't atomic, but we have no need at all for an
-       atomic operation here... */
-    int res = ps->interrupt;
-    ps->interrupt = 0;
-    return res;
+    return erts_atomic32_read_nob(&ps->wakeup_state) == ERTS_POLL_WOKEN_IO_READY;
 }
 
+static ERTS_INLINE void
+woke_up(ErtsPollSet ps)
+{
+    if (erts_atomic32_read_nob(&ps->wakeup_state) == ERTS_POLL_NOT_WOKEN)
+	erts_atomic32_cmpxchg_nob(&ps->wakeup_state,
+				  ERTS_POLL_WOKEN_TIMEDOUT,
+				  ERTS_POLL_NOT_WOKEN);
+#ifdef DEBUG
+    {
+	erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
+	switch (wakeup_state) {
+	case ERTS_POLL_WOKEN_IO_READY:
+	case ERTS_POLL_WOKEN_INTR:
+	case ERTS_POLL_WOKEN_TIMEDOUT:
+	    break;
+	default:
+	    ASSERT(0);
+	    break;
+	}
+    }
 #endif
+}
+
+static ERTS_INLINE int
+wakeup_cause(ErtsPollSet ps)
+{
+    int res;
+    erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
+    switch (wakeup_state) {
+    case ERTS_POLL_WOKEN_IO_READY:
+	res = 0;
+	break;
+    case ERTS_POLL_WOKEN_INTR:
+	res = EINTR;
+	break;
+    case ERTS_POLL_WOKEN_TIMEDOUT:
+	res = ETIMEDOUT;
+	break;
+    default:
+	res = 0;
+	erl_exit(ERTS_ABORT_EXIT,
+		 "%s:%d: Internal error: Invalid wakeup_state=%d\n",
+		 __FILE__, __LINE__, (int) wakeup_state);
+    }
+    return res;
+}
+
+static ERTS_INLINE DWORD
+poll_wait_timeout(ErtsPollSet ps, SysTimeval *tvp)
+{
+    time_t timeout = tvp->tv_sec * 1000 + tvp->tv_usec / 1000;
+
+    if (timeout <= 0) {
+	woke_up(ps);
+	return (DWORD) 0;
+    }
+
+    ResetEvent(ps->event_io_ready);
+    /*
+     * Since we don't know the internals of ResetEvent() we issue
+     * a memory barrier as a safety precaution ensuring that
+     * the load of wakeup_state wont be reordered with stores made
+     * by ResetEvent().
+     */
+    ERTS_THR_MEMORY_BARRIER;
+    if (erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN)
+	return (DWORD) 0;
+
+    if (timeout > ((time_t) ERTS_AINT32_T_MAX))
+	timeout = ERTS_AINT32_T_MAX; /* Also prevents DWORD overflow */
+
+    erts_smp_atomic32_set_relb(&ps->timeout, (erts_aint32_t) timeout);
+    return (DWORD) timeout;
+}
 
-#ifdef ERTS_SMP
 static ERTS_INLINE void
-wake_poller(ErtsPollSet ps)
+wake_poller(ErtsPollSet ps, int io_ready)
 {
-    if (!ERTS_POLLSET_SET_POLLER_WOKEN_CHK(ps)) {
+    erts_aint32_t wakeup_state;
+    if (io_ready) {
+	/* We may set the event multiple times. This is, however, harmless. */
+	wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
+	erts_atomic32_set_relb(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY);
+    }
+    else {
+	ERTS_THR_MEMORY_BARRIER;
+	wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
+	while (wakeup_state != ERTS_POLL_WOKEN_IO_READY
+	       && wakeup_state != ERTS_POLL_WOKEN_INTR) {
+	    erts_aint32_t act = erts_atomic32_cmpxchg_nob(&ps->wakeup_state,
+							  ERTS_POLL_WOKEN_INTR,
+							  wakeup_state);
+	    if (act == wakeup_state) {
+		wakeup_state = act;
+		break;
+	    }
+	    wakeup_state = act;
+	}
+    }
+    if (wakeup_state == ERTS_POLL_NOT_WOKEN) {
+	/*
+	 * Since we don't know the internals of SetEvent() we issue
+	 * a memory barrier as a safety precaution ensuring that
+	 * the store we just made to wakeup_state wont be reordered
+	 * with loads in SetEvent().
+	 */
+	ERTS_THR_MEMORY_BARRIER;
 	SetEvent(ps->event_io_ready);
     }
 }
-#endif
+
+static ERTS_INLINE void
+reset_io_ready(ErtsPollSet ps)
+{
+    erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
+}
+
+static ERTS_INLINE void
+restore_io_ready(ErtsPollSet ps)
+{
+    erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY);
+}
+
+/*
+ * notify_io_ready() is used by threads waiting for events, when
+ * notifying a poller thread about I/O ready.
+ */
+static ERTS_INLINE void
+notify_io_ready(ErtsPollSet ps)
+{
+    wake_poller(ps, 1);
+}
+
+static ERTS_INLINE void
+reset_interrupt(ErtsPollSet ps)
+{
+    /* We need to keep io-ready if set */
+    erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
+    while (wakeup_state != ERTS_POLL_WOKEN_IO_READY
+	   && wakeup_state != ERTS_POLL_NOT_WOKEN) {
+	erts_aint32_t act = erts_atomic32_cmpxchg_nob(&ps->wakeup_state,
+						      ERTS_POLL_NOT_WOKEN,
+						      wakeup_state);
+	if (wakeup_state == act)
+	    break;
+	wakeup_state = act;
+    }
+    ERTS_THR_MEMORY_BARRIER;
+}
+
+static ERTS_INLINE void
+set_interrupt(ErtsPollSet ps)
+{
+    wake_poller(ps, 0);
+}
 
 static void setup_standby_wait(ErtsPollSet ps, int num_threads)
 {
@@ -626,14 +692,16 @@ static void *break_waiter(void *param)
 	case WAIT_OBJECT_0:
 	    ResetEvent(harr[0]);
 	    erts_mtx_lock(&break_waiter_lock);
-	    erts_atomic_set(&break_waiter_state,BREAK_WAITER_GOT_BREAK);
+	    erts_atomic32_set_nob(&break_waiter_state,BREAK_WAITER_GOT_BREAK);
+	    ERTS_THR_MEMORY_BARRIER;
 	    SetEvent(break_happened_event);
 	    erts_mtx_unlock(&break_waiter_lock);
 	    break;
 	case (WAIT_OBJECT_0+1):
 	    ResetEvent(harr[1]);
 	    erts_mtx_lock(&break_waiter_lock);
-	    erts_atomic_set(&break_waiter_state,BREAK_WAITER_GOT_HALT);
+	    erts_atomic32_set_nob(&break_waiter_state,BREAK_WAITER_GOT_HALT);
+	    ERTS_THR_MEMORY_BARRIER;
 	    SetEvent(break_happened_event);
 	    erts_mtx_unlock(&break_waiter_lock);
 	    break;
@@ -740,12 +808,7 @@ event_happened:
 	    consistency_check(w);
 #endif
 	    ASSERT(WAIT_OBJECT_0 < i && i < WAIT_OBJECT_0+w->active_events);
-	    if (!erts_atomic_xchg(&ps->sys_io_ready,1)) { 
-		HARDDEBUGF(("SET EventIoReady (%d)",erts_atomic_read(&ps->sys_io_ready)));
-		SetEvent(ps->event_io_ready);
-	    } else {
-		HARDDEBUGF(("DONT SET EventIoReady"));
-	    }
+	    notify_io_ready(ps);
 
 	    /* 
 	     * The main thread wont start working on our arrays untill we're
@@ -940,34 +1003,22 @@ static int cancel_driver_select(ErtsPollSet ps, HANDLE event)
 void  erts_poll_interrupt(ErtsPollSet ps, int set /* bool */)
 {
     HARDTRACEF(("In erts_poll_interrupt(%d)",set));
-#ifdef ERTS_SMP
-    if (set) {
-	ERTS_POLLSET_SET_INTERRUPTED(ps);
-	wake_poller(ps);
-    }
-    else {
-	ERTS_POLLSET_UNSET_INTERRUPTED(ps);
-    }
-#endif
+    if (!set)
+	reset_interrupt(ps);
+    else
+	set_interrupt(ps);
     HARDTRACEF(("Out erts_poll_interrupt(%d)",set));
 }
 
 void erts_poll_interrupt_timed(ErtsPollSet ps,
 			       int set /* bool */,
-			       long msec)
+			       erts_short_time_t msec)
 {
     HARDTRACEF(("In erts_poll_interrupt_timed(%d,%ld)",set,msec));
-#ifdef ERTS_SMP
-    if (set) {
-	if (erts_smp_atomic_read(&ps->timeout) > msec) {
-	    ERTS_POLLSET_SET_INTERRUPTED(ps);
-	    wake_poller(ps);
-	}
-    }
-    else {
-	ERTS_POLLSET_UNSET_INTERRUPTED(ps);
-    }
-#endif
+    if (!set)
+	reset_interrupt(ps);
+    else if (erts_smp_atomic32_read_acqb(&ps->timeout) > (erts_aint32_t) msec)
+	set_interrupt(ps);
     HARDTRACEF(("Out erts_poll_interrupt_timed"));
 }
 
@@ -1041,10 +1092,8 @@ void erts_poll_controlv(ErtsPollSet ps,
 int erts_poll_wait(ErtsPollSet ps,
 		   ErtsPollResFd pr[],
 		   int *len,
-		   SysTimeval *utvp)
+		   SysTimeval *tvp)
 {
-    SysTimeval *tvp = utvp;
-    SysTimeval itv;
     int no_fds;
     DWORD timeout;
     EventData* ev;
@@ -1057,7 +1106,7 @@ int erts_poll_wait(ErtsPollSet ps,
     HARDTRACEF(("In erts_poll_wait"));
     ERTS_POLLSET_LOCK(ps);
 
-    if (!erts_atomic_read(&ps->sys_io_ready) && ps->restore_events) {
+    if (!is_io_ready(ps) && ps->restore_events) {
 	HARDDEBUGF(("Restore events: %d",ps->num_waiters));
 	ps->restore_events = 0;
 	for (i = 0; i < ps->num_waiters; ++i) {
@@ -1075,7 +1124,7 @@ int erts_poll_wait(ErtsPollSet ps,
 	       if (w->highwater != w->active_events) {
 		   HARDDEBUGF(("Oups!"));
 		   /* Oups, got signalled before we took the lock, can't reset */
-		   if(erts_atomic_read(&ps->sys_io_ready) == 0) {
+		   if(!is_io_ready(ps)) {
 		       erl_exit(1,"Internal error: "
 				"Inconsistent io structures in erl_poll.\n");
 		   }
@@ -1100,39 +1149,33 @@ int erts_poll_wait(ErtsPollSet ps,
 	no_fds = ERTS_POLL_MAX_RES;
 #endif
 
+    timeout = poll_wait_timeout(ps, tvp);
 
-    ResetEvent(ps->event_io_ready);
-    ERTS_POLLSET_UNSET_POLLER_WOKEN(ps);
-
-#ifdef ERTS_SMP
-    if (ERTS_POLLSET_IS_INTERRUPTED(ps)) {
-	/* Interrupt use zero timeout */
-	itv.tv_sec = 0;
-	itv.tv_usec = 0;
-	tvp = &itv;
-    }
-#endif
-
-    timeout = tvp->tv_sec * 1000 + tvp->tv_usec / 1000;
     /*HARDDEBUGF(("timeout = %ld",(long) timeout));*/
-    erts_smp_atomic_set(&ps->timeout, timeout);
 
-    if (timeout > 0 && ! erts_atomic_read(&ps->sys_io_ready) && ! erts_atomic_read(&break_waiter_state)) {
+    if (timeout > 0 && !erts_atomic32_read_nob(&break_waiter_state)) {
 	HANDLE harr[2] = {ps->event_io_ready, break_happened_event};
 	int num_h = 2;
 
-	HARDDEBUGF(("Start waiting %d [%d]",num_h, (long) timeout));
+	HARDDEBUGF(("Start waiting %d [%d]",num_h, (int) timeout));
 	ERTS_POLLSET_UNLOCK(ps);
+#ifdef ERTS_SMP
+	erts_thr_progress_prepare_wait(NULL);
+#endif
 	WaitForMultipleObjects(num_h, harr, FALSE, timeout);
+#ifdef ERTS_SMP
+	erts_thr_progress_finalize_wait(NULL);
+#endif
 	ERTS_POLLSET_LOCK(ps);
-	HARDDEBUGF(("Stop waiting %d [%d]",num_h, (long) timeout));
+	HARDDEBUGF(("Stop waiting %d [%d]",num_h, (int) timeout));
+	woke_up(ps);
     }
 
     ERTS_UNSET_BREAK_REQUESTED;
-    if(erts_atomic_read(&break_waiter_state)) {
+    if(erts_atomic32_read_nob(&break_waiter_state)) {
 	erts_mtx_lock(&break_waiter_lock);
-	break_state = erts_atomic_read(&break_waiter_state);
-	erts_atomic_set(&break_waiter_state,0);
+	break_state = erts_atomic32_read_nob(&break_waiter_state);
+	erts_atomic32_set_nob(&break_waiter_state,0);
 	ResetEvent(break_happened_event);
 	erts_mtx_unlock(&break_waiter_lock);
 	switch (break_state) {
@@ -1147,15 +1190,13 @@ int erts_poll_wait(ErtsPollSet ps,
 	}
     }
 
-    ERTS_POLLSET_SET_POLLER_WOKEN(ps);
-
-    if (!erts_atomic_read(&ps->sys_io_ready)) {
-	res = EINTR;
-	HARDDEBUGF(("EINTR!"));
-	goto done; 
+    res = wakeup_cause(ps);
+    if (res != 0) {
+	HARDDEBUGF(("%s!", res == EINTR ? "EINTR" : "ETIMEDOUT"));
+	goto done;
     }
 
-    erts_atomic_set(&ps->sys_io_ready,0);
+    reset_io_ready(ps);
 
     n = ps->num_waiters;	
 
@@ -1177,9 +1218,9 @@ int erts_poll_wait(ErtsPollSet ps,
 	    if (num >= no_fds) {
 		w->highwater=j+1;
 		erts_mtx_unlock(&w->mtx);
-		/* This might mean we still have data to report, set
-		   back the global flag! */
-		erts_atomic_set(&ps->sys_io_ready,1);
+		/* This might mean we still have data to report,
+		   restore flag indicating I/O ready! */
+		restore_io_ready(ps);
 		HARDDEBUGF(("To many FD's to report!"));
 		goto done;
 	    }
@@ -1201,7 +1242,7 @@ int erts_poll_wait(ErtsPollSet ps,
 	erts_mtx_unlock(&w->mtx);
     }
  done:
-    erts_smp_atomic_set(&ps->timeout, LONG_MAX);
+    erts_smp_atomic32_set_nob(&ps->timeout, ERTS_AINT32_T_MAX);
     *len = num;
     ERTS_POLLSET_UNLOCK(ps);
     HARDTRACEF(("Out erts_poll_wait"));
@@ -1279,15 +1320,13 @@ ErtsPollSet erts_poll_create_pollset(void)
     ps->standby_wait_counter = 0;
     ps->event_io_ready = CreateManualEvent(FALSE);
     ps->standby_wait_event = CreateManualEvent(FALSE); 
-    erts_atomic_init(&ps->sys_io_ready,0);
     ps->restore_events = 0;
 
+    erts_atomic32_init_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
 #ifdef ERTS_SMP
-    erts_smp_atomic_init(&ps->woken, 0);
     erts_smp_mtx_init(&ps->mtx, "pollset");
-    erts_smp_atomic_init(&ps->interrupt, 0);
 #endif
-    erts_smp_atomic_init(&ps->timeout, LONG_MAX);
+    erts_smp_atomic32_init_nob(&ps->timeout, ERTS_AINT32_T_MAX);
 
     HARDTRACEF(("Out erts_poll_create_pollset"));
     return ps;
@@ -1339,7 +1378,7 @@ void  erts_poll_init(void)
 
     erts_mtx_init(&break_waiter_lock,"break_waiter_lock");
     break_happened_event = CreateManualEvent(FALSE);
-    erts_atomic_init(&break_waiter_state, 0); 
+    erts_atomic32_init_nob(&break_waiter_state, 0); 
 
     erts_thr_create(&thread, &break_waiter, NULL, NULL);
     ERTS_UNSET_BREAK_REQUESTED;
diff --git a/erts/emulator/sys/win32/erl_win32_sys_ddll.c b/erts/emulator/sys/win32/erl_win32_sys_ddll.c
index a19f49af10..d00eed932b 100644
--- a/erts/emulator/sys/win32/erl_win32_sys_ddll.c
+++ b/erts/emulator/sys/win32/erl_win32_sys_ddll.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -25,6 +25,9 @@
 #include <windows.h>
 
 #define GET_ERTS_ALC_TEST
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
 #include "sys.h"
 #include "global.h"
 #include "erl_alloc.h"
diff --git a/erts/emulator/sys/win32/erl_win_dyn_driver.h b/erts/emulator/sys/win32/erl_win_dyn_driver.h
index 4949998abc..ec5141838a 100644
--- a/erts/emulator/sys/win32/erl_win_dyn_driver.h
+++ b/erts/emulator/sys/win32/erl_win_dyn_driver.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -39,11 +39,11 @@ WDD_TYPEDEF(int, driver_exit, (ErlDrvPort, int));
 WDD_TYPEDEF(int, driver_failure_eof, (ErlDrvPort));
 WDD_TYPEDEF(int, driver_select, (ErlDrvPort, ErlDrvEvent, int, int));
 WDD_TYPEDEF(int, driver_event, (ErlDrvPort, ErlDrvEvent,ErlDrvEventData));
-WDD_TYPEDEF(int, driver_output, (ErlDrvPort, char *, int));
-WDD_TYPEDEF(int, driver_output2, (ErlDrvPort, char *, int,char *, int));
-WDD_TYPEDEF(int, driver_output_binary, (ErlDrvPort, char *, int,ErlDrvBinary*, int, int));
-WDD_TYPEDEF(int, driver_outputv, (ErlDrvPort, char*, int, ErlIOVec *,int));
-WDD_TYPEDEF(int, driver_vec_to_buf, (ErlIOVec *, char *, int));
+WDD_TYPEDEF(int, driver_output, (ErlDrvPort, char *, ErlDrvSizeT));
+WDD_TYPEDEF(int, driver_output2, (ErlDrvPort, char *, ErlDrvSizeT ,char *, ErlDrvSizeT));
+WDD_TYPEDEF(int, driver_output_binary, (ErlDrvPort, char *, ErlDrvSizeT, ErlDrvBinary*, ErlDrvSizeT, ErlDrvSizeT));
+WDD_TYPEDEF(int, driver_outputv, (ErlDrvPort, char*, ErlDrvSizeT, ErlIOVec *, ErlDrvSizeT));
+WDD_TYPEDEF(ErlDrvSizeT, driver_vec_to_buf, (ErlIOVec *, char *, ErlDrvSizeT));
 WDD_TYPEDEF(int, driver_set_timer, (ErlDrvPort, unsigned long));
 WDD_TYPEDEF(int, driver_cancel_timer, (ErlDrvPort));
 WDD_TYPEDEF(int, driver_read_timer, (ErlDrvPort, unsigned long *));
@@ -51,22 +51,22 @@ WDD_TYPEDEF(char *, erl_errno_id, (int));
 WDD_TYPEDEF(void, set_busy_port, (ErlDrvPort, int));
 WDD_TYPEDEF(void, set_port_control_flags, (ErlDrvPort, int));
 WDD_TYPEDEF(int, get_port_flags, (ErlDrvPort));
-WDD_TYPEDEF(ErlDrvBinary *, driver_alloc_binary, (int));
-WDD_TYPEDEF(ErlDrvBinary *, driver_realloc_binary, (ErlDrvBinary *, int));
+WDD_TYPEDEF(ErlDrvBinary *, driver_alloc_binary, (ErlDrvSizeT));
+WDD_TYPEDEF(ErlDrvBinary *, driver_realloc_binary, (ErlDrvBinary *, ErlDrvSizeT));
 WDD_TYPEDEF(void, driver_free_binary, (ErlDrvBinary *));
-WDD_TYPEDEF(void *, driver_alloc, (size_t));
-WDD_TYPEDEF(void *, driver_realloc, (void *, size_t));
+WDD_TYPEDEF(void *, driver_alloc, (ErlDrvSizeT));
+WDD_TYPEDEF(void *, driver_realloc, (void *, ErlDrvSizeT));
 WDD_TYPEDEF(void, driver_free, (void *));
-WDD_TYPEDEF(int, driver_enq, (ErlDrvPort, char*, int));
-WDD_TYPEDEF(int, driver_pushq, (ErlDrvPort, char*, int));
-WDD_TYPEDEF(int, driver_deq, (ErlDrvPort, int));
-WDD_TYPEDEF(int, driver_sizeq, (ErlDrvPort));
-WDD_TYPEDEF(int, driver_enq_bin, (ErlDrvPort, ErlDrvBinary *, int,int));
-WDD_TYPEDEF(int, driver_pushq_bin, (ErlDrvPort, ErlDrvBinary *, int,int));
-WDD_TYPEDEF(int, driver_peekqv, (ErlDrvPort, ErlIOVec *));
+WDD_TYPEDEF(int, driver_enq, (ErlDrvPort, char*, ErlDrvSizeT));
+WDD_TYPEDEF(int, driver_pushq, (ErlDrvPort, char*, ErlDrvSizeT));
+WDD_TYPEDEF(ErlDrvSizeT, driver_deq, (ErlDrvPort, ErlDrvSizeT));
+WDD_TYPEDEF(ErlDrvSizeT, driver_sizeq, (ErlDrvPort));
+WDD_TYPEDEF(int, driver_enq_bin, (ErlDrvPort, ErlDrvBinary *, ErlDrvSizeT, ErlDrvSizeT));
+WDD_TYPEDEF(int, driver_pushq_bin, (ErlDrvPort, ErlDrvBinary *, ErlDrvSizeT, ErlDrvSizeT));
+WDD_TYPEDEF(ErlDrvSizeT, driver_peekqv, (ErlDrvPort, ErlIOVec *));
 WDD_TYPEDEF(SysIOVec *, driver_peekq, (ErlDrvPort, int *));
-WDD_TYPEDEF(int, driver_enqv, (ErlDrvPort, ErlIOVec *, int));
-WDD_TYPEDEF(int, driver_pushqv, (ErlDrvPort, ErlIOVec *, int));
+WDD_TYPEDEF(int, driver_enqv, (ErlDrvPort, ErlIOVec *, ErlDrvSizeT));
+WDD_TYPEDEF(int, driver_pushqv, (ErlDrvPort, ErlIOVec *, ErlDrvSizeT));
 WDD_TYPEDEF(void, add_driver_entry, (ErlDrvEntry *));
 WDD_TYPEDEF(int, remove_driver_entry, (ErlDrvEntry *));
 WDD_TYPEDEF(ErlDrvTermData, driver_mk_atom, (char*));
@@ -83,19 +83,19 @@ WDD_TYPEDEF(void *, driver_dl_open, (char *));
 WDD_TYPEDEF(void *, driver_dl_sym, (void *, char *));
 WDD_TYPEDEF(int, driver_dl_close, (void *));
 WDD_TYPEDEF(char *, driver_dl_error, (void));
-WDD_TYPEDEF(unsigned long, erts_alc_test, (unsigned long,
-					   unsigned long,
-					   unsigned long,
-					   unsigned long));
-WDD_TYPEDEF(long, driver_binary_get_refc, (ErlDrvBinary *dbp));
-WDD_TYPEDEF(long, driver_binary_inc_refc, (ErlDrvBinary *dbp));
-WDD_TYPEDEF(long, driver_binary_dec_refc, (ErlDrvBinary *dbp));
+WDD_TYPEDEF(ErlDrvUInt, erts_alc_test, (ErlDrvUInt,
+					ErlDrvUInt,
+					ErlDrvUInt,
+					ErlDrvUInt));
+WDD_TYPEDEF(ErlDrvSInt, driver_binary_get_refc, (ErlDrvBinary *dbp));
+WDD_TYPEDEF(ErlDrvSInt, driver_binary_inc_refc, (ErlDrvBinary *dbp));
+WDD_TYPEDEF(ErlDrvSInt, driver_binary_dec_refc, (ErlDrvBinary *dbp));
 WDD_TYPEDEF(ErlDrvPDL, driver_pdl_create, (ErlDrvPort));
 WDD_TYPEDEF(void, driver_pdl_lock, (ErlDrvPDL));
 WDD_TYPEDEF(void, driver_pdl_unlock, (ErlDrvPDL));
-WDD_TYPEDEF(long, driver_pdl_get_refc, (ErlDrvPDL));
-WDD_TYPEDEF(long, driver_pdl_inc_refc, (ErlDrvPDL));
-WDD_TYPEDEF(long, driver_pdl_dec_refc, (ErlDrvPDL));
+WDD_TYPEDEF(ErlDrvSInt, driver_pdl_get_refc, (ErlDrvPDL));
+WDD_TYPEDEF(ErlDrvSInt, driver_pdl_inc_refc, (ErlDrvPDL));
+WDD_TYPEDEF(ErlDrvSInt, driver_pdl_dec_refc, (ErlDrvPDL));
 WDD_TYPEDEF(void, driver_system_info, (ErlDrvSysInfo *, size_t));
 WDD_TYPEDEF(int, driver_get_now, (ErlDrvNowData *));
 WDD_TYPEDEF(int, driver_monitor_process, (ErlDrvPort port, 
diff --git a/erts/emulator/sys/win32/erl_win_sys.h b/erts/emulator/sys/win32/erl_win_sys.h
index 92d8577537..03298a6c54 100644
--- a/erts/emulator/sys/win32/erl_win_sys.h
+++ b/erts/emulator/sys/win32/erl_win_sys.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -117,9 +117,30 @@ int erts_check_io_debug(void);
 #define SYS_CLK_TCK 1000
 #define SYS_CLOCK_RESOLUTION 1
 
+#if SIZEOF_TIME_T != 8
+#  error "Unexpected sizeof(time_t)"
+#endif
+
+/*
+ * gcc uses a 4 byte time_t and vc++ uses an 8 byte time_t.
+ * Types seen in beam_emu.c *need* to have the same size
+ * as in the rest of the system...
+ */
+typedef __int64 erts_time_t;
+
+struct tm *sys_localtime_r(time_t *epochs, struct tm *ptm);
+struct tm *sys_gmtime_r(time_t *epochs, struct tm *ptm);
+time_t sys_mktime( struct tm *ptm);
+
+#define localtime_r sys_localtime_r
+#define HAVE_LOCALTIME_R 1
+#define gmtime_r sys_gmtime_r
+#define HAVE_GMTIME_R
+#define mktime sys_mktime
+
 typedef struct {
-    long tv_sec;
-    long tv_usec;
+    erts_time_t tv_sec;
+    erts_time_t tv_usec;
 } SysTimeval;
 
 typedef struct {
@@ -169,10 +190,12 @@ void erts_sys_env_init(void);
 extern volatile int erl_fp_exception;
 
 #include <float.h>
-#if defined (__GNUC__)
+/* I suspect this test isn't right, it might depend on the version of GCC
+   rather than if it's a MINGW gcc, but I havent been able to pinpoint the
+   exact point where _finite was added to the headers in cygwin... */
+#if defined (__GNUC__) && !defined(__MINGW32__)
 int _finite(double x);
 #endif
-#endif
 
 /*#define NO_FPE_SIGNALS*/
 #define erts_get_current_fp_exception() NULL
@@ -191,13 +214,6 @@ int _finite(double x);
 #define erts_sys_block_fpe() 0
 #define erts_sys_unblock_fpe(x) do{}while(0)
 
-#define SIZEOF_SHORT   2
-#define SIZEOF_INT     4
-#define SIZEOF_LONG    4
-#define SIZEOF_VOID_P  4
-#define SIZEOF_SIZE_T  4
-#define SIZEOF_OFF_T   4
-
 /*
  * Seems to be missing.
  */
@@ -210,3 +226,4 @@ typedef long ssize_t;
 int init_async(int);
 int exit_async(void);
 #endif
+#endif
diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c
index 3194493ac8..acbbfc2ce9 100644..100755
--- a/erts/emulator/sys/win32/sys.c
+++ b/erts/emulator/sys/win32/sys.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /*
@@ -31,12 +31,13 @@
 #include "global.h"
 #include "erl_threads.h"
 #include "../../drivers/win32/win_con.h"
+#include "erl_cpu_topology.h"
 
 
 void erts_sys_init_float(void);
 
 void erl_start(int, char**);
-void erl_exit(int n, char*, _DOTS_);
+void erl_exit(int n, char*, ...);
 void erl_error(char*, va_list);
 void erl_crash_dump(char*, int, char*, ...);
 
@@ -66,14 +67,17 @@ static void async_read_file(struct async_io* aio, LPVOID buf, DWORD numToRead);
 static int async_write_file(struct async_io* aio, LPVOID buf, DWORD numToWrite);
 static int get_overlapped_result(struct async_io* aio,
 				 LPDWORD pBytesRead, BOOL wait);
-static FUNCTION(BOOL, CreateChildProcess, (char *, HANDLE, HANDLE,
-					   HANDLE, LPHANDLE, BOOL,
-					   LPVOID, LPTSTR, unsigned, 
-					   char **, int *));
+static BOOL create_child_process(char *, HANDLE, HANDLE,
+				 HANDLE, LPHANDLE, LPDWORD, BOOL,
+				 LPVOID, LPTSTR, unsigned,
+				 char **, int *);
 static int create_pipe(LPHANDLE, LPHANDLE, BOOL, BOOL);
-static int ApplicationType(const char* originalName, char fullPath[MAX_PATH],
+static int application_type(const char* originalName, char fullPath[MAX_PATH],
 			   BOOL search_in_path, BOOL handle_quotes,
 			   int *error_return);
+static int application_type_w(const WCHAR *originalName, WCHAR fullPath[MAX_PATH],
+			      BOOL search_in_path, BOOL handle_quotes,
+			      int *error_return);
 
 HANDLE erts_service_event;
 
@@ -86,15 +90,19 @@ static erts_smp_atomic_t pipe_creation_counter;
 static erts_smp_mtx_t sys_driver_data_lock;
 
 
-/* Results from ApplicationType is one of */
+/* Results from application_type(_w) is one of */
 #define APPL_NONE 0
 #define APPL_DOS  1
 #define APPL_WIN3X 2
 #define APPL_WIN32 3
 
-static FUNCTION(int, driver_write, (long, HANDLE, byte*, int));
+static int driver_write(long, HANDLE, byte*, int);
 static void common_stop(int);
 static int create_file_thread(struct async_io* aio, int mode);
+#ifdef ERTS_SMP
+static void close_active_handle(ErlDrvPort, HANDLE handle);
+static DWORD WINAPI threaded_handle_closer(LPVOID param);
+#endif
 static DWORD WINAPI threaded_reader(LPVOID param);
 static DWORD WINAPI threaded_writer(LPVOID param);
 static DWORD WINAPI threaded_exiter(LPVOID param);
@@ -132,6 +140,13 @@ static BOOL win_console = FALSE;
 
 static OSVERSIONINFO int_os_version;	/* Version information for Win32. */
 
+/*#define USE_CANCELIOEX
+    Disabled the use of CancelIoEx as its been seen to cause problem with some
+    drivers. Not sure what to blame; faulty drivers or some form of invalid use.
+*/
+#if defined(ERTS_SMP) && defined(USE_CANCELIOEX)
+static BOOL (WINAPI *fpCancelIoEx)(HANDLE,LPOVERLAPPED);
+#endif
 
 /* This is the system's main function (which may or may not be called "main")
    - do general system-dependent initialization
@@ -183,13 +198,27 @@ Uint
 erts_sys_misc_mem_sz(void)
 {
     Uint res = (Uint) erts_check_io_size();
-    res += (Uint) erts_smp_atomic_read(&sys_misc_mem_sz);
+    res += (Uint) erts_smp_atomic_read_mb(&sys_misc_mem_sz);
     return res;
 }
 
+/*
+ * Reset the terminal to the original settings on exit
+ */
+void sys_tty_reset(int exit_code)
+{
+    if (exit_code > 0)
+	ConWaitForExit();
+    else
+	ConNormalExit();
+}
+
 void erl_sys_args(int* argc, char** argv)
 {
     char *event_name;
+
+    erts_sys_env_init();
+
     nohup = get_and_remove_option(argc, argv, "-nohup");
 
 #ifdef DEBUG
@@ -234,7 +263,7 @@ erts_sys_prepare_crash_dump(void)
 }
 
 static void
-init_console()
+init_console(void)
 {
     char* mode = erts_read_env("ERL_CONSOLE_MODE");
 
@@ -254,7 +283,7 @@ init_console()
     erts_free_read_env(mode);
 }
 
-int sys_max_files() 
+int sys_max_files(void) 
 {
     return max_files;
 }
@@ -270,10 +299,7 @@ int sys_max_files()
  */
 
 static int
-get_and_remove_option(argc, argv, option)
-    int* argc;			/* Number of arguments. */
-    char* argv[];		/* The argument vector. */
-    const char* option;		/* Option to search for and remove. */
+get_and_remove_option(int* argc, char* argv[], const char *option)
 {
     int i;
 
@@ -323,9 +349,7 @@ static char *get_and_remove_option2(int *argc, char **argv,
 char os_type[] = "win32";
 
 void
-os_flavor(namebuf, size)
-char* namebuf;			/* Where to return the name. */
-unsigned size;			/* Size of name buffer. */
+os_flavor(char *namebuf, unsigned size)
 {
     switch (int_os_version.dwPlatformId) {
     case VER_PLATFORM_WIN32_WINDOWS:
@@ -450,7 +474,7 @@ static int spawn_init(void);
 static int fd_init(void);
 static void fd_stop(ErlDrvData);
 static void stop(ErlDrvData);
-static void output(ErlDrvData, char*, int);
+static void output(ErlDrvData, char*, ErlDrvSizeT);
 static void ready_input(ErlDrvData, ErlDrvEvent);
 static void ready_output(ErlDrvData, ErlDrvEvent);
 static void stop_select(ErlDrvEvent, void*);
@@ -545,51 +569,6 @@ struct erl_drv_entry vanilla_driver_entry = {
     stop_select
 };
 
-#if defined(USE_THREADS) && !defined(ERTS_SMP)
-
-static int  async_drv_init(void);
-static ErlDrvData async_drv_start(ErlDrvPort, char*, SysDriverOpts*);
-static void async_drv_stop(ErlDrvData);
-static void async_drv_input(ErlDrvData, ErlDrvEvent);
-
-/* INTERNAL use only */
-
-void null_output(ErlDrvData drv_data, char* buf, int len)
-{
-}
-
-void null_ready_output(ErlDrvData drv_data, ErlDrvEvent event)
-{
-}
-
-struct erl_drv_entry async_driver_entry = {
-    async_drv_init,
-    async_drv_start,
-    async_drv_stop,
-    null_output,
-    async_drv_input,
-    null_ready_output,
-    "async",
-    NULL, /* finish */
-    NULL, /* handle */
-    NULL, /* control */
-    NULL, /* timeout */
-    NULL, /* outputv */
-    NULL, /* ready_async */
-    NULL, /* flush */
-    NULL, /* call */
-    NULL, /* event */
-    ERL_DRV_EXTENDED_MARKER,
-    ERL_DRV_EXTENDED_MAJOR_VERSION,
-    ERL_DRV_EXTENDED_MINOR_VERSION,
-    0,	/* ERL_DRV_FLAGs */
-    NULL,
-    NULL, /* process_exit */
-    stop_select
-};
-
-#endif
-
 /*
  * Initialises a DriverData structure.
  *
@@ -598,12 +577,7 @@ struct erl_drv_entry async_driver_entry = {
  */
 
 static DriverData*
-new_driver_data(port_num, packet_bytes, wait_objs_required, use_threads)
-    int port_num;		/* The port number. */
-    int packet_bytes;		/* Number of bytes in header. */
-    int wait_objs_required;	/* The number objects this port is going
-				/* wait for (typically 1 or 2). */
-    int use_threads;		/* TRUE if threads are intended to be used. */
+new_driver_data(int port_num, int packet_bytes, int wait_objs_required, int use_threads)
 {
     DriverData* dp;
     
@@ -632,7 +606,7 @@ new_driver_data(port_num, packet_bytes, wait_objs_required, use_threads)
 		erts_smp_mtx_unlock(&sys_driver_data_lock);
 		return NULL;
 	    }
-	    erts_smp_atomic_add(&sys_misc_mem_sz, dp->inBufSize);
+	    erts_smp_atomic_add_nob(&sys_misc_mem_sz, dp->inBufSize);
 	    dp->outBufSize = 0;
 	    dp->outbuf = NULL;
 	    dp->port_num = port_num;
@@ -665,25 +639,50 @@ release_driver_data(DriverData* dp)
     erts_smp_mtx_lock(&sys_driver_data_lock);
 
 #ifdef ERTS_SMP
-    /* This is a workaround for the fact that CancelIo cant cancel
-       requests issued by another thread and that we still cant use
-       CancelIoEx as that's only availabele in Vista etc. */
-    if(dp->in.async_io_active && dp->in.fd != INVALID_HANDLE_VALUE) {
-	CloseHandle(dp->in.fd);
-	dp->in.fd = INVALID_HANDLE_VALUE;
-	DEBUGF(("Waiting for the in event thingie"));
-	WaitForSingleObject(dp->in.ov.hEvent,INFINITE);
-	DEBUGF(("...done\n"));
-    }
-    if(dp->out.async_io_active && dp->out.fd != INVALID_HANDLE_VALUE) {
-	CloseHandle(dp->out.fd);
-	dp->out.fd = INVALID_HANDLE_VALUE;
-	DEBUGF(("Waiting for the out event thingie"));
-	WaitForSingleObject(dp->out.ov.hEvent,INFINITE);
-	DEBUGF(("...done\n"));
+#ifdef USE_CANCELIOEX
+    if (fpCancelIoEx != NULL) {
+	if (dp->in.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) {
+	    (*fpCancelIoEx)(dp->in.fd, NULL);
+	}
+	if (dp->out.thread == (HANDLE) -1 && dp->out.fd != INVALID_HANDLE_VALUE) {
+	    (*fpCancelIoEx)(dp->out.fd, NULL);
+	}
+    }
+    else
+#endif
+    {
+	/* This is a workaround for the fact that CancelIo cant cancel
+	   requests issued by another thread and that we cant use
+	   CancelIoEx as that's only available in Vista etc.
+	   R14: Avoid scheduler deadlock by only wait for 10ms, and then spawn
+	    a thread that will keep waiting in in order to close handles. */
+	HANDLE handles[2];
+	int i = 0;
+	int timeout = 10;
+	if(dp->in.async_io_active && dp->in.fd != INVALID_HANDLE_VALUE) {
+	    CloseHandle(dp->in.fd);
+	    dp->in.fd = INVALID_HANDLE_VALUE;
+	    DEBUGF(("Waiting for the in event thingie"));
+	    if (WaitForSingleObject(dp->in.ov.hEvent,timeout) == WAIT_TIMEOUT) {
+		close_active_handle(dp->port_num, dp->in.ov.hEvent);
+		dp->in.ov.hEvent = NULL;
+		timeout = 0;
+	    }
+	    DEBUGF(("...done\n"));
+	}
+	if(dp->out.async_io_active && dp->out.fd != INVALID_HANDLE_VALUE) {
+	    CloseHandle(dp->out.fd);
+	    dp->out.fd = INVALID_HANDLE_VALUE;
+	    DEBUGF(("Waiting for the out event thingie"));
+	    if (WaitForSingleObject(dp->out.ov.hEvent,timeout) == WAIT_TIMEOUT) {
+		close_active_handle(dp->port_num, dp->out.ov.hEvent);
+		dp->out.ov.hEvent = NULL;
+	    }
+	    DEBUGF(("...done\n"));
+	}
     }
 #else
-    	if (dp->out.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) {
+	if (dp->in.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) {
 	     CancelIo(dp->in.fd); 
 	}
 	if (dp->out.thread == (HANDLE) -1 && dp->out.fd != INVALID_HANDLE_VALUE) {
@@ -692,8 +691,8 @@ release_driver_data(DriverData* dp)
 #endif
 
     if (dp->inbuf != NULL) {
-	ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->inBufSize);
-	erts_smp_atomic_add(&sys_misc_mem_sz, -1*dp->inBufSize);
+	ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize);
+	erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->inBufSize);
 	DRV_BUF_FREE(dp->inbuf);
 	dp->inBufSize = 0;
 	dp->inbuf = NULL;
@@ -701,8 +700,8 @@ release_driver_data(DriverData* dp)
     ASSERT(dp->inBufSize == 0);
 
     if (dp->outbuf != NULL) {
-	ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->outBufSize);
-	erts_smp_atomic_add(&sys_misc_mem_sz, -1*dp->outBufSize);
+	ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize);
+	erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize);
 	DRV_BUF_FREE(dp->outbuf);
 	dp->outBufSize = 0;
 	dp->outbuf = NULL;
@@ -726,6 +725,88 @@ release_driver_data(DriverData* dp)
     erts_smp_mtx_unlock(&sys_driver_data_lock);
 }
 
+#ifdef ERTS_SMP
+
+struct handles_to_be_closed {
+    HANDLE handles[MAXIMUM_WAIT_OBJECTS];
+    unsigned cnt;
+};
+static struct handles_to_be_closed* htbc_curr = NULL;
+CRITICAL_SECTION htbc_lock;
+
+static void close_active_handle(ErlDrvPort port_num, HANDLE handle)
+{
+    struct handles_to_be_closed* htbc;
+    int i;
+    EnterCriticalSection(&htbc_lock);
+    htbc = htbc_curr;
+    if (htbc == NULL || htbc->cnt >= MAXIMUM_WAIT_OBJECTS) {
+	DWORD tid;
+	HANDLE thread;
+
+	htbc = (struct handles_to_be_closed*) erts_alloc(ERTS_ALC_T_DRV_TAB,
+							 sizeof(*htbc));
+	htbc->handles[0] = CreateAutoEvent(FALSE);
+	htbc->cnt = 1;
+	thread = (HANDLE *) _beginthreadex(NULL, 0, threaded_handle_closer, htbc, 0, &tid);
+	CloseHandle(thread);
+    }
+    htbc->handles[htbc->cnt++] = handle;
+    driver_select(port_num, (ErlDrvEvent)handle, ERL_DRV_USE_NO_CALLBACK, 0);
+    SetEvent(htbc->handles[0]);
+    htbc_curr = htbc;
+    LeaveCriticalSection(&htbc_lock);
+}
+
+static DWORD WINAPI
+threaded_handle_closer(LPVOID param)
+{
+    struct handles_to_be_closed* htbc = (struct handles_to_be_closed*) param;
+    unsigned ix;
+    DWORD res;
+    DEBUGF(("threaded_handle_closer %p started\r\n", htbc));
+    EnterCriticalSection(&htbc_lock);
+    for (;;) {
+	{
+	    HANDLE* handles = htbc->handles;
+	    unsigned cnt = htbc->cnt;
+	    DWORD timeout = (htbc == htbc_curr) ? INFINITE : 10*1000;
+
+	    LeaveCriticalSection(&htbc_lock);
+	    DEBUGF(("threaded_handle_closer %p waiting for %d handles\r\n", htbc, cnt));
+	    res = WaitForMultipleObjects(cnt, handles, FALSE, timeout);
+	}
+	EnterCriticalSection(&htbc_lock);
+	switch (res) {
+	case WAIT_OBJECT_0:
+	case WAIT_TIMEOUT:
+	    break; /* got some more handles to wait for maybe */
+	default:
+	    ix = res - WAIT_OBJECT_0;
+	    if (ix > 0 && ix < htbc->cnt) {
+		CloseHandle(htbc->handles[ix]);
+		htbc->handles[ix] = htbc->handles[--htbc->cnt];
+	    }
+	}
+	if (htbc != htbc_curr) {
+	    if (htbc->cnt == 1) { /* no real handles left */
+		break;
+	    }
+	    /* The thread with most free slots will be "current" */
+	    if (htbc->cnt < htbc_curr->cnt) {
+		htbc_curr = htbc;
+		DEBUGF(("threaded_handle_closer %p made current\r\n", htbc));
+	    }
+	}
+    }
+    LeaveCriticalSection(&htbc_lock);
+    CloseHandle(htbc->handles[0]);
+    erts_free(ERTS_ALC_T_DRV_TAB, htbc);
+    DEBUGF(("threaded_handle_closer %p terminating\r\n", htbc));
+    return 0;
+}
+#endif /* ERTS_SMP */
+
 /*
  * Stores input and output file descriptors in the DriverData structure,
  * and calls driver_select().
@@ -734,12 +815,7 @@ release_driver_data(DriverData* dp)
  */
 
 static ErlDrvData
-set_driver_data(dp, ifd, ofd, read_write, report_exit)
-    DriverData* dp;
-    HANDLE ifd;
-    HANDLE ofd;
-    int read_write;
-    int report_exit;
+set_driver_data(DriverData* dp, HANDLE ifd, HANDLE ofd, int read_write, int report_exit)
 {
     int index = dp - driver_data;
     int result;
@@ -763,6 +839,31 @@ set_driver_data(dp, ifd, ofd, read_write, report_exit)
     return (ErlDrvData)index;
 }
 
+static ErlDrvData
+reuse_driver_data(DriverData *dp, HANDLE ifd, HANDLE ofd, int read_write, ErlDrvPort port_num)
+{
+    int index = dp - driver_data;
+    int result;
+
+    dp->port_num = port_num;
+    dp->in.fd = ifd;
+    dp->out.fd = ofd;
+    dp->report_exit = 0;
+
+    if (read_write & DO_READ) {
+	result = driver_select(dp->port_num, (ErlDrvEvent)dp->in.ov.hEvent,
+			       ERL_DRV_READ|ERL_DRV_USE, 1);
+	ASSERT(result != -1);
+    }
+
+    if (read_write & DO_WRITE) {
+	result = driver_select(dp->port_num, (ErlDrvEvent)dp->out.ov.hEvent,
+			       ERL_DRV_WRITE|ERL_DRV_USE, 1);
+	ASSERT(result != -1);
+    }
+    return (ErlDrvData)index;
+}
+
 /*
  * Initialises an AsyncIo structure.
  */
@@ -836,10 +937,7 @@ release_async_io(AsyncIo* aio, ErlDrvPort port_num)
  */
 
 static void
-async_read_file(aio, buf, numToRead)
-    AsyncIo* aio;		/* Pointer to driver data. */
-    LPVOID buf;			/* Pointer to buffer to receive data. */
-    DWORD numToRead;		/* Number of bytes to read. */
+async_read_file(AsyncIo* aio, LPVOID buf, DWORD numToRead)
 {
     aio->pendingError = NO_ERROR;
 #ifdef HARD_POLL_DEBUG
@@ -890,10 +988,9 @@ async_read_file(aio, buf, numToRead)
  * ----------------------------------------------------------------------
  */
 static int
-async_write_file(aio, buf, numToWrite)
-    AsyncIo* aio;		/* Pointer to async control block. */
-    LPVOID buf;			/* Pointer to buffer with data to write. */
-    DWORD numToWrite;		/* Number of bytes to write. */
+async_write_file(AsyncIo* aio,		/* Pointer to async control block. */
+		 LPVOID buf,		/* Pointer to buffer with data to write. */
+		 DWORD numToWrite)	/* Number of bytes to write. */
 {
     aio->pendingError = NO_ERROR;
     if (aio->thread != (HANDLE) -1) {
@@ -937,12 +1034,12 @@ async_write_file(aio, buf, numToWrite)
  * ----------------------------------------------------------------------
  */
 static int
-get_overlapped_result(aio, pBytesRead, wait)
-    AsyncIo* aio;		/* Pointer to async control block. */
-    LPDWORD pBytesRead;		/* Where to place the number of bytes
-				 * transferred.
-				 */
-    BOOL wait;			/* If true, wait until result is ready. */
+get_overlapped_result(AsyncIo* aio,		/* Pointer to async control block. */
+		      LPDWORD pBytesRead,	/* Where to place the number of bytes
+						 * transferred.
+						 */
+		      BOOL wait			/* If true, wait until result is ready. */
+		      )
 {
     DWORD error = NO_ERROR;	/* Error status from last function. */
 
@@ -1012,15 +1109,22 @@ fd_init(void)
     return 0;
 }
 static int
-spawn_init()
+spawn_init(void)
 {
     int i;
-  
+#if defined(ERTS_SMP) && defined(USE_CANCELIOEX)
+    HMODULE module = GetModuleHandle("kernel32");
+    fpCancelIoEx = (BOOL (WINAPI *)(HANDLE,LPOVERLAPPED))
+	((module != NULL) ? GetProcAddress(module,"CancelIoEx") : NULL);
+    DEBUGF(("fpCancelIoEx = %p\r\n", fpCancelIoEx));
+#endif
     driver_data = (struct driver_data *)
 	erts_alloc(ERTS_ALC_T_DRV_TAB, max_files * sizeof(struct driver_data));
-    erts_smp_atomic_add(&sys_misc_mem_sz, max_files*sizeof(struct driver_data));
+    erts_smp_atomic_add_nob(&sys_misc_mem_sz,
+			    max_files*sizeof(struct driver_data));
     for (i = 0; i < max_files; i++)
 	driver_data[i].port_num = PORT_FREE;
+
     return 0;
 }
 
@@ -1032,6 +1136,7 @@ spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* opts)
     HANDLE hChildStdin = INVALID_HANDLE_VALUE;		/* Child's stdin. */
     HANDLE hChildStdout = INVALID_HANDLE_VALUE;	/* Child's stout. */
     HANDLE hChildStderr = INVALID_HANDLE_VALUE;	/* Child's sterr. */
+    DWORD pid;
     int close_child_stderr = 0;
     DriverData* dp;		/* Pointer to driver data. */
     ErlDrvData retval = ERL_DRV_ERROR_GENERAL; /* Return value. */
@@ -1099,12 +1204,15 @@ spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* opts)
      */
 
     DEBUGF(("Spawning \"%s\"\n", name));
-    envir = win_build_environment(envir);
-    ok = CreateChildProcess(name, 
+    envir = win_build_environment(envir); /* Still an ansi environment, could be
+					     converted to unicode for spawn_executable, but
+					     that is not done (yet) */ 
+    ok = create_child_process(name, 
 			    hChildStdin, 
 			    hChildStdout,
 			    hChildStderr,
 			    &dp->port_pid,
+			    &pid,
 			    opts->hide_window,
 			    (LPVOID) envir,
 			    (LPTSTR) opts->wd,
@@ -1148,6 +1256,9 @@ spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* opts)
 #endif
 	retval = set_driver_data(dp, hFromChild, hToChild, opts->read_write,
 				 opts->exit_status);
+	if (retval != ERL_DRV_ERROR_GENERAL && retval != ERL_DRV_ERROR_ERRNO)
+	    /* We assume that this cannot generate a negative number */
+	    erts_port[port_num].os_pid = (SWord) pid;
     }
     
     if (retval != ERL_DRV_ERROR_GENERAL && retval != ERL_DRV_ERROR_ERRNO)
@@ -1179,7 +1290,7 @@ create_file_thread(AsyncIo* aio, int mode)
 }
 
 /* 
- *  A helper function used by CreateChildProcess().
+ *  A helper function used by create_child_process().
  *  Parses a command line with arguments and returns the length of the
  *  first part containing the program name.
  *  Example: input = "\"Program Files\"\\erl arg1 arg2"
@@ -1220,24 +1331,25 @@ int parse_command(char* cmd){
     return i;
 }
 
-BOOL need_quotes(char *str)
+static BOOL need_quotes(WCHAR *str) 
 {
     int in_quote = 0;
     int backslashed = 0;
     int naked_space = 0;
-    while (*str != '\0') {
+
+    while (*str != L'\0') {
 	switch (*str) {
-	case '\\' :
+	case L'\\' :
 	    backslashed = !backslashed;
 	    break;
-	case '"':
+	case L'"':
 	    if (backslashed) {
 		backslashed=0;
 	    } else {
 		in_quote = !in_quote;
 	    }
 	    break;
-	case ' ':
+	case L' ':
 	    backslashed = 0;
 	    if (!(backslashed || in_quote)) {
 		naked_space++;
@@ -1256,7 +1368,7 @@ BOOL need_quotes(char *str)
 /*
  *----------------------------------------------------------------------
  *
- * CreateChildProcess --
+ * create_child_process --
  *
  *	Create a child process that has pipes as its 
  *	standard input, output, and error.  The child process runs
@@ -1281,7 +1393,7 @@ BOOL need_quotes(char *str)
  */
 
 static BOOL
-CreateChildProcess
+create_child_process
 (
  char *origcmd,  /* Command line for child process (including
 		  * name of executable). Or whole executable if st is
@@ -1290,7 +1402,8 @@ CreateChildProcess
  HANDLE hStdin,  /* The standard input handle for child. */
  HANDLE hStdout, /* The standard output handle for child. */ 
  HANDLE hStderr, /* The standard error handle for child. */
- LPHANDLE phPid, /* Pointer to variable to received PID. */
+ LPHANDLE phPid, /* Pointer to variable to received Process handle. */
+ LPDWORD pdwID,   /* Pointer to variable to received Process ID */
  BOOL hide,      /* Hide the window unconditionally. */
  LPVOID env,     /* Environment for the child */
  LPTSTR wd,      /* Working dir for the child */
@@ -1300,14 +1413,12 @@ CreateChildProcess
  )
 { 
     PROCESS_INFORMATION piProcInfo = {0};
-    STARTUPINFO siStartInfo = {0};
     BOOL ok = FALSE;
     int applType;
     /* Not to be changed for different types of executables */
     int staticCreateFlags = GetPriorityClass(GetCurrentProcess()); 
     int createFlags = DETACHED_PROCESS;
     char *newcmdline = NULL;
-    char execPath[MAX_PATH];
     int cmdlength;
     char* thecommand;
     LPTSTR appname = NULL;
@@ -1315,14 +1426,17 @@ CreateChildProcess
     
     *errno_return = -1;
 
-    siStartInfo.cb = sizeof(STARTUPINFO); 
-    siStartInfo.dwFlags = STARTF_USESTDHANDLES;
-    siStartInfo.hStdInput = hStdin;
-    siStartInfo.hStdOutput = hStdout;
-    siStartInfo.hStdError = hStderr;
-
 
     if (st != ERTS_SPAWN_EXECUTABLE) {
+	STARTUPINFO siStartInfo = {0};
+	char execPath[MAX_PATH];
+
+	siStartInfo.cb = sizeof(STARTUPINFO); 
+	siStartInfo.dwFlags = STARTF_USESTDHANDLES;
+	siStartInfo.hStdInput = hStdin;
+	siStartInfo.hStdOutput = hStdout;
+	siStartInfo.hStdError = hStderr;
+
 	/*
 	 * Parse out the program name from the command line (it can be quoted and
 	 * contain spaces).
@@ -1334,9 +1448,9 @@ CreateChildProcess
 	thecommand[cmdlength] = '\0';
 	DEBUGF(("spawn command: %s\n", thecommand));
     
-	applType = ApplicationType(thecommand, execPath, TRUE, 
+	applType = application_type(thecommand, execPath, TRUE, 
 				   TRUE, errno_return);
-	DEBUGF(("ApplicationType returned for (%s) is %d\n", thecommand, applType));
+	DEBUGF(("application_type returned for (%s) is %d\n", thecommand, applType));
 	erts_free(ERTS_ALC_T_TMP, (void *) thecommand);
 	if (applType == APPL_NONE) {
 	    erts_free(ERTS_ALC_T_TMP,newcmdline);
@@ -1365,126 +1479,147 @@ CreateChildProcess
 
 	strcat(newcmdline, execPath);
 	strcat(newcmdline, origcmd+cmdlength);
-    } else { /* ERTS_SPAWN_EXECUTABLE */
+	DEBUGF(("Creating child process: %s, createFlags = %d\n", newcmdline, createFlags));
+	ok = CreateProcessA(appname, 
+			    newcmdline, 
+			    NULL, 
+			    NULL, 
+			    TRUE, 
+			    createFlags | staticCreateFlags, 
+			    env, 
+			    wd, 
+			    &siStartInfo, 
+			    &piProcInfo);
+
+    } else { /* ERTS_SPAWN_EXECUTABLE, filename and args are in unicode ({utf16,little}) */
 	int run_cmd = 0;
-	applType = ApplicationType(origcmd, execPath, FALSE, FALSE, 
-				   errno_return);
+	STARTUPINFOW siStartInfo = {0};
+	WCHAR execPath[MAX_PATH];
+
+
+	siStartInfo.cb = sizeof(STARTUPINFOW); 
+	siStartInfo.dwFlags = STARTF_USESTDHANDLES;
+	siStartInfo.hStdInput = hStdin;
+	siStartInfo.hStdOutput = hStdout;
+	siStartInfo.hStdError = hStderr;
+
+	applType = application_type_w((WCHAR *) origcmd, execPath, FALSE, FALSE, 
+				      errno_return);
 	if (applType == APPL_NONE) {
 	    return FALSE;
 	} 
 	if (applType == APPL_DOS) {
-		/*
-		 * See comment above
-		 */
+	    /*
+	     * See comment above
+	     */
 		
-		siStartInfo.wShowWindow = SW_HIDE;
-		siStartInfo.dwFlags |= STARTF_USESHOWWINDOW;
-		createFlags = CREATE_NEW_CONSOLE;
-		run_cmd = 1;
+	    siStartInfo.wShowWindow = SW_HIDE;
+	    siStartInfo.dwFlags |= STARTF_USESHOWWINDOW;
+	    createFlags = CREATE_NEW_CONSOLE;
+	    run_cmd = 1;
 	} else if (hide) {
-		DEBUGF(("hiding window\n"));
-		siStartInfo.wShowWindow = SW_HIDE;
-		siStartInfo.dwFlags |= STARTF_USESHOWWINDOW;
-		createFlags = 0;
+	    DEBUGF(("hiding window\n"));
+	    siStartInfo.wShowWindow = SW_HIDE;
+	    siStartInfo.dwFlags |= STARTF_USESHOWWINDOW;
+	    createFlags = 0;
 	}
 	if (run_cmd) {
-	    char cmdPath[MAX_PATH];
+	    WCHAR cmdPath[MAX_PATH];
 	    int cmdType;
-	    cmdType = ApplicationType("cmd.exe", cmdPath, TRUE, FALSE, errno_return);
+	    cmdType = application_type_w(L"cmd.exe", cmdPath, TRUE, FALSE, errno_return);
 	    if (cmdType == APPL_NONE || cmdType == APPL_DOS) {
 		return FALSE;
 	    }
-	    appname = (char *) erts_alloc(ERTS_ALC_T_TMP, strlen(cmdPath)+1);
-	    strcpy(appname,cmdPath);
+	    appname = (char *) erts_alloc(ERTS_ALC_T_TMP, (wcslen(cmdPath)+1)*sizeof(WCHAR));
+	    wcscpy((WCHAR *) appname,cmdPath);
 	} else {
-	    appname = (char *) erts_alloc(ERTS_ALC_T_TMP, strlen(execPath)+1);
-	    strcpy(appname,execPath);
+	    appname = (char *) erts_alloc(ERTS_ALC_T_TMP, (wcslen(execPath)+1)*sizeof(WCHAR));
+	    wcscpy((WCHAR *) appname, execPath);
 	}
-	if (argv == NULL) {
+	if (argv == NULL) { 
 	    BOOL orig_need_q = need_quotes(execPath);
-	    char *ptr;
-	    int ocl = strlen(execPath);
+	    WCHAR *ptr;
+	    int ocl = wcslen(execPath);
 	    if (run_cmd) {
 		newcmdline = (char *) erts_alloc(ERTS_ALC_T_TMP, 
-						 ocl + ((orig_need_q) ? 3 : 1)
-						 + 11);
-		memcpy(newcmdline,"cmd.exe /c ",11);
-		ptr = newcmdline + 11;
+						 (ocl + ((orig_need_q) ? 3 : 1)
+						  + 11)*sizeof(WCHAR));
+		memcpy(newcmdline,L"cmd.exe /c ",11*sizeof(WCHAR));
+		ptr = (WCHAR *) (newcmdline + (11*sizeof(WCHAR)));
 	    } else {
 		newcmdline = (char *) erts_alloc(ERTS_ALC_T_TMP, 
-						 ocl + ((orig_need_q) ? 3 : 1));
-		ptr = newcmdline;
+						 (ocl + ((orig_need_q) ? 3 : 1))*sizeof(WCHAR));
+		ptr = (WCHAR *) newcmdline;
 	    }
 	    if (orig_need_q) {
-		*ptr++ = '"';
+		*ptr++ = L'"';
 	    }
-	    memcpy(ptr,execPath,ocl);
+	    memcpy(ptr,execPath,ocl*sizeof(WCHAR));
 	    ptr += ocl;
 	    if (orig_need_q) {
-		*ptr++ = '"';
+		*ptr++ = L'"';
 	    }
-	    *ptr = '\0';
+	    *ptr = L'\0';
 	} else {
 	    int sum = 1; /* '\0' */
-	    char **ar = argv;
-	    char *n;
+	    WCHAR **ar = (WCHAR **) argv;
+	    WCHAR *n;
 	    char *save_arg0 = NULL;
 	    if (argv[0] == erts_default_arg0 || run_cmd) {
 		save_arg0 = argv[0];
-		argv[0] = execPath;
+		argv[0] = (char *) execPath;
 	    }
 	    if (run_cmd) {
 		sum += 11; /* cmd.exe /c */
 	    }
 	    while (*ar != NULL) {
-		sum += strlen(*ar);
+		sum += wcslen(*ar);
 		if (need_quotes(*ar)) {
 		    sum += 2; /* quotes */
 		}
 		sum++; /* space */
 		++ar;
 	    }
-	    ar = argv;
-	    newcmdline = erts_alloc(ERTS_ALC_T_TMP, sum);
-	    n = newcmdline;
+	    ar = (WCHAR **) argv;
+	    newcmdline = erts_alloc(ERTS_ALC_T_TMP, sum*sizeof(WCHAR));
+	    n = (WCHAR *) newcmdline;
 	    if (run_cmd) {
-		memcpy(n,"cmd.exe /c ",11);
+		memcpy(n,L"cmd.exe /c ",11*sizeof(WCHAR));
 		n += 11;
 	    }
 	    while (*ar != NULL) {
 		int q = need_quotes(*ar);
-		sum = strlen(*ar);
+		sum = wcslen(*ar);
 		if (q) {
-		    *n++ = '"';
+		    *n++ = L'"';
 		}
-		memcpy(n,*ar,sum);
+		memcpy(n,*ar,sum*sizeof(WCHAR));
 		n += sum;
 		if (q) {
-		    *n++ = '"';
+		    *n++ = L'"';
 		}
-		*n++ = ' ';
+		*n++ = L' ';
 		++ar;
 	    }
-	    ASSERT(n > newcmdline);
-	    *(n-1) = '\0';
+	    *(n-1) = L'\0';
 	    if (save_arg0 != NULL) {
 		argv[0] = save_arg0;
 	    }
 	}	    
 	    
-    }
-    DEBUGF(("Creating child process: %s, createFlags = %d\n", newcmdline, createFlags));
-    ok = CreateProcess(appname, 
-		       newcmdline, 
-		       NULL, 
-		       NULL, 
-		       TRUE, 
-		       createFlags | staticCreateFlags, 
-		       env, 
-		       wd, 
-		       &siStartInfo, 
-		       &piProcInfo);
-
+	DEBUGF(("Creating child process: %s, createFlags = %d\n", newcmdline, createFlags));
+	ok = CreateProcessW((WCHAR *) appname, 
+			    (WCHAR *) newcmdline, 
+			    NULL, 
+			    NULL, 
+			    TRUE, 
+			    createFlags | staticCreateFlags, 
+			    env, 
+			    (WCHAR *) wd, 
+			    &siStartInfo, 
+			    &piProcInfo);
+
+    } /* end SPAWN_EXECUTABLE */
     if (newcmdline != NULL) {
 	    erts_free(ERTS_ALC_T_TMP,newcmdline);
     }	
@@ -1500,22 +1635,12 @@ CreateChildProcess
     }
     CloseHandle(piProcInfo.hThread); /* Necessary to avoid resource leak. */
     *phPid = piProcInfo.hProcess;
-    
+    *pdwID = piProcInfo.dwProcessId;
+
     if (applType == APPL_DOS) {
 	WaitForSingleObject(hProcess, 50);
     }
     
-    /* 
-     * When an application spawns a process repeatedly, a new thread 
-     * instance will be created for each process but the previous 
-     * instances may not be cleaned up.  This results in a significant 
-     * virtual memory loss each time the process is spawned.  If there 
-     * is a WaitForInputIdle() call between CreateProcess() and
-     * CloseHandle(), the problem does not occur. PSS ID Number: Q124121
-     */
-    
-    WaitForInputIdle(piProcInfo.hProcess, 5000);
-    
     return ok;
 }
 
@@ -1528,7 +1653,7 @@ CreateChildProcess
 static int create_pipe(HANDLE *phRead, HANDLE *phWrite, BOOL inheritRead, BOOL overlapped_io)
 {
     SECURITY_ATTRIBUTES sa = {sizeof(SECURITY_ATTRIBUTES), NULL, TRUE};
-    char pipe_name[128];	/* Name of pipe. */
+    char pipe_name[256];	/* Name of pipe. */
     Uint calls;
 
     /*
@@ -1565,9 +1690,9 @@ static int create_pipe(HANDLE *phRead, HANDLE *phWrite, BOOL inheritRead, BOOL o
      * Otherwise, create named pipes.
      */
 
-    calls = (Uint) erts_smp_atomic_inctest(&pipe_creation_counter);
-    sprintf(pipe_name, "\\\\.\\pipe\\erlang44_%d_%d",
-	    getpid(), calls);
+    calls = (UWord) erts_smp_atomic_inc_read_nob(&pipe_creation_counter);
+    erts_snprintf(pipe_name, sizeof(pipe_name),
+		  "\\\\.\\pipe\\erlang44_%d_%bpu", getpid(), calls);
 
     DEBUGF(("Creating pipe %s\n", pipe_name));
     sa.bInheritHandle = inheritRead;
@@ -1603,7 +1728,7 @@ static int create_pipe(HANDLE *phRead, HANDLE *phWrite, BOOL inheritRead, BOOL o
 
 
 
-static int ApplicationType
+static int application_type
 (
  const char *originalName, /* Name of the application to find. */ 
  char fullPath[MAX_PATH],  /* Filled with complete path to 
@@ -1757,6 +1882,145 @@ static int ApplicationType
     return applType;
 }
 
+static int application_type_w (const WCHAR *originalName, /* Name of the application to find. */ 
+			       WCHAR wfullpath[MAX_PATH],/* Filled with complete path to 
+							  * application. */
+			       BOOL search_in_path,      /* If we should search the system wide path */
+			       BOOL handle_quotes,       /* If we should handle quotes around executable */
+			       int *error_return)         /* A place to put an error code */
+{
+    int applType, i;
+    HANDLE hFile;
+    WCHAR *ext, *rest;
+    char buf[2];
+    DWORD read;
+    IMAGE_DOS_HEADER header;
+    static WCHAR extensions[][5] = {L"", L".com", L".exe", L".bat"};
+    int is_quoted;
+    int len;
+    WCHAR xfullpath[MAX_PATH];
+
+    len = wcslen(originalName);
+    is_quoted = handle_quotes && len > 0 && originalName[0] == L'"' && 
+	originalName[len-1] == L'"';
+
+    applType = APPL_NONE;
+    *error_return = ENOENT;
+    for (i = 0; i < (int) (sizeof(extensions) / sizeof(extensions[0])); i++) {
+	if(is_quoted) {
+	   lstrcpynW(xfullpath, originalName+1, MAX_PATH - 7); /* Cannot start using StringCchCopy yet, we support
+							   older platforms */
+	   len = wcslen(xfullpath);
+	   if(len > 0) {
+	       xfullpath[len-1] = L'\0';
+	   }
+	} else {
+	    lstrcpynW(xfullpath, originalName, MAX_PATH - 5);
+	}
+	wcscat(xfullpath, extensions[i]);
+	/* It seems that the Unicode version does not allow in and out parameter to overlap. */
+	SearchPathW((search_in_path) ? NULL : L".", xfullpath, NULL, MAX_PATH, wfullpath, &rest);
+
+	/*
+	 * Ignore matches on directories or data files, return if identified
+	 * a known type.
+	 */
+
+	if (GetFileAttributesW(wfullpath) & FILE_ATTRIBUTE_DIRECTORY) {
+	    continue;
+	}
+
+	ext = wcsrchr(wfullpath, L'.');
+	if ((ext != NULL) && (_wcsicmp(ext, L".bat") == 0)) {
+	    *error_return = EACCES;
+	    applType = APPL_DOS;
+	    break;
+	}
+
+	hFile = CreateFileW(wfullpath, GENERIC_READ, FILE_SHARE_READ, NULL, 
+		OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+	if (hFile == INVALID_HANDLE_VALUE) {
+	    continue;
+	}
+
+	*error_return = EACCES; /* If considered an error, 
+				    it's an access error */
+	header.e_magic = 0;
+	ReadFile(hFile, (void *) &header, sizeof(header), &read, NULL);
+	if (header.e_magic != IMAGE_DOS_SIGNATURE) {
+	    /* 
+	     * Doesn't have the magic number for relocatable executables.  If 
+	     * filename ends with .com, assume it's a DOS application anyhow.
+	     * Note that we didn't make this assumption at first, because some
+	     * supposed .com files are really 32-bit executables with all the
+	     * magic numbers and everything.  
+	     */
+
+	    CloseHandle(hFile);
+	    if ((ext != NULL) && (_wcsicmp(ext, L".com") == 0)) {
+		applType = APPL_DOS;
+		break;
+	    }
+	    continue;
+	}
+	if (header.e_lfarlc != sizeof(header)) {
+	    /* 
+	     * All Windows 3.X and Win32 and some DOS programs have this value
+	     * set here.  If it doesn't, assume that since it already had the 
+	     * other magic number it was a DOS application.
+	     */
+
+	    CloseHandle(hFile);
+	    applType = APPL_DOS;
+	    break;
+	}
+
+	/* 
+	 * The DWORD at header.e_lfanew points to yet another magic number.
+	 */
+
+	buf[0] = '\0';
+	SetFilePointer(hFile, header.e_lfanew, NULL, FILE_BEGIN);
+	ReadFile(hFile, (void *) buf, 2, &read, NULL);
+	CloseHandle(hFile);
+
+	if ((buf[0] == 'L') && (buf[1] == 'E')) {
+	    applType = APPL_DOS;
+	} else if ((buf[0] == 'N') && (buf[1] == 'E')) {
+	    applType = APPL_WIN3X;
+	} else if ((buf[0] == 'P') && (buf[1] == 'E')) {
+	    applType = APPL_WIN32;
+	} else {
+	    continue;
+	}
+	break;
+    }
+
+    if (applType == APPL_NONE) {
+	return APPL_NONE;
+    }
+
+    if ((applType == APPL_DOS) || (applType == APPL_WIN3X)) {
+	/* 
+	 * Replace long path name of executable with short path name for 
+	 * 16-bit applications.  Otherwise the application may not be able
+	 * to correctly parse its own command line to separate off the 
+	 * application name from the arguments.
+	 */
+
+	GetShortPathNameW(wfullpath, wfullpath, MAX_PATH);
+    }
+    if (is_quoted) {
+	/* restore quotes on quoted program name */
+	len = wcslen(wfullpath);
+	memmove(wfullpath+1,wfullpath,len*sizeof(WCHAR));
+	wfullpath[0]=L'"';
+	wfullpath[len+1]=L'"';
+	wfullpath[len+2]=L'\0';
+    }
+    return applType;
+}
+
 /*
  * Thread function used to emulate overlapped reading.
  */
@@ -1776,9 +2040,10 @@ threaded_reader(LPVOID param)
 	buf = OV_BUFFER_PTR(aio);
 	numToRead = OV_NUM_TO_READ(aio);
 	aio->pendingError = 0;
-	if (!ReadFile(aio->fd, buf, numToRead, &aio->bytesTransferred, NULL))
-	    aio->pendingError = GetLastError();
-	else if (aio->flags & DF_XLAT_CR) {
+	if (!ReadFile(aio->fd, buf, numToRead, &aio->bytesTransferred, NULL)) {
+	    int error = GetLastError();
+	    aio->pendingError = error;
+	} else if (aio->flags & DF_XLAT_CR) {
 	    char *s;
 	    int n;
 	    
@@ -1905,56 +2170,79 @@ translate_fd(int fd)
     return handle;
 }
 
+/* Driver level locking, start function is serialized */
+static DriverData *save_01_port = NULL;
+static DriverData *save_22_port = NULL;
+
 static ErlDrvData
 fd_start(ErlDrvPort port_num, char* name, SysDriverOpts* opts)
 {
     DriverData* dp;
     int is_std_error = (opts->ofd == 2);
-    
-    opts->ifd = (int) translate_fd(opts->ifd);
-    opts->ofd = (int) translate_fd(opts->ofd);
-    if ((dp = new_driver_data(port_num, opts->packet_bytes, 2, TRUE)) == NULL)
-	return ERL_DRV_ERROR_GENERAL;
-    
-    if (!create_file_thread(&dp->in, DO_READ)) {
-	dp->port_num = PORT_FREE;
-	return ERL_DRV_ERROR_GENERAL;
-    }
-
-    if (!create_file_thread(&dp->out, DO_WRITE)) {
-	dp->port_num = PORT_FREE;
-	return ERL_DRV_ERROR_GENERAL;
-    }
-    
-    fd_driver_input = &(dp->in);
-    dp->in.flags = DF_XLAT_CR;
-    if (is_std_error) {
-	dp->out.flags |= DF_DROP_IF_INVH; /* Just drop messages if stderror
-					     is an invalid handle */
+    int in = opts->ifd, out = opts->ofd;
+
+    opts->ifd = (Uint) translate_fd(in);
+    opts->ofd = (Uint) translate_fd(out);
+    if ( in == 0 && out == 1 && save_01_port != NULL) {
+	dp = save_01_port;
+	return reuse_driver_data(dp, (HANDLE) opts->ifd, (HANDLE) opts->ofd, opts->read_write, port_num);
+    } else if (in == 2 && out == 2 && save_22_port != NULL) {
+	dp = save_22_port;
+	return reuse_driver_data(dp, (HANDLE) opts->ifd, (HANDLE) opts->ofd, opts->read_write, port_num);
+    } else {
+	if ((dp = new_driver_data(port_num, opts->packet_bytes, 2, TRUE)) == NULL)
+	    return ERL_DRV_ERROR_GENERAL;
+	
+	if (!create_file_thread(&dp->in, DO_READ)) {
+	    dp->port_num = PORT_FREE;
+	    return ERL_DRV_ERROR_GENERAL;
+	}
+	
+	if (!create_file_thread(&dp->out, DO_WRITE)) {
+	    dp->port_num = PORT_FREE;
+	    return ERL_DRV_ERROR_GENERAL;
+	}
+	
+	fd_driver_input = &(dp->in);
+	dp->in.flags = DF_XLAT_CR;
+	if (is_std_error) {
+	    dp->out.flags |= DF_DROP_IF_INVH; /* Just drop messages if stderror
+						 is an invalid handle */
+	}
+	
+	if ( in == 0 && out == 1) {
+	    save_01_port = dp;
+	} else if (in == 2 && out == 2) {
+	    save_22_port = dp;
+	}
+	return set_driver_data(dp, (HANDLE) opts->ifd, (HANDLE) opts->ofd, opts->read_write, 0);
     }
-    return set_driver_data(dp, opts->ifd, opts->ofd, opts->read_write, 0);
 }
 
 static void fd_stop(ErlDrvData d)
 {
   int fd = (int)d;
+  DriverData* dp = driver_data+fd;
   /*
-   * I don't know a clean way to terminate the threads
-   * (TerminateThread() doesn't release the stack),
-   * so will we'll let the threads live.  Normally, the fd
-   * driver is only used to support the -oldshell option,
-   * so this shouldn't be a problem in practice.
-   *
-   * Since we will not attempt to terminate the threads,
-   * better not close the input or output files either.
+   * There's no way we can terminate an fd port in a consistent way.
+   * Instead we let it live until it's opened again (which it is,
+   * as the only FD-drivers are for 0,1 and 2 adn the only time they
+   * get closed is by init:reboot).
+   * So - just deselect them and let everything be as is. 
+   * They get woken up in fd_start again, where the DriverData is
+   * remembered. /PaN
    */
+  if (dp->in.ov.hEvent != NULL) {
+      (void) driver_select(dp->port_num,
+			   (ErlDrvEvent)dp->in.ov.hEvent,
+			   ERL_DRV_READ, 0);
+  }
+  if (dp->out.ov.hEvent != NULL) {
+      (void) driver_select(dp->port_num,
+			   (ErlDrvEvent)dp->out.ov.hEvent,
+			   ERL_DRV_WRITE, 0);
+  }    
 
-  driver_data[fd].in.thread = (HANDLE) -1;
-  driver_data[fd].out.thread = (HANDLE) -1;
-  driver_data[fd].in.fd = INVALID_HANDLE_VALUE;
-  driver_data[fd].out.fd = INVALID_HANDLE_VALUE;
-
-  /*return */ common_stop(fd);
 }
 
 static ErlDrvData
@@ -2046,7 +2334,6 @@ threaded_exiter(LPVOID param)
      * because it is an auto reset event.  Therefore, always set the
      * exit flag and signal the event.
      */
-
     i = 0;
     if (dp->out.thread != (HANDLE) -1) {
 	dp->out.flags = DF_EXIT_THREAD;
@@ -2151,13 +2438,13 @@ threaded_exiter(LPVOID param)
  */
 
 static void
-output(ErlDrvData drv_data, char* buf, int len)
+output(ErlDrvData drv_data, char* buf, ErlDrvSizeT len)
 /*     long drv_data;		/* The slot to use in the driver data table.
 				 * For Windows NT, this is *NOT* a file handle.
 				 * The handle is found in the driver data.
 				 */
-/*     char *buf;			/* Pointer to data to write to the port program. */
-/*     int len;			/* Number of bytes to write. */
+/*     char *buf;		/* Pointer to data to write to the port program. */
+/*     ErlDrvSizeT len;		/* Number of bytes to write. */
 {
     DriverData* dp;
     int pb;			/* The header size for this port. */
@@ -2197,7 +2484,7 @@ output(ErlDrvData drv_data, char* buf, int len)
     }
 
     dp->outBufSize = pb+len;
-    erts_smp_atomic_add(&sys_misc_mem_sz, dp->outBufSize);
+    erts_smp_atomic_add_nob(&sys_misc_mem_sz, dp->outBufSize);
 
     /*
      * Store header bytes (if any).
@@ -2226,8 +2513,8 @@ output(ErlDrvData drv_data, char* buf, int len)
     } else {
 	dp->out.ov.Offset += pb+len; /* For vanilla driver. */
 	/* XXX OffsetHigh should be changed too. */
-	ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->outBufSize);
-	erts_smp_atomic_add(&sys_misc_mem_sz, -1*dp->outBufSize);
+	ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize);
+	erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize);
 	DRV_BUF_FREE(dp->outbuf);
 	dp->outBufSize = 0;
 	dp->outbuf = NULL;
@@ -2341,9 +2628,9 @@ ready_input(ErlDrvData drv_data, ErlDrvEvent ready_event)
 			    error = ERROR_NOT_ENOUGH_MEMORY;
 			    break; /* Break out of loop into error handler. */
 			}
-			ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->inBufSize);
-			erts_smp_atomic_add(&sys_misc_mem_sz,
-					    dp->totalNeeded - dp->inBufSize);
+			ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->inBufSize);
+			erts_smp_atomic_add_nob(&sys_misc_mem_sz,
+						dp->totalNeeded - dp->inBufSize);
 			dp->inBufSize = dp->totalNeeded;
 			dp->inbuf = new_buf;
 		    }
@@ -2414,6 +2701,7 @@ ready_input(ErlDrvData drv_data, ErlDrvEvent ready_event)
 	    driver_failure_eof(dp->port_num);
 	} else {			/* Report real errors. */
 	    int error = GetLastError();
+
 	    (void) driver_select(dp->port_num, ready_event, ERL_DRV_READ, 0);
 	    _dosmaperr(error);
 	    driver_failure_posix(dp->port_num, errno);
@@ -2438,12 +2726,12 @@ ready_output(ErlDrvData drv_data, ErlDrvEvent ready_event)
     DEBUGF(("ready_output(%d, 0x%x)\n", drv_data, ready_event));
     set_busy_port(dp->port_num, 0);
     if (!(dp->outbuf)) {
-	/* Happens because event sometimes get signalled during a succesful
+	/* Happens because event sometimes get signalled during a successful
 	   write... */
 	return;
     }
-    ASSERT(erts_smp_atomic_read(&sys_misc_mem_sz) >= dp->outBufSize);
-    erts_smp_atomic_add(&sys_misc_mem_sz, -1*dp->outBufSize);
+    ASSERT(erts_smp_atomic_read_nob(&sys_misc_mem_sz) >= dp->outBufSize);
+    erts_smp_atomic_add_nob(&sys_misc_mem_sz, -1*dp->outBufSize);
     DRV_BUF_FREE(dp->outbuf);
     dp->outBufSize = 0;
     dp->outbuf = NULL;
@@ -2491,30 +2779,6 @@ sys_init_io(void)
        We estimate the number to twice the amount of ports. 
        We really dont know on windows, do we? */
     max_files = 2*erts_max_ports;
-    
-#ifdef USE_THREADS
-#ifdef ERTS_SMP
-    if (init_async(-1) < 0)
-	erl_exit(1, "Failed to initialize async-threads\n");
-#else
-    {
-	/* This is special stuff, starting a driver from the 
-	 * system routines, but is a nice way of handling stuff
-	 * the erlang way
-	 */
-	SysDriverOpts dopts;
-	int ret;
-
-	sys_memset((void*)&dopts, 0, sizeof(SysDriverOpts));
-	add_driver_entry(&async_driver_entry);
-	ret = erts_open_driver(NULL, NIL, "async", &dopts, NULL);
-	DEBUGF(("open_driver = %d\n", ret));
-	if (ret < 0)
-	    erl_exit(1, "Failed to open async driver\n");
-	erts_port[ret].status |= ERTS_PORT_SFLG_IMMORTAL;
-    }
-#endif
-#endif
 }
 
 #ifdef ERTS_SMP
@@ -2534,7 +2798,6 @@ erts_sys_main_thread(void)
 
 void erts_sys_alloc_init(void)
 {
-    elib_ensure_initialized();
 }
 
 void *erts_sys_alloc(ErtsAlcType_t t, void *x, Uint sz)
@@ -2594,8 +2857,8 @@ Preload* sys_preloaded(void)
 			   (num_preloaded+1)*sizeof(Preload));
     res_name = erts_alloc(ERTS_ALC_T_PRELOADED,
 			  (num_preloaded+1)*sizeof(unsigned));
-    erts_smp_atomic_add(&sys_misc_mem_sz,
-			(num_preloaded+1)*sizeof(Preload)
+    erts_smp_atomic_add_nob(&sys_misc_mem_sz,
+			    (num_preloaded+1)*sizeof(Preload)
 			    + (num_preloaded+1)*sizeof(unsigned));
     for (i = 0; i < num_preloaded; i++) {
 	int n;
@@ -2607,7 +2870,7 @@ Preload* sys_preloaded(void)
 	n = GETWORD(data);
 	data += 2;
 	preloaded[i].name = erts_alloc(ERTS_ALC_T_PRELOADED, n+1);
-	erts_smp_atomic_add(&sys_misc_mem_sz, n+1);
+	erts_smp_atomic_add_nob(&sys_misc_mem_sz, n+1);
 	sys_memcpy(preloaded[i].name, data, n);
 	preloaded[i].name[n] = '\0';
 	data += n;
@@ -2881,19 +3144,51 @@ check_supported_os_version(void)
 }
 
 #ifdef USE_THREADS
-static void *ethr_internal_alloc(size_t size)
+
+typedef struct {
+    int sched_bind_data;
+} erts_thr_create_data_t;
+
+/*
+ * thr_create_prepare() is called in parent thread before thread creation.
+ * Returned value is passed as argument to thr_create_cleanup().
+ */
+static void *
+thr_create_prepare(void)
 {
-    return erts_alloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, (Uint) size);
+    erts_thr_create_data_t *tcdp;
+
+    tcdp = erts_alloc(ERTS_ALC_T_TMP, sizeof(erts_thr_create_data_t));
+    tcdp->sched_bind_data = erts_sched_bind_atthrcreate_prepare();
+
+    return (void *) tcdp;
 }
-static void *ethr_internal_realloc(void *ptr, size_t size)
+
+
+/* thr_create_cleanup() is called in parent thread after thread creation. */
+static void
+thr_create_cleanup(void *vtcdp)
 {
-    return erts_realloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, ptr, (Uint) size);
+    erts_thr_create_data_t *tcdp = (erts_thr_create_data_t *) vtcdp;
+
+    erts_sched_bind_atthrcreate_parent(tcdp->sched_bind_data);
+
+    erts_free(ERTS_ALC_T_TMP, tcdp);
 }
-static void ethr_internal_free(void *ptr)
+
+static void
+thr_create_prepare_child(void *vtcdp)
 {
-    erts_free(ERTS_ALC_T_ETHR_INTERNAL, ptr);
+    erts_thr_create_data_t *tcdp = (erts_thr_create_data_t *) vtcdp;
+
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_thread_setup();
+#endif /* ERTS_ENABLE_LOCK_COUNT */
+
+    erts_sched_bind_atthrcreate_child(tcdp->sched_bind_data);
 }
-#endif
+
+#endif /* USE_THREADS */
 
 void
 erts_sys_pre_init(void)
@@ -2904,25 +3199,22 @@ erts_sys_pre_init(void)
 #ifdef USE_THREADS
     {
 	erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER;
-	eid.alloc = ethr_internal_alloc;
-	eid.realloc = ethr_internal_realloc;
-	eid.free = ethr_internal_free;
+
+	eid.thread_create_child_func = thr_create_prepare_child;
+	/* Before creation in parent */
+	eid.thread_create_prepare_func = thr_create_prepare;
+	/* After creation in parent */
+	eid.thread_create_parent_func = thr_create_cleanup,
+
 	erts_thr_init(&eid);
 #ifdef ERTS_ENABLE_LOCK_COUNT
 	erts_lcnt_init();
 #endif
     }
 #endif
-    erts_smp_atomic_init(&sys_misc_mem_sz, 0);
-    erts_sys_env_init();
+    erts_smp_atomic_init_nob(&sys_misc_mem_sz, 0);
 }
 
-/*
- * the last two only used for standalone erlang
- * they should are used by sae_main in beam dll to
- * enable standalone execution via erl_api-routines
- */
-
 void noinherit_std_handle(DWORD type)
 {
     HANDLE h = GetStdHandle(type);
@@ -2945,8 +3237,9 @@ void erl_sys_init(void)
 
 #ifdef ERTS_SMP
     erts_smp_tsd_key_create(&win32_errstr_key);
+    InitializeCriticalSection(&htbc_lock);
 #endif
-    erts_smp_atomic_init(&pipe_creation_counter,0);
+    erts_smp_atomic_init_nob(&pipe_creation_counter,0);
     /*
      * Test if we have named pipes or not.
      */
@@ -2996,15 +3289,15 @@ void erl_sys_init(void)
 		 SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX); 
 }
 
-#ifdef ERTS_SMP
 void
 erts_sys_schedule_interrupt(int set)
 {
     erts_check_io_interrupt(set);
 }
 
+#ifdef ERTS_SMP
 void
-erts_sys_schedule_interrupt_timed(int set, long msec)
+erts_sys_schedule_interrupt_timed(int set, erts_short_time_t msec)
 {
     erts_check_io_interrupt_timed(set, msec);
 }
@@ -3018,76 +3311,7 @@ erts_sys_schedule_interrupt_timed(int set, long msec)
 void
 erl_sys_schedule(int runnable)
 {
-#ifdef ERTS_SMP
     erts_check_io(!runnable);
-    ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING);
-#else
-    erts_check_io_interrupt(0);
-    if (runnable) {
-	erts_check_io(0);	/* Poll for I/O */
-	check_async_ready();	/* Check async completions */
-    } else {
-	erts_check_io(check_async_ready() ? 0 : 1);
-    }
-#endif
-}
-
-#if defined(USE_THREADS) && !defined(ERTS_SMP)
-/*
- * Async operation support.
- */
-
-static ErlDrvEvent async_drv_event;
-
-void
-sys_async_ready(int fd)
-{
-    SetEvent((HANDLE)async_drv_event);
-}
-
-static int
-async_drv_init(void)
-{
-    async_drv_event = (ErlDrvEvent) NULL;
-    return 0;
-}
-
-static ErlDrvData
-async_drv_start(ErlDrvPort port_num, char* name, SysDriverOpts* opts)
-{
-    if (async_drv_event != (ErlDrvEvent) NULL) {
-	return ERL_DRV_ERROR_GENERAL;
-    }
-    if ((async_drv_event = (ErlDrvEvent)CreateAutoEvent(FALSE)) == (ErlDrvEvent) NULL) {
-	return ERL_DRV_ERROR_GENERAL;
-    }
-
-    driver_select(port_num, async_drv_event, ERL_DRV_READ|ERL_DRV_USE, 1);
-    if (init_async(async_drv_event) < 0) {
-	return ERL_DRV_ERROR_GENERAL;
-    }
-    return (ErlDrvData)port_num;
-}
-
-static void
-async_drv_stop(ErlDrvData port_num)
-{
-    exit_async();
-    driver_select((ErlDrvPort)port_num, async_drv_event, ERL_DRV_READ|ERL_DRV_USE, 0);
-    /*CloseHandle((HANDLE)async_drv_event);*/
-    async_drv_event = (ErlDrvEvent) NULL;
+    ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());
 }
 
-
-static void
-async_drv_input(ErlDrvData port_num, ErlDrvEvent e) 
-{
-    check_async_ready();
-
-    /*
-     * Our event is auto-resetting.
-     */
-}
-
-#endif
-
diff --git a/erts/emulator/sys/win32/sys_env.c b/erts/emulator/sys/win32/sys_env.c
index ac4be3f316..064745d418 100644
--- a/erts/emulator/sys/win32/sys_env.c
+++ b/erts/emulator/sys/win32/sys_env.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2002-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -55,19 +55,17 @@ erts_sys_putenv(char *key_value, int sep_ix)
 }
 
 int
-erts_sys_getenv(char *key, char *value, size_t *size)
+erts_sys_getenv__(char *key, char *value, size_t *size)
 {
     size_t req_size = 0;
     int res = 0;
     DWORD new_size;
 
-    erts_smp_rwmtx_rlock(&environ_rwmtx);
     SetLastError(0);
     new_size = GetEnvironmentVariable((LPCTSTR) key,
 				      (LPTSTR) value,
 				      (DWORD) *size);
     res = !new_size && GetLastError() == ERROR_ENVVAR_NOT_FOUND ? -1 : 0;
-    erts_smp_rwmtx_runlock(&environ_rwmtx);
     if (res < 0)
 	return res;
     res = new_size > *size ? 1 : 0;
@@ -75,6 +73,16 @@ erts_sys_getenv(char *key, char *value, size_t *size)
     return res;
 }
 
+int
+erts_sys_getenv(char *key, char *value, size_t *size)
+{
+    int res;
+    erts_smp_rwmtx_rlock(&environ_rwmtx);
+    res = erts_sys_getenv__(key, value, size);
+    erts_smp_rwmtx_runlock(&environ_rwmtx);
+    return res;
+}
+
 struct win32_getenv_state {
     char *env;
     char *next;
@@ -145,15 +153,17 @@ merge_environment(char *old, char *add)
     for(j = 0; a_arg[j] != NULL; ++j){
 	char **tmp;
 	char *current = a_arg[j];
+	char *eq_p = strchr(current,'=');
+	int unset = (eq_p!=NULL && eq_p[1]=='\0');
 
 	if ((tmp = find_arg(c_arg, current)) != NULL) {
-	    if (current[strlen(current)-1] != '=') {
+	    if (!unset) {
 		*tmp = current;
 	    } else {
 		*tmp = c_arg[--i];
 		c_arg[i] = NULL;
 	    }
-	} else if (current[strlen(current)-1] != '=') {
+	} else if (!unset) {
 	    c_arg[i++] = current;
 	    c_arg[i] = NULL;
 	}
diff --git a/erts/emulator/sys/win32/sys_float.c b/erts/emulator/sys/win32/sys_float.c
index 9e67ca7f48..6558ad2d99 100644
--- a/erts/emulator/sys/win32/sys_float.c
+++ b/erts/emulator/sys/win32/sys_float.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -18,6 +18,9 @@
  */
 /* Float conversions */
 
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
 #include "sys.h"
 #include "signal.h"
 
diff --git a/erts/emulator/sys/win32/sys_interrupt.c b/erts/emulator/sys/win32/sys_interrupt.c
index d2449a1bdb..347c31053b 100644
--- a/erts/emulator/sys/win32/sys_interrupt.c
+++ b/erts/emulator/sys/win32/sys_interrupt.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -19,8 +19,12 @@
 /*
  * Purpose: Interrupt handling in windows.
  */
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
 #include "sys.h"
 #include "erl_alloc.h"
+#include "erl_thr_progress.h"
 #include "erl_driver.h"
 #include "../../drivers/win32/win_con.h"
 
@@ -31,11 +35,11 @@
 #endif
 
 #ifdef ERTS_SMP
-erts_smp_atomic_t erts_break_requested;
+erts_smp_atomic32_t erts_break_requested;
 #define ERTS_SET_BREAK_REQUESTED \
-  erts_smp_atomic_set(&erts_break_requested, (long) 1)
+  erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1)
 #define ERTS_UNSET_BREAK_REQUESTED \
-  erts_smp_atomic_set(&erts_break_requested, (long) 0)
+  erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0)
 #else
 volatile int erts_break_requested = 0;
 #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1)
@@ -52,14 +56,14 @@ void erts_do_break_handling(void)
      * therefore, make sure that all threads but this one are blocked before
      * proceeding!
      */
-    erts_smp_block_system(0);
+    erts_smp_thr_progress_block();
     /* call the break handling function, reset the flag */
     do_break();
 
     ResetEvent(erts_sys_break_event);
     ERTS_UNSET_BREAK_REQUESTED;
 
-    erts_smp_release_system();
+    erts_smp_thr_progress_unblock();
 }
 
 
diff --git a/erts/emulator/sys/win32/sys_time.c b/erts/emulator/sys/win32/sys_time.c
index 50e43065b5..b5123dc45d 100644
--- a/erts/emulator/sys/win32/sys_time.c
+++ b/erts/emulator/sys/win32/sys_time.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -20,6 +20,9 @@
  * Purpose: System-dependent time functions.
  */
 
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
 #include "sys.h"
 #include "assert.h"
 
@@ -32,28 +35,336 @@
 /******************* Routines for time measurement *********************/
 
 #define EPOCH_JULIAN_DIFF LL_LITERAL(11644473600)
+#define TICKS_PER_SECOND LL_LITERAL(10000000)
+#define SECONDS_PER_DAY LL_LITERAL(86400)
+
+#define ULI_TO_FILETIME(ft,ull)			\
+ do {						\
+     (ft).dwLowDateTime = (ull).LowPart;	\
+     (ft).dwHighDateTime = (ull).HighPart;	\
+ } while (0)
+
+#define FILETIME_TO_ULI(ull,ft)			\
+ do {						\
+     (ull).LowPart = (ft).dwLowDateTime;	\
+     (ull).HighPart = (ft).dwHighDateTime;	\
+ } while (0)
+
 
+#define EPOCH_TO_FILETIME(ft, epoch) \
+    do { \
+	ULARGE_INTEGER ull; \
+	ull.QuadPart = (((epoch) + EPOCH_JULIAN_DIFF) * TICKS_PER_SECOND); \
+	ULI_TO_FILETIME(ft,ull); \
+    } while(0)
+
+#define FILETIME_TO_EPOCH(epoch, ft) \
+    do { \
+	ULARGE_INTEGER ull; \
+	FILETIME_TO_ULI(ull,ft); \
+	(epoch) = ((ull.QuadPart / TICKS_PER_SECOND) - EPOCH_JULIAN_DIFF); \
+    } while(0)
+ 
 static SysHrTime wrap = 0;
 static DWORD last_tick_count = 0;
 
+/* Getting timezone information is a heavy operation, so we want to do this 
+   only once */
+
+static TIME_ZONE_INFORMATION static_tzi;
+static int have_static_tzi = 0;
+
+static int days_in_month[2][13] = {
+    {0,31,28,31,30,31,30,31,31,30,31,30,31},
+    {0,31,29,31,30,31,30,31,31,30,31,30,31}};
+
 int 
 sys_init_time(void)
 {
+    if(GetTimeZoneInformation(&static_tzi) && 
+       static_tzi.StandardDate.wMonth != 0 &&
+       static_tzi.DaylightDate.wMonth != 0) {
+	have_static_tzi = 1;
+    }
     return 1;
 }
 
+/* Returns a switchtimes for DST as UTC filetimes given data from a 
+   TIME_ZONE_INFORMATION, see sys_localtime_r for usage. */ 
+static void 
+get_dst_switchtime(DWORD year, 
+		   SYSTEMTIME dstinfo, LONG bias,
+		   FILETIME *utc_switchtime) 
+{
+    DWORD occu;
+    DWORD weekday,wday_1st;
+    DWORD day, days_in;
+    FILETIME tmp,tmp2;
+    ULARGE_INTEGER ull;
+    int leap_year = 0;
+    if (dstinfo.wYear != 0) {
+	/* A year specific transition, in which case the data in the structure
+	   is already properly set for a specific year. Compare year
+	   with parameter and see if they correspond, in that case generate a 
+	   filetime directly, otherwise set the filetime to 0 */
+	if (year != dstinfo.wYear) {
+	    utc_switchtime->dwLowDateTime = utc_switchtime->dwHighDateTime = 0;
+	    return;
+	} 
+    } else {
+	occu = dstinfo.wDay;
+	weekday = dstinfo.wDayOfWeek;
+
+	dstinfo.wDayOfWeek = 0;
+	dstinfo.wDay = 1;
+	dstinfo.wYear = year;
+	
+	SystemTimeToFileTime(&dstinfo,&tmp);
+	ull.LowPart = tmp.dwLowDateTime;
+	ull.HighPart = tmp.dwHighDateTime;
+    
+	ull.QuadPart /= (TICKS_PER_SECOND*SECONDS_PER_DAY); /* Julian Day */
+	wday_1st = (DWORD) ((ull.QuadPart + LL_LITERAL(1)) % LL_LITERAL(7));
+	day = (weekday >= wday_1st) ? 
+	    weekday - wday_1st + 1 :
+	    weekday - wday_1st + 8;
+	--occu;
+	if (((dstinfo.wYear % 4) == 0 && (dstinfo.wYear % 100) > 0) ||
+	    ((dstinfo.wYear % 400) == 0)) {
+	    leap_year = 1;
+	}
+	days_in = days_in_month[leap_year][dstinfo.wMonth];
+	while (occu > 0 && (day + 7 <= days_in)) {
+	    --occu;
+	    day += 7;
+	}
+	dstinfo.wDay = day;
+    }
+    SystemTimeToFileTime(&dstinfo,&tmp);
+    /* correct for bias */
+    ull.LowPart = tmp.dwLowDateTime;
+    ull.HighPart = tmp.dwHighDateTime;
+    ull.QuadPart += (((LONGLONG) bias) * LL_LITERAL(60) * TICKS_PER_SECOND);
+    utc_switchtime->dwLowDateTime = ull.LowPart;
+    utc_switchtime->dwHighDateTime = ull.HighPart;
+    return;
+}
+
+/* This function gives approximately the correct year from a FILETIME
+   Around the actual new year, it may return the wrong value, but that's OK
+   as DST never switches around new year. */
+static DWORD 
+approx_year(FILETIME ft)
+{
+    ULARGE_INTEGER ull;
+    FILETIME_TO_ULI(ull,ft);
+    ull.QuadPart /= LL_LITERAL(1000);
+    ull.QuadPart /= SECONDS_PER_DAY;
+    ull.QuadPart /= LL_LITERAL(3652425);
+    ull.QuadPart += 1601;
+    return (DWORD) ull.QuadPart;
+}
+
+struct tm *
+sys_localtime_r(time_t *epochs, struct tm *ptm)
+{
+    FILETIME ft,lft;
+    SYSTEMTIME st;
+    
+    if ((((*epochs) + EPOCH_JULIAN_DIFF) * TICKS_PER_SECOND) < 0LL) {
+	fprintf(stderr,"1\r\n"); fflush(stderr);
+	return NULL;
+    }
+    
+    EPOCH_TO_FILETIME(ft,*epochs);
+    ptm->tm_isdst = 0;
+    if (have_static_tzi) {
+	FILETIME dst_start, dst_stop;
+	ULARGE_INTEGER ull;
+	DWORD year = approx_year(ft);
+	get_dst_switchtime(year,static_tzi.DaylightDate,
+			   static_tzi.Bias+static_tzi.StandardBias,&dst_start);
+	get_dst_switchtime(year,static_tzi.StandardDate,
+			   static_tzi.Bias+static_tzi.StandardBias+
+			   static_tzi.DaylightBias,
+			   &dst_stop);
+	FILETIME_TO_ULI(ull,ft);
+
+	if (CompareFileTime(&ft,&dst_start) >= 0 && 
+	    CompareFileTime(&ft,&dst_stop) < 0) {
+	    ull.QuadPart -= 
+		((LONGLONG) static_tzi.Bias+static_tzi.StandardBias+
+		 static_tzi.DaylightBias) * 
+		LL_LITERAL(60) * TICKS_PER_SECOND;
+	    ptm->tm_isdst = 1;
+	} else {
+	    ull.QuadPart -= 
+		((LONGLONG) static_tzi.Bias+static_tzi.StandardBias) 
+		* LL_LITERAL(60) * TICKS_PER_SECOND;
+	}
+	ULI_TO_FILETIME(ft,ull);
+    } else {
+	if (!FileTimeToLocalFileTime(&ft,&lft)) {
+	    return NULL;
+	}
+	ft = lft;
+    }
+    
+    if (!FileTimeToSystemTime(&ft,&st)) {
+	return NULL;
+    }
+    
+    ptm->tm_year = (int) st.wYear - 1900;
+    ptm->tm_mon  = (int) st.wMonth - 1;
+    ptm->tm_mday = (int) st.wDay;
+    ptm->tm_hour = (int) st.wHour;
+    ptm->tm_min  = (int) st.wMinute;
+    ptm->tm_sec  = (int) st.wSecond;
+    ptm->tm_wday  = (int) st.wDayOfWeek;
+    {
+	int yday = ptm->tm_mday - 1;
+	int m =  ptm->tm_mon;
+	int leap_year = 0;
+	if (((st.wYear % 4) == 0 && (st.wYear % 100) > 0) ||
+	    ((st.wYear % 400) == 0)) {
+	    leap_year = 1;
+	}
+	while (m > 0) {
+	    yday +=days_in_month[leap_year][m];
+	    --m;
+	}
+	ptm->tm_yday = yday;
+    }
+    return ptm;
+}
+
+struct tm * 
+sys_gmtime_r(time_t *epochs, struct tm *ptm)
+{
+    FILETIME ft;
+    SYSTEMTIME st;
+    
+    if ((((*epochs) + EPOCH_JULIAN_DIFF) * TICKS_PER_SECOND) < 0LL) {
+	return NULL;
+    }
+    
+    EPOCH_TO_FILETIME(ft,*epochs);
+    
+    if (!FileTimeToSystemTime(&ft,&st)) {
+	return NULL;
+    }
+    
+    ptm->tm_year = (int) st.wYear - 1900;
+    ptm->tm_mon  = (int) st.wMonth - 1;
+    ptm->tm_mday = (int) st.wDay;
+    ptm->tm_hour = (int) st.wHour;
+    ptm->tm_min  = (int) st.wMinute;
+    ptm->tm_sec  = (int) st.wSecond;
+    ptm->tm_wday  = (int) st.wDayOfWeek;
+    ptm->tm_isdst  = 0;
+    {
+	int yday = ptm->tm_mday - 1;
+	int m =  ptm->tm_mon;
+	int leap_year = 0;
+	if (((st.wYear % 4) == 0 && (st.wYear % 100) > 0) ||
+	    ((st.wYear % 400) == 0)) {
+	    leap_year = 1;
+	}
+	while (m > 0) {
+	    yday +=days_in_month[leap_year][m];
+	    --m;
+	}
+	ptm->tm_yday = yday;
+    }
+
+    return ptm;
+}
+
+time_t 
+sys_mktime(struct tm *ptm) 
+{
+    FILETIME ft;
+    SYSTEMTIME st;
+    int dst = 0;
+    time_t epochs;
+
+    memset(&st,0,sizeof(st));
+    /* Convert relevant parts of truct tm to SYSTEMTIME */
+    st.wYear =  (USHORT) (ptm->tm_year + 1900);
+    st.wMonth =  (USHORT) (ptm->tm_mon + 1);
+    st.wDay =  (USHORT) ptm->tm_mday;
+    st.wHour =  (USHORT) ptm->tm_hour;
+    st.wMinute =  (USHORT) ptm->tm_min;
+    st.wSecond = (USHORT) ptm->tm_sec;
+
+    SystemTimeToFileTime(&st,&ft);
+    
+    /* ft is now some kind of local file time, but it may be wrong depending 
+       on what is in the tm_dst field. We need to manually convert it to
+       UTC before turning it into epochs */
+
+    if (have_static_tzi) {
+	FILETIME dst_start, dst_stop;
+	ULARGE_INTEGER ull_start,ull_stop,ull_ft;
+
+	FILETIME_TO_ULI(ull_ft,ft);
+
+	/* Correct everything except DST */
+	ull_ft.QuadPart += (static_tzi.Bias+static_tzi.StandardBias) 
+	    * LL_LITERAL(60) * TICKS_PER_SECOND;
+
+	/* Determine if DST is active */
+	if (ptm->tm_isdst >= 0) {
+	    dst = ptm->tm_isdst;
+	} else if (static_tzi.DaylightDate.wMonth != 0){
+	    /* This is how windows mktime does it, meaning it does not
+	       take nonexisting local times into account */
+	    get_dst_switchtime(st.wYear,static_tzi.DaylightDate,
+			       static_tzi.Bias+static_tzi.StandardBias,
+			       &dst_start);
+	    get_dst_switchtime(st.wYear,static_tzi.StandardDate,
+			       static_tzi.Bias+static_tzi.StandardBias+
+			       static_tzi.DaylightBias,
+			       &dst_stop);
+	    FILETIME_TO_ULI(ull_start,dst_start);
+	    FILETIME_TO_ULI(ull_stop,dst_stop);
+	    if ((ull_ft.QuadPart >= ull_start.QuadPart) &&
+		(ull_ft.QuadPart < ull_stop.QuadPart)) {
+		/* We are in DST */
+		dst = 1;
+	    }
+	}
+	/* Correct for DST */
+	if (dst) {
+	    ull_ft.QuadPart += static_tzi.DaylightBias * 
+		LL_LITERAL(60) * TICKS_PER_SECOND;
+	}
+	epochs = ((ull_ft.QuadPart / TICKS_PER_SECOND) - EPOCH_JULIAN_DIFF);
+    } else {
+	/* No DST, life is easy... */
+	FILETIME lft;
+	LocalFileTimeToFileTime(&ft,&lft);
+	FILETIME_TO_EPOCH(epochs,lft);
+    }
+    /* Normalize the struct tm */
+    sys_localtime_r(&epochs,ptm);
+    return epochs;
+}
+
 void 
 sys_gettimeofday(SysTimeval *tv)
 {
     SYSTEMTIME t;
     FILETIME ft;
-    LONGLONG lft;
+    ULARGE_INTEGER ull;
 
     GetSystemTime(&t);
     SystemTimeToFileTime(&t, &ft);
-    memcpy(&lft, &ft, sizeof(lft));
-    tv->tv_usec = (long) ((lft / LL_LITERAL(10)) % LL_LITERAL(1000000));
-    tv->tv_sec = (long) ((lft / LL_LITERAL(10000000)) - EPOCH_JULIAN_DIFF);
+    FILETIME_TO_ULI(ull,ft);
+    tv->tv_usec = (long) ((ull.QuadPart / LL_LITERAL(10)) % 
+			  LL_LITERAL(1000000));
+    tv->tv_sec = (long) ((ull.QuadPart / LL_LITERAL(10000000)) - 
+			 EPOCH_JULIAN_DIFF);
 }
 
 SysHrTime 
@@ -88,9 +399,3 @@ sys_times(SysTimes *buffer) {
     buffer->tms_stime = (clock_t) (system & LL_LITERAL(0x7FFFFFFF));
     return kernel_ticks;
 }
-
-
-
-
-
-
diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile
index b1374950b2..03311423f0 100644
--- a/erts/emulator/test/Makefile
+++ b/erts/emulator/test/Makefile
@@ -1,19 +1,19 @@
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 1997-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 1997-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
 
@@ -61,7 +61,7 @@ MODULES= \
 	exception_SUITE \
 	float_SUITE \
 	fun_SUITE \
-	fun_r11_SUITE \
+	fun_r13_SUITE \
 	gc_SUITE \
 	guard_SUITE \
 	hash_SUITE \
@@ -75,18 +75,20 @@ MODULES= \
 	node_container_SUITE \
 	nofrag_SUITE \
 	num_bif_SUITE \
-	obsolete_SUITE \
 	op_SUITE \
 	port_SUITE \
 	port_bif_SUITE \
 	process_SUITE \
 	pseudoknot_SUITE \
+	receive_SUITE \
 	ref_SUITE \
 	register_SUITE \
+	mtx_SUITE \
 	save_calls_SUITE \
 	send_term_SUITE \
 	sensitive_SUITE \
 	signal_SUITE \
+	smoke_test_SUITE \
 	statistics_SUITE \
 	system_info_SUITE \
 	system_profile_SUITE \
@@ -100,6 +102,7 @@ MODULES= \
 	trace_local_SUITE \
 	trace_meta_SUITE \
 	trace_call_count_SUITE \
+	trace_call_time_SUITE \
 	scheduler_SUITE \
 	old_scheduler_SUITE \
 	z_SUITE \
@@ -117,12 +120,17 @@ NO_OPT= bs_bincomp \
 	bs_match_int \
 	bs_match_tail \
 	bs_match_misc \
-	bs_utf
+	bs_utf \
+	guard
 
+NATIVE= hibernate
 
 NO_OPT_MODULES= $(NO_OPT:%=%_no_opt_SUITE)
 NO_OPT_ERL_FILES= $(NO_OPT_MODULES:%=%.erl)
 
+NATIVE_MODULES= $(NATIVE:%=%_native_SUITE)
+NATIVE_ERL_FILES= $(NATIVE_MODULES:%=%.erl)
+
 ERL_FILES= $(MODULES:%=%.erl)
 
 TARGET_FILES = $(MODULES:%=$(EBIN)/%.$(EMULATOR))
@@ -148,7 +156,7 @@ ERL_COMPILE_FLAGS += -I$(ERL_TOP)/lib/test_server/include
 # Targets
 # ----------------------------------------------------
 
-make_emakefile: $(NO_OPT_ERL_FILES)
+make_emakefile: $(NO_OPT_ERL_FILES) $(NATIVE_ERL_FILES)
 	# This special rule can be removed when communication with R7B nodes 
 	# is no longer supported.
 	$(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) +compressed -o$(EBIN) \
@@ -157,6 +165,8 @@ make_emakefile: $(NO_OPT_ERL_FILES)
 	$(MODULES) >> $(EMAKEFILE)
 	$(ERL_TOP)/make/make_emakefile +no_copt +no_postopt $(ERL_COMPILE_FLAGS) \
 	-o$(EBIN) $(NO_OPT_MODULES) >> $(EMAKEFILE)
+	$(ERL_TOP)/make/make_emakefile +native $(ERL_COMPILE_FLAGS) \
+	-o$(EBIN) $(NATIVE_MODULES) >> $(EMAKEFILE)
 
 tests debug opt: make_emakefile
 	erl $(ERL_MAKE_FLAGS) -make
@@ -175,6 +185,9 @@ docs:
 %_no_opt_SUITE.erl: %_SUITE.erl
 	sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@
 
+%_native_SUITE.erl: %_SUITE.erl
+	sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@
+
 # ----------------------------------------------------
 # Release Target
 # ---------------------------------------------------- 
@@ -183,12 +196,13 @@ include $(ERL_TOP)/make/otp_release_targets.mk
 release_spec:
 
 release_tests_spec: make_emakefile
-	$(INSTALL_DIR) $(RELSYSDIR)
+	$(INSTALL_DIR) "$(RELSYSDIR)"
 	$(INSTALL_DATA) $(EMAKEFILE) $(TEST_SPEC_FILES) \
-		$(ERL_FILES) $(RELSYSDIR)
-	$(INSTALL_DATA) $(NO_OPT_ERL_FILES) $(RELSYSDIR)
-	chmod -f -R u+w $(RELSYSDIR)
-	tar cf - *_SUITE_data | (cd $(RELSYSDIR); tar xf -)
+		$(ERL_FILES) "$(RELSYSDIR)"
+	$(INSTALL_DATA) $(NO_OPT_ERL_FILES) "$(RELSYSDIR)"
+	$(INSTALL_DATA) $(NATIVE_ERL_FILES) "$(RELSYSDIR)"
+	chmod -R u+w "$(RELSYSDIR)"
+	tar cf - *_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -)
 
 release_docs_spec:
 
diff --git a/erts/emulator/test/a_SUITE.erl b/erts/emulator/test/a_SUITE.erl
index e9d653a7c4..b541be3df6 100644
--- a/erts/emulator/test/a_SUITE.erl
+++ b/erts/emulator/test/a_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -26,15 +26,32 @@
 %%%-------------------------------------------------------------------
 -module(a_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1, long_timers/1, pollset_size/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, long_timers/1, pollset_size/1]).
 
-all(doc) ->
-    [];
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     [long_timers, pollset_size].
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 long_timers(doc) ->
     [];
 long_timers(suite) ->
diff --git a/erts/emulator/test/a_SUITE_data/timer_driver.c b/erts/emulator/test/a_SUITE_data/timer_driver.c
index ef4dcdf501..44be94e0f0 100644
--- a/erts/emulator/test/a_SUITE_data/timer_driver.c
+++ b/erts/emulator/test/a_SUITE_data/timer_driver.c
@@ -17,7 +17,9 @@
 #define CANCELLED 4
 
 static ErlDrvData timer_start(ErlDrvPort, char*);
-static void timer_stop(ErlDrvData), timer_read(ErlDrvData, char*, int), timer(ErlDrvData);
+static void timer_stop(ErlDrvData),
+    timer_read(ErlDrvData, char*, ErlDrvSizeT),
+    timer(ErlDrvData);
 
 static ErlDrvEntry timer_driver_entry =
 {
@@ -33,6 +35,16 @@ static ErlDrvEntry timer_driver_entry =
     NULL,
     timer,
     NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
     NULL
 };
 
@@ -47,8 +59,9 @@ static ErlDrvData timer_start(ErlDrvPort port, char *buf)
 }
 
 /* set the timer, this is monitored from erlang measuring the time */
-static void timer_read(ErlDrvData port, char *buf, int len)
+static void timer_read(ErlDrvData p, char *buf, ErlDrvSizeT len)
 {
+    ErlDrvPort port = (ErlDrvPort) p;
     char reply[1];
 
     if (buf[0] == START_TIMER) { 
@@ -62,8 +75,9 @@ static void timer_read(ErlDrvData port, char *buf, int len)
     }
 }
 
-static void timer_stop(ErlDrvData port)
+static void timer_stop(ErlDrvData p)
 {
+    ErlDrvPort port = (ErlDrvPort) p;
     driver_cancel_timer(port);
 }
 
diff --git a/erts/emulator/test/after_SUITE.erl b/erts/emulator/test/after_SUITE.erl
index 3e1a871408..7cc329cc69 100644
--- a/erts/emulator/test/after_SUITE.erl
+++ b/erts/emulator/test/after_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -21,27 +21,48 @@
 
 %% Tests receive after.
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1, t_after/1, receive_after/1, receive_after_big/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 t_after/1, receive_after/1, receive_after_big/1,
 	 receive_after_errors/1, receive_var_zero/1, receive_zero/1,
 	 multi_timeout/1, receive_after_32bit/1]).
 
--export([init_per_testcase/2, fin_per_testcase/2]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
 %% Internal exports.
 
 -export([timeout_g/0]).
 
-all(suite) ->
-    [t_after, receive_after, receive_after_big, receive_after_errors,
-     receive_var_zero, receive_zero, multi_timeout, receive_after_32bit].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [t_after, receive_after, receive_after_big,
+     receive_after_errors, receive_var_zero, receive_zero,
+     multi_timeout, receive_after_32bit].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?t:minutes(3)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl
index 94766dc6e9..22b5d93983 100644
--- a/erts/emulator/test/alloc_SUITE.erl
+++ b/erts/emulator/test/alloc_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2003-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2003-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,7 +18,8 @@
 
 -module(alloc_SUITE).
 -author('[email protected]').
--export([all/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 
 -export([basic/1,
 	 coalesce/1,
@@ -29,28 +30,40 @@
 	 rbtree/1,
 	 mseg_clear_cache/1]).
 
--export([init_per_testcase/2, fin_per_testcase/2]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -define(DEFAULT_TIMETRAP_SECS, 240).
 
-all(doc) -> [];
-all(suite) -> [basic,
-	       coalesce,
-	       threads,
-	       realloc_copy,
-	       bucket_index,
-	       bucket_mask,
-	       rbtree,
-	       mseg_clear_cache].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [basic, coalesce, threads, realloc_copy, bucket_index,
+     bucket_mask, rbtree, mseg_clear_cache].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 
 init_per_testcase(Case, Config) when is_list(Config) ->
     Dog = ?t:timetrap(?t:seconds(?DEFAULT_TIMETRAP_SECS)),
     [{watchdog, Dog},{testcase, Case}|Config].
 
-fin_per_testcase(_Case, Config) when is_list(Config) ->
+end_per_testcase(_Case, Config) when is_list(Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
diff --git a/erts/emulator/test/alloc_SUITE_data/allocator_test.h b/erts/emulator/test/alloc_SUITE_data/allocator_test.h
index b869a4079c..cd4a91d34a 100644
--- a/erts/emulator/test/alloc_SUITE_data/allocator_test.h
+++ b/erts/emulator/test/alloc_SUITE_data/allocator_test.h
@@ -19,7 +19,7 @@
 #ifndef ALLOCATOR_TEST_H__
 #define ALLOCATOR_TEST_H__
 
-typedef unsigned long Ulong;
+typedef ErlDrvUInt Ulong;
 
 #ifndef __WIN32__
 Ulong erts_alc_test(Ulong, Ulong, Ulong, Ulong);
@@ -82,15 +82,17 @@ typedef void* erts_cond;
 #define NO_OF_BKTS		((Ulong)	ALC_TEST0(0x102))
 #define FIND_BKT(A, I)		((int)		ALC_TEST2(0x103, (A), (I)))
 
-/* From erl_bestfit_alloc.c */
-#define IS_AOBF(A)		((Ulong)	ALC_TEST1(0x200, (A)))
-#define RBT_ROOT(A)		((RBT_t *)	ALC_TEST1(0x201, (A)))
-#define RBT_PARENT(T)		((RBT_t *)	ALC_TEST1(0x202, (T)))
-#define RBT_LEFT(T)		((RBT_t *)	ALC_TEST1(0x203, (T)))
-#define RBT_RIGHT(T)		((RBT_t *)	ALC_TEST1(0x204, (T)))
-#define RBT_NEXT(T)		((RBTL_t *)	ALC_TEST1(0x205, (T)))
-#define RBT_IS_BLACK(T)		((Ulong)	ALC_TEST1(0x206, (T)))
-#define RBT_IS_TREE(T)		((Ulong)	ALC_TEST1(0x207, (T)))
+/* From erl_bestfit_alloc.c and erl_ao_firstfit_alloc.c */
+#define IS_AOBF(A)		((Ulong)	ALC_TEST1(RBT_OP(0), (A)))
+#define RBT_ROOT(A)		((RBT_t *)	ALC_TEST1(RBT_OP(1), (A)))
+#define RBT_PARENT(T)		((RBT_t *)	ALC_TEST1(RBT_OP(2), (T)))
+#define RBT_LEFT(T)		((RBT_t *)	ALC_TEST1(RBT_OP(3), (T)))
+#define RBT_RIGHT(T)		((RBT_t *)	ALC_TEST1(RBT_OP(4), (T)))
+#define RBT_NEXT(T)		((RBTL_t *)	ALC_TEST1(RBT_OP(5), (T)))
+#define RBT_IS_BLACK(T)		((Ulong)	ALC_TEST1(RBT_OP(6), (T)))
+#define RBT_IS_TREE(T)		((Ulong)	ALC_TEST1(RBT_OP(7), (T)))
+#define IS_AOFF(A)		((Ulong)	ALC_TEST1(RBT_OP(8), (A)))
+#define RBT_MAX_SZ(T)		((Ulong)	ALC_TEST1(RBT_OP(9), (T)))
 
 /* From erl_mseg.c */
 #define HAVE_MSEG()		((int)		ALC_TEST0(0x400))
diff --git a/erts/emulator/test/alloc_SUITE_data/coalesce.c b/erts/emulator/test/alloc_SUITE_data/coalesce.c
index c84da97d35..6f35d3279b 100644
--- a/erts/emulator/test/alloc_SUITE_data/coalesce.c
+++ b/erts/emulator/test/alloc_SUITE_data/coalesce.c
@@ -267,7 +267,7 @@ void
 testcase_run(TestCaseState_t *tcs)
 {
     char *argv_org[] = {"-tmmbcs1024", "-tsbct2048", "-trmbcmt100", "-tas", NULL, NULL};
-    char *alg[] = {"af", "gf", "bf", "aobf", NULL};
+    char *alg[] = {"af", "gf", "bf", "aobf", "aoff", NULL};
     int i;
 
     for (i = 0; alg[i]; i++) {
diff --git a/erts/emulator/test/alloc_SUITE_data/rbtree.c b/erts/emulator/test/alloc_SUITE_data/rbtree.c
index c97e0aac1a..4e7f821baf 100644
--- a/erts/emulator/test/alloc_SUITE_data/rbtree.c
+++ b/erts/emulator/test/alloc_SUITE_data/rbtree.c
@@ -34,6 +34,14 @@ typedef struct {
 #define PRINT_TREE
 #endif
 
+/* Ugly hack to steer the test code towards the right allocator */
+#define RBT_OP(CMD) (current_rbt_type_op_base + (CMD))
+static enum {
+    BESTFIT_OP_BASE = 0x200,
+    AO_FIRSTFIT_OP_BASE = 0x500
+}current_rbt_type_op_base;
+
+
 #ifdef PRINT_TREE
 
 #define INDENT_STEP 5
@@ -65,12 +73,11 @@ print_tree_aux(TestCaseState_t *tcs, RBT_t *x, int indent)
 
 
 static void
-print_tree(TestCaseState_t *tcs, RBT_t *root, int aobf)
+print_tree(TestCaseState_t *tcs, RBT_t *root)
 {
-    char *type = aobf ? "Size-Adress" : "Size";
-    testcase_printf(tcs, " --- %s tree begin ---\r\n", type);
+    testcase_printf(tcs, " --- Tree begin ---\r\n");
     print_tree_aux(tcs, root, 0);
-    testcase_printf(tcs, " --- %s tree end ---\r\n", type);
+    testcase_printf(tcs, " --- Tree end ---\r\n");
 }
 
 #endif
@@ -78,7 +85,8 @@ print_tree(TestCaseState_t *tcs, RBT_t *root, int aobf)
 static RBT_t *
 check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size)
 {
-    int i, max_i, address_order;
+    enum { BF, AOBF, AOFF }type;
+    int i, max_i;
     char stk[128];
     RBT_t *root, *x, *y, *res;
     Ulong x_sz, y_sz, is_x_black;
@@ -86,11 +94,14 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size)
 
     res = NULL;
 
-    address_order = IS_AOBF(alc);
+    if      (IS_AOBF(alc)) type = AOBF;
+    else if (IS_AOFF(alc)) type = AOFF;
+    else type = BF;
+
     root = RBT_ROOT(alc);
 
 #ifdef PRINT_TREE
-    print_tree(tcs, root, address_order);
+    print_tree(tcs, root);
 #endif
 
     max_i = i = -1;
@@ -165,12 +176,18 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size)
 	if (y) {
 	    y_sz = BLK_SZ(y);
 	    ASSERT(tcs, RBT_PARENT(y) == x);
-	    if (address_order) {
+	    switch (type) {
+	    case AOBF:
 		ASSERT(tcs, y_sz < x_sz || (y_sz == x_sz && y < x));
-	    }
-	    else {
+		break;
+	    case BF:
 		ASSERT(tcs, RBT_IS_TREE(y));
 		ASSERT(tcs, y_sz < x_sz);
+		break;
+	    case AOFF:
+		ASSERT(tcs, y < x);
+		ASSERT(tcs, RBT_MAX_SZ(y) <= RBT_MAX_SZ(x));
+		break;
 	    }
 	}
 
@@ -178,16 +195,22 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size)
 	if (y) {
 	    y_sz = BLK_SZ(y);
 	    ASSERT(tcs, RBT_PARENT(y) == x);
-	    if (address_order) {
+	    switch (type) {
+	    case AOBF:
 		ASSERT(tcs, y_sz > x_sz || (y_sz == x_sz && y > x));
-	    }
-	    else {
+		break;
+	    case BF:
 		ASSERT(tcs, RBT_IS_TREE(y));
 		ASSERT(tcs, y_sz > x_sz);
+		break;
+	    case AOFF:
+		ASSERT(tcs, y > x);
+		ASSERT(tcs, RBT_MAX_SZ(y) <= RBT_MAX_SZ(x));
+		break;
 	    }
 	}
 
-	if (!address_order) {
+	if (type == BF) {
 	    Ulong l_sz;
 	    RBTL_t *l = RBT_NEXT(x);
 	    for (l = RBT_NEXT(x); l; l = RBT_NEXT(l)) {
@@ -202,13 +225,20 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size)
 		res = x;
 	    else {
 		y_sz = BLK_SZ(res);
-		if (address_order) {
+		switch (type) {
+		case AOBF:
 		    if (x_sz < y_sz || (x_sz == y_sz && x < res))
 			res = x;
-		}
-		else {
-		    if (!res || x_sz < y_sz)
+		    break;
+		case BF:
+		    if (x_sz < y_sz)
 			res = x;
+		    break;
+		case AOFF:
+		    if (x < res) {
+			res = x;
+		    }
+		    break;
 		}
 	    }
 	}
@@ -257,7 +287,7 @@ static void
 test_it(TestCaseState_t *tcs)
 {
     int i;
-    Allctr_t a = ((rbtree_test_data *) tcs->extra)->allocator;
+    Allctr_t* a = ((rbtree_test_data *) tcs->extra)->allocator;
     void **blk = ((rbtree_test_data *) tcs->extra)->blk;
     void **fence = ((rbtree_test_data *) tcs->extra)->fence;
     Ulong min_blk_sz;
@@ -338,6 +368,7 @@ testcase_run(TestCaseState_t *tcs)
 {
     char *argv1[] = {"-tasbf", NULL};
     char *argv2[] = {"-tasaobf", NULL};
+    char *argv3[] = {"-tasaoff", NULL};
     Allctr_t *a;
     rbtree_test_data *td;
 
@@ -355,6 +386,7 @@ testcase_run(TestCaseState_t *tcs)
 
     testcase_printf(tcs, "Starting test of best fit...\n");
 
+    current_rbt_type_op_base = BESTFIT_OP_BASE;
     td->allocator = a = START_ALC("rbtree_bf_", 0, argv1);
 
     ASSERT(tcs, a);
@@ -371,6 +403,7 @@ testcase_run(TestCaseState_t *tcs)
 
     testcase_printf(tcs, "Starting test of address order best fit...\n");
 
+    current_rbt_type_op_base = BESTFIT_OP_BASE;
     td->allocator = a = START_ALC("rbtree_aobf_", 0, argv2);
 
     ASSERT(tcs, a);
@@ -383,4 +416,19 @@ testcase_run(TestCaseState_t *tcs)
 
     testcase_printf(tcs, "Address order best fit test succeeded!\n");
 
+    /* Address order first fit... */
+
+    testcase_printf(tcs, "Starting test of address order first fit...\n");
+
+    current_rbt_type_op_base = AO_FIRSTFIT_OP_BASE;
+    td->allocator = a = START_ALC("rbtree_aoff_", 0, argv3);
+
+    ASSERT(tcs, a);
+
+    test_it(tcs);
+
+    STOP_ALC(a);
+    td->allocator = NULL;
+
+    testcase_printf(tcs, "Address order first fit test succeeded!\n");
 }
diff --git a/erts/emulator/test/alloc_SUITE_data/testcase_driver.c b/erts/emulator/test/alloc_SUITE_data/testcase_driver.c
index 1e98844838..66971654a2 100644
--- a/erts/emulator/test/alloc_SUITE_data/testcase_driver.c
+++ b/erts/emulator/test/alloc_SUITE_data/testcase_driver.c
@@ -50,13 +50,32 @@ typedef struct {
 
 ErlDrvData testcase_drv_start(ErlDrvPort port, char *command);
 void testcase_drv_stop(ErlDrvData drv_data);
-void testcase_drv_run(ErlDrvData drv_data, char *buf, int len);
+void testcase_drv_run(ErlDrvData drv_data, char *buf, ErlDrvSizeT len);
 
 static ErlDrvEntry testcase_drv_entry = { 
     NULL,
     testcase_drv_start,
     testcase_drv_stop,
-    testcase_drv_run
+    testcase_drv_run,
+    NULL,
+    NULL,
+    "testcase_drv",
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
+    NULL
 };
 
 
@@ -92,7 +111,7 @@ testcase_drv_stop(ErlDrvData drv_data)
 }
 
 void
-testcase_drv_run(ErlDrvData drv_data, char *buf, int len)
+testcase_drv_run(ErlDrvData drv_data, char *buf, ErlDrvSizeT len)
 {
     InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) drv_data;
     ErlDrvTermData result_atom;
diff --git a/erts/emulator/test/beam_SUITE.erl b/erts/emulator/test/beam_SUITE.erl
index cc1626630b..02c6e19686 100644
--- a/erts/emulator/test/beam_SUITE.erl
+++ b/erts/emulator/test/beam_SUITE.erl
@@ -1,34 +1,55 @@
 %%
 %% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 1998-2009. All Rights Reserved.
-%% 
+%%
+%% Copyright Ericsson AB 1998-2011. All Rights Reserved.
+%%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
 %% compliance with the License. You should have received a copy of the
 %% Erlang Public License along with this software. If not, it can be
 %% retrieved online at http://www.erlang.org/.
-%% 
+%%
 %% Software distributed under the License is distributed on an "AS IS"
 %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 %% the License for the specific language governing rights and limitations
 %% under the License.
-%% 
+%%
 %% %CopyrightEnd%
 %%
 
 -module(beam_SUITE).
 
--export([all/1, packed_registers/1, apply_last/1, apply_last_bif/1,
-	 buildo_mucho/1, heap_sizes/1, big_lists/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 packed_registers/1, apply_last/1, apply_last_bif/1,
+	 buildo_mucho/1, heap_sizes/1, big_lists/1, fconv/1,
+	 select_val/1]).
 
 -export([applied/2]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [packed_registers, apply_last, apply_last_bif,
+     buildo_mucho, heap_sizes, big_lists, select_val].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [packed_registers, apply_last, apply_last_bif, buildo_mucho,
-     heap_sizes, big_lists].
 
 
 %% Verify that apply(M, F, A) is really tail recursive.
@@ -279,3 +300,42 @@ b() ->
          _} ->
 	    ok
     end.
+
+fconv(Config) when is_list(Config) ->
+    ?line do_fconv(atom),
+    ?line do_fconv(nil),
+    ?line do_fconv(tuple_literal),
+    ?line 3.0 = do_fconv(1.0, 2.0),
+    ok.
+
+do_fconv(Type) ->
+    try
+	do_fconv(Type, 1.0),
+	test_server:fail()
+    catch
+	error:badarith ->
+	    ok
+    end.
+
+do_fconv(atom, Float) when is_float(Float) ->
+    Float + a;
+do_fconv(nil, Float) when is_float(Float) ->
+    Float + [];
+do_fconv(tuple_literal, Float) when is_float(Float) ->
+    Float + {a,b}.
+
+select_val(Config) when is_list(Config) ->
+    ?line zero = do_select_val(0),
+    ?line big = do_select_val(1 bsl 64),
+    ?line integer = do_select_val(42),
+    ok.
+
+do_select_val(X) ->
+    case X of
+	0 ->
+	    zero;
+	1 bsl 64 ->
+	    big;
+	Int when is_integer(Int) ->
+	    integer
+    end.
diff --git a/erts/emulator/test/beam_literals_SUITE.erl b/erts/emulator/test/beam_literals_SUITE.erl
index 75841adbfc..85236e4203 100644
--- a/erts/emulator/test/beam_literals_SUITE.erl
+++ b/erts/emulator/test/beam_literals_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,21 +18,41 @@
 %%
 
 -module(beam_literals_SUITE).
--export([all/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 -export([putting/1, matching_smalls/1, matching_smalls_jt/1,
 	 matching_bigs/1, matching_more_bigs/1,
 	 matching_bigs_and_smalls/1, badmatch/1, case_clause/1,
 	 receiving/1, literal_type_tests/1,
-	 put_list/1, fconv/1, literal_case_expression/1]).
+	 put_list/1, fconv/1, literal_case_expression/1,
+	 increment/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     [putting, matching_smalls, matching_smalls_jt,
      matching_bigs, matching_more_bigs,
      matching_bigs_and_smalls, badmatch, case_clause,
-     receiving, literal_type_tests,
-     put_list, fconv, literal_case_expression].
+     receiving, literal_type_tests, put_list, fconv,
+     literal_case_expression, increment].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 putting(doc) -> "Test creating lists and tuples containing big number literals.";
 putting(Config) when is_list(Config) ->
@@ -48,6 +68,7 @@ matching_bigs(doc) -> "Test matching of a few big number literals (in Beam,"
 matching_bigs(Config) when is_list(Config) ->
     a = matching1(3972907842873739),
     b = matching1(-389789298378939783333333333333333333784),
+    other = matching1(3141699999999999999999999999999999999),
     other = matching1(42).
 
 matching_smalls(doc) -> "Test matching small numbers (both positive and negative).";
@@ -236,14 +257,14 @@ make_test([{T,L}|Ts]) ->
 make_test([]) -> [].
 
 test(T, L) ->
-    S = lists:flatten(io_lib:format("begin io:format(\"~~p~~n\", [{~p,~p}]), if ~w(~w) -> true; true -> false end end. ", [T, L, T, L])),
+    S = lists:flatten(io_lib:format("begin io:format(\"~~p~n\", [{~p,~p}]), if ~w(~w) -> true; true -> false end end. ", [T, L, T, L])),
     {ok,Toks,_Line} = erl_scan:string(S),
     {ok,E} = erl_parse:parse_exprs(Toks),
     {value,Val,_Bs} = erl_eval:exprs(E, []),
     {match,0,{atom,0,Val},hd(E)}.
 
 test(T, A, L) ->
-    S = lists:flatten(io_lib:format("begin io:format(\"~~p~~n\", [{~p,~p,~p}]), if ~w(~w, ~w) -> true; true -> false end end. ",
+    S = lists:flatten(io_lib:format("begin io:format(\"~~p~n\", [{~p,~p,~p}]), if ~w(~w, ~w) -> true; true -> false end end. ",
 				    [T,L,A,T,L,A])),
     {ok,Toks,_Line} = erl_scan:string(S),
     {ok,E} = erl_parse:parse_exprs(Toks),
@@ -405,14 +426,51 @@ fconv_2(F) when is_float(F) ->
 literal_case_expression(Config) when is_list(Config) ->
     ?line DataDir = ?config(data_dir, Config),
     ?line Src = filename:join(DataDir, "literal_case_expression"),
-    ?line {ok,literal_case_expression=Mod,Code} = compile:file(Src, [from_asm,binary]),
+    ?line {ok,literal_case_expression=Mod,Code} =
+	compile:file(Src, [from_asm,binary]),
     ?line {module,Mod} = code:load_binary(Mod, Src, Code),
     ?line ok = Mod:x(),
     ?line ok = Mod:y(),
+    ?line ok = Mod:zi1(),
+    ?line ok = Mod:zi2(),
+    ?line ok = Mod:za1(),
+    ?line ok = Mod:za2(),
     ?line true = code:delete(Mod),
     ?line code:purge(Mod),
     ok.
 
+%% Test the i_increment instruction.
+increment(Config) when is_list(Config) ->
+    %% In the 32-bit emulator, Neg32 can be represented as a small,
+    %% but -Neg32 cannot. Therefore the i_increment instruction must
+    %% not be used in the subtraction that follows (since i_increment
+    %% cannot handle a bignum literal).
+    Neg32 = -(1 bsl 27),
+    Big32 = id(1 bsl 32),
+    Result32 = (1 bsl 32) + (1 bsl 27),
+    ?line Result32 = Big32 + (1 bsl 27),
+    ?line Result32 = Big32 - Neg32,
+
+    %% Same thing, but for the 64-bit emulator.
+    Neg64 = -(1 bsl 59),
+    Big64 = id(1 bsl 64),
+    Result64 = (1 bsl 64) + (1 bsl 59),
+    ?line Result64 = Big64 + (1 bsl 59),
+    ?line Result64 = Big64 - Neg64,
+
+    %% Test error handling for the i_increment instruction.
+    Bad = id(bad),
+    ?line {'EXIT',{badarith,_}} = (catch Bad + 42),
+
+    %% Small operands, but a big result.
+    Res32 = 1 bsl 27,
+    Small32 = id(Res32-1),
+    ?line Res32 = Small32 + 1,
+    Res64 = 1 bsl 59,
+    Small64 = id(Res64-1),
+    ?line Res64 = Small64 + 1,
+    ok.
+
 %% Help functions.
 
 chksum(Term) ->
diff --git a/erts/emulator/test/beam_literals_SUITE_data/literal_case_expression.S b/erts/emulator/test/beam_literals_SUITE_data/literal_case_expression.S
index c0ffe9ab53..bfdfc079dc 100644
--- a/erts/emulator/test/beam_literals_SUITE_data/literal_case_expression.S
+++ b/erts/emulator/test/beam_literals_SUITE_data/literal_case_expression.S
@@ -1,10 +1,11 @@
 {module, literal_case_expression}.  %% version = 0
 
-{exports, [{module_info,0},{module_info,1},{x,0},{y,0}]}.
+{exports, [{module_info,0},{module_info,1},{x,0},{y,0},
+	   {zi1,0},{zi2,0},{za1,0},{za2,0}]}.
 
 {attributes, []}.
 
-{labels, 15}.
+{labels, 32}.
 
 
 {function, x, 0, 2}.
@@ -52,6 +53,81 @@
   {label,10}.
     {case_end,{float,34.0000}}.
 
+{function, zi1, 0, 16}.
+  {label,15}.
+    {func_info,{atom,literal_case_expression},{atom,zi1},0}.
+  {label,16}.
+    {test,is_integer,{f,19},[{integer,42}]}.
+    {select_val,{integer,42},
+                {f,18},
+                {list,[{integer,42},
+                       {f,17},
+                       {integer,1000},
+                       {f,18}]}}.
+  {label,17}.
+    {move,{atom,ok},{x,0}}.
+    return.
+  {label,18}.
+    {move,{atom,error},{x,0}}.
+    return.
+  {label,19}.
+    {case_end,{integer,42}}.
+
+{function, zi2, 0, 16}.
+  {label,20}.
+    {func_info,{atom,literal_case_expression},{atom,zi2},0}.
+  {label,21}.
+    {test,is_integer,{f,23},[{integer,42}]}.
+    {select_val,{integer,42},
+                {f,23},
+                {list,[{integer,42},
+                       {f,22},
+                       {integer,1000},
+                       {f,23}]}}.
+  {label,22}.
+    {move,{atom,ok},{x,0}}.
+    return.
+  {label,23}.
+    {move,{atom,error},{x,0}}.
+    return.
+
+{function, za1, 0, 25}.
+  {label,24}.
+    {func_info,{atom,literal_case_expression},{atom,za1},0}.
+  {label,25}.
+    {test,is_atom,{f,28},[{atom,x}]}.
+    {select_val,{atom,x},
+                {f,27},
+                {list,[{atom,a},
+                       {f,27},
+                       {atom,x},
+                       {f,26}]}}.
+  {label,26}.
+    {move,{atom,ok},{x,0}}.
+    return.
+  {label,27}.
+    {move,{atom,error},{x,0}}.
+    return.
+  {label,28}.
+    {case_end,{atom,x}}.
+
+{function, za2, 0, 30}.
+  {label,29}.
+    {func_info,{atom,literal_case_expression},{atom,za2},0}.
+  {label,30}.
+    {test,is_atom,{f,32},[{atom,x}]}.
+    {select_val,{atom,x},
+                {f,32},
+                {list,[{atom,a},
+                       {f,32},
+                       {atom,x},
+                       {f,31}]}}.
+  {label,31}.
+    {move,{atom,ok},{x,0}}.
+    return.
+  {label,32}.
+    {move,{atom,error},{x,0}}.
+    return.
 
 {function, module_info, 0, 12}.
   {label,11}.
diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl
index cfbc5dfe81..99ed8f1748 100644
--- a/erts/emulator/test/bif_SUITE.erl
+++ b/erts/emulator/test/bif_SUITE.erl
@@ -1,44 +1,159 @@
 %%
 %% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 2005-2009. All Rights Reserved.
-%% 
+%%
+%% Copyright Ericsson AB 2005-2012. All Rights Reserved.
+%%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
 %% compliance with the License. You should have received a copy of the
 %% Erlang Public License along with this software. If not, it can be
 %% retrieved online at http://www.erlang.org/.
-%% 
+%%
 %% Software distributed under the License is distributed on an "AS IS"
 %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 %% the License for the specific language governing rights and limitations
 %% under the License.
-%% 
+%%
 %% %CopyrightEnd%
 %%
 
 -module(bif_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
+	 display/1, display_huge/0,
+	 types/1,
 	 t_list_to_existing_atom/1,os_env/1,otp_7526/1,
 	 binary_to_atom/1,binary_to_existing_atom/1,
-	 atom_to_binary/1,min_max/1]).
+	 atom_to_binary/1,min_max/1, erlang_halt/1]).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [types, t_list_to_existing_atom, os_env, otp_7526,
+     display,
+     atom_to_binary, binary_to_atom, binary_to_existing_atom,
+     min_max, erlang_halt].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [t_list_to_existing_atom,os_env,otp_7526,
-     atom_to_binary,binary_to_atom,binary_to_existing_atom,
-     min_max].
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?t:minutes(1)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
+
+display(suite) ->
+    [];
+display(doc) ->
+    ["Uses erlang:display to test that erts_printf does not do deep recursion"];
+display(Config) when is_list(Config) ->
+    Pa = filename:dirname(code:which(?MODULE)),
+    {ok, Node} = test_server:start_node(display_huge_term,peer,
+					[{args, "-pa \""++Pa++"\""}]),
+    true = rpc:call(Node,?MODULE,display_huge,[]),
+    test_server:stop_node(Node),
+    ok.
+
+display_huge() ->
+    erlang:display(deeep(100000)).
+
+deeep(0,Acc) ->
+    Acc;
+deeep(N,Acc) ->
+    deeep(N-1,[Acc|[]]).
+
+deeep(N) ->
+    deeep(N,[hello]).
+
+
+types(Config) when is_list(Config) ->
+    c:l(erl_bif_types),
+    case erlang:function_exported(erl_bif_types, module_info, 0) of
+	false ->
+	    %% Fail cleanly.
+	    ?line ?t:fail("erl_bif_types not compiled");
+	true ->
+	    types_1()
+    end.
+
+types_1() ->
+    ?line List0 = erlang:system_info(snifs),
+
+    %% Ignore missing type information for hipe BIFs.
+    ?line List = [MFA || {M,_,_}=MFA <- List0, M =/= hipe_bifs],
+
+    case [MFA || MFA <- List, not known_types(MFA)] of
+	[] ->
+	    types_2(List);
+	BadTypes ->
+	    io:put_chars("No type information:\n"),
+	    io:format("~p\n", [lists:sort(BadTypes)]),
+	    ?line ?t:fail({length(BadTypes),bifs_without_types})
+    end.
+
+types_2(List) ->
+    BadArity = [MFA || {M,F,A}=MFA <- List,
+		       begin
+			   Types = erl_bif_types:arg_types(M, F, A),
+			   length(Types) =/= A
+		       end],
+    case BadArity of
+	[] ->
+	    types_3(List);
+	[_|_] ->
+	    io:put_chars("Bifs with bad arity\n"),
+	    io:format("~p\n", [BadArity]),
+	    ?line ?t:fail({length(BadArity),bad_arity})
+    end.
+
+types_3(List) ->
+    BadSmokeTest = [MFA || {M,F,A}=MFA <- List,
+			   begin
+			       try erl_bif_types:type(M, F, A) of
+				   Type ->
+				       %% Test that type is returned.
+				       not erl_types:is_erl_type(Type)
+			       catch
+				   Class:Error ->
+				       io:format("~p: ~p ~p\n",
+						 [MFA,Class,Error]),
+				       true
+			       end
+			   end],
+    case BadSmokeTest of
+	[] ->
+	    ok;
+	[_|_] ->
+	    io:put_chars("Bifs with failing calls to erlang_bif_types:type/3 "
+			 "(or with bogus return values):\n"),
+	    io:format("~p\n", [BadSmokeTest]),
+	    ?line ?t:fail({length(BadSmokeTest),bad_smoke_test})
+    end.
+
+known_types({M,F,A}) ->
+    erl_bif_types:is_known(M, F, A).
+
 t_list_to_existing_atom(Config) when is_list(Config) ->
     ?line all = list_to_existing_atom("all"),
     ?line ?MODULE = list_to_existing_atom(?MODULE_STRING),
@@ -308,10 +423,70 @@ min_max(Config) when is_list(Config) ->
 
     ?line 42.0 = erlang:min(42.0, 42),
     ?line 42.0 = erlang:max(42.0, 42),
+    %% And now (R14) they are also autoimported!
+    ?line a = min(id(a), a),
+    ?line a = min(id(a), b),
+    ?line a = min(id(b), a),
+    ?line b = min(id(b), b),
+    ?line a = max(id(a), a),
+    ?line b = max(id(a), b),
+    ?line b = max(id(b), a),
+    ?line b = max(id(b), b),
+
+    ?line 42.0 = min(42.0, 42),
+    ?line 42.0 = max(42.0, 42),
 
     ok.
 
+
+
+erlang_halt(Config) when is_list(Config) ->
+    try erlang:halt(undefined) of
+	_-> ?t:fail({erlang,halt,{undefined}})
+    catch error:badarg -> ok end,
+    try halt(undefined) of
+	_-> ?t:fail({halt,{undefined}})
+    catch error:badarg -> ok end,
+    try erlang:halt(undefined, []) of
+	_-> ?t:fail({erlang,halt,{undefined,[]}})
+    catch error:badarg -> ok end,
+    try halt(undefined, []) of
+	_-> ?t:fail({halt,{undefined,[]}})
+    catch error:badarg -> ok end,
+    try halt(0, undefined) of
+	_-> ?t:fail({halt,{0,undefined}})
+    catch error:badarg -> ok end,
+    try halt(0, [undefined]) of
+	_-> ?t:fail({halt,{0,[undefined]}})
+    catch error:badarg -> ok end,
+    try halt(0, [{undefined,true}]) of
+	_-> ?t:fail({halt,{0,[{undefined,true}]}})
+    catch error:badarg -> ok end,
+    try halt(0, [{flush,undefined}]) of
+	_-> ?t:fail({halt,{0,[{flush,undefined}]}})
+    catch error:badarg -> ok end,
+    try halt(0, [{flush,true,undefined}]) of
+	_-> ?t:fail({halt,{0,[{flush,true,undefined}]}})
+    catch error:badarg -> ok end,
+    H = hostname(),
+    {ok,N1} = slave:start(H, halt_node1),
+    {badrpc,nodedown} = rpc:call(N1, erlang, halt, []),
+    {ok,N2} = slave:start(H, halt_node2),
+    {badrpc,nodedown} = rpc:call(N2, erlang, halt, [0]),
+    {ok,N3} = slave:start(H, halt_node3),
+    {badrpc,nodedown} = rpc:call(N3, erlang, halt, [0,[]]),
+    ok.
+
+
+
 %% Helpers
     
 id(I) -> I.
 
+hostname() ->
+    hostname(atom_to_list(node())).
+
+hostname([$@ | Hostname]) ->
+    list_to_atom(Hostname);
+hostname([_C | Cs]) ->
+    hostname(Cs).
diff --git a/erts/emulator/test/big_SUITE.erl b/erts/emulator/test/big_SUITE.erl
index 6cedd39009..413bd3bcae 100644
--- a/erts/emulator/test/big_SUITE.erl
+++ b/erts/emulator/test/big_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,30 +19,50 @@
 -module(big_SUITE).
 
 
--export([all/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 -export([t_div/1, eq_28/1, eq_32/1, eq_big/1, eq_math/1, big_literals/1,
-	 borders/1, negative/1, big_float/1, big_float_1/1, big_float_2/1,
+	 borders/1, negative/1, big_float_1/1, big_float_2/1,
 	 shift_limit_1/1, powmod/1, system_limit/1, otp_6692/1]).
 
 %% Internal exports.
--export([eval/1, funcall/2]).
+-export([eval/1]).
 -export([init/3]).
 
 -export([fac/1, fib/1, pow/2, gcd/2, lcm/2]).
 
--export([init_per_testcase/2, fin_per_testcase/2]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [t_div, eq_28, eq_32, eq_big, eq_math, big_literals,
+     borders, negative, {group, big_float}, shift_limit_1,
+     powmod, system_limit, otp_6692].
+
+groups() -> 
+    [{big_float, [], [big_float_1, big_float_2]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [t_div, eq_28, eq_32, eq_big, eq_math, big_literals, borders,
-     negative, big_float, shift_limit_1, powmod, system_limit, otp_6692].
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?t:minutes(3)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
@@ -142,12 +162,15 @@ multi_match([Node|Ns], Expr, Rs) ->
 multi_match([], _, Rs) -> Rs.
 
 eval(Expr) ->
-    Fun = {?MODULE,funcall},
-    {value,V,_} = erl_eval:expr(Expr, [], Fun),	%Applied arithmetic BIFs.
-    V = eval(Expr, Fun),			%Real arithmetic instructions.
-    V.
+    LFH = fun(Name, As) -> apply(?MODULE, Name, As) end,
 
-funcall(F, As) -> apply(?MODULE, F, As).
+    %% Applied arithmetic BIFs.
+    {value,V,_} = erl_eval:expr(Expr, [], {value,LFH}),
+
+    %% Real arithmetic instructions.
+    V = eval(Expr, LFH),
+
+    V.
 
 %% Like a subset of erl_eval:expr/3, but uses real arithmetic instructions instead of
 %% applying them (it does make a difference).
@@ -260,10 +283,6 @@ big_literals(Config) when is_list(Config) ->
     ?line ok = Mod:t(),
     ok.
 
-big_float(doc) ->
-    ["Test cases for mixing bignums and floats"];
-big_float(suite) ->
-    [big_float_1, big_float_2].
 
 big_float_1(doc) ->
     ["OTP-2436, part 1"];
diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl
index db2b3e10db..c9f43bb00b 100644
--- a/erts/emulator/test/binary_SUITE.erl
+++ b/erts/emulator/test/binary_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -23,29 +23,30 @@
 %% Tests binaries and the BIFs:
 %%	list_to_binary/1
 %%      iolist_to_binary/1
-%%      bitstr_to_list/1
+%%      list_to_bitstring/1
 %%	binary_to_list/1
 %%	binary_to_list/3
 %%	binary_to_term/1
 %%  	binary_to_term/2
-%%      bitstr_to_list/1
+%%      bitstring_to_list/1
 %%	term_to_binary/1
 %%      erlang:external_size/1
 %%	size(Binary)
 %%      iolist_size/1
-%%	concat_binary/1
 %%	split_binary/2
 %%      hash(Binary, N)
 %%      phash(Binary, N)
 %%      phash2(Binary, N)
 %%
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1, init_per_testcase/2, fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2,
 	 copy_terms/1, conversions/1, deep_lists/1, deep_bitstr_lists/1,
 	 bad_list_to_binary/1, bad_binary_to_list/1,
-	 t_split_binary/1, bad_split/1, t_concat_binary/1,
+	 t_split_binary/1, bad_split/1,
 	 terms/1, terms_float/1, external_size/1, t_iolist_size/1,
 	 t_hash/1,
 	 bad_size/1,
@@ -55,31 +56,47 @@
 	 otp_5484/1,otp_5933/1,
 	 ordering/1,unaligned_order/1,gc_test/1,
 	 bit_sized_binary_sizes/1,
-	 bitlevel_roundtrip/1,
 	 otp_6817/1,deep/1,obsolete_funs/1,robustness/1,otp_8117/1,
 	 otp_8180/1]).
 
 %% Internal exports.
 -export([sleeper/0]).
 
-all(suite) ->
-    [copy_terms,conversions,deep_lists,deep_bitstr_lists,
-     t_split_binary, bad_split, t_concat_binary,
-     bad_list_to_binary, bad_binary_to_list, terms, terms_float,
-     external_size, t_iolist_size,
-     bad_binary_to_term_2,safe_binary_to_term2,
-     bad_binary_to_term, bad_terms, t_hash, bad_size, bad_term_to_binary,
-     more_bad_terms, otp_5484, otp_5933, ordering, unaligned_order,
-     gc_test, bit_sized_binary_sizes, bitlevel_roundtrip, otp_6817, otp_8117,
-     deep,obsolete_funs,robustness,otp_8180].
+suite() -> [{ct_hooks,[ts_install_cth]},
+	    {timetrap,{minutes,2}}].
+
+all() -> 
+    [copy_terms, conversions, deep_lists, deep_bitstr_lists,
+     t_split_binary, bad_split,
+     bad_list_to_binary, bad_binary_to_list, terms,
+     terms_float, external_size, t_iolist_size,
+     bad_binary_to_term_2, safe_binary_to_term2,
+     bad_binary_to_term, bad_terms, t_hash, bad_size,
+     bad_term_to_binary, more_bad_terms, otp_5484, otp_5933,
+     ordering, unaligned_order, gc_test,
+     bit_sized_binary_sizes, otp_6817, otp_8117, deep,
+     obsolete_funs, robustness, otp_8180].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
-    Dog=?t:timetrap(?t:minutes(2)),
-    [{watchdog, Dog}|Config].
+    Config.
 
-fin_per_testcase(_Func, Config) ->
-    Dog=?config(watchdog, Config),
-    ?t:timetrap_cancel(Dog).
+end_per_testcase(_Func, _Config) ->
+    ok.
 
 -define(heap_binary_size, 64).
 
@@ -256,12 +273,33 @@ bad_list_to_binary(Config) when is_list(Config) ->
     ?line test_bad_bin(fun(X, Y) -> X*Y end),
     ?line test_bad_bin([1,fun(X) -> X + 1 end,2|fun() -> 0 end]),
     ?line test_bad_bin([fun(X) -> X + 1 end]),
+
+    %% Test iolists that do not fit in the address space.
+    %% Unfortunately, it would be too slow to test in a 64-bit emulator.
+    case erlang:system_info(wordsize) of
+	4 -> huge_iolists();
+	_ -> ok
+    end.
+
+huge_iolists() ->
+    FourGigs = 1 bsl 32,
+    ?line Sizes = [FourGigs+N || N <- lists:seq(0, 64)] ++
+	[1 bsl N || N <- lists:seq(33, 37)],
+    ?line Base = <<0:(1 bsl 20)/unit:8>>,
+    [begin
+	 L = build_iolist(Sz, Base),
+	 ?line {'EXIT',{system_limit,_}} = (catch list_to_binary([L])),
+	 ?line {'EXIT',{system_limit,_}} = (catch list_to_bitstring([L])),
+	 ?line {'EXIT',{system_limit,_}} = (catch binary:list_to_bin([L])),
+	 ?line {'EXIT',{system_limit,_}} = (catch iolist_to_binary(L))
+	 end || Sz <- Sizes],
     ok.
 
 test_bad_bin(List) ->
     {'EXIT',{badarg,_}} = (catch list_to_binary(List)),
     {'EXIT',{badarg,_}} = (catch iolist_to_binary(List)),
-    {'EXIT',{badarg,_}} = (catch list_to_bitstring(List)).
+    {'EXIT',{badarg,_}} = (catch list_to_bitstring(List)),
+    {'EXIT',{badarg,_}} = (catch iolist_size(List)).
 
 bad_binary_to_list(doc) -> "Tries binary_to_list/1,3 with bad arguments.";
 bad_binary_to_list(Config) when is_list(Config) ->
@@ -341,41 +379,6 @@ bad_split(Config) when is_list(Config) ->
 bad_split(Bin, Pos) ->
     {'EXIT',{badarg,_}} = (catch split_binary(Bin, Pos)).
 
-%% Tests concat_binary/2 and size/1.
-
-t_concat_binary(suite) -> [];
-t_concat_binary(Config) when is_list(Config) ->
-    test_concat([]),
-
-    test_concat([[]]),
-    test_concat([[], []]),
-    test_concat([[], [], []]),
-
-    test_concat([[1], []]),
-    test_concat([[], [2]]),
-    test_concat([[], [3], []]),
-
-    test_concat([[1, 2, 3], [4, 5, 6, 7]]),
-    test_concat([[1, 2, 3], [4, 5, 6, 7], [9, 10]]),
-
-    test_concat([lists:seq(0, 255), lists:duplicate(1024, $@),
-		 lists:duplicate(2048, $a),
-		 lists:duplicate(4000, $b)]),
-    ok.
-
-test_concat(Lists) ->
-    test_concat(Lists, 0, [], []).
-
-test_concat([List|Rest], Size, Combined, Binaries) ->
-    ?line Bin = list_to_binary(List),
-    ?line test_concat(Rest, Size+length(List), Combined++List, [Bin|Binaries]);
-test_concat([], Size, Combined, Binaries0) ->
-    ?line Binaries = lists:reverse(Binaries0),
-    ?line Bin = concat_binary(Binaries),
-    ?line Size = size(Bin),
-    ?line Size = iolist_size(Bin),
-    ?line Combined = binary_to_list(Bin).
-
 t_hash(doc) -> "Test hash/2 with different type of binaries.";
 t_hash(Config) when is_list(Config) ->
     test_hash([]),
@@ -438,12 +441,17 @@ terms(Config) when is_list(Config) ->
 			  Sz when is_integer(Sz), size(Bin) =< Sz ->
 			      ok
 		      end,
+              Bin1 = term_to_binary(Term, [{minor_version, 1}]),
+              case erlang:external_size(Bin1, [{minor_version, 1}]) of
+              Sz1 when is_integer(Sz1), size(Bin1) =< Sz1 ->
+                  ok
+              end,
 		      Term = binary_to_term(Bin),
-		      Term = erlang:binary_to_term(Bin, [safe]),
+		      Term = binary_to_term(Bin, [safe]),
 		      Unaligned = make_unaligned_sub_binary(Bin),
 		      Term = binary_to_term(Unaligned),
-		      Term = erlang:binary_to_term(Unaligned, []),
-		      Term = erlang:binary_to_term(Bin, [safe]),
+		      Term = binary_to_term(Unaligned, []),
+		      Term = binary_to_term(Bin, [safe]),
 		      BinC = erlang:term_to_binary(Term, [compressed]),
 		      Term = binary_to_term(BinC),
 		      true = size(BinC) =< size(Bin),
@@ -470,7 +478,12 @@ terms_float(Config) when is_list(Config) ->
 			      Term = binary_to_term(Bin0),
 			      Bin1 = term_to_binary(Term, [{minor_version,1}]),
 			      Term = binary_to_term(Bin1),
-			      true = size(Bin1) < size(Bin0)
+			      true = size(Bin1) < size(Bin0),
+                  Size0 = erlang:external_size(Term),
+                  Size00 = erlang:external_size(Term, [{minor_version, 0}]),
+                  Size1 = erlang:external_size(Term, [{minor_version, 1}]),
+                  true = (Size0 =:= Size00),
+                  true = Size1 < Size0
 		      end).
 
 external_size(Config) when is_list(Config) ->
@@ -486,7 +499,9 @@ external_size(Config) when is_list(Config) ->
 	    io:format("  Aligned size: ~p\n", [Sz1]),
 	    io:format("Unaligned size: ~p\n", [Sz2]),
 	    ?line ?t:fail()
-    end.
+    end,
+    ?line erlang:external_size(Bin) =:= erlang:external_size(Bin, [{minor_version, 1}]),
+    ?line erlang:external_size(Unaligned) =:= erlang:external_size(Unaligned, [{minor_version, 1}]).
 
 external_size_1(Term, Size0, Limit) when Size0 < Limit ->
     case erlang:external_size(Term) of
@@ -497,18 +512,65 @@ external_size_1(Term, Size0, Limit) when Size0 < Limit ->
 external_size_1(_, _, _) -> ok.
 
 t_iolist_size(Config) when is_list(Config) ->
-    %% Build a term whose external size only fits in a big num (on 32-bit CPU).
-    Bin = iolist_to_binary(lists:seq(0, 254)),
-    ?line ok = t_iolist_size_1(Bin, 0, 16#7FFFFFFF),
-    ?line ok = t_iolist_size_1(make_unaligned_sub_binary(Bin), 0, 16#7FFFFFFF).
+    ?line Seed = now(),
+    ?line io:format("Seed: ~p", [Seed]),
+    ?line random:seed(Seed),
+    ?line Base = <<0:(1 bsl 20)/unit:8>>,
+    ?line Powers = [1 bsl N || N <- lists:seq(2, 37)],
+    ?line Sizes0 = [[N - random:uniform(N div 2),
+		     lists:seq(N-2, N+2),
+		     N+N div 2,
+		     N + random:uniform(N div 2)] ||
+		       N <- Powers],
+    %% Test sizes around 1^32 more thoroughly.
+    FourGigs = 1 bsl 32,
+    ?line Sizes1 = [FourGigs+N || N <- lists:seq(-8, 40)] ++ Sizes0,
+    ?line Sizes2 = lists:flatten(Sizes1),
+    ?line Sizes = lists:usort(Sizes2),
+    io:format("~p sizes:", [length(Sizes)]),
+    io:format("~p\n", [Sizes]),
+    ?line [Sz = iolist_size(build_iolist(Sz, Base)) || Sz <- Sizes],
+    ok.
 
-t_iolist_size_1(IOList, Size0, Limit) when Size0 < Limit ->
-    case iolist_size(IOList) of
-	Size when is_integer(Size), Size0 < Size ->
-	    io:format("~p", [Size]),
-	    t_iolist_size_1([IOList|IOList], Size, Limit)
+build_iolist(N, Base) when N < 16 ->
+    case random:uniform(3) of
+	1 ->
+	    <<Bin:N/binary,_/binary>> = Base,
+	    Bin;
+	_ ->
+	    lists:seq(1, N)
+    end;
+build_iolist(N, Base) when N =< byte_size(Base) ->
+    case random:uniform(3) of
+	1 ->
+	    <<Bin:N/binary,_/binary>> = Base,
+	    Bin;
+	2 ->
+	    <<Bin:N/binary,_/binary>> = Base,
+	    [Bin];
+	3 ->
+	    case N rem 2 of
+		0 ->
+		    L = build_iolist(N div 2, Base),
+		    [L,L];
+		1 ->
+		    L = build_iolist(N div 2, Base),
+		    [L,L,45]
+	    end
     end;
-t_iolist_size_1(_, _, _) -> ok.
+build_iolist(N0, Base) ->
+    Small = random:uniform(15),
+    Seq = lists:seq(1, Small),
+    N = N0 - Small,
+    case N rem 2 of
+	0 ->
+	    L = build_iolist(N div 2, Base),
+	    [L,L|Seq];
+	1 ->
+	    L = build_iolist(N div 2, Base),
+	    [47,L,L|Seq]
+    end.
+
 
 bad_binary_to_term_2(doc) -> "OTP-4053.";
 bad_binary_to_term_2(suite) -> [];
@@ -543,7 +605,7 @@ bad_bin_to_term(BadBin) ->
     {'EXIT',{badarg,_}} = (catch binary_to_term(BadBin)).
 
 bad_bin_to_term(BadBin,Opts) ->
-    {'EXIT',{badarg,_}} = (catch erlang:binary_to_term(BadBin,Opts)).
+    {'EXIT',{badarg,_}} = (catch binary_to_term(BadBin,Opts)).
 
 safe_binary_to_term2(doc) -> "Test safety options for binary_to_term/2";
 safe_binary_to_term2(Config) when is_list(Config) ->
@@ -554,7 +616,7 @@ safe_binary_to_term2(Config) when is_list(Config) ->
     BadRef = <<131,114,0,3,BadHostAtom/binary,0,<<0,0,0,255>>/binary,
 	      Empty/binary,Empty/binary>>,
     ?line bad_bin_to_term(BadRef, [safe]), % good ref, with a bad atom
-    ?line fullsweep_after = erlang:binary_to_term(<<131,100,0,15,"fullsweep_after">>, [safe]), % should be a good atom
+    ?line fullsweep_after = binary_to_term(<<131,100,0,15,"fullsweep_after">>, [safe]), % should be a good atom
     BadExtFun = <<131,113,100,0,4,98,108,117,101,100,0,4,109,111,111,110,97,3>>,
     ?line bad_bin_to_term(BadExtFun, [safe]),
     ok.
@@ -1042,7 +1104,7 @@ test_terms(Test_Func) ->
     ?line Test_Func(F = fun(A) -> 42*A end),
     ?line Test_Func(lists:duplicate(32, F)),
 
-    ?line Test_Func(FF = fun binary_SUITE:all/1),
+    ?line Test_Func(FF = fun binary_SUITE:all/0),
     ?line Test_Func(lists:duplicate(32, FF)),
 
     ok.
@@ -1073,15 +1135,8 @@ sleeper() ->
     ?line receive after infinity -> ok end.
 
 
-gc_test(doc) -> "Test that binaries are garbage collected properly.";
-gc_test(suite) -> [];
+%% Test that binaries are garbage collected properly.
 gc_test(Config) when is_list(Config) ->
-    case erlang:system_info(heap_type) of
-	private -> gc_test_1();
-	hybrid -> {skip,"Hybrid heap"}
-    end.
-
-gc_test_1() ->
     %% Note: This test is only relevant for REFC binaries.
     %% Therefore, we take care that all binaries are REFC binaries.
     B = list_to_binary(lists:seq(0, ?heap_binary_size)),
@@ -1150,35 +1205,6 @@ bsbs_1(A) ->
     Bin = binary_to_term(<<131,$M,5:32,A,0,0,0,0,0>>),
     BinSize = bit_size(Bin).
 
-bitlevel_roundtrip(Config) when is_list(Config) ->
-    case ?t:is_release_available("r11b") of
-	true -> bitlevel_roundtrip_1();
-	false -> {skip,"No R11B found"}
-    end.
-
-bitlevel_roundtrip_1() ->
-    Name = bitlevelroundtrip,
-    ?line N = list_to_atom(atom_to_list(Name) ++ "@" ++ hostname()),
-    ?line ?t:start_node(Name, slave, [{erl,[{release,"r11b"}]}]),
-
-    ?line {<<128>>,1} = roundtrip(N, <<1:1>>),
-    ?line {<<64>>,2} = roundtrip(N, <<1:2>>),
-    ?line {<<16#E0>>,3} = roundtrip(N, <<7:3>>),
-    ?line {<<16#70>>,4} = roundtrip(N, <<7:4>>),
-    ?line {<<16#10>>,5} = roundtrip(N, <<2:5>>),
-    ?line {<<16#8>>,6} = roundtrip(N, <<2:6>>),
-    ?line {<<16#2>>,7} = roundtrip(N, <<1:7>>),
-    ?line {<<8,128>>,1} = roundtrip(N, <<8,1:1>>),
-    ?line {<<42,248>>,5} = roundtrip(N, <<42,31:5>>),
-
-    ?line ?t:stop_node(N),
-    ok.
-
-roundtrip(Node, Term) ->
-    {badrpc,{'EXIT',Res}} = rpc:call(Node, erlang, exit, [Term]),
-    io:format("<<~p bits>> => ~w", [bit_size(Term),Res]),
-    Res.
-
 deep(Config) when is_list(Config) ->
     ?line deep_roundtrip(lists:foldl(fun(E, A) ->
 					     [E,A]
@@ -1193,34 +1219,7 @@ deep(Config) when is_list(Config) ->
 
 deep_roundtrip(T) ->
     B = term_to_binary(T),
-    true = deep_eq(T, binary_to_term(B)).
-
-%%
-%% FIXME: =:= runs out of stack.
-%%
-deep_eq([H1|T1], [H2|T2]) ->
-    deep_eq(H1, H2) andalso deep_eq(T1, T2);
-deep_eq(T1, T2) when tuple_size(T1) =:= tuple_size(T2) ->
-    deep_eq_tup(T1, T2, tuple_size(T1));
-deep_eq(T1, T2) when is_function(T1), is_function(T2) ->
-    {uniq,U1} = erlang:fun_info(T1, uniq),
-    {index,I1} = erlang:fun_info(T1, index),
-    {arity,A1} = erlang:fun_info(T1, arity),
-    {env,E1} = erlang:fun_info(T1, env),
-    {uniq,U2} = erlang:fun_info(T2, uniq),
-    {index,I2} = erlang:fun_info(T2, index),
-    {arity,A2} = erlang:fun_info(T2, arity),
-    {env,E2} = erlang:fun_info(T2, env),
-    U1 =:= U2 andalso I1 =:= I2 andalso A1 =:= A2 andalso
-	deep_eq(E1, E2);
-deep_eq(T1, T2) ->
-    T1 =:= T2.
-
-deep_eq_tup(_T1, _T2, 0) ->
-    true;
-deep_eq_tup(T1, T2, N) ->
-    deep_eq(element(N, T1), element(N, T2)) andalso
-	deep_eq_tup(T1, T2, N-1).
+    T = binary_to_term(B).
 
 obsolete_funs(Config) when is_list(Config) ->
     erts_debug:set_internal_state(available_internal_state, true),
@@ -1331,11 +1330,4 @@ unaligned_sub_bin(Bin0, Offs) ->
     <<_:Offs,Bin:Sz/binary,_:Roffs>> = id(Bin1),
     Bin.
 
-hostname() ->
-    from($@, atom_to_list(node())).
-
-from(H, [H | T]) -> T;
-from(H, [_ | T]) -> from(H, T);
-from(_, []) -> [].
-
 id(I) -> I.
diff --git a/erts/emulator/test/bs_bincomp_SUITE.erl b/erts/emulator/test/bs_bincomp_SUITE.erl
index 4e83d97689..f1c2dff560 100644
--- a/erts/emulator/test/bs_bincomp_SUITE.erl
+++ b/erts/emulator/test/bs_bincomp_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -22,15 +22,34 @@
 
 -module(bs_bincomp_SUITE).
 
--export([all/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
 	 byte_aligned/1,bit_aligned/1,extended_byte_aligned/1,
 	 extended_bit_aligned/1,mixed/1,tracing/1]).
 
--include("test_server.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [byte_aligned, bit_aligned, extended_byte_aligned,
+     extended_bit_aligned, mixed, tracing].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [byte_aligned,bit_aligned,extended_byte_aligned,
-     extended_bit_aligned,mixed,tracing].
 
 
 byte_aligned(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/bs_bit_binaries_SUITE.erl b/erts/emulator/test/bs_bit_binaries_SUITE.erl
index 52bb925385..ff1088118d 100644
--- a/erts/emulator/test/bs_bit_binaries_SUITE.erl
+++ b/erts/emulator/test/bs_bit_binaries_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -22,18 +22,38 @@
 
 -module(bs_bit_binaries_SUITE).
 
--export([all/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
 	 misc/1,horrid_match/1,test_bitstr/1,test_bit_size/1,asymmetric_tests/1,
 	 big_asymmetric_tests/1,binary_to_and_from_list/1,
 	 big_binary_to_and_from_list/1,send_and_receive/1,
 	 send_and_receive_alot/1,append/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [misc, horrid_match, test_bitstr, test_bit_size,
+     asymmetric_tests, big_asymmetric_tests,
+     binary_to_and_from_list, big_binary_to_and_from_list,
+     send_and_receive, send_and_receive_alot, append].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [misc,horrid_match,test_bitstr,test_bit_size,asymmetric_tests,
-     big_asymmetric_tests,binary_to_and_from_list,big_binary_to_and_from_list,
-     send_and_receive,send_and_receive_alot,append].
 
 misc(Config) when is_list(Config) ->
     ?line <<1:100>> = id(<<1:100>>),
diff --git a/erts/emulator/test/bs_construct_SUITE.erl b/erts/emulator/test/bs_construct_SUITE.erl
index 3d9b51d278..dea02b1f4b 100644
--- a/erts/emulator/test/bs_construct_SUITE.erl
+++ b/erts/emulator/test/bs_construct_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -21,22 +21,39 @@
 
 -module(bs_construct_SUITE).
 
--export([all/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
 	 test1/1, test2/1, test3/1, test4/1, test5/1, testf/1,
 	 not_used/1, in_guard/1,
 	 mem_leak/1, coerce_to_float/1, bjorn/1,
 	 huge_float_field/1, huge_binary/1, system_limit/1, badarg/1,
 	 copy_writable_binary/1, kostis/1, dynamic/1, bs_add/1,
-	 otp_7422/1]).
+	 otp_7422/1, zero_width/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
-all(suite) ->
-    [test1, test2, test3, test4, test5, testf,
-     not_used, in_guard, mem_leak, coerce_to_float, bjorn,
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [test1, test2, test3, test4, test5, testf, not_used,
+     in_guard, mem_leak, coerce_to_float, bjorn,
      huge_float_field, huge_binary, system_limit, badarg,
-     copy_writable_binary, kostis, dynamic, bs_add,
-     otp_7422].
+     copy_writable_binary, kostis, dynamic, bs_add, otp_7422, zero_width].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
 big(1) ->
     57285702734876389752897683.
@@ -443,7 +460,6 @@ mem_leak(0, _) -> ok;
 mem_leak(N, B) ->
     ?line big_bin(B, <<23>>),
     ?line {'EXIT',{badarg,_}} = (catch big_bin(B, bad)),
-    maybe_gc(),
     mem_leak(N-1, B).
 
 big_bin(B1, B2) ->
@@ -456,13 +472,6 @@ big_bin(B1, B2) ->
 make_bin(0, Acc) -> Acc;
 make_bin(N, Acc) -> make_bin(N-1, <<Acc/binary,Acc/binary>>).
 
-maybe_gc() ->
-    case erlang:system_info(heap_type) of
-	shared -> erlang:garbage_collect();
-	hybrid -> erlang:garbage_collect();
-	private -> ok
-    end.
-
 -define(COF(Int0),
 	?line (fun(Int) ->
 		       true = <<Int:32/float>> =:= <<(float(Int)):32/float>>,
@@ -536,6 +545,11 @@ huge_float_check({'EXIT',{badarg,_}}) -> ok.
 
 huge_binary(Config) when is_list(Config) ->
     ?line 16777216 = size(<<0:(id(1 bsl 26)),(-1):(id(1 bsl 26))>>),
+    ?line garbage_collect(),
+    ?line id(<<0:((1 bsl 32)-1)>>),
+    ?line garbage_collect(),
+    ?line id(<<0:(id((1 bsl 32)-1))>>),
+    ?line garbage_collect(),
     ok.
 
 system_limit(Config) when is_list(Config) ->
@@ -548,6 +562,10 @@ system_limit(Config) when is_list(Config) ->
     ?line {'EXIT',{system_limit,_}} =
 	(catch <<(id(<<>>))/binary,0:(id(1 bsl 100))>>),
 
+    %% Would fail to load.
+    ?line {'EXIT',{system_limit,_}} = (catch <<0:(1 bsl 67)>>),
+    ?line {'EXIT',{system_limit,_}} = (catch <<0:((1 bsl 64)+1)>>),
+
     case WordSize of
 	4 ->
 	    system_limit_32();
@@ -564,6 +582,14 @@ system_limit_32() ->
     ?line {'EXIT',{system_limit,_}} = (catch <<0:(id(8)),42:536870912/unit:8>>),
     ?line {'EXIT',{system_limit,_}} =
 	(catch <<0:(id(8)),42:(id(536870912))/unit:8>>),
+
+    %% The size would be silently truncated, resulting in a crash.
+    ?line {'EXIT',{system_limit,_}} = (catch <<0:(1 bsl 35)>>),
+    ?line {'EXIT',{system_limit,_}} = (catch <<0:((1 bsl 32)+1)>>),
+
+    %% Would fail to load.
+    ?line {'EXIT',{system_limit,_}} = (catch <<0:(1 bsl 43)>>),
+    ?line {'EXIT',{system_limit,_}} = (catch <<0:((1 bsl 40)+1)>>),
     ok.
 
 badarg(Config) when is_list(Config) ->
@@ -786,5 +812,20 @@ otp_7422_bin(N) when N < 512 ->
 	       end),
     otp_7422_bin(N+1);
 otp_7422_bin(_) -> ok.
+
+zero_width(Config) when is_list(Config) ->
+    ?line Z = id(0),
+    Small = id(42),
+    Big = id(1 bsl 128),
+    ?line <<>> = <<Small:Z>>,
+    ?line <<>> = <<Small:0>>,
+    ?line <<>> = <<Big:Z>>,
+    ?line <<>> = <<Big:0>>,
+    
+    ?line {'EXIT',{badarg,_}} = (catch <<not_a_number:0>>),
+    ?line {'EXIT',{badarg,_}} = (catch <<(id(not_a_number)):Z>>),
+    ?line {'EXIT',{badarg,_}} = (catch <<(id(not_a_number)):0>>),
+
+    ok.
     
 id(I) -> I.
diff --git a/erts/emulator/test/bs_match_bin_SUITE.erl b/erts/emulator/test/bs_match_bin_SUITE.erl
index 3d054a279f..96e69dbc0b 100644
--- a/erts/emulator/test/bs_match_bin_SUITE.erl
+++ b/erts/emulator/test/bs_match_bin_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,12 +19,32 @@
 
 -module(bs_match_bin_SUITE).
 
--export([all/1,byte_split_binary/1,bit_split_binary/1,match_huge_bin/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 byte_split_binary/1,bit_split_binary/1,match_huge_bin/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [byte_split_binary, bit_split_binary, match_huge_bin].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [byte_split_binary,bit_split_binary,match_huge_bin].
 
 byte_split_binary(doc) -> "Tries to split a binary at all byte-aligned positions.";
 byte_split_binary(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/bs_match_int_SUITE.erl b/erts/emulator/test/bs_match_int_SUITE.erl
index 99dee7c7bc..ce03ecb548 100644
--- a/erts/emulator/test/bs_match_int_SUITE.erl
+++ b/erts/emulator/test/bs_match_int_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,16 +18,36 @@
 
 -module(bs_match_int_SUITE).
 
--export([all/1,integer/1,signed_integer/1,dynamic/1,more_dynamic/1,mml/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 integer/1,signed_integer/1,dynamic/1,more_dynamic/1,mml/1,
 	 match_huge_int/1,bignum/1,unaligned_32_bit/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -import(lists, [seq/2]).
 
-all(suite) ->
-    [integer,signed_integer,dynamic,more_dynamic,mml,match_huge_int,bignum,
-     unaligned_32_bit].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [integer, signed_integer, dynamic, more_dynamic, mml,
+     match_huge_int, bignum, unaligned_32_bit].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 integer(Config) when is_list(Config) ->
     ?line 0 = get_int(mkbin([])),
diff --git a/erts/emulator/test/bs_match_misc_SUITE.erl b/erts/emulator/test/bs_match_misc_SUITE.erl
index 6de2ef67e5..15427661f3 100644
--- a/erts/emulator/test/bs_match_misc_SUITE.erl
+++ b/erts/emulator/test/bs_match_misc_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2000-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2000-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,18 +18,38 @@
 %%
 -module(bs_match_misc_SUITE).
 
--export([all/1,bound_var/1,bound_tail/1,t_float/1,little_float/1,sean/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 bound_var/1,bound_tail/1,t_float/1,little_float/1,sean/1,
 	 kenneth/1,encode_binary/1,native/1,happi/1,
 	 size_var/1,wiger/1,x0_context/1,huge_float_field/1,
-	 writable_binary_matched/1,otp_7198/1]).
+	 writable_binary_matched/1,otp_7198/1,unordered_bindings/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [bound_var, bound_tail, t_float, little_float, sean,
+     kenneth, encode_binary, native, happi, size_var, wiger,
+     x0_context, huge_float_field, writable_binary_matched,
+     otp_7198, unordered_bindings].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [bound_var,bound_tail,t_float,little_float,sean,
-     kenneth,encode_binary,native,happi,
-     size_var,wiger,x0_context,huge_float_field,
-     writable_binary_matched,otp_7198].
 
 bound_var(doc) -> "Test matching of bound variables.";
 bound_var(Config) when is_list(Config) ->
@@ -533,5 +553,15 @@ otp_7198_scan(<<C, Rest/binary>>, TokAcc) when
                         otp_7198_scan(Rest, [{'KEYWORD', C} | TokAcc])
         end.
 
+unordered_bindings(Config) when is_list(Config) ->
+    {<<1,2,3,4>>,<<42,42>>,<<3,3,3>>} =
+	unordered_bindings(4, 2, 3, <<1,2,3,4, 42,42, 3,3,3, 3>>),
+    ok.
+
+unordered_bindings(CompressedLength, HashSize, PadLength, T) ->
+    <<Content:CompressedLength/binary,Mac:HashSize/binary,
+     Padding:PadLength/binary,PadLength>> = T,
+    {Content,Mac,Padding}.
+
 
 id(I) -> I.
diff --git a/erts/emulator/test/bs_match_tail_SUITE.erl b/erts/emulator/test/bs_match_tail_SUITE.erl
index b0b0779b65..1397f2069c 100644
--- a/erts/emulator/test/bs_match_tail_SUITE.erl
+++ b/erts/emulator/test/bs_match_tail_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -20,11 +20,31 @@
 -module(bs_match_tail_SUITE).
 
 -author('[email protected]').
--export([all/1,aligned/1,unaligned/1,zero_tail/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,aligned/1,unaligned/1,zero_tail/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [aligned, unaligned, zero_tail].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) -> [aligned,unaligned,zero_tail].
 
 aligned(doc) -> "Test aligned tails.";
 aligned(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/bs_utf_SUITE.erl b/erts/emulator/test/bs_utf_SUITE.erl
index 87adc5197b..4ab7d674a6 100644
--- a/erts/emulator/test/bs_utf_SUITE.erl
+++ b/erts/emulator/test/bs_utf_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2008-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2008-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,13 +19,15 @@
 
 -module(bs_utf_SUITE).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 utf8_roundtrip/1,utf16_roundtrip/1,utf32_roundtrip/1,
 	 utf8_illegal_sequences/1,utf16_illegal_sequences/1,
 	 utf32_illegal_sequences/1,
 	 bad_construction/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -define(FAIL(Expr), ?line fail_check(catch Expr, ??Expr, [])).
 
@@ -33,19 +35,36 @@ init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog = ?t:timetrap(?t:minutes(6)),
     [{watchdog,Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
-all(suite) ->
-    [utf8_roundtrip,utf16_roundtrip,utf32_roundtrip,
-     utf8_illegal_sequences,utf16_illegal_sequences,
-     utf32_illegal_sequences,bad_construction].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [utf8_roundtrip, utf16_roundtrip, utf32_roundtrip,
+     utf8_illegal_sequences, utf16_illegal_sequences,
+     utf32_illegal_sequences, bad_construction].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 utf8_roundtrip(Config) when is_list(Config) ->
     ?line utf8_roundtrip(0, 16#D7FF),
-    ?line utf8_roundtrip(16#E000, 16#FFFD),
-    ?line utf8_roundtrip(16#10000, 16#10FFFF),
+    ?line utf8_roundtrip(16#E000, 16#10FFFF),
     ok.
 
 utf8_roundtrip(First, Last) when First =< Last ->
@@ -71,8 +90,7 @@ utf16_roundtrip(Config) when is_list(Config) ->
 
 do_utf16_roundtrip(Fun) ->
     do_utf16_roundtrip(0, 16#D7FF, Fun),
-    do_utf16_roundtrip(16#E000, 16#FFFD, Fun),
-    do_utf16_roundtrip(16#10000, 16#10FFFF, Fun).
+    do_utf16_roundtrip(16#E000, 16#10FFFF, Fun).
 
 do_utf16_roundtrip(First, Last, Fun) when First =< Last ->
     Fun(First),
@@ -109,8 +127,7 @@ utf32_roundtrip(Config) when is_list(Config) ->
 
 do_utf32_roundtrip(Fun) ->
     do_utf32_roundtrip(0, 16#D7FF, Fun),
-    do_utf32_roundtrip(16#E000, 16#FFFD, Fun),
-    do_utf32_roundtrip(16#10000, 16#10FFFF, Fun).
+    do_utf32_roundtrip(16#E000, 16#10FFFF, Fun).
 
 do_utf32_roundtrip(First, Last, Fun) when First =< Last ->
     Fun(First),
@@ -138,7 +155,6 @@ utf32_little_roundtrip(Char) ->
 utf8_illegal_sequences(Config) when is_list(Config) ->
     ?line fail_range(16#10FFFF+1, 16#10FFFF+512), %Too large.
     ?line fail_range(16#D800, 16#DFFF),		%Reserved for UTF-16.
-    ?line fail_range(16#FFFE, 16#FFFF),		%Non-characters.
 
     %% Illegal first character.
     ?line [fail(<<I,16#8F,16#8F,16#8F>>) || I <- lists:seq(16#80, 16#BF)],
@@ -231,7 +247,6 @@ fail_1(_) -> ok.
 utf16_illegal_sequences(Config) when is_list(Config) ->
     ?line utf16_fail_range(16#10FFFF+1, 16#10FFFF+512), %Too large.
     ?line utf16_fail_range(16#D800, 16#DFFF),		%Reserved for UTF-16.
-    ?line utf16_fail_range(16#FFFE, 16#FFFF),		%Non-characters.
 
     ?line lonely_hi_surrogate(16#D800, 16#DFFF),
     ?line leading_lo_surrogate(16#DC00, 16#DFFF),
@@ -280,7 +295,6 @@ leading_lo_surrogate(_, _, _) -> ok.
 utf32_illegal_sequences(Config) when is_list(Config) ->
     ?line utf32_fail_range(16#10FFFF+1, 16#10FFFF+512), %Too large.
     ?line utf32_fail_range(16#D800, 16#DFFF),		%Reserved for UTF-16.
-    ?line utf32_fail_range(16#FFFE, 16#FFFF),		%Non-characters.
     ?line utf32_fail_range(-100, -1),
     ok.
 
diff --git a/erts/emulator/test/busy_port_SUITE.erl b/erts/emulator/test/busy_port_SUITE.erl
index 9b16170293..3a29fd4d68 100644
--- a/erts/emulator/test/busy_port_SUITE.erl
+++ b/erts/emulator/test/busy_port_SUITE.erl
@@ -1,39 +1,72 @@
 %%
 %% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
-%% 
+%%
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
+%%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
 %% compliance with the License. You should have received a copy of the
 %% Erlang Public License along with this software. If not, it can be
 %% retrieved online at http://www.erlang.org/.
-%% 
+%%
 %% Software distributed under the License is distributed on an "AS IS"
 %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 %% the License for the specific language governing rights and limitations
 %% under the License.
-%% 
+%%
 %% %CopyrightEnd%
 %%
 
 -module(busy_port_SUITE).
 
--export([all/1, io_to_busy/1, message_order/1, send_3/1, 
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,end_per_testcase/2,
+	 io_to_busy/1, message_order/1, send_3/1, 
 	 system_monitor/1, no_trap_exit/1,
 	 no_trap_exit_unlinked/1, trap_exit/1, multiple_writers/1,
 	 hard_busy_driver/1, soft_busy_driver/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %% Internal exports.
 -export([init/2]).
 
-all(suite) -> {req, [dynamic_loading],
-	       [io_to_busy, message_order, send_3, 
-		system_monitor, no_trap_exit,
-		no_trap_exit_unlinked, trap_exit, multiple_writers,
-		hard_busy_driver, soft_busy_driver]}.
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [io_to_busy, message_order, send_3, system_monitor,
+     no_trap_exit, no_trap_exit_unlinked, trap_exit,
+     multiple_writers, hard_busy_driver, soft_busy_driver].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_testcase(_Case, Config) when is_list(Config) ->
+    case whereis(busy_drv_server) of
+	undefined ->
+	    ok;
+	Pid when is_pid(Pid) ->
+	    Ref = monitor(process, Pid),
+	    unlink(Pid),
+	    exit(Pid, kill),
+	    receive
+		{'DOWN',Ref,process,Pid,_} ->
+		    ok
+	    end
+    end,
+    Config.
 
 %% Tests I/O operations to a busy port, to make sure a suspended send
 %% operation is correctly restarted.  This used to crash Beam.
@@ -182,7 +215,7 @@ system_monitor(Config) when is_list(Config) ->
     ?line Master ! {Owner, {command, "u"}},
     ?line {Busy,beta} = rec(Void),
     ?line Void = rec(Void),
-    ?line NewMonitor = erlang:system_monitor(OldMonitor),
+    ?line _NewMonitor = erlang:system_monitor(OldMonitor),
     ?line OldMonitor = erlang:system_monitor(),
     ?line OldMonitor = erlang:system_monitor(OldMonitor),
     %%
@@ -361,7 +394,6 @@ soft_busy_driver(Config) when is_list(Config) ->
     hs_test(Config, false).
 
 hs_test(Config, HardBusy) when is_list(Config) ->
-    ?line Me = self(),
     ?line DrvName = case HardBusy of
 			true -> 'hard_busy_drv';
 			false -> 'soft_busy_drv'
@@ -477,12 +509,12 @@ hs_busy_pcmd(Prt, Opts, StartFun, EndFun) ->
     P = spawn_link(fun () ->
 			   erlang:yield(),
 			   Tester ! {self(), doing_port_command},
-			   Start = os:timestamp(),
+			   Start = now(),
 			   Res = try {return,
-				      erlang:port_command(Prt, [], Opts)}
+				      port_command(Prt, [], Opts)}
 				 catch Exception:Error -> {Exception, Error}
 				 end,
-			   End = os:timestamp(),
+			   End = now(),
 			   Time = round(timer:now_diff(End, Start)/1000),
 			   Tester ! {self(), port_command_result, Res, Time}
 		   end),
diff --git a/erts/emulator/test/busy_port_SUITE_data/busy_drv.c b/erts/emulator/test/busy_port_SUITE_data/busy_drv.c
index 1273d610ba..75106d3757 100644
--- a/erts/emulator/test/busy_port_SUITE_data/busy_drv.c
+++ b/erts/emulator/test/busy_port_SUITE_data/busy_drv.c
@@ -11,7 +11,8 @@
 #define YES 1
 
 static ErlDrvData busy_start(ErlDrvPort, char*);
-static void busy_stop(ErlDrvData), busy_from_erlang(ErlDrvData, char*, int);
+static void busy_stop(ErlDrvData),
+    busy_from_erlang(ErlDrvData, char*, ErlDrvSizeT);
 
 ErlDrvEntry busy_driver_entry =
 {
@@ -23,6 +24,20 @@ ErlDrvEntry busy_driver_entry =
     NULL,
     "busy_drv",
     NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
     NULL
 };
 
@@ -73,7 +88,7 @@ static void busy_stop(ErlDrvData port)
 }
 
 static void
-busy_from_erlang(ErlDrvData port, char* buf, int count)
+busy_from_erlang(ErlDrvData port, char* buf, ErlDrvSizeT count)
 {
     if ((ErlDrvPort)port == slave_port) {
 	set_busy_port(slave_port, next_slave_state);
diff --git a/erts/emulator/test/busy_port_SUITE_data/hs_busy_drv.c b/erts/emulator/test/busy_port_SUITE_data/hs_busy_drv.c
index 35919da2d0..9f6bd310c6 100644
--- a/erts/emulator/test/busy_port_SUITE_data/hs_busy_drv.c
+++ b/erts/emulator/test/busy_port_SUITE_data/hs_busy_drv.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2009. All Rights Reserved.
+ * Copyright Ericsson AB 2009-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -21,9 +21,9 @@
 #include "erl_driver.h"
 
 ErlDrvData start(ErlDrvPort port, char *command);
-void output(ErlDrvData drv_data, char *buf, int len);
-int control(ErlDrvData drv_data, unsigned int command, char *buf, 
-	    int len, char **rbuf, int rlen); 
+void output(ErlDrvData drv_data, char *buf, ErlDrvSizeT len);
+ErlDrvSSizeT control(ErlDrvData drv_data, unsigned int command, char *buf,
+		     ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen);
 
 static ErlDrvEntry busy_drv_entry = { 
     NULL /* init */,
@@ -61,7 +61,7 @@ ErlDrvData start(ErlDrvPort port, char *command)
     return (ErlDrvData) port;
 }
 
-void output(ErlDrvData drv_data, char *buf, int len)
+void output(ErlDrvData drv_data, char *buf, ErlDrvSizeT len)
 {
     int res;
     ErlDrvPort port = (ErlDrvPort) drv_data;
@@ -76,8 +76,8 @@ void output(ErlDrvData drv_data, char *buf, int len)
 	driver_failure_atom(port, "driver_output_term failed");
 }
 
-int control(ErlDrvData drv_data, unsigned int command, char *buf, 
-	    int len, char **rbuf, int rlen)
+ErlDrvSSizeT control(ErlDrvData drv_data, unsigned int command, char *buf,
+		     ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen)
 {
     switch (command) {
     case 'B': /* busy */
diff --git a/erts/emulator/test/call_trace_SUITE.erl b/erts/emulator/test/call_trace_SUITE.erl
index e0528955b0..9d80b01748 100644
--- a/erts/emulator/test/call_trace_SUITE.erl
+++ b/erts/emulator/test/call_trace_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2012. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -20,7 +20,9 @@
 
 -module(call_trace_SUITE).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 hipe/1,process_specs/1,basic/1,flags/1,errors/1,pam/1,change_pam/1,
 	 return_trace/1,exception_trace/1,on_load/1,deep_exception/1,
 	 exception_nocatch/1,bit_syntax/1]).
@@ -35,25 +37,44 @@
 -export([abbr/1,abbr/2]).
 
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -define(P, 20).
 
-all(suite) ->
-    Common = [errors,on_load],
-    NotHipe = [process_specs,basic,flags,pam,change_pam,return_trace,
-	       exception_trace,deep_exception,exception_nocatch,bit_syntax],
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    Common = [errors, on_load],
+    NotHipe = [process_specs, basic, flags, pam, change_pam,
+	       return_trace, exception_trace, deep_exception,
+	       exception_nocatch, bit_syntax],
     Hipe = [hipe],
-    case test_server:is_native(?MODULE) of
+    case test_server:is_native(call_trace_SUITE) of
 	true -> Hipe ++ Common;
 	false -> NotHipe ++ Common
     end.
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog = ?t:timetrap(?t:seconds(30)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
@@ -144,10 +165,14 @@ worker_loop() ->
 worker_foo(_Arg) ->
     ok.
 
-basic(doc) ->    
-    "Basic test of the call tracing (we trace one process).";
-basic(suite) -> [];
-basic(Config) when is_list(Config) ->
+%% Basic test of the call tracing (we trace one process).
+basic(_Config) ->
+    case test_server:is_native(lists) of
+	true -> {skip,"lists is native"};
+	false -> basic()
+    end.
+
+basic() ->
     ?line start_tracer(),
     ?line trace_info(self(), flags),
     ?line trace_info(self(), tracer),
@@ -242,9 +267,15 @@ foo() -> foo0.
 foo(X) -> X+1.
 foo(X, Y) -> X+Y.
 
-flags(doc) -> "Test flags (arity, timestamp) for call_trace/3. "
-		  "Also, test the '{tracer,Pid}' option.";
-flags(Config) when is_list(Config) ->
+%% Test flags (arity, timestamp) for call_trace/3.
+%% Also, test the '{tracer,Pid}' option.
+flags(_Config) ->
+    case test_server:is_native(filename) of
+	true -> {skip,"filename is native"};
+	false -> flags()
+    end.
+
+flags() ->
     ?line Tracer = start_tracer_loop(),
     ?line trace_pid(self(), true, [call,{tracer,Tracer}]),
 
@@ -407,9 +438,14 @@ pam_foo(A, B) ->
     {ok,A,B}.
 
 
-change_pam(doc) -> "Test changing PAM programs for a function.";
-change_pam(suite) -> [];
-change_pam(Config) when is_list(Config) ->
+%% Test changing PAM programs for a function.
+change_pam(_Config) ->
+    case test_server:is_native(lists) of
+	true -> {skip,"lists is native"};
+	false -> change_pam()
+    end.
+
+change_pam() ->
     ?line start_tracer(),
     ?line Self = self(),
 
@@ -447,10 +483,11 @@ change_pam_trace(Prog) ->
     {match_spec,Prog} = trace_info({erlang,process_info,2}, match_spec),
     ok.
 
-return_trace(doc) -> "Test the new return trace.";
-return_trace(suite) -> [];
-return_trace(Config) when is_list(Config) ->
-    return_trace().
+return_trace(_Config) ->
+    case test_server:is_native(lists) of
+	true -> {skip,"lists is native"};
+	false -> return_trace()
+    end.
 
 return_trace() ->
     X = {save,me},
@@ -500,7 +537,7 @@ return_trace() ->
     ?line {match_spec,Prog2} = trace_info({erlang,atom_to_list,1}, match_spec),
 
     ?line lists:seq(2, 7),
-    ?line atom_to_list(non_literal(nisse)),
+    ?line _ = atom_to_list(non_literal(nisse)),
     ?line expect({trace,Self,return_from,{lists,seq,2},[2,3,4,5,6,7]}),
     ?line expect({trace,Self,return_from,{erlang,atom_to_list,1},"nisse"}),
 
@@ -518,10 +555,11 @@ return_trace() ->
 nasty() ->
     exit(good_bye).
 
-exception_trace(doc) -> "Test the new exception trace.";
-exception_trace(suite) -> [];
-exception_trace(Config) when is_list(Config) ->
-    exception_trace().
+exception_trace(_Config) ->
+    case test_server:is_native(lists) of
+	true -> {skip,"lists is native"};
+	false -> exception_trace()
+    end.
 
 exception_trace() ->
     X = {save,me},
@@ -579,7 +617,7 @@ exception_trace() ->
 	trace_info({erlang,atom_to_list,1}, match_spec),
 
     ?line lists:seq(2, 7),
-    ?line atom_to_list(non_literal(nisse)),
+    ?line _ = atom_to_list(non_literal(nisse)),
     ?line expect({trace,Self,return_from,{lists,seq,2},[2,3,4,5,6,7]}),
     ?line expect({trace,Self,return_from,{erlang,atom_to_list,1},"nisse"}),
 
@@ -913,6 +951,10 @@ exception_nocatch(Config) when is_list(Config) ->
     exception_nocatch().
 
 exception_nocatch() ->
+    Deep4LocThrow = get_deep_4_loc({throw,[42]}),
+    Deep4LocError = get_deep_4_loc({error,[42]}),
+    Deep4LocBadmatch = get_deep_4_loc({'=',[a,b]}),
+
     Prog = [{'_',[],[{exception_trace}]}],
     ?line 1 = erlang:trace_pattern({?MODULE,deep_1,'_'}, Prog),
     ?line 1 = erlang:trace_pattern({?MODULE,deep_2,'_'}, Prog),
@@ -938,8 +980,9 @@ exception_nocatch() ->
 			   {trace,t2,exception_from,{erlang,throw,1},
 			    {error,{nocatch,Q2}}}],
 			  exception_from, {error,{nocatch,Q2}}),
-    ?line expect({trace,T2,exit,{{nocatch,Q2},[{erlang,throw,[Q2]},
-					       {?MODULE,deep_4,1}]}}),
+    ?line expect({trace,T2,exit,{{nocatch,Q2},[{erlang,throw,[Q2],[]},
+					       {?MODULE,deep_4,1,
+						Deep4LocThrow}]}}),
     ?line Q3 = {dump,[dump,{dump}]},
     ?line T3 = 
 	exception_nocatch(?LINE, error, [Q3], 4, 
@@ -947,18 +990,29 @@ exception_nocatch() ->
 			   {trace,t3,exception_from,{erlang,error,1},
 			    {error,Q3}}],
 			  exception_from, {error,Q3}),
-    ?line expect({trace,T3,exit,{Q3,[{erlang,error,[Q3]},
-				     {?MODULE,deep_4,1}]}}),
+    ?line expect({trace,T3,exit,{Q3,[{erlang,error,[Q3],[]},
+				     {?MODULE,deep_4,1,Deep4LocError}]}}),
     ?line T4 = 
 	exception_nocatch(?LINE, '=', [17,4711], 5, [], 
 			  exception_from, {error,{badmatch,4711}}),
-    ?line expect({trace,T4,exit,{{badmatch,4711},[{?MODULE,deep_4,1}]}}),
+    ?line expect({trace,T4,exit,{{badmatch,4711},
+				 [{?MODULE,deep_4,1,Deep4LocBadmatch}]}}),
     %%
     ?line erlang:trace_pattern({?MODULE,'_','_'}, false),
     ?line erlang:trace_pattern({erlang,'_','_'}, false),
     ?line expect(),
     ?line ok.
 
+get_deep_4_loc(Arg) ->
+    try
+	deep_4(Arg),
+	?t:fail(should_not_return_to_here)
+    catch
+	_:_ ->
+	    [{?MODULE,deep_4,1,Loc0}|_] = erlang:get_stacktrace(),
+	    Loc0
+    end.
+
 exception_nocatch(Line, B, Q, N, Extra, Tag, R) ->
     ?line io:format("== Subtest: ~w", [Line]),
     ?line Go = make_ref(),
diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index 33351a3cc9..74ce5e397a 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,18 +18,98 @@
 %%
 
 -module(code_SUITE).
--export([all/1,
-	 new_binary_types/1,t_check_process_code/1,t_check_process_code_ets/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 versions/1,new_binary_types/1,
+	 t_check_process_code/1,t_check_old_code/1,
+	 t_check_process_code_ets/1,
 	 external_fun/1,get_chunk/1,module_md5/1,make_stub/1,
-	 make_stub_many_funs/1,constant_pools/1,
-	 false_dependency/1,coverage/1]).
+	 make_stub_many_funs/1,constant_pools/1,constant_refc_binaries/1,
+	 false_dependency/1,coverage/1,fun_confusion/1]).
 
--include("test_server.hrl").
+-define(line_trace, 1).
+-include_lib("test_server/include/test_server.hrl").
 
-all(suite) ->
-    [new_binary_types,t_check_process_code,t_check_process_code_ets,
-     external_fun,get_chunk,module_md5,make_stub,make_stub_many_funs,
-     constant_pools,false_dependency,coverage].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [versions, new_binary_types, t_check_process_code,
+     t_check_process_code_ets, t_check_old_code, external_fun, get_chunk,
+     module_md5, make_stub, make_stub_many_funs,
+     constant_pools, constant_refc_binaries, false_dependency,
+     coverage, fun_confusion].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    erts_debug:set_internal_state(available_internal_state, true),
+    Config.
+
+end_per_suite(_Config) ->
+    catch erts_debug:set_internal_state(available_internal_state, false),
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+%% Make sure that only two versions of a module can be loaded.
+versions(Config) when is_list(Config) ->
+    V1 = compile_version(1, Config),
+    V2 = compile_version(2, Config),
+    V3 = compile_version(3, Config),
+
+    {ok,P1,1} = load_version(V1, 1),
+    {ok,P2,2} = load_version(V2, 2),
+    {error,not_purged} = load_version(V2, 2),
+    {error,not_purged} = load_version(V3, 3),
+
+    1 = check_version(P1),
+    2 = check_version(P2),
+    2 = versions:version(),
+
+    %% Kill processes, unload code.
+    P1 ! P2 ! done,
+    _ = monitor(process, P1),
+    _ = monitor(process, P2),
+    receive
+	{'DOWN',_,process,P1,normal} -> ok
+    end,
+    receive
+	{'DOWN',_,process,P2,normal} -> ok
+    end,
+    true = erlang:purge_module(versions),
+    true = erlang:delete_module(versions),
+    true = erlang:purge_module(versions),
+    ok.
+
+compile_version(Version, Config) ->
+    Data = ?config(data_dir, Config),
+    File = filename:join(Data, "versions"),
+    {ok,versions,Bin} = compile:file(File, [{d,'VERSION',Version},
+					    binary,report]),
+    Bin.
+
+load_version(Code, Ver) ->
+    case erlang:load_module(versions, Code) of
+	{module,versions} ->
+	    Pid = spawn_link(versions, loop, []),
+	    Ver = versions:version(),
+	    Ver = check_version(Pid),
+	    {ok,Pid,Ver};
+	Error ->
+	    Error
+    end.
+
+check_version(Pid) ->
+    Pid ! {self(),version},
+    receive
+	{Pid,version,Version} ->
+	    Version
+    end.
 
 new_binary_types(Config) when is_list(Config) ->
     ?line Data = ?config(data_dir, Config),
@@ -55,14 +135,7 @@ new_binary_types(Config) when is_list(Config) ->
 						      bit_sized_binary(Bin))),
     ok.
 
-t_check_process_code(doc) -> "Test check_process_code/2.";
 t_check_process_code(Config) when is_list(Config) ->
-    case erlang:system_info(heap_type) of
-	private -> t_check_process_code_1(Config);
-	hybrid -> {skip,"Hybrid heap"}
-    end.
-
-t_check_process_code_1(Config) ->
     ?line Priv = ?config(priv_dir, Config),
     ?line Data = ?config(data_dir, Config),
     ?line File = filename:join(Data, "my_code_test"),
@@ -167,12 +240,10 @@ gc1() -> ok.
 t_check_process_code_ets(doc) ->
     "Test check_process_code/2 in combination with a fun obtained from an ets table.";
 t_check_process_code_ets(Config) when is_list(Config) ->
-    case {test_server:is_native(?MODULE),erlang:system_info(heap_type)} of
-	{true,_} ->
-	    {skipped,"Native code"};
-	{_,hybrid} ->
-	    {skipped,"Hybrid heap"};
-	{false,private} ->
+    case test_server:is_native(?MODULE) of
+	true ->
+	    {skip,"Native code"};
+	false ->
 	    do_check_process_code_ets(Config)
     end.
 
@@ -228,9 +299,36 @@ fun_refc(F) ->
     Count.
 
 
+%% Test the erlang:check_old_code/1 BIF.
+t_check_old_code(Config) when is_list(Config) ->
+    ?line Data = ?config(data_dir, Config),
+    ?line File = filename:join(Data, "my_code_test"),
+
+    ?line erlang:purge_module(my_code_test),
+    ?line erlang:delete_module(my_code_test),
+    ?line catch erlang:purge_module(my_code_test),
+
+    ?line false = erlang:check_old_code(my_code_test),
+
+    ?line {ok,my_code_test,Code} = compile:file(File, [binary]),
+    ?line {module,my_code_test} = code:load_binary(my_code_test, File, Code),
+    
+    ?line false = erlang:check_old_code(my_code_test),
+    ?line {module,my_code_test} = code:load_binary(my_code_test, File, Code),
+    ?line true = erlang:check_old_code(my_code_test),
+
+    ?line true = erlang:purge_module(my_code_test),
+    ?line true = erlang:delete_module(my_code_test),
+    ?line true = erlang:purge_module(my_code_test),
+
+    ?line {'EXIT',_} = (catch erlang:check_old_code([])),
+    
+    ok.
+
 external_fun(Config) when is_list(Config) ->
     ?line false = erlang:function_exported(another_code_test, x, 1),
-    ?line ExtFun = erlang:make_fun(id(another_code_test), x, 1),
+    AnotherCodeTest = id(another_code_test),
+    ExtFun = fun AnotherCodeTest:x/1,
     ?line {'EXIT',{undef,_}} = (catch ExtFun(answer)),
     ?line false = erlang:function_exported(another_code_test, x, 1),
     ?line false = lists:member(another_code_test, erlang:loaded()),
@@ -290,9 +388,7 @@ module_md5_ok(Code) ->
 
 
 make_stub(Config) when is_list(Config) ->
-    %% No old code to purge if hybrid heap because of skipped test cases,
-    %% so we'll need a catch here.
-    ?line (catch erlang:purge_module(my_code_test)),
+    catch erlang:purge_module(my_code_test),
 
     ?line Data = ?config(data_dir, Config),
     ?line File = filename:join(Data, "my_code_test"),
@@ -320,12 +416,13 @@ make_stub(Config) when is_list(Config) ->
 	(catch code:make_stub_module(my_code_test,
 				     bit_sized_binary(Code),
 				     {[],[]})),
+    ?line {'EXIT',{badarg,_}} =
+	(catch code:make_stub_module(my_code_test_with_wrong_name,
+				     Code, {[],[]})),
     ok.
 
 make_stub_many_funs(Config) when is_list(Config) ->
-    %% No old code to purge if hybrid heap because of skipped test cases,
-    %% so we'll need a catch here.
-    ?line (catch erlang:purge_module(many_funs)),
+    catch erlang:purge_module(many_funs),
 
     ?line Data = ?config(data_dir, Config),
     ?line File = filename:join(Data, "many_funs"),
@@ -352,7 +449,7 @@ make_stub_many_funs(Config) when is_list(Config) ->
 constant_pools(Config) when is_list(Config) ->
     ?line Data = ?config(data_dir, Config),
     ?line File = filename:join(Data, "literals"),
-    ?line {ok,literals,Code} = compile:file(File, [report,binary,constant_pool]),
+    ?line {ok,literals,Code} = compile:file(File, [report,binary]),
     ?line {module,literals} = erlang:load_module(literals,
 						 make_sub_binary(Code)),
 
@@ -423,6 +520,131 @@ create_old_heap() ->
 	    create_old_heap()
     end.
 
+constant_refc_binaries(Config) when is_list(Config) ->
+    wait_for_memory_deallocations(),
+    Bef = memory_binary(),
+    io:format("Binary data (bytes) before test: ~p\n", [Bef]),
+
+    %% Compile the the literals module.
+    Data = ?config(data_dir, Config),
+    File = filename:join(Data, "literals"),
+    {ok,literals,Code} = compile:file(File, [report,binary]),
+
+    %% Load the code and make sure that the binary is a refc binary.
+    {module,literals} = erlang:load_module(literals, Code),
+    Bin = literals:binary(),
+    Sz = byte_size(Bin),
+    Check = erlang:md5(Bin),
+    io:format("Size of literal refc binary: ~p\n", [Sz]),
+    {refc_binary,Sz,_,_} = erts_debug:get_internal_state({binary_info,Bin}),
+    true = erlang:delete_module(literals),
+    false = erlang:check_process_code(self(), literals),
+    true = erlang:purge_module(literals),
+
+    %% Now try to provoke a memory leak.
+    provoke_mem_leak(10, Code, Check),
+
+    %% Calculate the change in allocated binary data.
+    erlang:garbage_collect(),
+    wait_for_memory_deallocations(),
+    Aft = memory_binary(),
+    io:format("Binary data (bytes) after test: ~p", [Aft]),
+    Diff = Aft - Bef,
+    if
+	Diff < 0 ->
+	    io:format("~p less bytes", [abs(Diff)]);
+	Diff > 0 ->
+	    io:format("~p more bytes", [Diff]);
+	true ->
+	    ok
+    end,
+
+    %% Test for leaks. We must accept some natural variations in
+    %% the size of allocated binaries.
+    if
+	Diff > 64*1024 ->
+	    ?t:fail(binary_leak);
+	true ->
+	    ok
+    end.
+
+memory_binary() ->
+    try
+	erlang:memory(binary)
+    catch
+	error:notsup ->
+	    0
+    end.
+
+provoke_mem_leak(0, _, _) -> ok;
+provoke_mem_leak(N, Code, Check) ->
+    {module,literals} = erlang:load_module(literals, Code),
+
+    %% Create several processes with references to the literal binary.
+    Self = self(),
+    Pids = [spawn_link(fun() ->
+			       create_binaries(Self, NumRefs, Check)
+		       end) || NumRefs <- lists:seq(1, 10)],
+    [receive {started,Pid} -> ok end || Pid <- Pids],
+
+    %% Make the code old and remove references to the constant pool
+    %% in all processes.
+    true = erlang:delete_module(literals),
+    Ms = [spawn_monitor(fun() ->
+				false = erlang:check_process_code(Pid, literals)
+			end) || Pid <- Pids],
+    [receive
+	 {'DOWN',R,process,P,normal} ->
+	     ok
+     end || {P,R} <- Ms],
+
+    %% Purge the code.
+    true = erlang:purge_module(literals),
+
+    %% Tell the processes that the code has been purged.
+    [begin
+	 monitor(process, Pid),
+	 Pid ! purged
+     end || Pid <- Pids],
+
+    %% Wait for all processes to terminate.
+    [receive
+	 {'DOWN',_,process,Pid,normal} ->
+	     ok
+     end || Pid <- Pids],
+
+    %% We now expect that the binary has been deallocated.
+    provoke_mem_leak(N-1, Code, Check).
+
+create_binaries(Parent, NumRefs, Check) ->
+    Bin = literals:binary(),
+    Bins = lists:duplicate(NumRefs, Bin),
+    {bits,Bits} = literals:bits(),
+    Parent ! {started,self()},
+    receive
+	purged ->
+	    %% The code has been purged. Now make sure that
+	    %% the binaries haven't been corrupted.
+	    Check = erlang:md5(Bin),
+	    [Bin = B || B <- Bins],
+	    <<42:13,Bin/binary>> = Bits,
+
+	    %% Remove all references to the binaries
+	    %% Doing it explicitly like this ensures that
+	    %% the binaries are gone when the parent process
+	    %% receives the 'DOWN' message.
+	    erlang:garbage_collect()
+    end.
+
+wait_for_memory_deallocations() ->
+    try
+	erts_debug:set_internal_state(wait, deallocations)
+    catch
+	error:undef ->
+	    erts_debug:set_internal_state(available_internal_state, true),
+	    wait_for_memory_deallocations()
+    end.
+
 %% OTP-7559: c_p->cp could contain garbage and create a false dependency
 %% to a module in a process. (Thanks to Richard Carlsson.)
 false_dependency(Config) when is_list(Config) ->
@@ -460,7 +682,7 @@ do_false_dependency(Init, Code) ->
     %% Spawn process. Make sure it has the appropriate init function
     %% and returned. CP should not contain garbage after the return.
     Parent = self(),
-    ?line Pid = spawn_link(fun() -> false_dependency_loop(Parent, Init) end),
+    ?line Pid = spawn_link(fun() -> false_dependency_loop(Parent, Init, true) end),
     ?line receive initialized -> ok end,
 
     %% Reload the module. Make sure the process is still alive.
@@ -475,14 +697,23 @@ do_false_dependency(Init, Code) ->
     ?line unlink(Pid), exit(Pid, kill),
     ?line true = erlang:purge_module(cpbugx),
     ?line true = erlang:delete_module(cpbugx),
+    ?line code:is_module_native(cpbugx),  % test is_module_native on deleted code
     ?line true = erlang:purge_module(cpbugx),
+    ?line code:is_module_native(cpbugx),  % test is_module_native on purged code
     ok.
     
-false_dependency_loop(Parent, Init) ->
+false_dependency_loop(Parent, Init, SendInitAck) ->
     Init(),
-    Parent ! initialized,
+    case SendInitAck of
+	true -> Parent ! initialized;
+	false -> void
+		 %% Just send one init-ack. I guess the point of this test
+		 %% wasn't to fill parents msg-queue (?). Seen to cause
+		 %% out-of-mem (on halfword-vm for some reason) by
+		 %% 91 million msg in queue. /sverker
+    end,
     receive
-	_ -> false_dependency_loop(Parent, Init)
+	_ -> false_dependency_loop(Parent, Init, false)
     end.
 
 coverage(Config) when is_list(Config) ->
@@ -495,6 +726,30 @@ coverage(Config) when is_list(Config) ->
     ?line {'EXIT',{badarg,_}} = (catch erlang:module_loaded(42)),
     ok.
 
+fun_confusion(Config) when is_list(Config) ->
+    Data = ?config(data_dir, Config),
+    Src = filename:join(Data, "fun_confusion"),
+    Mod = fun_confusion,
+
+    %% Load first version of module.
+    compile_load(Mod, Src, 1),
+    F1 = Mod:f(),
+    1 = F1(),
+
+    %% Load second version of module.
+    compile_load(Mod, Src, 2),
+    F2 = Mod:f(),
+
+    %% F1 should refer to the old code, not the newly loaded code.
+    1 = F1(),
+    2 = F2(),
+    ok.
+
+compile_load(Mod, Src, Ver) ->
+    {ok,Mod,Code1} = compile:file(Src, [binary,{d,version,Ver}]),
+    {module,Mod} = code:load_binary(Mod, "fun_confusion.beam", Code1),
+    ok.
+
 %% Utilities.
 
 make_sub_binary(Bin) when is_binary(Bin) ->
diff --git a/erts/emulator/test/code_SUITE_data/fun_confusion.erl b/erts/emulator/test/code_SUITE_data/fun_confusion.erl
new file mode 100644
index 0000000000..16000861df
--- /dev/null
+++ b/erts/emulator/test/code_SUITE_data/fun_confusion.erl
@@ -0,0 +1,31 @@
+%%
+%% %CopyrightBegin%
+%% 
+%% Copyright Ericsson AB 2011. All Rights Reserved.
+%% 
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%% 
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%% 
+%% %CopyrightEnd%
+%%
+
+-module(fun_confusion).
+
+-export([f/0]).
+
+f() ->
+    fun() -> version() end.
+
+version() ->
+    %% Changing the value returned here should change
+    %% the identity of the fun in f/0.
+    ?version.
+
diff --git a/erts/emulator/test/code_SUITE_data/literals.erl b/erts/emulator/test/code_SUITE_data/literals.erl
index 9f99b1a780..658427095e 100644
--- a/erts/emulator/test/code_SUITE_data/literals.erl
+++ b/erts/emulator/test/code_SUITE_data/literals.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2007-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2007-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,7 +18,7 @@
 %%
 
 -module(literals).
--export([a/0,b/0,huge_bignum/0]).
+-export([a/0,b/0,huge_bignum/0,binary/0,unused_binaries/0,bits/0]).
 
 a() ->
     {a,42.0,[7,38877938333399637266518333334747]}.
@@ -81,3 +81,22 @@ b() ->
 
 huge_bignum() ->
     36#9987333333392789234879423987243987423432879423879234897423879423874328794323248423872348742323487423987423879243872347824374238792437842374283926276478623462342363243SDKJFSDLEFHDSHJFE48H3838973879JFSDKJLFASLKJVBJKLEJKLDYEIOEHFEOU39873487SFHJSLDFASUIDFHSDHFEYR0R987YDFHDHFDLKHFSIDFHSIDFSIFDHSIFHWIHR07373767667987769707660766789076874238792437842374283926276478623462342363243SDKJFSDLEFHDSHJFE48H3838973879JFSDKJLFASLKJVBJKLEJKLDYEIOEHFEOU39873487SFHJSLDFASUIDFHSDHFEYR0R987YDFHDHFDLKHFSIDFHSIDFSIFDHSIFHWIHR0737376766798779987333333392789234879423987243987423432879423879234897423879423874328794323248423872348742323487423987423879243872347824374238792437842374283926276478623462342363243SDKJFSDLEFHDSHJFE48H3838973879JFSDKJLFASLKJVBJKLEJKLDYEIOEHFEOU39873487SFHJSLDFASUIDFHSDHFEYR0R987YDFHDHFDLKHFSIDFHSIDFSIFDHSIFHWIHR07373767667987769707660766789076874238792437842374283926276478623462342363243SDKJFSDLEFHDSHJFE48H3838973879JFSDKJLFASLKJVBJKLEJKLDYEIOEHFEOU39873487SFHJSLDFASUIDFHSDHFEYR0R987YDFHDHFDLKHFSIDFHSIDFSIFDHSIFHWIHR07373767667987779JFSDKJLFASLKJVBJKLEJKLDYEIOEHFEOU39873487SFHJSLDFASUIDFHSDHFEYR0R987YDFHDHFDLKHFSIDFHSIDFSIFDHSIFHWIHR07373767667987769707660766789076874238792437842374283926276478623462342363243SDKJFSDLEFHDSHJFE48H3838973879JFSDKJLFASLKJVBJKLEJKLDYEIOEHFEOU39873487SFHJSLDFASUIDFHSDHFEYR0R987YDFHDHFDLKHFSIDFHSIDFSIFDHSIFHWIHR0737376766798779987333333392789234879423987243987423432879423879234897423879423874328794323248423872348742323487423987423879243872347824374238792437842374283926276478623462342363243SDKJFSDLEFHDSHJFE48H3838973879JFSDKJLFASLKJVBJKLEJKLDYEIOEHFEOU39873487SFHJSLDFASUIDFHSDHFEYR0R987YDFHDHFDLKHFSIDFHSIDFSIFDHSIFHWIHR07373767667987769707660766789076874238792437842374283926276478623462342363243SDKJFSDLEFHDSHJFE48H3838973879JFSDKJLFASLKJVBJKLEJKLDYEIOEHFEOU39873487SFHJSLDFASUIDFHSDHFEYR0R987YDFHDHFDLKHFSIDFHSIDFSIFDHSIFHWIHR073737676679877.
+
+-define(TIMES_FOUR(X), X,X,X,X).
+-define(BYTES_256, 0:256,1:256,2:256,3:256, 4:256,5:256,6:256,7:256).
+-define(KB_1, ?TIMES_FOUR(?BYTES_256)).
+-define(KB_4, ?TIMES_FOUR(?KB_1)).
+-define(KB_16, ?TIMES_FOUR(?KB_4)).
+-define(KB_64, ?TIMES_FOUR(?KB_16)).
+-define(KB_128, ?TIMES_FOUR(?KB_64)).
+-define(MB_1, ?TIMES_FOUR(?KB_128)).
+
+binary() ->
+    %% Too big to be a heap binary.
+    <<?MB_1>>.
+
+unused_binaries() ->
+    {<<?KB_128>>,<<?BYTES_256>>}.
+
+bits() ->
+    {bits,<<42:13,?MB_1>>}.
diff --git a/erts/emulator/test/code_SUITE_data/versions.erl b/erts/emulator/test/code_SUITE_data/versions.erl
new file mode 100644
index 0000000000..7a6fd8847d
--- /dev/null
+++ b/erts/emulator/test/code_SUITE_data/versions.erl
@@ -0,0 +1,33 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(versions).
+-export([loop/0,version/0]).
+
+loop() ->
+    receive
+	{Pid,version} ->
+	    Pid ! {self(),version,version()},
+	    loop();
+	done ->
+	    ok
+    end.
+
+version() ->
+    ?VERSION.
diff --git a/erts/emulator/test/crypto_SUITE.erl b/erts/emulator/test/crypto_SUITE.erl
index e3d34b923d..a82bd4fe38 100644
--- a/erts/emulator/test/crypto_SUITE.erl
+++ b/erts/emulator/test/crypto_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,14 +19,34 @@
 
 -module(crypto_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
 	 t_md5/1,t_md5_update/1,error/1,unaligned_context/1,random_lists/1,
 	 misc_errors/1]).
 
-all(suite) ->
-    [t_md5,t_md5_update,error,unaligned_context,random_lists,misc_errors].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [t_md5, t_md5_update, error, unaligned_context,
+     random_lists, misc_errors].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 
 misc_errors(doc) ->
diff --git a/erts/emulator/test/crypto_reference.erl b/erts/emulator/test/crypto_reference.erl
index 99107e3b57..b91535a50e 100644
--- a/erts/emulator/test/crypto_reference.erl
+++ b/erts/emulator/test/crypto_reference.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2008-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
diff --git a/erts/emulator/test/ddll_SUITE.erl b/erts/emulator/test/ddll_SUITE.erl
index 79047d7de5..6e15c228cd 100644
--- a/erts/emulator/test/ddll_SUITE.erl
+++ b/erts/emulator/test/ddll_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -30,7 +30,8 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 
--export([all/1, ddll_test/1, errors/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, ddll_test/1, errors/1,
 	 reference_count/1,
 	 kill_port/1, dont_kill_port/1]).
 -export([unload_on_process_exit/1, delayed_unload_with_ports/1, 
@@ -50,35 +51,39 @@
 
 -import(ordsets, [subtract/2]).
 
--include("test_server.hrl").
-
-all(suite) ->
-    [ddll_test, errors,
-     reference_count,
-     kill_port,
-     dont_kill_port,
-     properties,
-     load_and_unload,
-     unload_on_process_exit, 
-     delayed_unload_with_ports, 
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [ddll_test, errors, reference_count, kill_port,
+     dont_kill_port, properties, load_and_unload,
+     unload_on_process_exit, delayed_unload_with_ports,
      unload_due_to_process_exit,
-     no_unload_due_to_process_exit, 
-     no_unload_due_to_process_exit_2,
-     unload_reload_thingie, 
-     unload_reload_thingie_2,
-     unload_reload_thingie_3,
-     reload_pending,
-     load_fail_init,
-     reload_pending_fail_init,
-     reload_pending_kill,
-     more_error_codes, 
-     forced_port_killing, 
-     no_trap_exit_and_kill_ports,
-     monitor_demonitor, 
-     monitor_demonitor_load,
-     new_interface,
-     lock_driver
-    ].
+     no_unload_due_to_process_exit,
+     no_unload_due_to_process_exit_2, unload_reload_thingie,
+     unload_reload_thingie_2, unload_reload_thingie_3,
+     reload_pending, load_fail_init,
+     reload_pending_fail_init, reload_pending_kill,
+     more_error_codes, forced_port_killing,
+     no_trap_exit_and_kill_ports, monitor_demonitor,
+     monitor_demonitor_load, new_interface, lock_driver].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 unload_on_process_exit(suite) ->
     [];
diff --git a/erts/emulator/test/ddll_SUITE_data/dummy_drv.c b/erts/emulator/test/ddll_SUITE_data/dummy_drv.c
index e0d5067743..86f2abf1b1 100644
--- a/erts/emulator/test/ddll_SUITE_data/dummy_drv.c
+++ b/erts/emulator/test/ddll_SUITE_data/dummy_drv.c
@@ -7,7 +7,7 @@
 
 static ErlDrvPort erlang_port;
 static ErlDrvData dummy_start(ErlDrvPort, char*);
-static void dummy_read(ErlDrvData port, char *buf, int count);
+static void dummy_read(ErlDrvData port, char *buf, ErlDrvSizeT count);
 static void dummy_stop(ErlDrvData), easy_read(ErlDrvData, char*, int);
 
 static ErlDrvEntry dummy_driver_entry = { 
@@ -18,6 +18,21 @@ static ErlDrvEntry dummy_driver_entry = {
     NULL,
     NULL,
     "dummy_drv",
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
     NULL
 };
 
@@ -37,7 +52,7 @@ static ErlDrvData dummy_start(ErlDrvPort port,char *buf)
     return (ErlDrvData)port;
 }
 
-static void dummy_read(ErlDrvData port, char *buf, int count)
+static void dummy_read(ErlDrvData port, char *buf, ErlDrvSizeT count)
 {
     driver_output(erlang_port, buf, count);
 }
diff --git a/erts/emulator/test/ddll_SUITE_data/echo_drv.c b/erts/emulator/test/ddll_SUITE_data/echo_drv.c
index edf78a979d..2b3510c641 100644
--- a/erts/emulator/test/ddll_SUITE_data/echo_drv.c
+++ b/erts/emulator/test/ddll_SUITE_data/echo_drv.c
@@ -3,9 +3,10 @@
 
 static ErlDrvPort erlang_port;
 static ErlDrvData echo_start(ErlDrvPort, char *);
-static void from_erlang(ErlDrvData, char*, int);
-static int echo_call(ErlDrvData drv_data, unsigned int command, char *buf, 
-		     int len, char **rbuf, int rlen, unsigned *ret_flags);
+static void from_erlang(ErlDrvData, char*, ErlDrvSizeT);
+static ErlDrvSSizeT echo_call(ErlDrvData drv_data, unsigned int command,
+			      char *buf, ErlDrvSizeT len,
+			      char **rbuf, ErlDrvSizeT rlen, unsigned *ret_flags);
 static ErlDrvEntry echo_driver_entry = { 
     NULL,			/* Init */
     echo_start,
@@ -21,7 +22,15 @@ static ErlDrvEntry echo_driver_entry = {
     NULL,
     NULL,
     NULL,
-    echo_call
+    echo_call,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
+    NULL
 };
 
 DRIVER_INIT(echo_drv)
@@ -36,14 +45,15 @@ echo_start(ErlDrvPort port, char *buf)
 }
 
 static void
-from_erlang(ErlDrvData data, char *buf, int count)
+from_erlang(ErlDrvData data, char *buf, ErlDrvSizeT count)
 {
     driver_output((ErlDrvPort) data, buf, count);
 }
 
-static int 
-echo_call(ErlDrvData drv_data, unsigned int command, char *buf, 
-	  int len, char **rbuf, int rlen, unsigned *ret_flags) 
+static ErlDrvSSizeT
+echo_call(ErlDrvData drv_data, unsigned int command,
+	  char *buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen,
+	  unsigned *ret_flags)
 {
     *rbuf = buf;
     *ret_flags |= DRIVER_CALL_KEEP_BUFFER;
diff --git a/erts/emulator/test/ddll_SUITE_data/echo_drv_fail_init.c b/erts/emulator/test/ddll_SUITE_data/echo_drv_fail_init.c
index 3b2a44d907..26aa03a012 100644
--- a/erts/emulator/test/ddll_SUITE_data/echo_drv_fail_init.c
+++ b/erts/emulator/test/ddll_SUITE_data/echo_drv_fail_init.c
@@ -3,9 +3,10 @@
 
 static ErlDrvPort erlang_port;
 static ErlDrvData echo_start(ErlDrvPort, char *);
-static void from_erlang(ErlDrvData, char*, int);
-static int echo_call(ErlDrvData drv_data, unsigned int command, char *buf, 
-		     int len, char **rbuf, int rlen, unsigned *ret_flags);
+static void from_erlang(ErlDrvData, char*, ErlDrvSizeT);
+static ErlDrvSSizeT echo_call(ErlDrvData drv_data, unsigned int command,
+			      char *buf, ErlDrvSizeT len,
+			      char **rbuf, ErlDrvSizeT rlen, unsigned *ret_flags);
 static int echo_failing_init(void);
 
 static ErlDrvEntry echo_driver_entry = { 
@@ -23,7 +24,15 @@ static ErlDrvEntry echo_driver_entry = {
     NULL,
     NULL,
     NULL,
-    echo_call
+    echo_call,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
+    NULL
 };
 
 DRIVER_INIT(echo_drv)
@@ -43,14 +52,15 @@ echo_start(ErlDrvPort port, char *buf)
 }
 
 static void
-from_erlang(ErlDrvData data, char *buf, int count)
+from_erlang(ErlDrvData data, char *buf, ErlDrvSizeT count)
 {
     driver_output((ErlDrvPort) data, buf, count);
 }
 
-static int 
-echo_call(ErlDrvData drv_data, unsigned int command, char *buf, 
-	  int len, char **rbuf, int rlen, unsigned *ret_flags) 
+static ErlDrvSSizeT
+echo_call(ErlDrvData drv_data, unsigned int command,
+	  char *buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen,
+	  unsigned *ret_flags)
 {
     *rbuf = buf;
     *ret_flags |= DRIVER_CALL_KEEP_BUFFER;
diff --git a/erts/emulator/test/ddll_SUITE_data/initfail_drv.c b/erts/emulator/test/ddll_SUITE_data/initfail_drv.c
index b676ff5121..ad241b9c4f 100644
--- a/erts/emulator/test/ddll_SUITE_data/initfail_drv.c
+++ b/erts/emulator/test/ddll_SUITE_data/initfail_drv.c
@@ -3,7 +3,7 @@
 
 static ErlDrvPort erlang_port;
 static ErlDrvData easy_start(ErlDrvPort, char*);
-static void easy_stop(ErlDrvData), easy_read(ErlDrvData, char*, int);
+static void easy_stop(ErlDrvData), easy_read(ErlDrvData, char*, ErlDrvSizeT);
 
 static ErlDrvEntry easy_driver_entry =
 {
@@ -14,6 +14,21 @@ static ErlDrvEntry easy_driver_entry =
   NULL,
   NULL,
   "easy",
+  NULL,
+  NULL,
+  NULL,
+  NULL,
+  NULL,
+  NULL,
+  NULL,
+  NULL,
+  NULL,
+  ERL_DRV_EXTENDED_MARKER,
+  ERL_DRV_EXTENDED_MAJOR_VERSION,
+  ERL_DRV_EXTENDED_MINOR_VERSION,
+  0,
+  NULL,
+  NULL,
   NULL
 };
 
@@ -34,7 +49,7 @@ static ErlDrvData easy_start(ErlDrvPort port, char *buf)
     return (ErlDrvData)port;
 }
 
-static void easy_read(ErlDrvData port, char *buf, int count)
+static void easy_read(ErlDrvData port, char *buf, ErlDrvSizeT count)
 {
     driver_output(erlang_port, buf, count);
 }
diff --git a/erts/emulator/test/ddll_SUITE_data/lock_drv.c b/erts/emulator/test/ddll_SUITE_data/lock_drv.c
index 2ec8fa3a29..d2605c5bfc 100644
--- a/erts/emulator/test/ddll_SUITE_data/lock_drv.c
+++ b/erts/emulator/test/ddll_SUITE_data/lock_drv.c
@@ -3,9 +3,10 @@
 
 static ErlDrvPort erlang_port;
 static ErlDrvData echo_start(ErlDrvPort, char *);
-static void from_erlang(ErlDrvData, char*, int);
-static int echo_call(ErlDrvData drv_data, unsigned int command, char *buf, 
-		     int len, char **rbuf, int rlen, unsigned *ret_flags);
+static void from_erlang(ErlDrvData, char*, ErlDrvSizeT);
+static ErlDrvSSizeT echo_call(ErlDrvData drv_data, unsigned int command,
+			      char *buf, ErlDrvSizeT len,
+			      char **rbuf, ErlDrvSizeT rlen, unsigned *ret_flags);
 static ErlDrvEntry echo_driver_entry = { 
     NULL,			/* Init */
     echo_start,
@@ -21,7 +22,15 @@ static ErlDrvEntry echo_driver_entry = {
     NULL,
     NULL,
     NULL,
-    echo_call
+    echo_call,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
+    NULL
 };
 
 DRIVER_INIT(echo_drv)
@@ -37,14 +46,15 @@ echo_start(ErlDrvPort port, char *buf)
 }
 
 static void
-from_erlang(ErlDrvData data, char *buf, int count)
+from_erlang(ErlDrvData data, char *buf, ErlDrvSizeT count)
 {
     driver_output((ErlDrvPort) data, buf, count);
 }
 
-static int 
-echo_call(ErlDrvData drv_data, unsigned int command, char *buf, 
-	  int len, char **rbuf, int rlen, unsigned *ret_flags) 
+static ErlDrvSSizeT
+echo_call(ErlDrvData drv_data, unsigned int command,
+	  char *buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen,
+	  unsigned *ret_flags)
 {
     ErlDrvPort port = (ErlDrvPort) drv_data;
     driver_lock_driver(port);
diff --git a/erts/emulator/test/ddll_SUITE_data/noinit_drv.c b/erts/emulator/test/ddll_SUITE_data/noinit_drv.c
index 931386a305..5abf5c4dc6 100644
--- a/erts/emulator/test/ddll_SUITE_data/noinit_drv.c
+++ b/erts/emulator/test/ddll_SUITE_data/noinit_drv.c
@@ -3,7 +3,7 @@
 
 static ErlDrvPort erlang_port;
 static ErlDrvData easy_start(ErlDrvPort, char*);
-static void easy_stop(ErlDrvData), easy_read(ErlDrvData, char*, int);
+static void easy_stop(ErlDrvData), easy_read(ErlDrvData, char*, ErlDrvSizeT);
 
 static ErlDrvEntry easy_driver_entry =
 {
@@ -14,6 +14,21 @@ static ErlDrvEntry easy_driver_entry =
     NULL,
     NULL,
     "easy",
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
     NULL
 };
 
@@ -28,7 +43,7 @@ DRIVER_INIT(noinit_drv)
 /*
  * Provoke an error when loading the module.
  */
-int no_driver_init(void *handle)
+ErlDrvEntry* no_driver_init(void *handle)
 #endif
 {
     erlang_port = (ErlDrvPort)-1;
@@ -46,7 +61,7 @@ static ErlDrvData easy_start(ErlDrvPort port,char *buf)
     return (ErlDrvData)port;
 }
 
-static void easy_read(ErlDrvData port, char *buf, int count)
+static void easy_read(ErlDrvData port, char *buf, ErlDrvSizeT count)
 {
     driver_output(erlang_port, buf, count);
 }
diff --git a/erts/emulator/test/ddll_SUITE_data/wrongname_drv.c b/erts/emulator/test/ddll_SUITE_data/wrongname_drv.c
index 3a35820ee7..ac7efa30de 100644
--- a/erts/emulator/test/ddll_SUITE_data/wrongname_drv.c
+++ b/erts/emulator/test/ddll_SUITE_data/wrongname_drv.c
@@ -7,7 +7,7 @@
 
 static ErlDrvPort erlang_port;
 static ErlDrvData easy_start(ErlDrvPort, char*);
-static void easy_stop(ErlDrvData), easy_read(ErlDrvData, char*, int);
+static void easy_stop(ErlDrvData), easy_read(ErlDrvData, char*, ErlDrvSizeT);
 
 static ErlDrvEntry easy_driver_entry =
 {
@@ -18,6 +18,21 @@ static ErlDrvEntry easy_driver_entry =
     NULL,
     NULL,
     "easy",
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
     NULL
 };
 
@@ -38,7 +53,7 @@ static ErlDrvData easy_start(ErlDrvPort port,char *buf)
     return (ErlDrvData)port;
 }
 
-static void easy_read(ErlDrvData port, char *buf, int count)
+static void easy_read(ErlDrvData port, char *buf, ErlDrvSizeT count)
 {
     driver_output(erlang_port, buf, count);
 }
diff --git a/erts/emulator/test/decode_packet_SUITE.erl b/erts/emulator/test/decode_packet_SUITE.erl
index 13f17e972c..4acbe8c6e0 100644
--- a/erts/emulator/test/decode_packet_SUITE.erl
+++ b/erts/emulator/test/decode_packet_SUITE.erl
@@ -1,19 +1,19 @@
 %%
 %% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 2008-2009. All Rights Reserved.
-%% 
+%%
+%% Copyright Ericsson AB 2008-2011. All Rights Reserved.
+%%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
 %% compliance with the License. You should have received a copy of the
 %% Erlang Public License along with this software. If not, it can be
 %% retrieved online at http://www.erlang.org/.
-%% 
+%%
 %% Software distributed under the License is distributed on an "AS IS"
 %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 %% the License for the specific language governing rights and limitations
 %% under the License.
-%% 
+%%
 %% %CopyrightEnd%
 %%
 
@@ -21,13 +21,35 @@
 
 -module(decode_packet_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
+	 basic/1, packet_size/1, neg/1, http/1, line/1, ssl/1, otp_8536/1,
+         otp_9389/1, otp_9389_line/1]).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [basic, packet_size, neg, http, line, ssl, otp_8536,
+     otp_9389, otp_9389_line].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
-	 basic/1, packet_size/1, neg/1, http/1, line/1, ssl/1]).
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [basic, packet_size, neg, http, line, ssl].
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Seed = {S1,S2,S3} = now(),
@@ -36,7 +58,7 @@ init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?t:minutes(1)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
@@ -231,6 +253,28 @@ packet_size(Config) when is_list(Config) ->
 			  ?line {error,_} = decode_pkt(4,<<Size:32,Packet/binary>>)
 		  end, 
 		  lists:seq(-10,-1)),
+
+    %% Test OTP-9389, long HTTP header lines.
+    Opts = [{packet_size, 128}],
+    Pkt = list_to_binary(["GET / HTTP/1.1\r\nHost: localhost\r\nLink: /",
+                          string:chars($Y, 64), "\r\n\r\n"]),
+    <<Pkt1:50/binary, Pkt2/binary>> = Pkt,
+    ?line {ok, {http_request,'GET',{abs_path,"/"},{1,1}}, Rest1} =
+        erlang:decode_packet(http, Pkt1, Opts),
+    ?line {ok, {http_header,_,'Host',_,"localhost"}, Rest2} =
+        erlang:decode_packet(httph, Rest1, Opts),
+    ?line {more, undefined} = erlang:decode_packet(httph, Rest2, Opts),
+    ?line {ok, {http_header,_,"Link",_,_}, _} =
+        erlang:decode_packet(httph, list_to_binary([Rest2, Pkt2]), Opts),
+
+    Pkt3 = list_to_binary(["GET / HTTP/1.1\r\nHost: localhost\r\nLink: /",
+                           string:chars($Y, 129), "\r\n\r\n"]),
+    ?line {ok, {http_request,'GET',{abs_path,"/"},{1,1}}, Rest3} =
+        erlang:decode_packet(http, Pkt3, Opts),
+    ?line {ok, {http_header,_,'Host',_,"localhost"}, Rest4} =
+        erlang:decode_packet(httph, Rest3, Opts),
+    ?line {error, invalid} = erlang:decode_packet(httph, Rest4, Opts),
+
     ok.
 
 
@@ -304,6 +348,10 @@ http(Config) when is_list(Config) ->
 		   {ok, {http_request, 'GET', ResB, {1,1}}, Rest} = decode_pkt(http_bin,Bin) 
 	   end,
     lists:foreach(UriF, http_uri_variants()),
+
+    %% Response with empty phrase
+    ?line {ok,{http_response,{1,1},200,[]},<<>>} = decode_pkt(http, <<"HTTP/1.1 200\r\n">>, []),
+    ?line {ok,{http_response,{1,1},200,<<>>},<<>>} = decode_pkt(http_bin, <<"HTTP/1.1 200\r\n">>, []),
     ok.
 	     
 http_with_bin(http) ->
@@ -504,6 +552,27 @@ ssl(Config) when is_list(Config) ->
     F(v2hello),
     ok.
 
+otp_8536(doc) -> ["Corrupt sub-binary-strings from httph_bin"];
+otp_8536(Config) when is_list(Config) ->
+    lists:foreach(fun otp_8536_do/1, lists:seq(1,50)),
+    ok.
+
+otp_8536_do(N) ->
+    Data = <<"some data 123">>,
+    Letters = <<"bcdefghijklmnopqrstuvwxyzyxwvutsrqponmlkjihgfedcba">>,
+    <<HdrTail:N/binary,_/binary>> = Letters,
+    Hdr = <<$A, HdrTail/binary>>,
+    Bin = <<Hdr/binary, ": ", Data/binary, "\r\n\r\n">>,
+
+    io:format("Bin='~p'\n",[Bin]),
+    ?line {ok,{http_header,0,Hdr2,undefined,Data2},<<"\r\n">>} = decode_pkt(httph_bin, Bin,  []),
+
+    %% Do something to trash the C-stack, how about another decode_packet:
+    decode_pkt(httph_bin,<<Letters/binary, ": ", Data/binary, "\r\n\r\n">>, []),
+
+    %% Now check that we got the expected binaries
+    {Hdr, Data} = {Hdr2, Data2}.
+
 decode_pkt(Type,Bin) ->
     decode_pkt(Type,Bin,[]).		       
 decode_pkt(Type,Bin,Opts) ->
@@ -512,3 +581,35 @@ decode_pkt(Type,Bin,Opts) ->
     %%io:format(" -> ~p\n",[Res]),
     Res.
 
+otp_9389(doc) -> ["Verify line_length works correctly for HTTP headers"];
+otp_9389(suite) -> [];
+otp_9389(Config) when is_list(Config) ->
+    Opts = [{packet_size, 16384}, {line_length, 3000}],
+    Pkt = list_to_binary(["GET / HTTP/1.1\r\nHost: localhost\r\nLink: /",
+                          string:chars($X, 8192),
+                          "\r\nContent-Length: 0\r\n\r\n"]),
+    <<Pkt1:5000/binary, Pkt2/binary>> = Pkt,
+    {ok, {http_request,'GET',{abs_path,"/"},{1,1}}, Rest1} =
+        erlang:decode_packet(http, Pkt1, Opts),
+    {ok, {http_header,_,'Host',_,"localhost"}, Rest2} =
+        erlang:decode_packet(httph, Rest1, Opts),
+    {more, undefined} = erlang:decode_packet(httph, Rest2, Opts),
+    {ok, {http_header,_,"Link",_,Link}, Rest3} =
+        erlang:decode_packet(httph, list_to_binary([Rest2, Pkt2]), Opts),
+    true = (length(Link) > 8000),
+    {ok, {http_header,_,'Content-Length',_,"0"}, <<"\r\n">>} =
+        erlang:decode_packet(httph, Rest3, Opts),
+    ok.
+
+otp_9389_line(doc) -> ["Verify packet_size works correctly for line mode"];
+otp_9389_line(suite) -> [];
+otp_9389_line(Config) when is_list(Config) ->
+    Opts = [{packet_size, 20}],
+    Line1 = <<"0123456789012345678\n">>,
+    Line2 = <<"0123456789\n">>,
+    Line3 = <<"01234567890123456789\n">>,
+    Pkt = list_to_binary([Line1, Line2, Line3]),
+    ?line {ok, Line1, Rest1} = erlang:decode_packet(line, Pkt, Opts),
+    ?line {ok, Line2, Rest2} = erlang:decode_packet(line, Rest1, Opts),
+    ?line {error, invalid} = erlang:decode_packet(line, Rest2, Opts),
+    ok.
diff --git a/erts/emulator/test/dgawd_handler.erl b/erts/emulator/test/dgawd_handler.erl
index 881354b9da..27085b7b7e 100644
--- a/erts/emulator/test/dgawd_handler.erl
+++ b/erts/emulator/test/dgawd_handler.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2010. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl
index 8f48d8a992..2ddb4d76d9 100644
--- a/erts/emulator/test/distribution_SUITE.erl
+++ b/erts/emulator/test/distribution_SUITE.erl
@@ -1,65 +1,92 @@
 %%
 %% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
-%% 
+%%
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
+%%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
 %% compliance with the License. You should have received a copy of the
 %% Erlang Public License along with this software. If not, it can be
 %% retrieved online at http://www.erlang.org/.
-%% 
+%%
 %% Software distributed under the License is distributed on an "AS IS"
 %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 %% the License for the specific language governing rights and limitations
 %% under the License.
-%% 
+%%
 %% %CopyrightEnd%
 %%
 
 -module(distribution_SUITE).
+-compile(r13).
 
 %% Tests distribution and the tcp driver.
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,
-	 ping/1, bulk_send/1, bulk_send_small/1,
-	 bulk_send_big/1,
-	 local_send/1, local_send_small/1, local_send_big/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 ping/1, bulk_send_small/1,
+	 bulk_send_big/1, bulk_send_bigbig/1,
+	 local_send_small/1, local_send_big/1,
 	 local_send_legal/1, link_to_busy/1, exit_to_busy/1,
 	 lost_exit/1, link_to_dead/1, link_to_dead_new_node/1,
 	 applied_monitor_node/1, ref_port_roundtrip/1, nil_roundtrip/1,
-	 trap_bif/1, trap_bif_1/1, trap_bif_2/1, trap_bif_3/1,
-	 stop_dist/1, dist_auto_connect/1, 
+	 trap_bif_1/1, trap_bif_2/1, trap_bif_3/1,
+	 stop_dist/1, 
 	 dist_auto_connect_never/1, dist_auto_connect_once/1,
 	 dist_parallel_send/1,
 	 atom_roundtrip/1,
-	 atom_roundtrip_r12b/1,
+	 atom_roundtrip_r13b/1,
 	 contended_atom_cache_entry/1,
-	 bad_dist_ext/1,
+	 bad_dist_structure/1,
 	 bad_dist_ext_receive/1,
 	 bad_dist_ext_process_info/1,
 	 bad_dist_ext_control/1,
 	 bad_dist_ext_connection_id/1]).
 
--export([init_per_testcase/2, fin_per_testcase/2]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
 %% Internal exports.
 -export([sender/3, receiver2/2, dummy_waiter/0, dead_process/0,
 	 roundtrip/1, bounce/1, do_dist_auto_connect/1, inet_rpc_server/1,
 	 dist_parallel_sender/3, dist_parallel_receiver/0,
-	 dist_evil_parallel_receiver/0]).
-
-all(suite) -> [
-	       ping, bulk_send, local_send, link_to_busy, exit_to_busy,
-	       lost_exit, link_to_dead, link_to_dead_new_node,
-	       applied_monitor_node, ref_port_roundtrip, nil_roundtrip,
-	       stop_dist, trap_bif, dist_auto_connect, dist_parallel_send,
-	       atom_roundtrip, atom_roundtrip_r12b,
-	       contended_atom_cache_entry,
-	       bad_dist_ext
-	      ].
+	 dist_evil_parallel_receiver/0,
+         sendersender/4, sendersender2/4]).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [ping, {group, bulk_send}, {group, local_send},
+     link_to_busy, exit_to_busy, lost_exit, link_to_dead,
+     link_to_dead_new_node, applied_monitor_node,
+     ref_port_roundtrip, nil_roundtrip, stop_dist,
+     {group, trap_bif}, {group, dist_auto_connect},
+     dist_parallel_send, atom_roundtrip, atom_roundtrip_r13b,
+     contended_atom_cache_entry, bad_dist_structure, {group, bad_dist_ext}].
+
+groups() -> 
+    [{bulk_send, [], [bulk_send_small, bulk_send_big, bulk_send_bigbig]},
+     {local_send, [],
+      [local_send_small, local_send_big, local_send_legal]},
+     {trap_bif, [], [trap_bif_1, trap_bif_2, trap_bif_3]},
+     {dist_auto_connect, [],
+      [dist_auto_connect_never, dist_auto_connect_once]},
+     {bad_dist_ext, [],
+      [bad_dist_ext_receive, bad_dist_ext_process_info,
+       bad_dist_ext_control, bad_dist_ext_connection_id]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
 -define(DEFAULT_TIMETRAP, 4*60*1000).
 
@@ -67,7 +94,7 @@ init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?DEFAULT_TIMETRAP),
     [{watchdog, Dog},{testcase, Func}|Config].
 
-fin_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
+end_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
@@ -115,19 +142,15 @@ ping(Config) when is_list(Config) ->
 
     ok.
 
-bulk_send(doc) ->
- ["Tests sending large amount of data to another node and measure",
-  "the time. This tests that a process that is suspended on a ",
-  "busy port will eventually be resumed."];
-bulk_send(suite) ->
-    [bulk_send_small, bulk_send_big].
-
 bulk_send_small(Config) when is_list(Config) ->
     ?line bulk_send(64, 32).
 
 bulk_send_big(Config) when is_list(Config) ->
     ?line bulk_send(32, 64).
 
+bulk_send_bigbig(Config) when is_list(Config) ->
+    ?line bulk_sendsend(32*5, 4).
+
 bulk_send(Terms, BinSize) ->
     ?line Dog = test_server:timetrap(test_server:seconds(30)),
 
@@ -144,6 +167,58 @@ bulk_send(Terms, BinSize) ->
     ?line test_server:timetrap_cancel(Dog),
     {comment, integer_to_list(trunc(Size/1024/Elapsed+0.5)) ++ " K/s"}.
 
+bulk_sendsend(Terms, BinSize) ->
+    {Rate1, MonitorCount1} = bulk_sendsend2(Terms, BinSize,   5),
+    {Rate2, MonitorCount2} = bulk_sendsend2(Terms, BinSize, 995),
+    Ratio = if MonitorCount2 == 0 -> MonitorCount1 / 1.0;
+               true               -> MonitorCount1 / MonitorCount2
+            end,
+    Comment = integer_to_list(Rate1) ++ " K/s, " ++
+	integer_to_list(Rate2) ++ " K/s, " ++
+	integer_to_list(MonitorCount1) ++ " monitor msgs, " ++
+	integer_to_list(MonitorCount2) ++ " monitor msgs, " ++
+	float_to_list(Ratio) ++ " monitor ratio",
+    if
+	%% A somewhat arbitrary ratio, but hopefully one that will
+	%% accommodate a wide range of CPU speeds.
+	Ratio > 8.0 ->
+	    {comment,Comment};
+	true ->
+	    io:put_chars(Comment),
+	    ?line ?t:fail(ratio_too_low)
+    end.
+
+bulk_sendsend2(Terms, BinSize, BusyBufSize) ->
+    ?line Dog = test_server:timetrap(test_server:seconds(30)),
+
+    ?line io:format("Sending ~w binaries, each of size ~w K",
+		    [Terms, BinSize]),
+    ?line {ok, NodeRecv} = start_node(bulk_receiver),
+    ?line Recv = spawn(NodeRecv, erlang, apply, [fun receiver/2, [0, 0]]),
+    ?line Bin = list_to_binary(lists:duplicate(BinSize*1024, 253)),
+    %%?line Size = Terms*size(Bin),
+
+    %% SLF LEFT OFF HERE.
+    %% When the caller uses small hunks, like 4k via
+    %% bulk_sendsend(32*5, 4), then (on my laptop at least), we get
+    %% zero monitor messages.  But if we use "+zdbbl 5", then we
+    %% get a lot of monitor messages.  So, if we can count up the
+    %% total number of monitor messages that we get when running both
+    %% default busy size and "+zdbbl 5", and if the 5 case gets
+    %% "many many more" monitor messages, then we know we're working.
+
+    ?line {ok, NodeSend} = start_node(bulk_sender, "+zdbbl " ++ integer_to_list(BusyBufSize)),
+    ?line _Send = spawn(NodeSend, erlang, apply, [fun sendersender/4, [self(), Recv, Bin, Terms]]),
+    ?line {Elapsed, {_TermsN, SizeN}, MonitorCount} =
+        receive {sendersender, BigRes} ->
+                BigRes
+        end,
+    ?line stop_node(NodeRecv),
+    ?line stop_node(NodeSend),
+
+    ?line test_server:timetrap_cancel(Dog),
+    {trunc(SizeN/1024/Elapsed+0.5), MonitorCount}.
+
 sender(To, _Bin, 0) ->
     To ! {done, self()},
     receive
@@ -154,6 +229,43 @@ sender(To, Bin, Left) ->
     To ! {term, Bin},
     sender(To, Bin, Left-1).
 
+%% Sender process to be run on a slave node
+
+sendersender(Parent, To, Bin, Left) ->
+    erlang:system_monitor(self(), [busy_dist_port]),
+    [spawn(fun() -> sendersender2(To, Bin, Left, false) end) ||
+        _ <- lists:seq(1,1)],
+    {USec, {Res, MonitorCount}} =
+        timer:tc(?MODULE, sendersender2, [To, Bin, Left, true]),
+    Parent ! {sendersender, {USec/1000000, Res, MonitorCount}}.
+
+sendersender2(To, Bin, Left, SendDone) ->
+    sendersender3(To, Bin, Left, SendDone, 0).
+
+sendersender3(To, _Bin, 0, SendDone, MonitorCount) ->
+    if SendDone ->
+            To ! {done, self()};
+       true ->
+            ok
+    end,
+    receive
+        {monitor, _Pid, _Type, _Info} ->
+            sendersender3(To, _Bin, 0, SendDone, MonitorCount + 1)
+    after 0 ->
+            if SendDone ->
+                    receive
+                        Any when is_tuple(Any), size(Any) == 2 ->
+                            {Any, MonitorCount}
+                    end;
+               true ->
+                    exit(normal)
+            end
+    end;
+sendersender3(To, Bin, Left, SendDone, MonitorCount) ->
+    To ! {term, Bin},
+    %%timer:sleep(50),
+    sendersender3(To, Bin, Left-1, SendDone, MonitorCount).
+
 %% Receiver process to be run on a slave node.
 
 receiver(Terms, Size) ->
@@ -165,17 +277,14 @@ receiver(Terms, Size) ->
     end.
 
 
-local_send(suite) ->
-    [local_send_small, local_send_big, local_send_legal];
-local_send(doc) ->
-    ["Tests sending small and big messages to a non-existing ",
-     "local registered process."].
 
 local_send_big(doc) ->
     ["Sends several big message to an non-registered process on ",
      "the local node."];
 local_send_big(Config) when is_list(Config) ->
-    Data0=local_send_big(doc)++local_send(doc),
+    Data0=local_send_big(doc)++
+	["Tests sending small and big messages to a non-existing ",
+        "local registered process."],
     Data1=[Data0,[Data0, Data0, [Data0], Data0],Data0],
     Data2=Data0++lists:flatten(Data1)++
 	list_to_binary(lists:flatten(Data1)),
@@ -227,7 +336,7 @@ receiver2(Num, TotSize) ->
 
 link_to_busy(doc) -> "Test that link/1 to a busy distribution port works.";
 link_to_busy(Config) when is_list(Config) ->
-    ?line Dog = test_server:timetrap(test_server:seconds(30)),
+    ?line Dog = test_server:timetrap(test_server:seconds(60)),
     ?line {ok, Node} = start_node(link_to_busy),
     ?line Recv = spawn(Node, erlang, apply, [fun sink/1, [link_to_busy_sink]]),
 
@@ -274,7 +383,7 @@ tail_applied_linker(Pid) ->
     
 exit_to_busy(doc) -> "Test that exit/2 to a busy distribution port works.";
 exit_to_busy(Config) when is_list(Config) ->
-    ?line Dog = test_server:timetrap(test_server:seconds(30)),
+    ?line Dog = test_server:timetrap(test_server:seconds(60)),
     ?line {ok, Node} = start_node(exit_to_busy),
 
     Tracer = case os:getenv("TRACE_BUSY_DIST_PORT") of
@@ -432,7 +541,7 @@ sink1() ->
 
 lost_exit(doc) ->
     "Test that EXIT and DOWN messages send to another node are not lost if "
-	"if the distribution port is busy.";
+	"the distribution port is busy.";
 lost_exit(Config) when is_list(Config) ->
     ?line {ok, Node} = start_node(lost_exit),
 
@@ -661,9 +770,6 @@ stop_dist(Config) when is_list(Config) ->
 
     ok.
 
-trap_bif(doc) ->
-    ["Verifies that BIFs which are traps to Erlang work (OTP-2680)."];
-trap_bif(suite) -> [trap_bif_1, trap_bif_2, trap_bif_3].
 
 trap_bif_1(doc) ->
     [""];
@@ -700,10 +806,6 @@ tr3() ->
 
 
 
-dist_auto_connect(doc) ->
-    ["Tests the kernel parameter 'dist_auto_connect'."];
-dist_auto_connect(suite) ->
-    [dist_auto_connect_never, dist_auto_connect_once].
 
 % This has to be done by nodes with differrent cookies, otherwise global
 % will connect nodes, which is correct, but makes it hard to test.
@@ -998,17 +1100,17 @@ atom_roundtrip(Config) when is_list(Config) ->
     ?line stop_node(Node),
     ?line ok.
 
-atom_roundtrip_r12b(Config) when is_list(Config) ->
-    case ?t:is_release_available("r12b") of
+atom_roundtrip_r13b(Config) when is_list(Config) ->
+    case ?t:is_release_available("r13b") of
 	true ->
 	    ?line AtomData = atom_data(),
 	    ?line verify_atom_data(AtomData),
-	    ?line {ok, Node} = start_node(Config, [], "r12b"),
+	    ?line {ok, Node} = start_node(Config, [], "r13b"),
 	    ?line do_atom_roundtrip(Node, AtomData),
 	    ?line stop_node(Node),
 	    ?line ok;
 	false ->
-	    ?line {skip,"No OTP R12B available"}
+	    ?line {skip,"No OTP R13B available"}
     end.
 
 do_atom_roundtrip(Node, AtomData) ->
@@ -1053,8 +1155,7 @@ contended_atom_cache_entry(Config) when is_list(Config) ->
     ?line {ok, SNode} = start_node(Config),
     ?line {ok, RNode} = start_node(Config),
     ?line Success = make_ref(),
-    ?line Mstr
-	= spawn_link(
+    ?line spawn_link(
 	    SNode,
 	    fun () ->
 		    erts_debug:set_internal_state(available_internal_state,
@@ -1111,13 +1212,13 @@ contended_atom_cache_entry(Config) when is_list(Config) ->
     ?line stop_node(RNode),
     ?line ok.
 
-send_ref_atom(To, Ref, Atom, 0) ->
+send_ref_atom(_To, _Ref, _Atom, 0) ->
     ok;
 send_ref_atom(To, Ref, Atom, N) ->
     To ! {Ref, Atom},
     send_ref_atom(To, Ref, Atom, N-1).
 
-receive_ref_atom(Ref, Atom, 0) ->
+receive_ref_atom(_Ref, _Atom, 0) ->
     ok;
 receive_ref_atom(Ref, Atom, N) ->
     receive
@@ -1152,7 +1253,7 @@ unwanted_cixs() ->
 	      nodes()).
     
     
-get_conflicting_atoms(CIX, 0) ->
+get_conflicting_atoms(_CIX, 0) ->
     [];
 get_conflicting_atoms(CIX, N) ->
     {A, B, C} = now(),
@@ -1166,13 +1267,187 @@ get_conflicting_atoms(CIX, N) ->
 	    get_conflicting_atoms(CIX, N)
     end.
 
+-define(COOKIE, '').
+-define(DOP_LINK,		1).
+-define(DOP_SEND,		2).
+-define(DOP_EXIT,		3).
+-define(DOP_UNLINK,		4).
+-define(DOP_REG_SEND,		6).
+-define(DOP_GROUP_LEADER,	7).
+-define(DOP_EXIT2,		8).
+
+-define(DOP_SEND_TT,		12).
+-define(DOP_EXIT_TT,		13).
+-define(DOP_REG_SEND_TT,	16).
+-define(DOP_EXIT2_TT,		18).
+
+-define(DOP_MONITOR_P,		19).
+-define(DOP_DEMONITOR_P,	20).
+-define(DOP_MONITOR_P_EXIT,	21).
+
+start_monitor(Offender,P) ->
+    ?line Parent = self(),
+    ?line Q = spawn(Offender,
+		    fun () -> 
+			    Ref = erlang:monitor(process,P),
+			    Parent ! {self(),ref,Ref},
+			    receive
+				just_stay_alive -> ok
+			    end
+		    end),
+    ?line Ref = receive
+		    {Q,ref,R} ->
+			R
+		after  5000 ->
+			error
+		end,
+    io:format("Ref is ~p~n",[Ref]),
+    ok.
+start_link(Offender,P) ->
+    ?line Parent = self(),
+    ?line Q = spawn(Offender,
+		    fun () ->
+			    process_flag(trap_exit,true),
+			    link(P),
+			    Parent ! {self(),ref,P},
+			    receive
+				just_stay_alive -> ok
+			    end
+		    end),
+    ?line Ref = receive
+		    {Q,ref,R} ->
+			R
+		after  5000 ->
+			error
+		end,
+    io:format("Ref is ~p~n",[Ref]),
+    ok.
+
+bad_dist_structure(suite) ->
+    [];
+bad_dist_structure(doc) ->
+    ["Test dist messages with valid structure (binary to term ok) but malformed"
+     "control content"];
+bad_dist_structure(Config) when is_list(Config) ->
+    %process_flag(trap_exit,true),
+    ODog = ?config(watchdog, Config),
+    ?t:timetrap_cancel(ODog),
+    Dog = ?t:timetrap(?t:seconds(15)),
+
+    ?line {ok, Offender} = start_node(bad_dist_structure_offender),
+    ?line {ok, Victim} = start_node(bad_dist_structure_victim),
+    ?line start_node_monitors([Offender,Victim]),
+    ?line Parent = self(),
+    ?line P = spawn(Victim,
+		    fun () ->
+			    process_flag(trap_exit,true),
+			    Parent ! {self(), started},
+			    receive check_msgs -> ok end,
+			    bad_dist_struct_check_msgs([one,
+							two]),
+			    Parent ! {self(), messages_checked},
+			    receive done -> ok end
+		    end),
+    ?line receive {P, started} -> ok end,
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line verify_up(Offender, Victim),
+    ?line true = lists:member(Offender, rpc:call(Victim, erlang, nodes, [])),
+    ?line start_monitor(Offender,P),
+    ?line P ! one,
+    ?line send_bad_structure(Offender, P,{?DOP_MONITOR_P_EXIT,'replace',P,normal},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_monitor(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_MONITOR_P_EXIT,'replace',P,normal,normal},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_link(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_LINK},0),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_link(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_UNLINK,'replace'},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_link(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_UNLINK,'replace',make_ref()},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_link(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_UNLINK,make_ref(),P},0),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_link(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_UNLINK,normal,normal},0),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_monitor(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_MONITOR_P,'replace',P},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_monitor(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_MONITOR_P,'replace',P,normal},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_monitor(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_DEMONITOR_P,'replace',P},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line start_monitor(Offender,P),
+    ?line send_bad_structure(Offender, P,{?DOP_DEMONITOR_P,'replace',P,normal},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_EXIT,'replace',P},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_EXIT,make_ref(),normal,normal},0),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_EXIT_TT,'replace',token,P},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_EXIT_TT,make_ref(),token,normal,normal},0),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_EXIT2,'replace',P},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_EXIT2,make_ref(),normal,normal},0),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_EXIT2_TT,'replace',token,P},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_EXIT2_TT,make_ref(),token,normal,normal},0),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_GROUP_LEADER,'replace'},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_GROUP_LEADER,'replace','atomic'},2),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_GROUP_LEADER,'replace',P},0),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_REG_SEND_TT,'replace','',name},2,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_REG_SEND_TT,'replace','',name,token},0,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace',''},2,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace','',P},0,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace','',name},0,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace','',name,{token}},2,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_SEND_TT,'',P},0,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_SEND_TT,'',name,token},0,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_SEND,''},0,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_SEND,'',name},0,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line send_bad_structure(Offender, P,{?DOP_SEND,'',P,{token}},0,{message}),
+    ?line pong = rpc:call(Victim, net_adm, ping, [Offender]),
+    ?line P ! two,
+    ?line P ! check_msgs,
+    ?line receive 
+	      {P, messages_checked} -> ok 
+	  after 5000 ->
+		  exit(victim_is_dead)
+	  end,
+
+    ?line {message_queue_len, 0}
+	= rpc:call(Victim, erlang, process_info, [P, message_queue_len]),
+    
+    ?line unlink(P),
+    ?line P ! done,
+    ?line stop_node(Offender),
+    ?line stop_node(Victim),
+    ?t:timetrap_cancel(Dog),
+    ok.
 
-bad_dist_ext(doc) -> [];
-bad_dist_ext(suite) ->
-    [bad_dist_ext_receive,
-     bad_dist_ext_process_info,
-     bad_dist_ext_control,
-     bad_dist_ext_connection_id].
 
 
 bad_dist_ext_receive(Config) when is_list(Config) ->
@@ -1327,8 +1602,8 @@ bad_dist_ext_control(Config) when is_list(Config) ->
     ?line stop_node(Victim).
 
 bad_dist_ext_connection_id(Config) when is_list(Config) ->
-    ?line {ok, Offender} = start_node(bad_dist_ext_receive_offender),
-    ?line {ok, Victim} = start_node(bad_dist_ext_receive_victim),
+    ?line {ok, Offender} = start_node(bad_dist_ext_connection_id_offender),
+    ?line {ok, Victim} = start_node(bad_dist_ext_connection_id_victim),
     ?line start_node_monitors([Offender,Victim]),
 
     ?line Parent = self(),
@@ -1393,6 +1668,22 @@ bad_dist_ext_connection_id(Config) when is_list(Config) ->
     ?line stop_node(Victim).
 
 
+bad_dist_struct_check_msgs([]) ->
+    receive
+	Msg ->
+	    exit({unexpected_message, Msg})
+    after 0 ->
+	    ok
+    end;
+bad_dist_struct_check_msgs([M|Ms]) ->
+    receive
+	{'EXIT',_,_} = EM ->
+	    io:format("Ignoring exit message: ~p~n",[EM]),
+	    bad_dist_struct_check_msgs([M|Ms]);
+	Msg ->
+	    M = Msg,
+	    bad_dist_struct_check_msgs(Ms)
+    end.
 bad_dist_ext_check_msgs([]) ->
     receive
 	Msg ->
@@ -1407,24 +1698,6 @@ bad_dist_ext_check_msgs([M|Ms]) ->
 	    bad_dist_ext_check_msgs(Ms)
     end.
     
--define(COOKIE, '').
--define(DOP_LINK,		1).
--define(DOP_SEND,		2).
--define(DOP_EXIT,		3).
--define(DOP_UNLINK,		4).
--define(DOP_NODE_LINK,		5).
--define(DOP_REG_SEND,		6).
--define(DOP_GROUP_LEADER,	7).
--define(DOP_EXIT2,		8).
-
--define(DOP_SEND_TT,		12).
--define(DOP_EXIT_TT,		13).
--define(DOP_REG_SEND_TT,	16).
--define(DOP_EXIT2_TT,		18).
-
--define(DOP_MONITOR_P,		19).
--define(DOP_DEMONITOR_P,	20).
--define(DOP_MONITOR_P_EXIT,	21).
 
 dport_reg_send(Node, Name, Msg) ->
     DPrt = case dport(Node) of
@@ -1456,6 +1729,39 @@ dport_send(To, Msg) ->
 				  ?COOKIE,
 				  To}),
 			dmsg_ext(Msg)]).
+send_bad_structure(Offender,Victim,Bad,WhereToPutSelf) ->
+    send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,[]).
+send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) ->
+    Parent = self(),
+    Done = make_ref(),
+    spawn(Offender,
+	  fun () ->
+		  Node = node(Victim),
+		  pong = net_adm:ping(Node),
+		  DPrt = dport(Node),
+		  Bad1 = case WhereToPutSelf of
+			     0 -> 
+				 Bad;
+			     N when N > 0 ->
+				 setelement(N,Bad,self())
+			 end,
+		  DData = [dmsg_hdr(),
+			   dmsg_ext(Bad1)] ++
+		      case PayLoad of
+			  [] -> [];
+			  _Other -> [dmsg_ext(PayLoad)]
+		      end,
+		  port_command(DPrt, DData),
+		  Parent ! {DData,Done}
+	  end),
+    receive 
+	{WhatSent,Done} -> 
+	    io:format("Offender sent ~p~n",[WhatSent]),
+	    ok
+    after 5000 ->
+	    exit(unable_to_send)
+    end.
+   
 
 %% send_bad_msgs():
 %% Send a valid distribution header and control message
@@ -1539,10 +1845,10 @@ dmsg_bad_hdr() ->
      255].  % 255 atom references
     
 
-dmsg_fake_hdr1() ->
-    A = <<"fake header atom 1">>,
-    [131, % Version Magic
-     $D, 1, 16#8, 0, size(A), A]. % Fake header
+%% dmsg_fake_hdr1() ->
+%%     A = <<"fake header atom 1">>,
+%%     [131, % Version Magic
+%%      $D, 1, 16#8, 0, size(A), A]. % Fake header
 
 dmsg_fake_hdr2() ->
     A1 = <<"fake header atom 1">>,
@@ -1584,7 +1890,7 @@ start_node(Name, Args, Rel) when is_atom(Name), is_list(Rel) ->
 	     end,
     test_server:start_node(Name, slave, 
 			   [{args,
-			     Args++" -setcookie "++Cookie++" -pa "++Pa}
+			     Args++" -setcookie "++Cookie++" -pa \""++Pa++"\""}
 			    | RelArg]);
 start_node(Config, Args, Rel) when is_list(Config), is_list(Rel) ->
     Name = list_to_atom((atom_to_list(?MODULE)
@@ -1727,7 +2033,7 @@ flush_node_changes() ->
 
 node_monitor_loop(Master) ->
     receive
-	{nodeup, Node, InfoList} = Msg ->
+	{nodeup, Node, _InfoList} = Msg ->
 	    Master ! {nodeup, node(), Node},
 	    ?t:format("~p ~p: ~p~n", [node(), erlang:now(), Msg]),
 	    node_monitor_loop(Master);
@@ -1764,9 +2070,9 @@ verify_no_down(A, B) ->
 	    ok
     end.
 
-verify_down(A, B) ->
-    receive {nodedown, A, B, _} -> ok end,
-    receive {nodedown, B, A, _} -> ok end.
+%% verify_down(A, B) ->
+%%     receive {nodedown, A, B, _} -> ok end,
+%%     receive {nodedown, B, A, _} -> ok end.
 
 verify_down(A, ReasonA, B, ReasonB) ->
     receive
@@ -1786,11 +2092,11 @@ from(H, [H | T]) -> T;
 from(H, [_ | T]) -> from(H, T);
 from(_, []) -> [].
 
-fun_spawn(Fun) ->
-    fun_spawn(Fun, []).
+%% fun_spawn(Fun) ->
+%%     fun_spawn(Fun, []).
 
-fun_spawn(Fun, Args) ->
-    spawn_link(erlang, apply, [Fun, Args]).
+%% fun_spawn(Fun, Args) ->
+%%     spawn_link(erlang, apply, [Fun, Args]).
 
 
 long_or_short() -> 
diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl
index 39b2ed395f..643357263c 100644
--- a/erts/emulator/test/driver_SUITE.erl
+++ b/erts/emulator/test/driver_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -27,18 +27,18 @@
 %%% - queueing
 
 -module(driver_SUITE).
--export([all/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, 
+	 end_per_suite/1, init_per_group/2,end_per_group/2,
 	 init_per_testcase/2,
-	 fin_per_testcase/2,
-	 end_per_suite/1,
+	 end_per_testcase/2,
 	 outputv_echo/1,
-	 timer/1,
+	
 	 timer_measure/1,
 	 timer_cancel/1,
 	 timer_change/1,
 	 timer_delay/1,
 	 queue_echo/1,
-	 fun_to_port/1,
+	 outputv_errors/1,
 	 driver_unloaded/1,
 	 io_ready_exit/1,
 	 use_fallback_pollset/1,
@@ -47,11 +47,11 @@
 	 fd_change/1,
 	 steal_control/1,
 	 otp_6602/1,
-	 'driver_system_info_ver1.0'/1,
-	 'driver_system_info_ver1.1'/1,
+	 driver_system_info_base_ver/1,
+	 driver_system_info_prev_ver/1,
 	 driver_system_info_current_ver/1,
 	 driver_monitor/1,
-	 ioq_exit/1,
+	
 	 ioq_exit_ready_input/1,
 	 ioq_exit_ready_output/1,
 	 ioq_exit_timeout/1,
@@ -74,11 +74,14 @@
 	 missing_callbacks/1,
 	 smp_select/1,
 	 driver_select_use/1,
-	 thread_mseg_alloc_cache_clean/1]).
+	 thread_mseg_alloc_cache_clean/1,
+	 otp_9302/1,
+	 thr_free_drv/1,
+	 async_blast/1]).
 
 -export([bin_prefix/2]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 
 % First byte in communication with the timer driver
@@ -120,80 +123,137 @@ init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) ->
     ?line 0 = erts_debug:get_internal_state(check_io_debug),
     [{watchdog, Dog},{testcase, Case}|Config].
 
-fin_per_testcase(Case, Config) ->
+end_per_testcase(Case, Config) ->
     Dog = ?config(watchdog, Config),
-    erlang:display({fin_per_testcase, Case}),
+    erlang:display({end_per_testcase, Case}),
     ?line 0 = erts_debug:get_internal_state(check_io_debug),
     ?t:timetrap_cancel(Dog).
 
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [outputv_errors, outputv_echo, queue_echo, {group, timer},
+     driver_unloaded, io_ready_exit, use_fallback_pollset,
+     bad_fd_in_pollset, driver_event, fd_change,
+     steal_control, otp_6602, driver_system_info_base_ver,
+     driver_system_info_prev_ver,
+     driver_system_info_current_ver, driver_monitor,
+     {group, ioq_exit}, zero_extended_marker_garb_drv,
+     invalid_extended_marker_drv, larger_major_vsn_drv,
+     larger_minor_vsn_drv, smaller_major_vsn_drv,
+     smaller_minor_vsn_drv, peek_non_existing_queue,
+     otp_6879, caller, many_events, missing_callbacks,
+     smp_select, driver_select_use,
+     thread_mseg_alloc_cache_clean,
+     otp_9302,
+     thr_free_drv,
+     async_blast].
+
+groups() -> 
+    [{timer, [],
+      [timer_measure, timer_cancel, timer_delay,
+       timer_change]},
+     {ioq_exit, [],
+      [ioq_exit_ready_input, ioq_exit_ready_output,
+       ioq_exit_timeout, ioq_exit_ready_async, ioq_exit_event,
+       ioq_exit_ready_input_async, ioq_exit_ready_output_async,
+       ioq_exit_timeout_async, ioq_exit_event_async]}].
+
+init_per_suite(Config) ->
+    Config.
+
 end_per_suite(_Config) ->
     catch erts_debug:set_internal_state(available_internal_state, false).
 
-all(suite) ->
-    [
-     fun_to_port,
-     outputv_echo,
-     queue_echo,
-     timer,
-     driver_unloaded,
-     io_ready_exit,
-     use_fallback_pollset,
-     bad_fd_in_pollset,
-     driver_event,
-     fd_change,
-     steal_control,
-     otp_6602,
-     'driver_system_info_ver1.0',
-     'driver_system_info_ver1.1',
-     driver_system_info_current_ver,
-     driver_monitor,
-     ioq_exit,
-     zero_extended_marker_garb_drv,
-     invalid_extended_marker_drv,
-     larger_major_vsn_drv,
-     larger_minor_vsn_drv,
-     smaller_major_vsn_drv,
-     smaller_minor_vsn_drv,
-     peek_non_existing_queue,
-     otp_6879,
-     caller,
-     many_events,
-     missing_callbacks,
-     smp_select,
-     driver_select_use,
-     thread_mseg_alloc_cache_clean
-    ].
-
-fun_to_port(doc) -> "Test sending a fun to port with an outputv-capable driver.";
-fun_to_port(Config) when is_list(Config) ->
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+outputv_errors(doc) -> "Test sending bad types to port with an outputv-capable driver.";
+outputv_errors(Config) when is_list(Config) ->
     ?line Path = ?config(data_dir, Config),
     ?line erl_ddll:start(),
     ?line ok = load_driver(Path, outputv_drv),
 
-    ?line fun_to_port_1(fun() -> 33 end),
-    ?line fun_to_port_1([fun() -> 42 end]),
-    ?line fun_to_port_1([1|fun() -> 42 end]),
-    L = build_io_list(65536),
-    ?line fun_to_port_1([L,fun() -> 42 end]),
-    ?line fun_to_port_1([L|fun() -> 42 end]),
+    outputv_bad_types(fun(T) ->
+			      ?line outputv_errors_1(T),
+			      ?line outputv_errors_1([1|T]),
+			      ?line L = [1,2,3],
+			      ?line outputv_errors_1([L,T]),
+			      ?line outputv_errors_1([L|T])
+		      end),
+    outputv_errors_1(42),
+
+    %% Test iolists that do not fit in the address space.
+    %% Unfortunately, it would be too slow to test in a 64-bit emulator.
+    case erlang:system_info(wordsize) of
+	4 -> outputv_huge_iolists();
+	_ -> ok
+    end.
+
+outputv_bad_types(Test) ->
+    Types = [-1,256,atom,42.0,{a,b,c},make_ref(),fun() -> 42 end,
+	     [1|2],<<1:1>>,<<1:9>>,<<1:15>>],
+    _ = [Test(Type) || Type <- Types],
     ok.
 
-fun_to_port_1(Term) ->
-    Port = open_port({spawn,outputv_drv}, []),
+outputv_huge_iolists() ->
+    FourGigs = 1 bsl 32,
+    ?line Sizes = [FourGigs+N || N <- lists:seq(0, 64)] ++
+	[1 bsl N || N <- lists:seq(33, 37)],
+    ?line Base = <<0:(1 bsl 20)/unit:8>>,
+    [begin
+	 ?line L = build_iolist(Sz, Base),
+	 ?line outputv_errors_1(L)
+     end || Sz <- Sizes],
+    ok.
+
+outputv_errors_1(Term) ->
+    Port = open_port({spawn_driver,outputv_drv}, []),
     {'EXIT',{badarg,_}} = (catch port_command(Port, Term)),
     port_close(Port).
 
-build_io_list(0) -> [];
-build_io_list(1) -> [7];
-build_io_list(N) ->
-    L = build_io_list(N div 2),
+build_iolist(N, Base) when N < 16 ->
+    case random:uniform(3) of
+	1 ->
+	    <<Bin:N/binary,_/binary>> = Base,
+	    Bin;
+	_ ->
+	    lists:seq(1, N)
+    end;
+build_iolist(N, Base) when N =< byte_size(Base) ->
+    case random:uniform(3) of
+	1 ->
+	    <<Bin:N/binary,_/binary>> = Base,
+	    Bin;
+	2 ->
+	    <<Bin:N/binary,_/binary>> = Base,
+	    [Bin];
+	3 ->
+	    case N rem 2 of
+		0 ->
+		    L = build_iolist(N div 2, Base),
+		    [L,L];
+		1 ->
+		    L = build_iolist(N div 2, Base),
+		    [L,L,45]
+	    end
+    end;
+build_iolist(N0, Base) ->
+    Small = random:uniform(15),
+    Seq = lists:seq(1, Small),
+    N = N0 - Small,
     case N rem 2 of
-	0 -> [L|L];
-	1 -> [7,L|L]
+	0 ->
+	    L = build_iolist(N div 2, Base),
+	    [L,L|Seq];
+	1 ->
+	    L = build_iolist(N div 2, Base),
+	    [47,L,L|Seq]
     end.
 
-
-
 outputv_echo(doc) -> ["Test echoing data with a driver that supports outputv."];
 outputv_echo(Config) when is_list(Config) ->
     ?line Dog = test_server:timetrap(test_server:minutes(10)),
@@ -308,7 +368,6 @@ compare(Got, Expected) ->
 %% 		Driver timer test suites
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-timer(suite) -> [timer_measure,timer_cancel,timer_delay,timer_change].
 
 timer_measure(doc) -> ["Check that timers time out in good time."];
 timer_measure(Config) when is_list(Config) ->
@@ -1024,19 +1083,19 @@ otp_6602(Config) when is_list(Config) ->
 
 -define(EXPECTED_SYSTEM_INFO_NAMES, ?EXPECTED_SYSTEM_INFO_NAMES2).
 
-'driver_system_info_ver1.0'(doc) ->    
+'driver_system_info_base_ver'(doc) ->
     [];
-'driver_system_info_ver1.0'(suite) ->
+'driver_system_info_base_ver'(suite) ->
     [];
-'driver_system_info_ver1.0'(Config) when is_list(Config) ->
-    ?line driver_system_info_test(Config, sys_info_1_0_drv).
+'driver_system_info_base_ver'(Config) when is_list(Config) ->
+    ?line driver_system_info_test(Config, sys_info_base_drv).
 
-'driver_system_info_ver1.1'(doc) ->    
+'driver_system_info_prev_ver'(doc) ->
     [];
-'driver_system_info_ver1.1'(suite) ->
+'driver_system_info_prev_ver'(suite) ->
     [];
-'driver_system_info_ver1.1'(Config) when is_list(Config) ->
-    ?line driver_system_info_test(Config, sys_info_1_1_drv).
+'driver_system_info_prev_ver'(Config) when is_list(Config) ->
+    ?line driver_system_info_test(Config, sys_info_prev_drv).
 
 driver_system_info_current_ver(doc) ->    
     [];
@@ -1299,17 +1358,6 @@ driver_monitor(Config) when is_list(Config) ->
     ?line stop_driver(Port, Name),
     ?line ok.
 
-ioq_exit(doc) -> [];
-ioq_exit(suite) ->
-    [ioq_exit_ready_input,
-     ioq_exit_ready_output,
-     ioq_exit_timeout,
-     ioq_exit_ready_async,
-     ioq_exit_event,
-     ioq_exit_ready_input_async,
-     ioq_exit_ready_output_async,
-     ioq_exit_timeout_async,
-     ioq_exit_event_async].
 
 -define(IOQ_EXIT_READY_INPUT, 1).
 -define(IOQ_EXIT_READY_OUTPUT, 2).
@@ -1546,7 +1594,7 @@ otp_6879(Config) when is_list(Config) ->
 				      end
 			end,
 			Procs),
-    %% Also try it when input exeeds default buffer (256 bytes)
+    %% Also try it when input exceeds default buffer (256 bytes)
     ?line Data = lists:seq(1, 1000),
     ?line case open_port({spawn, Drv}, []) of
 	      Port when is_port(Port) ->
@@ -1682,7 +1730,7 @@ smp_select0(Config) ->
     ProcFun = fun()-> io:format("Worker ~p starting\n",[self()]),	
 		      ?line Port = open_port({spawn, DrvName}, []),
 		      smp_select_loop(Port, 100000),
-		      sleep(500), % wait for driver to handle pending events
+		      sleep(1000), % wait for driver to handle pending events
 		      ?line true = erlang:port_close(Port),
 		      Master ! {ok,self()},
 		      io:format("Worker ~p finished\n",[self()])
@@ -1760,13 +1808,13 @@ thread_mseg_alloc_cache_clean(Config) when is_list(Config) ->
 	    ?line {skipped, "driver_alloc() using too large single block threshold"};
 	{_, _, 0} ->
 	    ?line {skipped, "driver_alloc() using too low single block threshold"};
-	{true, MsegAllocInfo, SBCT} ->
+	{true, _MsegAllocInfo, SBCT} ->
 	    ?line DrvName = 'thr_alloc_drv',
 	    ?line Path = ?config(data_dir, Config),
 	    ?line erl_ddll:start(),
 	    ?line ok = load_driver(Path, DrvName),   
 	    ?line Port = open_port({spawn, DrvName}, []),
-	    ?line CCI = mseg_alloc_cci(MsegAllocInfo),
+	    ?line CCI = 1000,
 	    ?line ?t:format("CCI = ~p~n", [CCI]),
 	    ?line CCC = mseg_alloc_ccc(),
 	    ?line ?t:format("CCC = ~p~n", [CCC]),
@@ -1787,25 +1835,48 @@ mseg_alloc_cci(MsegAllocInfo) ->
     ?line CCI.
 
 mseg_alloc_ccc() ->
-    mseg_alloc_ccc(erlang:system_info({allocator,mseg_alloc})).
+    mseg_alloc_ccc(mseg_inst_info(0)).
 
 mseg_alloc_ccc(MsegAllocInfo) ->
-    ?line {value,{calls, CL}}
-	= lists:keysearch(calls, 1, MsegAllocInfo),
+    ?line {value,{memkind, MKL}} = lists:keysearch(memkind,1,MsegAllocInfo),
+    ?line {value,{calls, CL}} = lists:keysearch(calls, 1, MKL),
     ?line {value,{mseg_check_cache, GigaCCC, CCC}}
 	= lists:keysearch(mseg_check_cache, 1, CL),
     ?line GigaCCC*1000000000 + CCC.
 
 mseg_alloc_cached_segments() ->
-    mseg_alloc_cached_segments(erlang:system_info({allocator,mseg_alloc})).
+    mseg_alloc_cached_segments(mseg_inst_info(0)).
 
 mseg_alloc_cached_segments(MsegAllocInfo) ->
+    MemName = case is_halfword_vm() of
+	true -> "high memory";
+	false -> "all memory"
+    end,
+    ?line [{memkind,DrvMem}]
+	= lists:filter(fun(E) -> case E of
+				    {memkind, [{name, MemName} | _]} -> true;
+				    _ -> false
+		       end end, MsegAllocInfo),
     ?line {value,{status, SL}}
-	= lists:keysearch(status, 1, MsegAllocInfo),
+	= lists:keysearch(status, 1, DrvMem),
     ?line {value,{cached_segments, CS}}
 	= lists:keysearch(cached_segments, 1, SL),
     ?line CS.
 
+mseg_inst_info(I) ->
+    {value, {instance, I, Value}}
+	= lists:keysearch(I,
+			  2,
+			  erlang:system_info({allocator,mseg_alloc})),
+    Value.
+
+is_halfword_vm() ->
+    case {erlang:system_info({wordsize, internal}),
+	  erlang:system_info({wordsize, external})} of
+	{4, 8} -> true;
+	{WS, WS} -> false
+    end.
+
 driver_alloc_sbct() ->
     {_, _, _, As} = erlang:system_info(allocator),
     case lists:keysearch(driver_alloc, 1, As) of
@@ -1832,13 +1903,126 @@ thread_mseg_alloc_cache_clean_test(Port, N, CCI, Size) ->
     ?line ?t:format("CCC = ~p~n", [CCC]),
     ?line true = CCC > OCCC,
     ?line thread_mseg_alloc_cache_clean_test(Port, N-1, CCI, Size).
-    
-    
+
+otp_9302(Config) when is_list(Config) ->
+    ?line Path = ?config(data_dir, Config),
+    ?line erl_ddll:start(),
+    ?line ok = load_driver(Path, otp_9302_drv),
+    ?line Port = open_port({spawn, otp_9302_drv}, []),
+    ?line true = is_port(Port),
+    ?line port_command(Port, ""),
+    ?line {msg, block} = get_port_msg(Port, infinity),
+    ?line {msg, job} = get_port_msg(Port, infinity),
+    ?line C = case erlang:system_info(thread_pool_size) of
+		  0 ->
+		      ?line {msg, cancel} = get_port_msg(Port, infinity),
+		      ?line {msg, job} = get_port_msg(Port, infinity),
+		      ?line false;
+		  _ ->
+		      case get_port_msg(Port, infinity) of
+			  {msg, cancel} -> %% Cancel always fail in Rel >= 15
+			      ?line {msg, job} = get_port_msg(Port, infinity),
+			      ?line false;
+			  {msg, job} ->
+			      ?line ok,
+			      ?line true
+		      end
+	      end,
+    ?line {msg, end_of_jobs} = get_port_msg(Port, infinity),
+    ?line no_msg = get_port_msg(Port, 2000),
+    ?line port_close(Port),
+    ?line case C of
+	      true ->
+		  ?line {comment, "Async job cancelled"};
+	      false ->
+		  ?line {comment, "Async job not cancelled"}
+	  end.
+
+thr_free_drv(Config) when is_list(Config) ->
+    ?line Path = ?config(data_dir, Config),
+    ?line erl_ddll:start(),
+    ?line ok = load_driver(Path, thr_free_drv),
+    ?line MemBefore = driver_alloc_size(),
+%    io:format("SID=~p", [erlang:system_info(scheduler_id)]),
+    ?line Port = open_port({spawn, thr_free_drv}, []),
+    ?line MemPeek = driver_alloc_size(),
+    ?line true = is_port(Port),
+    ?line ok = thr_free_drv_control(Port, 0),
+    ?line port_close(Port),
+    ?line MemAfter = driver_alloc_size(),
+    ?line io:format("MemPeek=~p~n", [MemPeek]),
+    ?line io:format("MemBefore=~p, MemAfter=~p~n", [MemBefore, MemAfter]),
+    ?line MemBefore = MemAfter,
+    ?line case MemPeek of
+	      undefined -> ok;
+	      _ ->
+		  ?line true = MemPeek > MemBefore
+	  end,
+    ?line ok.
+
+thr_free_drv_control(Port, N) ->
+    case erlang:port_control(Port, 0, "") of
+	"done" ->
+	    ok;
+	"more" ->
+	    erlang:yield(),
+%	    io:format("N=~p, SID=~p", [N, erlang:system_info(scheduler_id)]),
+	    thr_free_drv_control(Port, N+1)
+    end.
+	    
+async_blast(Config) when is_list(Config) ->
+    ?line Path = ?config(data_dir, Config),
+    ?line erl_ddll:start(),
+    ?line ok = load_driver(Path, async_blast_drv),
+    ?line SchedOnln = erlang:system_info(schedulers_online),
+    ?line MemBefore = driver_alloc_size(),
+    ?line Start = os:timestamp(),
+    ?line Blast = fun () ->
+			  Port = open_port({spawn, async_blast_drv}, []),
+			  true = is_port(Port),
+			  port_command(Port, ""),
+			  receive
+			      {Port, done} ->
+				  ok
+			  end,
+			  port_close(Port)
+		  end,
+    ?line Ps = lists:map(fun (N) ->
+				 spawn_opt(Blast,
+					   [{scheduler,
+					     (N rem SchedOnln)+ 1},
+					    monitor])
+			 end,
+			 lists:seq(1, 100)),
+    ?line MemMid = driver_alloc_size(),
+    ?line lists:foreach(fun ({Pid, Mon}) ->
+				receive
+				    {'DOWN',Mon,process,Pid,_} -> ok
+				end
+			end, Ps),
+    ?line End = os:timestamp(),
+    ?line MemAfter = driver_alloc_size(),
+    ?line io:format("MemBefore=~p, MemMid=~p, MemAfter=~p~n",
+		    [MemBefore, MemMid, MemAfter]),
+    ?line AsyncBlastTime = timer:now_diff(End,Start)/1000000,
+    ?line io:format("AsyncBlastTime=~p~n", [AsyncBlastTime]),
+    ?line MemBefore = MemAfter,
+    ?line erlang:display({async_blast_time, AsyncBlastTime}),
+    ?line ok.
+
+
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %% 		Utilities
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-    
+
+get_port_msg(Port, Timeout) ->
+    receive
+	{Port, What} ->
+	    {msg, What}
+    after Timeout ->
+	    no_msg
+    end.
 
 wait_until(Fun) ->
     case Fun() of
@@ -1991,3 +2175,41 @@ start_node(Config) when is_list(Config) ->
 
 stop_node(Node) ->
     ?t:stop_node(Node).
+
+wait_deallocations() ->
+    try
+	erts_debug:set_internal_state(wait, deallocations)
+    catch error:undef ->
+	    erts_debug:set_internal_state(available_internal_state, true),
+	    wait_deallocations()
+    end.
+
+driver_alloc_size() ->
+    case erlang:system_info(smp_support) of
+	true ->
+	    ok;
+	false ->
+	    %% driver_alloc also used by elements in lock-free queues,
+	    %% give these some time to be deallocated...
+	    receive after 100 -> ok end
+    end,
+    wait_deallocations(),
+    case erlang:system_info({allocator_sizes, driver_alloc}) of
+	false ->
+	    undefined;
+	MemInfo ->
+	    CS = lists:foldl(
+		   fun ({instance, _, L}, Acc) ->
+			   {value,{_,SBMBCS}} = lists:keysearch(sbmbcs, 1, L),
+			   {value,{_,MBCS}} = lists:keysearch(mbcs, 1, L),
+			   {value,{_,SBCS}} = lists:keysearch(sbcs, 1, L),
+			   [SBMBCS,MBCS,SBCS | Acc]
+		   end,
+		   [],
+		   MemInfo),
+	    lists:foldl(
+	      fun(L, Sz0) ->
+		      {value,{_,Sz,_,_}} = lists:keysearch(blocks_size, 1, L),
+		      Sz0+Sz
+	      end, 0, CS)
+    end.
diff --git a/erts/emulator/test/driver_SUITE_data/Makefile.src b/erts/emulator/test/driver_SUITE_data/Makefile.src
index 4ac7987d2f..9cc107cc66 100644
--- a/erts/emulator/test/driver_SUITE_data/Makefile.src
+++ b/erts/emulator/test/driver_SUITE_data/Makefile.src
@@ -11,10 +11,13 @@ MISC_DRVS =		outputv_drv@dll@ \
 			caller_drv@dll@ \
 			many_events_drv@dll@ \
 			missing_callback_drv@dll@ \
-			thr_alloc_drv@dll@
+			thr_alloc_drv@dll@ \
+			otp_9302_drv@dll@ \
+			thr_free_drv@dll@ \
+			async_blast_drv@dll@
 
-SYS_INFO_DRVS = 	sys_info_1_0_drv@dll@ \
-			sys_info_1_1_drv@dll@ \
+SYS_INFO_DRVS = 	sys_info_base_drv@dll@ \
+			sys_info_prev_drv@dll@ \
 			sys_info_curr_drv@dll@
 
 VSN_MISMATCH_DRVS =	zero_extended_marker_garb_drv@dll@ \
diff --git a/erts/emulator/test/driver_SUITE_data/async_blast_drv.c b/erts/emulator/test/driver_SUITE_data/async_blast_drv.c
new file mode 100644
index 0000000000..c2086c5860
--- /dev/null
+++ b/erts/emulator/test/driver_SUITE_data/async_blast_drv.c
@@ -0,0 +1,124 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#include "erl_driver.h"
+
+#define NO_ASYNC_JOBS 10000
+
+static void stop(ErlDrvData drv_data);
+static ErlDrvData start(ErlDrvPort port,
+			char *command);
+static void output(ErlDrvData drv_data,
+		   char *buf, ErlDrvSizeT len);
+static void ready_async(ErlDrvData drv_data,
+			ErlDrvThreadData thread_data);
+
+static ErlDrvEntry async_blast_drv_entry = { 
+    NULL /* init */,
+    start,
+    stop,
+    output,
+    NULL /* ready_input */,
+    NULL /* ready_output */,
+    "async_blast_drv",
+    NULL /* finish */,
+    NULL /* handle */,
+    NULL /* control */,
+    NULL /* timeout */,
+    NULL /* outputv */,
+    ready_async,
+    NULL /* flush */,
+    NULL /* call */,
+    NULL /* event */,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    ERL_DRV_FLAG_USE_PORT_LOCKING,
+    NULL /* handle2 */,
+    NULL /* handle_monitor */
+};
+
+typedef struct {
+    ErlDrvPort port;
+    ErlDrvTermData caller;    
+    int counter;
+} async_blast_data_t;
+
+
+DRIVER_INIT(async_blast_drv)
+{
+    return &async_blast_drv_entry;
+}
+
+static void stop(ErlDrvData drv_data)
+{
+    driver_free((void *) drv_data);
+}
+
+static ErlDrvData start(ErlDrvPort port,
+			char *command)
+{
+    async_blast_data_t *abd;
+
+    abd = driver_alloc(sizeof(async_blast_data_t));
+    if (!abd)
+	return ERL_DRV_ERROR_GENERAL;
+
+    abd->port = port;
+    abd->counter = 0;
+    return (ErlDrvData) abd;
+}
+
+static void async_invoke(void *data)
+{
+
+}
+#include <stdio.h>
+
+static void ready_async(ErlDrvData drv_data,
+			ErlDrvThreadData thread_data)
+{
+    async_blast_data_t *abd = (async_blast_data_t *) drv_data;
+    if (--abd->counter == 0) {
+	ErlDrvTermData spec[] = {
+	    ERL_DRV_PORT, driver_mk_port(abd->port),
+	    ERL_DRV_ATOM, driver_mk_atom("done"),
+	    ERL_DRV_TUPLE, 2
+	};
+	driver_send_term(abd->port, abd->caller,
+			 spec, sizeof(spec)/sizeof(spec[0]));
+    }
+}
+
+static void output(ErlDrvData drv_data,
+		   char *buf, ErlDrvSizeT len)
+{
+    async_blast_data_t *abd = (async_blast_data_t *) drv_data;
+    if (abd->counter == 0) {
+	int i;
+	abd->caller = driver_caller(abd->port);
+	abd->counter = NO_ASYNC_JOBS;
+	for (i = 0; i < NO_ASYNC_JOBS; i++) {
+	    if (0 > driver_async(abd->port, NULL, async_invoke, NULL, NULL)) {
+		driver_failure_atom(abd->port, "driver_async_failed");
+		break;
+	    }
+	}
+    }
+}
diff --git a/erts/emulator/test/driver_SUITE_data/caller_drv.c b/erts/emulator/test/driver_SUITE_data/caller_drv.c
index a78d51966f..1ed20b0638 100644
--- a/erts/emulator/test/driver_SUITE_data/caller_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/caller_drv.c
@@ -23,17 +23,17 @@
 static ErlDrvData start(ErlDrvPort port,
 			char *command);
 static void output(ErlDrvData drv_data,
-		   char *buf, int len);
+		   char *buf, ErlDrvSizeT len);
 static void outputv(ErlDrvData drv_data,
 		    ErlIOVec *ev);
-static int control(ErlDrvData drv_data,
-		   unsigned int command, char *buf, 
-		   int len, char **rbuf, int rlen);
-static int call(ErlDrvData drv_data,
-		unsigned int command,
-		char *buf, int len,
-		char **rbuf, int rlen,
-		unsigned int *flags);
+static ErlDrvSSizeT control(ErlDrvData drv_data,
+			    unsigned int command, char *buf,
+			    ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen);
+static ErlDrvSSizeT call(ErlDrvData drv_data,
+			 unsigned int command,
+			 char *buf, ErlDrvSizeT len,
+			 char **rbuf, ErlDrvSizeT rlen,
+			 unsigned int *flags);
 
 static ErlDrvEntry caller_drv_entry = { 
     NULL /* init */,
@@ -98,7 +98,7 @@ start(ErlDrvPort port, char *command)
 }
 
 static void
-output(ErlDrvData drv_data, char *buf, int len)
+output(ErlDrvData drv_data, char *buf, ErlDrvSizeT len)
 {
     send_caller(drv_data, "output");
 }
@@ -109,20 +109,20 @@ outputv(ErlDrvData drv_data, ErlIOVec *ev)
     send_caller(drv_data, "outputv");
 }
 
-static int
+static ErlDrvSSizeT
 control(ErlDrvData drv_data,
 	unsigned int command, char *buf, 
-	int len, char **rbuf, int rlen)
+	ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen)
 {
     send_caller(drv_data, "control");
     return 0;
 }
 
-static int
+static ErlDrvSSizeT
 call(ErlDrvData drv_data,
      unsigned int command,
-     char *buf, int len,
-     char **rbuf, int rlen,
+     char *buf, ErlDrvSizeT len,
+     char **rbuf, ErlDrvSizeT rlen,
      unsigned int *flags)
 {
     /* echo call */
diff --git a/erts/emulator/test/driver_SUITE_data/chkio_drv.c b/erts/emulator/test/driver_SUITE_data/chkio_drv.c
index 9e1e5e72c2..40f1ad4fea 100644
--- a/erts/emulator/test/driver_SUITE_data/chkio_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/chkio_drv.c
@@ -17,7 +17,7 @@
  */
 
 #ifndef UNIX
-#if !defined(__WIN32__) && !defined(_OSE_) && !defined(VXWORKS)
+#if !defined(__WIN32__) && !defined(VXWORKS)
 #define UNIX 1
 #endif
 #endif
@@ -102,6 +102,7 @@ typedef struct chkio_smp_select {
     int write_fd;
     int next_read;
     int next_write;
+    int first_write;
     enum {Closed, Opened, Selected, Waiting} state;
     int wasSelected;
     unsigned rand_state;
@@ -140,8 +141,8 @@ static void chkio_drv_stop(ErlDrvData);
 static void chkio_drv_ready_input(ErlDrvData, ErlDrvEvent);
 static void chkio_drv_ready_output(ErlDrvData, ErlDrvEvent);
 static void chkio_drv_ready_event(ErlDrvData, ErlDrvEvent, ErlDrvEventData);
-static int chkio_drv_control(ErlDrvData, unsigned int,
-			    char *, int, char **, int);
+static ErlDrvSSizeT chkio_drv_control(ErlDrvData, unsigned int,
+				      char *, ErlDrvSizeT, char **, ErlDrvSizeT);
 static void chkio_drv_timeout(ErlDrvData);
 static void chkio_drv_stop_select(ErlDrvEvent, void*);
 
@@ -187,7 +188,7 @@ stop_use_fallback_pollset(ChkioDrvData *cddp)
 	for (i = 0; i < CHKIO_FALLBACK_FDS; i++) {
 	    if (cbdp->dev_null[i].fd >= 0) {
 		if (driver_select(cddp->port,
-				  (ErlDrvEvent) cbdp->dev_null[i].fd,
+				  (ErlDrvEvent) (ErlDrvSInt) cbdp->dev_null[i].fd,
 				  DO_WRITE,
 				  0) != 0) {
 		    fprintf(stderr,
@@ -199,7 +200,7 @@ stop_use_fallback_pollset(ChkioDrvData *cddp)
 	    }
 	    if (cbdp->dev_zero[i].fd >= 0) {
 		if (driver_select(cddp->port,
-				  (ErlDrvEvent) cbdp->dev_zero[i].fd,
+				  (ErlDrvEvent) (ErlDrvSInt) cbdp->dev_zero[i].fd,
 				  DO_READ,
 				  0) != 0) {
 		    fprintf(stderr,
@@ -211,7 +212,7 @@ stop_use_fallback_pollset(ChkioDrvData *cddp)
 	    }
 	    if (cbdp->pipe_in[i].fd >= 0) {
 		if (driver_select(cddp->port,
-				  (ErlDrvEvent) cbdp->pipe_in[i].fd,
+				  (ErlDrvEvent) (ErlDrvSInt) cbdp->pipe_in[i].fd,
 				  DO_READ,
 				  0) != 0) {
 		    fprintf(stderr,
@@ -223,7 +224,7 @@ stop_use_fallback_pollset(ChkioDrvData *cddp)
 	    }
 	    if (cbdp->pipe_out[i].fd >= 0) {
 		if (driver_select(cddp->port,
-				  (ErlDrvEvent) cbdp->pipe_out[i].fd,
+				  (ErlDrvEvent) (ErlDrvSInt) cbdp->pipe_out[i].fd,
 				  DO_WRITE,
 				  0) != 0) {
 		    fprintf(stderr,
@@ -248,11 +249,11 @@ stop_driver_event(ChkioDrvData *cddp)
 	cddp->test_data = NULL;
 
 	if (cdep->in_fd >= 0) {
-	    driver_event(cddp->port, (ErlDrvEvent) cdep->in_fd, NULL);
+	    driver_event(cddp->port, (ErlDrvEvent) (ErlDrvSInt) cdep->in_fd, NULL);
 	    close(cdep->in_fd);
 	}
 	if (cdep->out_fd >= 0) {
-	    driver_event(cddp->port, (ErlDrvEvent) cdep->out_fd, NULL);
+	    driver_event(cddp->port, (ErlDrvEvent) (ErlDrvSInt) cdep->out_fd, NULL);
 	    close(cdep->out_fd);
 	}
 	driver_free(cdep);	    
@@ -267,7 +268,7 @@ stop_fd_change(ChkioDrvData *cddp)
 	cddp->test_data = NULL;
 	driver_cancel_timer(cddp->port);
 	if (cfcp->fds[0] >= 0) {
-	    driver_select(cddp->port, (ErlDrvEvent) cfcp->fds[0], DO_READ, 0);
+	    driver_select(cddp->port, (ErlDrvEvent) (ErlDrvSInt) cfcp->fds[0], DO_READ, 0);
 	    close(cfcp->fds[0]);
 	    close(cfcp->fds[1]);
 	}
@@ -281,8 +282,8 @@ stop_bad_fd_in_pollset(ChkioDrvData *cddp)
     if (cddp->test_data) {
 	ChkioBadFdInPollset *bfipp = (ChkioBadFdInPollset *) cddp->test_data;
 	cddp->test_data = NULL;
-	driver_select(cddp->port, (ErlDrvEvent) bfipp->fds[0], DO_WRITE, 0);
-	driver_select(cddp->port, (ErlDrvEvent) bfipp->fds[1], DO_READ, 0);
+	driver_select(cddp->port, (ErlDrvEvent) (ErlDrvSInt) bfipp->fds[0], DO_WRITE, 0);
+	driver_select(cddp->port, (ErlDrvEvent) (ErlDrvSInt) bfipp->fds[1], DO_READ, 0);
 	driver_free((void *) bfipp);
     }
 }
@@ -295,21 +296,21 @@ stop_steal(ChkioDrvData *cddp)
 	cddp->test_data = NULL;
 	if (csp->driver_select_fds[0] >= 0)
 	    driver_select(cddp->port,
-			  (ErlDrvEvent) csp->driver_select_fds[0],
+			  (ErlDrvEvent) (ErlDrvSInt) csp->driver_select_fds[0],
 			  DO_READ,
 			  0);
 	if (csp->driver_select_fds[1] >= 0)
 	    driver_select(cddp->port,
-			  (ErlDrvEvent) csp->driver_select_fds[1],
+			  (ErlDrvEvent) (ErlDrvSInt) csp->driver_select_fds[1],
 			  DO_WRITE,
 			  0);
 	if (csp->driver_event_fds[0] >= 0)
 	    driver_event(cddp->port,
-			 (ErlDrvEvent) csp->driver_event_fds[0],
+			 (ErlDrvEvent) (ErlDrvSInt) csp->driver_event_fds[0],
 			 NULL);
 	if (csp->driver_event_fds[1] >= 0)
 	    driver_event(cddp->port,
-			 (ErlDrvEvent) csp->driver_event_fds[1],
+			 (ErlDrvEvent) (ErlDrvSInt) csp->driver_event_fds[1],
 			 NULL);
 	driver_free(csp);
     }
@@ -352,7 +353,7 @@ static void free_smp_select(ChkioSmpSelect* pip, ErlDrvPort port)
 	abort();
     }
     case Selected:
-	driver_select(port, (ErlDrvEvent)pip->read_fd, DO_READ, 0);
+	driver_select(port, (ErlDrvEvent)(ErlDrvSInt)pip->read_fd, DO_READ, 0);
 	/*fall through*/ 
     case Opened:
 	close(pip->read_fd);
@@ -474,8 +475,8 @@ chkio_drv_stop(ErlDrvData drv_data) {
 	fprintf(stderr,	"%s:%d: Failed to open /dev/null\n",
 		__FILE__, __LINE__);
     }
-    driver_select(cddp->port, (ErlDrvEvent) fd, DO_WRITE, 1);
-    driver_select(cddp->port, (ErlDrvEvent) fd, DO_WRITE, 0);
+    driver_select(cddp->port, (ErlDrvEvent) (ErlDrvSInt) fd, DO_WRITE, 1);
+    driver_select(cddp->port, (ErlDrvEvent) (ErlDrvSInt) fd, DO_WRITE, 0);
     close(fd);
 
 
@@ -490,7 +491,7 @@ chkio_drv_ready_output(ErlDrvData drv_data, ErlDrvEvent event)
 {
 #ifdef UNIX
     ChkioDrvData *cddp = (ChkioDrvData *) drv_data;
-    int fd = (int) event;
+    int fd = (int) (ErlDrvSInt) event;
 
     switch (cddp->test) {
     case CHKIO_USE_FALLBACK_POLLSET: {
@@ -532,7 +533,7 @@ chkio_drv_ready_input(ErlDrvData drv_data, ErlDrvEvent event)
 {
 #ifdef UNIX
     ChkioDrvData *cddp = (ChkioDrvData *) drv_data;
-    int fd = (int) event;
+    int fd = (int) (ErlDrvSInt) event;
 
     switch (cddp->test) {
     case CHKIO_USE_FALLBACK_POLLSET: {
@@ -577,9 +578,16 @@ chkio_drv_ready_input(ErlDrvData drv_data, ErlDrvEvent event)
 	inPipe = (pip->next_write - pip->next_read);
 	if (inPipe == 0) {
 	    bytes = read(pip->read_fd, &word, sizeof(word));
-	    printf("Unexpected empty pipe, expected %u -> %u, bytes=%d, word=%d\n",
-		   pip->next_read, pip->next_write-1, bytes, word);
-	    abort();
+	    printf("Unexpected empty pipe, expected %u -> %u, bytes=%d, word=%d, written=%d\n",
+		   pip->next_read, pip->next_write-1, bytes, word,
+		   (pip->next_write - pip->first_write));
+	    /*abort();
+    	      Allow unexpected events as it's been seen to be triggered by epoll
+	      on Linux. Most of the time the unwanted events are filtered by
+	      the erl_check_io layer. But when fd's are reused the events may
+	      slip up to the driver.
+	    */
+	    break; 
 	}
 
 	n = rand_r(&pip->rand_state) % (inPipe*4);
@@ -622,7 +630,7 @@ chkio_drv_ready_event(ErlDrvData drv_data,
     case CHKIO_DRIVER_EVENT: {
 #ifdef HAVE_POLL_H
 	ChkioDriverEvent *cdep = cddp->test_data;
-	int fd = (int) event;
+	int fd = (int) (ErlDrvSInt) event;
 	if (fd == cdep->in_fd) {
 	    if (event_data->events == POLLIN
 		&& event_data->revents == POLLIN) {
@@ -671,7 +679,7 @@ chkio_drv_timeout(ErlDrvData drv_data)
 	int in_fd = cfcp->fds[0];
 	int out_fd = cfcp->fds[1];
 	if (in_fd >= 0) {
-	    if (driver_select(cddp->port, (ErlDrvEvent) in_fd, DO_READ, 0) < 0)
+	    if (driver_select(cddp->port, (ErlDrvEvent) (ErlDrvSInt) in_fd, DO_READ, 0) < 0)
 		driver_failure_atom(cddp->port, "deselect_failed");
 	    (void) write(out_fd, (void *) "!", 1);
 	    close(out_fd);
@@ -681,7 +689,7 @@ chkio_drv_timeout(ErlDrvData drv_data)
 	    driver_failure_posix(cddp->port, errno);
 	}
 	else {
-	    if (driver_select(cddp->port, (ErlDrvEvent) cfcp->fds[0],
+	    if (driver_select(cddp->port, (ErlDrvEvent) (ErlDrvSInt) cfcp->fds[0],
 			      DO_READ, 1) < 0)
 		driver_failure_atom(cddp->port, "select_failed");
 	    if (cfcp->fds[0] == in_fd)
@@ -701,14 +709,14 @@ chkio_drv_timeout(ErlDrvData drv_data)
 #endif /* UNIX */
 }
 
-static int
+static ErlDrvSSizeT
 chkio_drv_control(ErlDrvData drv_data,
 		 unsigned int command,
-		 char *buf, int len,
-		 char **rbuf, int rlen)
+		 char *buf, ErlDrvSizeT len,
+		 char **rbuf, ErlDrvSizeT rlen)
 {
     char *res_str;
-    int res_len = -1;
+    ErlDrvSSizeT res_len = -1;
 #ifndef UNIX
 #ifdef __WIN32__
     res_str = "skip: windows_different";
@@ -846,7 +854,7 @@ chkio_drv_control(ErlDrvData drv_data,
 		int fds[2];
 		cbdp->dev_null[i].fd = open("/dev/null", O_WRONLY);
 		if (driver_select(cddp->port,
-				  (ErlDrvEvent) cbdp->dev_null[i].fd,
+				  (ErlDrvEvent) (ErlDrvSInt) cbdp->dev_null[i].fd,
 				  DO_WRITE,
 				  1) != 0) {
 		    driver_failure_posix(cddp->port, errno);
@@ -854,7 +862,7 @@ chkio_drv_control(ErlDrvData drv_data,
 		}
 		cbdp->dev_zero[i].fd = open("/dev/zero", O_RDONLY);
 		if (driver_select(cddp->port,
-				  (ErlDrvEvent) cbdp->dev_zero[i].fd,
+				  (ErlDrvEvent) (ErlDrvSInt) cbdp->dev_zero[i].fd,
 				  DO_READ,
 				  1) != 0) {
 		    driver_failure_posix(cddp->port, errno);
@@ -865,7 +873,7 @@ chkio_drv_control(ErlDrvData drv_data,
 		cbdp->pipe_in[i].fd = fds[0];
 		cbdp->pipe_out[i].fd = fds[1];
 		if (driver_select(cddp->port,
-				  (ErlDrvEvent) cbdp->pipe_in[i].fd,
+				  (ErlDrvEvent) (ErlDrvSInt) cbdp->pipe_in[i].fd,
 				  DO_READ,
 				  1) != 0) {
 		    driver_failure_posix(cddp->port, EIO);
@@ -874,7 +882,7 @@ chkio_drv_control(ErlDrvData drv_data,
 		if (i % 2 == 0)
 		    (void) write(cbdp->pipe_out[i].fd, "!", 1);
 		if (driver_select(cddp->port,
-				  (ErlDrvEvent) cbdp->pipe_out[i].fd,
+				  (ErlDrvEvent) (ErlDrvSInt) cbdp->pipe_out[i].fd,
 				  DO_WRITE,
 				  1) != 0) {
 		    driver_failure_posix(cddp->port, EIO);
@@ -920,8 +928,8 @@ chkio_drv_control(ErlDrvData drv_data,
 		bfipp->fds[0] = fds[9];
 		bfipp->fds[1] = fds[10];
 		cddp->test_data = (void *) bfipp;
-		driver_select(cddp->port, (ErlDrvEvent) fds[9], DO_WRITE, 1);
-		driver_select(cddp->port, (ErlDrvEvent) fds[10], DO_READ, 1);
+		driver_select(cddp->port, (ErlDrvEvent) (ErlDrvSInt) fds[9], DO_WRITE, 1);
+		driver_select(cddp->port, (ErlDrvEvent) (ErlDrvSInt) fds[10], DO_READ, 1);
 	    }
 	}
 	res_str = "ok";
@@ -957,7 +965,7 @@ chkio_drv_control(ErlDrvData drv_data,
 		cdep->in_ok = 0;
 
 		res = driver_event(cddp->port,
-				   (ErlDrvEvent) in_fd,
+				   (ErlDrvEvent) (ErlDrvSInt) in_fd,
 				   &cdep->in_data);
 		if (res < 0) {
 		    res_str = "skip: driver_event() not supported";
@@ -977,7 +985,7 @@ chkio_drv_control(ErlDrvData drv_data,
 		    cdep->out_ok = 0;
 
 		    res = driver_event(cddp->port,
-				       (ErlDrvEvent) out_fd,
+				       (ErlDrvEvent) (ErlDrvSInt) out_fd,
 				       &cdep->out_data);
 		    if (res < 0) {
 			close(out_fd);
@@ -1054,7 +1062,7 @@ chkio_drv_control(ErlDrvData drv_data,
 		    csp->event_data[0].events = POLLIN;
 		    csp->event_data[0].revents = 0;
 		    res = driver_event(cddp->port,
-				       (ErlDrvEvent) csp->driver_event_fds[0],
+				       (ErlDrvEvent) (ErlDrvSInt) csp->driver_event_fds[0],
 				       &csp->event_data[0]);
 		    if (res < 0)
 			driver_failure_atom(cddp->port,
@@ -1063,7 +1071,7 @@ chkio_drv_control(ErlDrvData drv_data,
 			csp->event_data[1].events = POLLOUT;
 			csp->event_data[1].revents = 0;
 			res = driver_event(cddp->port,
-					   (ErlDrvEvent) csp->driver_event_fds[1],
+					   (ErlDrvEvent) (ErlDrvSInt) csp->driver_event_fds[1],
 					   &csp->event_data[1]);
 			if (res < 0)
 			    driver_failure_atom(cddp->port,
@@ -1075,7 +1083,7 @@ chkio_drv_control(ErlDrvData drv_data,
 		/* Steal with driver_select() */
 		if (res >= 0) {
 		    res = driver_select(cddp->port,
-					(ErlDrvEvent) csp->driver_select_fds[0],
+					(ErlDrvEvent) (ErlDrvSInt) csp->driver_select_fds[0],
 					DO_READ,
 					1);
 		    if (res < 0)
@@ -1084,7 +1092,7 @@ chkio_drv_control(ErlDrvData drv_data,
 		}
 		if (res >= 0) {
 		    res = driver_select(cddp->port,
-					(ErlDrvEvent) csp->driver_select_fds[1],
+					(ErlDrvEvent) (ErlDrvSInt) csp->driver_select_fds[1],
 					DO_WRITE,
 					1);
 		    if (res < 0)
@@ -1151,14 +1159,14 @@ chkio_drv_control(ErlDrvData drv_data,
 		csap->driver_event_fds[1] = write_fds[1];
 
 		res = driver_select(cddp->port,
-				    (ErlDrvEvent) csap->driver_select_fds[0],
+				    (ErlDrvEvent) (ErlDrvSInt) csap->driver_select_fds[0],
 				    DO_READ,
 				    1);
 		if (res < 0)
 		    driver_failure_atom(cddp->port, "driver_select_failed");
 		if (res >= 0) {
 		    res = driver_select(cddp->port,
-					(ErlDrvEvent) csap->driver_select_fds[1],
+					(ErlDrvEvent) (ErlDrvSInt) csap->driver_select_fds[1],
 					DO_WRITE,
 					1);
 		    if (res < 0)
@@ -1169,7 +1177,7 @@ chkio_drv_control(ErlDrvData drv_data,
 		    csap->event_data[0].events = POLLIN;
 		    csap->event_data[0].revents = 0;
 		    res = driver_event(cddp->port,
-				       (ErlDrvEvent) csap->driver_event_fds[0],
+				       (ErlDrvEvent) (ErlDrvSInt) csap->driver_event_fds[0],
 				       &csap->event_data[0]);
 		    if (res < 0) {
 			close(csap->driver_event_fds[0]);
@@ -1182,7 +1190,7 @@ chkio_drv_control(ErlDrvData drv_data,
 			csap->event_data[1].events = POLLOUT;
 			csap->event_data[1].revents = 0;
 			res = driver_event(cddp->port,
-					   (ErlDrvEvent) csap->driver_event_fds[1],
+					   (ErlDrvEvent) (ErlDrvSInt) csap->driver_event_fds[1],
 					   &csap->event_data[1]);
 			if (res < 0)
 			    driver_failure_atom(cddp->port,
@@ -1252,6 +1260,7 @@ chkio_drv_control(ErlDrvData drv_data,
 		pip->state = Opened;
 		pip->wasSelected = 0;
 		pip->next_write = pip->next_read = rand_r(&pip->rand_state) % 1024;
+		pip->first_write = pip->next_write;
 		if (op & 1) break;
 		op >>= 1;
 	    }/*fall through*/
@@ -1276,7 +1285,7 @@ chkio_drv_control(ErlDrvData drv_data,
 		}
 		else {
 		    TRACEF(("%T: Select on pipe [%d->%d]\n", cddp->id, pip->write_fd, pip->read_fd));
-		    if (driver_select(cddp->port, (ErlDrvEvent)pip->read_fd, DO_READ, 1)) {
+		    if (driver_select(cddp->port, (ErlDrvEvent)(ErlDrvSInt)pip->read_fd, DO_READ, 1)) {
 			fprintf(stderr, "driver_select failed for fd=%d\n", pip->read_fd);
 			abort();
 		    }
@@ -1305,7 +1314,7 @@ chkio_drv_control(ErlDrvData drv_data,
 		op >>= 1;
 		if (op & 1) {
 		    TRACEF(("%T: Deselect on pipe [%d->%d]\n", cddp->id, pip->write_fd, pip->read_fd));
-		    if (driver_select(cddp->port, (ErlDrvEvent)pip->read_fd, DO_READ, 0)) {
+		    if (driver_select(cddp->port, (ErlDrvEvent)(ErlDrvSInt)pip->read_fd, DO_READ, 0)) {
 			fprintf(stderr, "driver_(de)select failed for fd=%d\n", pip->read_fd);
 			abort();
 		    }
diff --git a/erts/emulator/test/driver_SUITE_data/io_ready_exit_drv.c b/erts/emulator/test/driver_SUITE_data/io_ready_exit_drv.c
index 25d4b17001..e6a3edcd74 100644
--- a/erts/emulator/test/driver_SUITE_data/io_ready_exit_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/io_ready_exit_drv.c
@@ -17,7 +17,7 @@
  */
 
 #ifndef UNIX
-#if !defined(__WIN32__) && !defined(_OSE_) && !defined(VXWORKS)
+#if !defined(__WIN32__) && !defined(VXWORKS)
 #define UNIX 1
 #endif
 #endif
@@ -40,8 +40,9 @@ static void io_ready_exit_ready_input(ErlDrvData, ErlDrvEvent);
 static void io_ready_exit_ready_output(ErlDrvData, ErlDrvEvent);
 static void io_ready_exit_drv_output(ErlDrvData, char *, int);
 static void io_ready_exit_drv_finish(void);
-static int io_ready_exit_drv_control(ErlDrvData, unsigned int,
-				   char *, int, char **, int);
+static ErlDrvSSizeT io_ready_exit_drv_control(ErlDrvData, unsigned int,
+					      char *, ErlDrvSizeT,
+					      char **, ErlDrvSizeT);
 
 static ErlDrvEntry io_ready_exit_drv_entry = { 
     NULL, /* init */
@@ -56,7 +57,17 @@ static ErlDrvEntry io_ready_exit_drv_entry = {
     io_ready_exit_drv_control,
     NULL, /* timeout */
     NULL, /* outputv */
-    NULL  /* ready_async */
+    NULL, /* ready_async */
+    NULL, /* flush */
+    NULL, /* call */
+    NULL, /* event */
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0, /* ERL_DRV_FLAGs */
+    NULL, /* handle2 */
+    NULL, /* process_exit */
+    NULL  /* stop_select */
 };
 
 /* -------------------------------------------------------------------------
@@ -83,7 +94,7 @@ io_ready_exit_drv_stop(ErlDrvData drv_data) {
 #ifdef UNIX
     if (oeddp->fds[0] >= 0) {
 	driver_select(oeddp->port,
-		      (ErlDrvEvent) oeddp->fds[0],
+		      (ErlDrvEvent) (ErlDrvSInt) oeddp->fds[0],
 		      DO_READ|DO_WRITE,
 		      0);
 	close(oeddp->fds[0]);
@@ -109,15 +120,15 @@ io_ready_exit_ready_input(ErlDrvData drv_data, ErlDrvEvent event)
     driver_failure_atom(oeddp->port, "ready_input_driver_failure");
 }
 
-static int
+static ErlDrvSSizeT
 io_ready_exit_drv_control(ErlDrvData drv_data,
 			  unsigned int command,
-			  char *buf, int len,
-			  char **rbuf, int rlen)
+			  char *buf, ErlDrvSizeT len,
+			  char **rbuf, ErlDrvSizeT rlen)
 {
     char *abuf;
     char *res_str;
-    int res_len;
+    ErlDrvSSizeT res_len;
     IOReadyExitDrvData *oeddp = (IOReadyExitDrvData *) drv_data;
 #ifndef UNIX
     res_str = "nyiftos";
@@ -127,9 +138,9 @@ io_ready_exit_drv_control(ErlDrvData drv_data,
     }
     else {
 	res_str = "ok";
-	write(oeddp->fds[1], "!", 1);
+	(void) write(oeddp->fds[1], "!", 1);
 	driver_select(oeddp->port,
-		      (ErlDrvEvent) oeddp->fds[0],
+		      (ErlDrvEvent) (ErlDrvSInt) oeddp->fds[0],
 		      DO_READ|DO_WRITE,
 		      1);
     }
diff --git a/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c b/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c
index c7a42aa687..b2cc1e785a 100644
--- a/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/ioq_exit_drv.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2007-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -29,7 +29,7 @@
  */
 
 #ifndef UNIX
-#if !defined(__WIN32__) && !defined(_OSE_) && !defined(VXWORKS)
+#if !defined(__WIN32__) && !defined(VXWORKS)
 #define UNIX 1
 #endif
 #endif
@@ -91,7 +91,8 @@ static ErlDrvData start(ErlDrvPort port, char *command);
 static void stop(ErlDrvData drv_data);
 static void ready_input(ErlDrvData drv_data, ErlDrvEvent event); 
 static void ready_output(ErlDrvData drv_data, ErlDrvEvent event);  
-static int control(ErlDrvData, unsigned int, char *, int, char **, int);
+static ErlDrvSSizeT control(ErlDrvData, unsigned int,
+			    char *, ErlDrvSizeT, char **, ErlDrvSizeT);
 static void timeout(ErlDrvData drv_data);
 static void ready_async(ErlDrvData drv_data, ErlDrvThreadData thread_data);
 static void flush(ErlDrvData drv_data);
@@ -155,10 +156,10 @@ start(ErlDrvPort port, char *command)
     return (ErlDrvData) ddp;
 }
 
-static int control(ErlDrvData drv_data,
-		   unsigned int command,
-		   char *buf, int len,
-		   char **rbuf, int rlen)
+static ErlDrvSSizeT control(ErlDrvData drv_data,
+			    unsigned int command,
+			    char *buf, ErlDrvSizeT len,
+			    char **rbuf, ErlDrvSizeT rlen)
 {
     IOQExitDrvData *ddp = (IOQExitDrvData *) drv_data;
     char *res_str = "nyiftos";
@@ -227,7 +228,7 @@ static int control(ErlDrvData drv_data,
     res_str = "ok";
 
  done: {
-	int res_len = strlen(res_str);
+	ErlDrvSSizeT res_len = strlen(res_str);
 	if (res_len > rlen) {
 	    char *abuf = driver_alloc(sizeof(char)*res_len);
 	    if (!abuf)
diff --git a/erts/emulator/test/driver_SUITE_data/many_events_drv.c b/erts/emulator/test/driver_SUITE_data/many_events_drv.c
index 7417dbf7f8..a34432a8fe 100644
--- a/erts/emulator/test/driver_SUITE_data/many_events_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/many_events_drv.c
@@ -3,14 +3,17 @@
 #endif
 
 #include <stdio.h>
+#include <string.h>
 #include "erl_driver.h"
 
 static ErlDrvPort erlang_port;
 static ErlDrvData many_events_start(ErlDrvPort, char *);
-static void from_erlang(ErlDrvData, char*, int);
+static void from_erlang(ErlDrvData, char*, ErlDrvSizeT);
 static void from_port(ErlDrvData drv_data, ErlDrvEvent event);
-static int many_events_call(ErlDrvData drv_data, unsigned int command, char *buf, 
-		     int len, char **rbuf, int rlen, unsigned *ret_flags);
+static ErlDrvSSizeT many_events_call(ErlDrvData drv_data, unsigned int command,
+				     char *buf, ErlDrvSizeT len,
+				     char **rbuf, ErlDrvSizeT rlen,
+				     unsigned *ret_flags);
 static ErlDrvEntry many_events_driver_entry = { 
     NULL,			/* Init */
     many_events_start,
@@ -26,7 +29,15 @@ static ErlDrvEntry many_events_driver_entry = {
     NULL,
     NULL,
     NULL,
-    many_events_call
+    many_events_call,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
+    NULL
 };
 
 DRIVER_INIT(many_events_drv)
@@ -41,7 +52,7 @@ many_events_start(ErlDrvPort port, char *buf)
 }
 
 static void
-from_erlang(ErlDrvData data, char *buf, int count)
+from_erlang(ErlDrvData data, char *buf, ErlDrvSizeT count)
 {
     int i;
     int num;
@@ -87,9 +98,10 @@ static void from_port(ErlDrvData data, ErlDrvEvent ev)
     return;
 }
 
-static int 
-many_events_call(ErlDrvData drv_data, unsigned int command, char *buf, 
-	  int len, char **rbuf, int rlen, unsigned *ret_flags) 
+static ErlDrvSSizeT
+many_events_call(ErlDrvData drv_data, unsigned int command,
+		 char *buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen,
+		 unsigned *ret_flags)
 {
     *rbuf = buf;
     *ret_flags |= DRIVER_CALL_KEEP_BUFFER;
diff --git a/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c b/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c
index c80e492e3f..e7d9a294fa 100644
--- a/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/missing_callback_drv.c
@@ -17,7 +17,7 @@
  */
 
 #ifndef UNIX
-#if !defined(__WIN32__) && !defined(_OSE_) && !defined(VXWORKS)
+#if !defined(__WIN32__) && !defined(VXWORKS)
 #define UNIX 1
 #endif
 #endif
diff --git a/erts/emulator/test/driver_SUITE_data/monitor_drv.c b/erts/emulator/test/driver_SUITE_data/monitor_drv.c
index 1da6a56a72..3da067fd09 100644
--- a/erts/emulator/test/driver_SUITE_data/monitor_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/monitor_drv.c
@@ -21,8 +21,9 @@
 #include "erl_driver.h"
 
 static ErlDrvData monitor_drv_start(ErlDrvPort, char *);
-static int monitor_drv_control(ErlDrvData, unsigned int,
-				char *, int, char **, int);
+static void monitor_drv_stop(ErlDrvData data);
+static ErlDrvSSizeT monitor_drv_control(ErlDrvData, unsigned int,
+					char *, ErlDrvSizeT, char **, ErlDrvSizeT);
 static void handle_monitor(ErlDrvData drv_data, ErlDrvMonitor *monitor);
 
 #define OP_I_AM_IPID 1
@@ -50,7 +51,7 @@ typedef struct {
 static ErlDrvEntry monitor_drv_entry = { 
     NULL /* init */,
     monitor_drv_start,
-    NULL /* stop */,
+    monitor_drv_stop,
     NULL /* output */,
     NULL /* ready_input */,
     NULL /* ready_output */,
@@ -122,16 +123,16 @@ static void handle_monitor(ErlDrvData drv_data, ErlDrvMonitor *monitor)
     return;
 }
 
-static int
+static ErlDrvSSizeT
 monitor_drv_control(ErlDrvData drv_data,
 		     unsigned int command,
-		     char *ibuf, int ilen,
-		     char **rbuf, int rlen)
+		     char *ibuf, ErlDrvSizeT ilen,
+		     char **rbuf, ErlDrvSizeT rlen)
 {
     MyDrvData *data = (MyDrvData *) drv_data;
     char *answer = NULL;
     char buff[64];
-    int alen;
+    ErlDrvSSizeT alen;
 
     switch (command) {
     case OP_I_AM_IPID:
diff --git a/erts/emulator/test/driver_SUITE_data/otp_6879_drv.c b/erts/emulator/test/driver_SUITE_data/otp_6879_drv.c
index 8c0a9aadfd..ff44145ca7 100644
--- a/erts/emulator/test/driver_SUITE_data/otp_6879_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/otp_6879_drv.c
@@ -20,11 +20,11 @@
 #include <string.h>
 #include "erl_driver.h"
 
-static int call(ErlDrvData drv_data,
-		unsigned int command,
-		char *buf, int len,
-		char **rbuf, int rlen,
-		unsigned int *flags);
+static ErlDrvSSizeT call(ErlDrvData drv_data,
+			 unsigned int command,
+			 char *buf, ErlDrvSizeT len,
+			 char **rbuf, ErlDrvSizeT rlen,
+			 unsigned int *flags);
 
 static ErlDrvEntry otp_6879_drv_entry = { 
     NULL /* init */,
@@ -57,11 +57,11 @@ DRIVER_INIT(otp_6879_drv)
 }
 
 
-static int call(ErlDrvData drv_data,
-		unsigned int command,
-		char *buf, int len,
-		char **rbuf, int rlen,
-		unsigned int *flags)
+static ErlDrvSSizeT call(ErlDrvData drv_data,
+			 unsigned int command,
+			 char *buf, ErlDrvSizeT len,
+			 char **rbuf, ErlDrvSizeT rlen,
+			 unsigned int *flags)
 {
     /* echo call */
     if (len > rlen)
diff --git a/erts/emulator/test/driver_SUITE_data/otp_9302_drv.c b/erts/emulator/test/driver_SUITE_data/otp_9302_drv.c
new file mode 100644
index 0000000000..221fd0ce51
--- /dev/null
+++ b/erts/emulator/test/driver_SUITE_data/otp_9302_drv.c
@@ -0,0 +1,232 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+#ifdef __WIN32__
+#include <windows.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include "erl_driver.h"
+
+static void stop(ErlDrvData drv_data);
+static ErlDrvData start(ErlDrvPort port,
+			char *command);
+static void output(ErlDrvData drv_data,
+		   char *buf, ErlDrvSizeT len);
+static void ready_async(ErlDrvData drv_data,
+			ErlDrvThreadData thread_data);
+
+static ErlDrvEntry otp_9302_drv_entry = { 
+    NULL /* init */,
+    start,
+    stop,
+    output,
+    NULL /* ready_input */,
+    NULL /* ready_output */,
+    "otp_9302_drv",
+    NULL /* finish */,
+    NULL /* handle */,
+    NULL /* control */,
+    NULL /* timeout */,
+    NULL /* outputv */,
+    ready_async,
+    NULL /* flush */,
+    NULL /* call */,
+    NULL /* event */,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    ERL_DRV_FLAG_USE_PORT_LOCKING,
+    NULL /* handle2 */,
+    NULL /* handle_monitor */
+};
+
+typedef struct Otp9302AsyncData_ Otp9302AsyncData;
+
+typedef struct {
+    ErlDrvMutex *mtx;
+    Otp9302AsyncData *start;
+    Otp9302AsyncData *end;
+} Otp9302MsgQ;
+
+typedef struct {
+    ErlDrvPort port;
+    int smp;
+    Otp9302MsgQ msgq;
+} Otp9302Data;
+
+struct Otp9302AsyncData_ {
+    Otp9302AsyncData *next;
+    ErlDrvPort port;
+    int smp;
+    int refc;
+    int block;
+    struct {
+	ErlDrvTermData port;
+	ErlDrvTermData receiver;
+	ErlDrvTermData msg;
+    } term_data;
+    Otp9302MsgQ *msgq;
+};
+
+
+DRIVER_INIT(otp_9302_drv)
+{
+    return &otp_9302_drv_entry;
+}
+
+static void stop(ErlDrvData drv_data)
+{
+    Otp9302Data *data = (Otp9302Data *) drv_data;
+    if (!data->smp)
+	erl_drv_mutex_destroy(data->msgq.mtx);
+    driver_free(data);
+}
+
+static ErlDrvData start(ErlDrvPort port,
+			char *command)
+{
+    Otp9302Data *data;
+    ErlDrvSysInfo sys_info;
+
+    data = driver_alloc(sizeof(Otp9302Data));
+    if (!data)
+	return ERL_DRV_ERROR_GENERAL;
+
+    data->port = port;
+
+    driver_system_info(&sys_info, sizeof(ErlDrvSysInfo));
+    data->smp = sys_info.smp_support;
+
+    if (!data->smp) {
+	data->msgq.start = NULL;
+	data->msgq.end = NULL;
+	data->msgq.mtx = erl_drv_mutex_create("");
+	if (!data->msgq.mtx) {
+	    driver_free(data);
+	    return ERL_DRV_ERROR_GENERAL;
+	}
+    }
+
+    return (ErlDrvData) data;
+}
+
+static void send_reply(Otp9302AsyncData *adata)
+{
+    ErlDrvTermData spec[] = {
+	ERL_DRV_PORT, adata->term_data.port,
+	ERL_DRV_ATOM, adata->term_data.msg,
+	ERL_DRV_TUPLE, 2
+    };
+    driver_send_term(adata->port, adata->term_data.receiver,
+		     spec, sizeof(spec)/sizeof(spec[0]));
+}
+
+static void enqueue_reply(Otp9302AsyncData *adata)
+{
+    Otp9302MsgQ *msgq = adata->msgq;
+    adata->next = NULL;
+    adata->refc++;
+    erl_drv_mutex_lock(msgq->mtx);
+    if (msgq->end)
+	msgq->end->next = adata;
+    else
+	msgq->end = msgq->start = adata;
+    msgq->end = adata;
+    erl_drv_mutex_unlock(msgq->mtx);
+}
+
+static void dequeue_replies(Otp9302AsyncData *adata)
+{
+    Otp9302MsgQ *msgq = adata->msgq;
+    erl_drv_mutex_lock(msgq->mtx);
+    if (--adata->refc == 0)
+	driver_free(adata);
+    while (msgq->start) {
+	send_reply(msgq->start);
+	adata = msgq->start;
+	msgq->start = msgq->start->next;
+	if (--adata->refc == 0)
+	    driver_free(adata);
+    }
+    msgq->start = msgq->end = NULL;
+    erl_drv_mutex_unlock(msgq->mtx);
+}
+
+static void async_invoke(void *data)
+{
+    Otp9302AsyncData *adata = (Otp9302AsyncData *) data;
+    if (adata->block) {
+#ifdef __WIN32__
+	Sleep((DWORD) 2000);
+#else
+	sleep(2);
+#endif
+    }
+    if (adata->smp)
+	send_reply(adata);
+    else
+	enqueue_reply(adata);
+}
+
+static void ready_async(ErlDrvData drv_data,
+			ErlDrvThreadData thread_data)
+{
+    Otp9302AsyncData *adata = (Otp9302AsyncData *) thread_data;
+    if (adata->smp)
+	driver_free(adata);
+    else
+	dequeue_replies(adata);
+}
+
+static void output(ErlDrvData drv_data,
+		   char *buf, ErlDrvSizeT len)
+{
+    Otp9302Data *data = (Otp9302Data *) drv_data;
+    ErlDrvTermData td_port = driver_mk_port(data->port);
+    ErlDrvTermData td_receiver = driver_caller(data->port);
+    ErlDrvTermData td_job = driver_mk_atom("job");
+    unsigned int key = (unsigned int) (ErlDrvSInt) data->port;
+    long id[5];
+    Otp9302AsyncData *ad[5];
+    int i;
+
+    for (i = 0; i < sizeof(ad)/sizeof(ad[0]); i++) {
+	ad[i] = driver_alloc(sizeof(Otp9302AsyncData));
+	if (!ad[i])
+	    abort();
+
+	ad[i]->smp = data->smp;
+	ad[i]->port = data->port;
+	ad[i]->block = 0;
+	ad[i]->refc = 1;
+	ad[i]->term_data.port = td_port;
+	ad[i]->term_data.receiver = td_receiver;
+	ad[i]->term_data.msg = td_job;
+	ad[i]->msgq = &data->msgq;
+    }
+    ad[0]->block = 1;
+    ad[0]->term_data.msg = driver_mk_atom("block");
+    ad[2]->term_data.msg = driver_mk_atom("cancel");
+    ad[4]->term_data.msg = driver_mk_atom("end_of_jobs");
+    for (i = 0; i < sizeof(id)/sizeof(id[0]); i++)
+	id[i] = driver_async(data->port, &key, async_invoke, ad[i], driver_free);
+    if (id[2] > 0)
+	driver_async_cancel(id[2]);
+}
diff --git a/erts/emulator/test/driver_SUITE_data/outputv_drv.c b/erts/emulator/test/driver_SUITE_data/outputv_drv.c
index 87f66ae413..3e3d4a3a03 100644
--- a/erts/emulator/test/driver_SUITE_data/outputv_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/outputv_drv.c
@@ -3,7 +3,9 @@
 
 static ErlDrvPort erlang_port;
 static ErlDrvData outputv_start(ErlDrvPort, char*);
-static void outputv_stop(ErlDrvData), outputv_read(ErlDrvData, char*, int), outputv(ErlDrvData, ErlIOVec*);
+static void outputv_stop(ErlDrvData),
+    outputv_read(ErlDrvData, char*, ErlDrvSizeT),
+    outputv(ErlDrvData, ErlIOVec*);
 
 static ErlDrvEntry outputv_driver_entry =
 {
@@ -19,6 +21,16 @@ static ErlDrvEntry outputv_driver_entry =
     NULL,
     NULL,
     outputv,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
     NULL
 };
 
@@ -38,7 +50,7 @@ static ErlDrvData outputv_start(ErlDrvPort port, char *buf)
     return (ErlDrvData)port;
 }
 
-static void outputv_read(ErlDrvData port, char *buf, int count)
+static void outputv_read(ErlDrvData port, char *buf, ErlDrvSizeT count)
 {
     erlang_port = (ErlDrvPort)-1;
 }
diff --git a/erts/emulator/test/driver_SUITE_data/peek_non_existing_queue_drv.c b/erts/emulator/test/driver_SUITE_data/peek_non_existing_queue_drv.c
index f429a5b51e..8e203f74ec 100644
--- a/erts/emulator/test/driver_SUITE_data/peek_non_existing_queue_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/peek_non_existing_queue_drv.c
@@ -28,7 +28,7 @@
  */
 
 #ifndef UNIX
-#if !defined(__WIN32__) && !defined(_OSE_) && !defined(VXWORKS)
+#if !defined(__WIN32__) && !defined(VXWORKS)
 #define UNIX 1
 #endif
 #endif
@@ -66,7 +66,8 @@ typedef struct {
 
 static ErlDrvData start(ErlDrvPort, char *);
 static void stop(ErlDrvData);
-static int control(ErlDrvData, unsigned int, char *, int, char **, int);
+static ErlDrvSSizeT control(ErlDrvData, unsigned int,
+			    char *, ErlDrvSizeT, char **, ErlDrvSizeT);
 static void ready_async(ErlDrvData, ErlDrvThreadData);
 static void async_test(void *);
 static void async_wait(void *);
@@ -121,10 +122,10 @@ static void stop(ErlDrvData drv_data)
     driver_free(drv_data);
 }
 
-static int control(ErlDrvData drv_data,
-		   unsigned int command,
-		   char *buf, int len,
-		   char **rbuf, int rlen)
+static ErlDrvSSizeT control(ErlDrvData drv_data,
+			    unsigned int command,
+			    char *buf, ErlDrvSizeT len,
+			    char **rbuf, ErlDrvSizeT rlen)
 {
     PeekNonXQDrvData *dp = (PeekNonXQDrvData *) drv_data;
     unsigned int key = 0;
@@ -158,7 +159,7 @@ static int control(ErlDrvData drv_data,
     }
 
  done: {
-	int res_len = strlen(res_str);
+	ErlDrvSSizeT res_len = strlen(res_str);
 	if (res_len > rlen) {
 	    char *abuf = driver_alloc(sizeof(char)*res_len);
 	    if (!abuf)
diff --git a/erts/emulator/test/driver_SUITE_data/queue_drv.c b/erts/emulator/test/driver_SUITE_data/queue_drv.c
index ded69f89f9..a02b57dc9a 100644
--- a/erts/emulator/test/driver_SUITE_data/queue_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/queue_drv.c
@@ -1,4 +1,5 @@
 #include <stdio.h>
+#include <string.h>
 #include "erl_driver.h"
 
 #define put_int32(i, s) {((char*)(s))[0] = (char)((i) >> 24) & 0xff; \
@@ -33,9 +34,10 @@
 static ErlDrvPort erlang_port;
 static unsigned opcode;		/* Opcode for next operation. */
 static ErlDrvData queue_start(ErlDrvPort, char*);
-static void queue_stop(ErlDrvData), queue_read(ErlDrvData, char*, int);
+static void queue_stop(ErlDrvData), queue_read(ErlDrvData, char*, ErlDrvSizeT);
 static void queue_outputv(ErlDrvData, ErlIOVec*);
-static int control(ErlDrvData, unsigned int, char*, int, char**, int);
+static ErlDrvSSizeT control(ErlDrvData, unsigned int,
+			    char*, ErlDrvSizeT, char**, ErlDrvSizeT);
 static ErlDrvBinary* read_head(ErlDrvPort, int bytes);
 
 static ErlDrvEntry queue_driver_entry =
@@ -52,6 +54,16 @@ static ErlDrvEntry queue_driver_entry =
     control,
     NULL,
     queue_outputv,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
     NULL
 };
 
@@ -73,7 +85,7 @@ static ErlDrvData queue_start(ErlDrvPort port, char *buf)
 }
 
 /* messages from Erlang */
-static void queue_read(ErlDrvData port, char *buf, int len)
+static void queue_read(ErlDrvData port, char *buf, ErlDrvSizeT len)
 {
 }
 
@@ -82,8 +94,9 @@ static void queue_stop(ErlDrvData port)
     erlang_port = (ErlDrvPort) -1;
 }
 
-static int
-control(ErlDrvData drv_data, unsigned command, char* buf, int len, char** rbuf, int rlen)
+static ErlDrvSSizeT
+control(ErlDrvData drv_data, unsigned command,
+	char* buf, ErlDrvSizeT len, char** rbuf, ErlDrvSizeT rlen)
 {
     ErlDrvBinary* b;
 
diff --git a/erts/emulator/test/driver_SUITE_data/sys_info_1_0_drv.c b/erts/emulator/test/driver_SUITE_data/sys_info_base_drv.c
index 0504778086..c22a415c59 100644
--- a/erts/emulator/test/driver_SUITE_data/sys_info_1_0_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/sys_info_base_drv.c
@@ -3,34 +3,35 @@
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved via the world wide web at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * The Initial Developer of the Original Code is Ericsson Utvecklings AB.
  * Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings
  * AB. All Rights Reserved.''
- * 
+ *
  *     $Id$
  */
 
 /*
  * Author: Rickard Green
  *
- * Description: Driver that fakes driver version 1.0 and tests
+ * Description: Driver that fakes driver version 2.0 and tests
  *              driver_system_info().
  *
  */
 
 #include "sys_info_drv_impl.h"
 
-#define SYS_INFO_DRV_MAJOR_VSN		1
+#define SYS_INFO_DRV_MAJOR_VSN		2
 #define SYS_INFO_DRV_MINOR_VSN		0
-#define SYS_INFO_DRV_NAME_STR		"sys_info_1_0_drv"
-#define SYS_INFO_DRV_NAME		sys_info_1_0_drv
+#define SYS_INFO_DRV_NAME_STR		"sys_info_base_drv"
+#define SYS_INFO_DRV_NAME		sys_info_base_drv
 #define SYS_INFO_DRV_LAST_FIELD		smp_support
+#define ERL_DRV_SYS_INFO_SIZE		sizeof(ErlDrvSysInfo)
 
 #define SYS_INFO_DRV_RES_FORMAT "ok: "	\
   "drv_drv_vsn=%d.%d "			\
@@ -38,8 +39,9 @@
   "erts_vsn=%s "			\
   "otp_vsn=%s "				\
   "thread=%s "				\
-  "smp=%s"
-
+  "smp=%s "				\
+  "async_thrs=%d "			\
+  "sched_thrs=%d"
 
 static size_t
 sys_info_drv_max_res_len(ErlDrvSysInfo *sip)
@@ -51,6 +53,8 @@ sys_info_drv_max_res_len(ErlDrvSysInfo *sip)
     slen += strlen(sip->otp_release) + 1;
     slen += 5;		/* threads */
     slen += 5;		/* smp */
+    slen += 20;		/* async_thrs */
+    slen += 20;		/* sched_thrs */
     return slen;
 }
 
@@ -66,7 +70,9 @@ sys_info_drv_sprintf_sys_info(ErlDrvSysInfo *sip, char *str)
 		   sip->erts_version,
 		   sip->otp_release,
 		   sip->thread_support ? "true" : "false",
-		   sip->smp_support ? "true" : "false");
+		   sip->smp_support ? "true" : "false",
+		   sip->async_threads,
+		   sip->scheduler_threads);
 }
 
 #include "sys_info_drv_impl.c"
diff --git a/erts/emulator/test/driver_SUITE_data/sys_info_drv_impl.c b/erts/emulator/test/driver_SUITE_data/sys_info_drv_impl.c
index 2d3203ae5d..c6c70a2075 100644
--- a/erts/emulator/test/driver_SUITE_data/sys_info_drv_impl.c
+++ b/erts/emulator/test/driver_SUITE_data/sys_info_drv_impl.c
@@ -46,7 +46,8 @@
 #endif
 
 static ErlDrvData start(ErlDrvPort, char *);
-static int control(ErlDrvData, unsigned int, char *, int, char **, int);
+static ErlDrvSSizeT control(ErlDrvData, unsigned int,
+			    char *, ErlDrvSizeT, char **, ErlDrvSizeT);
 
 static ErlDrvEntry drv_entry = { 
     NULL /* init */,
@@ -84,13 +85,13 @@ start(ErlDrvPort port, char *command)
     return (ErlDrvData) port;
 }
 
-static int
+static ErlDrvSSizeT
 control(ErlDrvData drv_data,
 	unsigned int command,
-	char *buf, int len,
-	char **rbuf, int rlen)
+	char *buf, ErlDrvSizeT len,
+	char **rbuf, ErlDrvSizeT rlen)
 {
-    int res;
+    ErlDrvSSizeT res;
     char *str;
     size_t slen, slen2;
     ErlDrvPort port = (ErlDrvPort) drv_data;
diff --git a/erts/emulator/test/driver_SUITE_data/sys_info_1_1_drv.c b/erts/emulator/test/driver_SUITE_data/sys_info_prev_drv.c
index fa21828284..815d96cc97 100644
--- a/erts/emulator/test/driver_SUITE_data/sys_info_1_1_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/sys_info_prev_drv.c
@@ -3,34 +3,35 @@
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved via the world wide web at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * The Initial Developer of the Original Code is Ericsson Utvecklings AB.
  * Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings
  * AB. All Rights Reserved.''
- * 
+ *
  *     $Id$
  */
 
 /*
  * Author: Rickard Green
  *
- * Description: Driver that fakes driver version 1.1 and tests
+ * Description: Driver that fakes driver version 2.0 and tests
  *              driver_system_info().
  *
  */
 
 #include "sys_info_drv_impl.h"
 
-#define SYS_INFO_DRV_MAJOR_VSN		1
-#define SYS_INFO_DRV_MINOR_VSN		1
-#define SYS_INFO_DRV_NAME_STR		"sys_info_1_1_drv"
-#define SYS_INFO_DRV_NAME		sys_info_1_1_drv
+#define SYS_INFO_DRV_MAJOR_VSN		2
+#define SYS_INFO_DRV_MINOR_VSN		0
+#define SYS_INFO_DRV_NAME_STR		"sys_info_prev_drv"
+#define SYS_INFO_DRV_NAME		sys_info_prev_drv
 #define SYS_INFO_DRV_LAST_FIELD		scheduler_threads
+#define ERL_DRV_SYS_INFO_SIZE		sizeof(ErlDrvSysInfo)
 
 #define SYS_INFO_DRV_RES_FORMAT "ok: "	\
   "drv_drv_vsn=%d.%d "			\
@@ -42,7 +43,6 @@
   "async_thrs=%d "			\
   "sched_thrs=%d"
 
-
 static size_t
 sys_info_drv_max_res_len(ErlDrvSysInfo *sip)
 {
@@ -76,5 +76,3 @@ sys_info_drv_sprintf_sys_info(ErlDrvSysInfo *sip, char *str)
 }
 
 #include "sys_info_drv_impl.c"
-
-
diff --git a/erts/emulator/test/driver_SUITE_data/thr_alloc_drv.c b/erts/emulator/test/driver_SUITE_data/thr_alloc_drv.c
index c7edbba7f6..95a6ae9bdf 100644
--- a/erts/emulator/test/driver_SUITE_data/thr_alloc_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/thr_alloc_drv.c
@@ -21,14 +21,8 @@
 #include "erl_driver.h"
 
 ErlDrvData start(ErlDrvPort port, char *command);
-int control(ErlDrvData drv_data, unsigned int command, char *buf,
-	    int len, char **rbuf, int rlen);
-
-static int call(ErlDrvData drv_data,
-		unsigned int command,
-		char *buf, int len,
-		char **rbuf, int rlen,
-		unsigned int *flags);
+ErlDrvSSizeT control(ErlDrvData drv_data, unsigned int command, char *buf,
+		     ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen);
 
 static ErlDrvEntry thr_alloc_drv_entry = { 
     NULL /* init */,
@@ -76,12 +70,12 @@ ErlDrvData start(ErlDrvPort port, char *command)
     return (ErlDrvData) port;
 }
 
-int control(ErlDrvData drv_data, unsigned int command, char *buf,
-	    int len, char **rbuf, int rlen)
+ErlDrvSSizeT control(ErlDrvData drv_data, unsigned int command, char *buf,
+		     ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen)
 {
     ErlDrvPort port = (ErlDrvPort) drv_data;
     char *result = "failure";
-    int result_len;
+    ErlDrvSSizeT result_len;
     if (len <= 20) {
 	int res;
 	ErlDrvTid tid;
diff --git a/erts/emulator/test/driver_SUITE_data/thr_free_drv.c b/erts/emulator/test/driver_SUITE_data/thr_free_drv.c
new file mode 100644
index 0000000000..439fe6a184
--- /dev/null
+++ b/erts/emulator/test/driver_SUITE_data/thr_free_drv.c
@@ -0,0 +1,241 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+#include <stdlib.h>
+#include <errno.h>
+#include <string.h>
+#include "erl_driver.h"
+
+#define BLOCKS_PER_THREAD 100000
+#define NO_THREADS 10
+#define BLOCKS_PER_CTRL 1000
+
+typedef struct {
+    ErlDrvMutex *mtx;
+    ErlDrvCond *cnd;
+    int b;
+    int *go;
+    int *skip;
+    void *blocks[BLOCKS_PER_THREAD];
+} test_thread_data;
+
+typedef struct {
+    ErlDrvPort port;
+    int b;
+    int go;
+    int skip;
+    test_thread_data ttd[NO_THREADS+1];
+    ErlDrvTid tids[NO_THREADS+1];    
+} test_data;
+
+static ErlDrvData start(ErlDrvPort port, char *command);
+static void stop(ErlDrvData data);
+static ErlDrvSSizeT control(ErlDrvData drv_data, unsigned int command, char *buf,
+			    ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen);
+
+static ErlDrvEntry thr_free_drv_entry = { 
+    NULL /* init */,
+    start,
+    stop,
+    NULL /* output */,
+    NULL /* ready_input */,
+    NULL /* ready_output */,
+    "thr_free_drv",
+    NULL /* finish */,
+    NULL /* handle */,
+    control,
+    NULL /* timeout */,
+    NULL /* outputv */,
+    NULL /* ready_async */,
+    NULL /* flush */,
+    NULL /* call */,
+    NULL /* event */,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    ERL_DRV_FLAG_USE_PORT_LOCKING,
+    NULL /* handle2 */,
+    NULL /* handle_monitor */
+};
+
+DRIVER_INIT(thr_free_drv)
+{
+    return &thr_free_drv_entry;
+}
+
+void *
+test_thread(void *vttd)
+{
+    test_thread_data *ttd = (test_thread_data *) vttd;
+    int i, skip;
+
+    erl_drv_mutex_lock(ttd->mtx);
+
+    while (!*ttd->go)
+	erl_drv_cond_wait(ttd->cnd, ttd->mtx);
+    skip = *ttd->skip;
+    erl_drv_mutex_unlock(ttd->mtx);
+
+    if (!skip) {
+	for (i = 0; i < BLOCKS_PER_THREAD; i++)
+	    driver_free(ttd->blocks[i]);
+    }
+    return NULL;
+}
+
+ErlDrvData start(ErlDrvPort port, char *command)
+{
+    int join = 0, t, b, res;
+    test_thread_data *ttd;
+    test_data *td = driver_alloc(sizeof(test_data));
+    if (!td)
+	return ERL_DRV_ERROR_GENERAL;
+    ttd = td->ttd;
+    for (b = 0; b < BLOCKS_PER_THREAD; b++)
+	for (t = 0; t <= NO_THREADS; t++)
+	    ttd[t].blocks[b] = NULL;
+    ttd[0].mtx = NULL;
+    ttd[0].cnd = NULL;
+
+    for (b = 0; b < BLOCKS_PER_THREAD; b++) {
+	for (t = 0; t <= NO_THREADS; t++) {
+	    ttd[t].blocks[b] = driver_alloc(1);
+	    if (ttd[t].blocks[b] == NULL)
+		goto fail;
+	}
+    }
+
+    td->b = -1;
+    td->go = 0;
+    td->skip = 0;
+
+    ttd[0].mtx = erl_drv_mutex_create("test_mutex");
+    if (!ttd[0].mtx)
+	goto fail;
+    ttd[0].cnd = erl_drv_cond_create("test_cnd");
+    if (!ttd[0].cnd)
+	goto fail;
+    ttd[0].go = &td->go;
+    ttd[0].skip = &td->skip;
+
+    for (t = 1; t <= NO_THREADS; t++) {
+	ttd[t].mtx = ttd[0].mtx;
+	ttd[t].cnd = ttd[0].cnd;
+	ttd[t].go = ttd[0].go;
+	ttd[t].skip = ttd[0].skip;
+	res = erl_drv_thread_create("test_thread",
+				    &td->tids[t],
+				    test_thread,
+				    &ttd[t],
+				    NULL);
+	if (res != 0)
+	    goto fail;
+	join = t;
+    }
+
+    td->port = port;
+
+    return (ErlDrvData) td;
+
+fail:
+
+    if (join) {
+	erl_drv_mutex_lock(ttd[0].mtx);
+	td->go = 1;
+	td->skip = 1;
+	erl_drv_cond_broadcast(ttd[0].cnd);
+	erl_drv_mutex_unlock(ttd[0].mtx);
+	for (t = 1; t <= join; t++)
+	    erl_drv_thread_join(td->tids[t], NULL);
+    }
+
+    if (ttd[0].mtx)
+	erl_drv_mutex_destroy(ttd[0].mtx);
+    if (ttd[0].cnd)
+	erl_drv_cond_destroy(ttd[0].cnd);
+
+    for (b = 0; b < BLOCKS_PER_THREAD; b++) {
+	for (t = 0; t <= NO_THREADS; t++) {
+	    if (ttd[t].blocks[b] != NULL)
+		driver_free(ttd[t].blocks[b]);
+	}
+    }
+    driver_free(td);
+    return ERL_DRV_ERROR_GENERAL;
+}
+
+static void stop(ErlDrvData drv_data)
+{
+    test_data *td = (test_data *) drv_data;
+    int t, b;
+    for (t = 1; t <= NO_THREADS; t++)
+	erl_drv_thread_join(td->tids[t], NULL);
+    for (b = 0; b < BLOCKS_PER_THREAD; b++) {
+	if (td->ttd[0].blocks[b])
+	    driver_free(td->ttd[0].blocks[b]);
+    }
+    erl_drv_mutex_destroy(td->ttd[0].mtx);
+    erl_drv_cond_destroy(td->ttd[0].cnd);
+    driver_free(td);
+}
+
+static ErlDrvSSizeT control(ErlDrvData drv_data, unsigned int command, char *buf,
+			    ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen)
+{
+    test_data *td = (test_data *) drv_data;
+    char *result = "failure";
+    int i, b;
+    int res;
+    ErlDrvSSizeT result_len;
+
+    if (td->b == -1) {
+	erl_drv_mutex_lock(td->ttd[0].mtx);
+	td->go = 1;
+	erl_drv_cond_broadcast(td->ttd[0].cnd);
+	erl_drv_mutex_unlock(td->ttd[0].mtx);
+	td->b = 0;
+    }
+
+    for (i = 0, b = td->b; i < BLOCKS_PER_CTRL && b < BLOCKS_PER_THREAD; i++, b++) {
+	driver_free(td->ttd[0].blocks[b]);
+	td->ttd[0].blocks[b] = NULL;
+    }
+
+    td->b = b;
+    if (b >= BLOCKS_PER_THREAD)
+	result = "done";
+    else
+	result = "more";
+
+    result_len = strlen(result);
+    if (result_len <= rlen) {
+	memcpy(*rbuf, result, result_len);
+	return result_len;
+    }
+    else {
+	*rbuf = driver_alloc(result_len);
+	if (!*rbuf) {
+	    driver_failure_posix(td->port, ENOMEM);
+	    return 0;
+	}
+	else {
+	    memcpy(*rbuf, result, result_len);
+	    return result_len;
+	}
+    }
+}
diff --git a/erts/emulator/test/driver_SUITE_data/timer_drv.c b/erts/emulator/test/driver_SUITE_data/timer_drv.c
index b96a95dd4c..8c3f203a64 100644
--- a/erts/emulator/test/driver_SUITE_data/timer_drv.c
+++ b/erts/emulator/test/driver_SUITE_data/timer_drv.c
@@ -22,7 +22,9 @@
 
 static ErlDrvPort erlang_port;
 static ErlDrvData timer_start(ErlDrvPort, char*);
-static void timer_stop(ErlDrvData), timer_read(ErlDrvData, char*, int), timer(ErlDrvData);
+static void timer_stop(ErlDrvData);
+static void timer_read(ErlDrvData, char*, ErlDrvSizeT);
+static void timer(ErlDrvData);
 
 static ErlDrvEntry timer_driver_entry =
 {
@@ -38,6 +40,16 @@ static ErlDrvEntry timer_driver_entry =
     NULL,
     timer,
     NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
     NULL
 };
 
@@ -57,8 +69,9 @@ static ErlDrvData timer_start(ErlDrvPort port, char *buf)
 }
 
 /* set the timer, this is monitored from erlang measuring the time */
-static void timer_read(ErlDrvData port, char *buf, int len)
+static void timer_read(ErlDrvData p, char *buf, ErlDrvSizeT len)
 {
+    ErlDrvPort port = (ErlDrvPort) p;
     char reply[1];
 
     if (buf[0] == START_TIMER) { 
diff --git a/erts/emulator/test/efile_SUITE.erl b/erts/emulator/test/efile_SUITE.erl
index 1d66b6ef70..9ac004200e 100644
--- a/erts/emulator/test/efile_SUITE.erl
+++ b/erts/emulator/test/efile_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -17,12 +17,32 @@
 %% %CopyrightEnd%
 
 -module(efile_SUITE).
--export([all/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 -export([iter_max_files/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [iter_max_files].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) -> [iter_max_files].
 
 %%
 %% Open as many files as possible. Do this several times and check 
diff --git a/erts/emulator/test/emulator.spec b/erts/emulator/test/emulator.spec
index ed5bd48e84..7a6dd83020 100644
--- a/erts/emulator/test/emulator.spec
+++ b/erts/emulator/test/emulator.spec
@@ -1 +1,2 @@
-{topcase, {dir, "../emulator_test"}}.
+{enable_builtin_hooks, false}.
+{suites,"../emulator_test",all}.
diff --git a/erts/emulator/test/erl_drv_thread_SUITE.erl b/erts/emulator/test/erl_drv_thread_SUITE.erl
index ea618e9feb..84a82cced0 100644
--- a/erts/emulator/test/erl_drv_thread_SUITE.erl
+++ b/erts/emulator/test/erl_drv_thread_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2007-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2007-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,18 +19,36 @@
 
 -module(erl_drv_thread_SUITE).
 -author('[email protected]').
--export([all/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 
 -export([basic/1, rwlock/1, tsd/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -define(DEFAULT_TIMETRAP_SECS, 240).
 
-all(doc) -> [];
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     [basic, rwlock, tsd].
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%                                                                        %%
 %% Testcases                                                              %%
diff --git a/erts/emulator/test/erl_drv_thread_SUITE_data/testcase_driver.c b/erts/emulator/test/erl_drv_thread_SUITE_data/testcase_driver.c
index 1e98844838..b4542f3e36 100644
--- a/erts/emulator/test/erl_drv_thread_SUITE_data/testcase_driver.c
+++ b/erts/emulator/test/erl_drv_thread_SUITE_data/testcase_driver.c
@@ -50,13 +50,33 @@ typedef struct {
 
 ErlDrvData testcase_drv_start(ErlDrvPort port, char *command);
 void testcase_drv_stop(ErlDrvData drv_data);
-void testcase_drv_run(ErlDrvData drv_data, char *buf, int len);
+void testcase_drv_run(ErlDrvData drv_data, char *buf, ErlDrvSizeT len);
 
 static ErlDrvEntry testcase_drv_entry = { 
     NULL,
     testcase_drv_start,
     testcase_drv_stop,
-    testcase_drv_run
+    testcase_drv_run,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL, /* handle */
+    NULL, /* control */
+    NULL, /* timeout */
+    NULL, /* outputv */
+    NULL, /* ready_async */
+    NULL,
+    NULL,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
+    NULL,
+
 };
 
 
@@ -92,7 +112,7 @@ testcase_drv_stop(ErlDrvData drv_data)
 }
 
 void
-testcase_drv_run(ErlDrvData drv_data, char *buf, int len)
+testcase_drv_run(ErlDrvData drv_data, char *buf, ErlDrvSizeT len)
 {
     InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) drv_data;
     ErlDrvTermData result_atom;
diff --git a/erts/emulator/test/erl_link_SUITE.erl b/erts/emulator/test/erl_link_SUITE.erl
index 542c8dffbe..435c0872e6 100644
--- a/erts/emulator/test/erl_link_SUITE.erl
+++ b/erts/emulator/test/erl_link_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2001-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2001-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -28,9 +28,10 @@
 -author('[email protected]').
 
 %-define(line_trace, 1).
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 
 % Test cases
 -export([links/1,
@@ -46,7 +47,7 @@
 	 otp_5772_dist_monitor/1,
 	 otp_7946/1]).
 
--export([init_per_testcase/2, fin_per_testcase/2, end_per_suite/1]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
 % Internal exports
 -export([test_proc/0]).
@@ -77,11 +78,29 @@
 
 
 
-all(suite) -> [links, dist_links, monitor_nodes, process_monitors,
-	       dist_process_monitors, busy_dist_port_monitor,
-	       busy_dist_port_link, otp_5772_link, otp_5772_dist_link,
-	       otp_5772_monitor, otp_5772_dist_monitor,
-	       otp_7946].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [links, dist_links, monitor_nodes, process_monitors,
+     dist_process_monitors, busy_dist_port_monitor,
+     busy_dist_port_link, otp_5772_link, otp_5772_dist_link,
+     otp_5772_monitor, otp_5772_dist_monitor, otp_7946].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    catch erts_debug:set_internal_state(available_internal_state, false).
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 links(doc) -> ["Tests node local links"];
 links(suite) -> [];
@@ -678,13 +697,10 @@ init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     end,
     ?line [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     ?line Dog = ?config(watchdog, Config),
     ?line ?t:timetrap_cancel(Dog).
 
-end_per_suite(_Config) ->
-    catch erts_debug:set_internal_state(available_internal_state, false).
-
 tp_call(Tp, Fun) ->
     ?line R = make_ref(),
     ?line Tp ! {call, self(), R, Fun},
@@ -1050,7 +1066,6 @@ stop_node(Node) ->
 -define(DOP_SEND,		2).
 -define(DOP_EXIT,		3).
 -define(DOP_UNLINK,		4).
--define(DOP_NODE_LINK,		5).
 -define(DOP_REG_SEND,		6).
 -define(DOP_GROUP_LEADER,	7).
 -define(DOP_EXIT2,		8).
diff --git a/erts/emulator/test/erts_debug_SUITE.erl b/erts/emulator/test/erts_debug_SUITE.erl
index e60a999df1..87778dd0c2 100644
--- a/erts/emulator/test/erts_debug_SUITE.erl
+++ b/erts/emulator/test/erts_debug_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2005-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2012. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,32 +18,95 @@
 %%
 
 -module(erts_debug_SUITE).
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
-	 flat_size/1,flat_size_big/1,df/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
+	 test_size/1,flat_size_big/1,df/1,
+	 instructions/1]).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [test_size, flat_size_big, df, instructions].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [flat_size,flat_size_big,df].
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?t:minutes(2)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
-flat_size(Config) when is_list(Config) ->
-    0 = erts_debug:flat_size([]),
-    0 = erts_debug:flat_size(42),
-    2 = erts_debug:flat_size([a|b]),
-    1 = erts_debug:flat_size({}),
-    2 = erts_debug:flat_size({[]}),
-    3 = erts_debug:flat_size({a,b}),
-    7 = erts_debug:flat_size({a,[b,c]}),
+test_size(Config) when is_list(Config) ->
+    ConsCell1 = id([a|b]),
+    ConsCell2 = id(ConsCell1),
+    ConsCellSz = 2,
+
+    0 = do_test_size([]),
+    0 = do_test_size(42),
+    ConsCellSz = do_test_size(ConsCell1),
+    1 = do_test_size({}),
+    2 = do_test_size({[]}),
+    3 = do_test_size({a,b}),
+    7 = do_test_size({a,[b,c]}),
+
+    %% Test internal consistency of sizes, but without testing
+    %% exact sizes.
+    Const = id(42),
+    AnotherConst = id(7),
+
+    %% Fun environment size = 0 (the smallest fun possible)
+    SimplestFun = fun() -> ok end,
+    FunSz0 = do_test_size(SimplestFun),
+
+    %% Fun environment size = 1
+    FunSz1 = do_test_size(fun() -> Const end),
+    FunSz1 = FunSz0 + 1,
+
+    %% Fun environment size = 2
+    FunSz2 = do_test_size(fun() -> Const+AnotherConst end),
+    FunSz2 = FunSz1 + 1,
+
+    FunSz1 = do_test_size(fun() -> ConsCell1 end) - do_test_size(ConsCell1),
+
+    %% Test shared data structures.
+    do_test_size([ConsCell1|ConsCell1],
+		 3*ConsCellSz,
+		 2*ConsCellSz),
+    do_test_size(fun() -> {ConsCell1,ConsCell2} end,
+		 FunSz2 + 2*ConsCellSz,
+		 FunSz2 + ConsCellSz),
+    do_test_size({SimplestFun,SimplestFun},
+		 2*FunSz0+do_test_size({a,b}),
+		 FunSz0+do_test_size({a,b})),
     ok.
 
+do_test_size(Term) ->
+    Sz = erts_debug:flat_size(Term),
+    Sz = erts_debug:size(Term).
+
+do_test_size(Term, FlatSz, Sz) ->
+    FlatSz = erts_debug:flat_size(Term),
+    Sz = erts_debug:size(Term).
+
 flat_size_big(Config) when is_list(Config) ->
     %% Build a term whose external size only fits in a big num (on 32-bit CPU).
     flat_size_big_1(16#11111111111111117777777777777777888889999, 0, 16#FFFFFFF).
@@ -70,3 +133,11 @@ df(Config) when is_list(Config) ->
 
 pps() ->
     {erlang:ports()}.
+
+instructions(Config) when is_list(Config) ->
+    ?line Is = erts_debug:instructions(),
+    ?line _ = [list_to_atom(I) || I <- Is],
+    ok.
+
+id(I) ->
+    I.
diff --git a/erts/emulator/test/estone_SUITE.erl b/erts/emulator/test/estone_SUITE.erl
index 7fb92faf0d..2417d4bcfe 100644
--- a/erts/emulator/test/estone_SUITE.erl
+++ b/erts/emulator/test/estone_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2002-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2002-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,8 +18,9 @@
 
 -module(estone_SUITE).
 %% Test functions
--export([all/1,estone/1]).
--export([init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,estone/1]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
 %% Internal exports for EStone tests
 -export([lists/1,
@@ -30,7 +31,7 @@
 	 trav/1,
 	 port_io/1,
 	 large_dataset_work/1,
-	 large_local_dataset_work/1,mk_big_procs/1,big_proc/0,
+	 large_local_dataset_work/1,mk_big_procs/1,big_proc/0, very_big/1,
 	 alloc/1,
 	 bif_dispatch/1,
 	 binary_h/1,echo/1,
@@ -44,7 +45,7 @@
 	 run_micro/3,p1/1,ppp/3,macro/2,micros/0]).
 
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %% Test suite defines
 -define(default_timeout, ?t:minutes(10)).
@@ -68,12 +69,31 @@
 init_per_testcase(_Case, Config) ->
     ?line Dog=test_server:timetrap(?default_timeout),
     [{watchdog, Dog}|Config].
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
 
-all(suite) -> [estone].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [estone].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 estone(suite) ->
     [];
diff --git a/erts/emulator/test/evil_SUITE.erl b/erts/emulator/test/evil_SUITE.erl
index a8288584f4..f982b9d4ff 100644
--- a/erts/emulator/test/evil_SUITE.erl
+++ b/erts/emulator/test/evil_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2002-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2002-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,7 +18,9 @@
 
 -module(evil_SUITE).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 heap_frag/1,
 	 encode_decode_ext/1,
 	 decode_integer_ext/1,
@@ -30,26 +32,37 @@
 	 decode_pos_neg_zero/1
 	]).
 
--include("test_server.hrl").
-
-all(suite) ->
-    [
-     heap_frag,
-     encode_decode_ext,
-     decode_integer_ext,
-     decode_small_big_ext,
-     decode_large_big_ext,
-     decode_small_big_ext_neg,
-     decode_large_big_ext_neg,
-     decode_too_small,
-     decode_pos_neg_zero
-    ].
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [heap_frag, encode_decode_ext, decode_integer_ext,
+     decode_small_big_ext, decode_large_big_ext,
+     decode_small_big_ext_neg, decode_large_big_ext_neg,
+     decode_too_small, decode_pos_neg_zero].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(_Case, Config) ->
     ?line Dog = test_server:timetrap(?t:minutes(0.5)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog=?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
diff --git a/erts/emulator/test/exception_SUITE.erl b/erts/emulator/test/exception_SUITE.erl
index f1e6e004ad..109cec25cb 100644
--- a/erts/emulator/test/exception_SUITE.erl
+++ b/erts/emulator/test/exception_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,19 +19,40 @@
 
 -module(exception_SUITE).
 
--export([all/1, badmatch/1, pending_errors/1, nil_arith/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 badmatch/1, pending_errors/1, nil_arith/1,
          stacktrace/1, nested_stacktrace/1, raise/1, gunilla/1, per/1,
-	 exception_with_heap_frag/1]).
+	 exception_with_heap_frag/1, line_numbers/1]).
 
 -export([bad_guy/2]).
+-export([crash/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 -import(lists, [foreach/2]).
 
-all(suite) ->
-    [badmatch, pending_errors, nil_arith, 
-     stacktrace, nested_stacktrace, raise, gunilla, per,
-     exception_with_heap_frag].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [badmatch, pending_errors, nil_arith, stacktrace,
+     nested_stacktrace, raise, gunilla, per,
+     exception_with_heap_frag, line_numbers].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 -define(try_match(E),
 	catch ?MODULE:bar(),
@@ -121,14 +142,20 @@ pending_exit_message(Args, Expected) ->
     end,
     process_flag(trap_exit, false).
 
-pending({badarg, [{erlang,Bif,BifArgs},{?MODULE,Func,Arity}|_]}, Func, Args, _Code)
-  when is_atom(Bif), is_list(BifArgs), length(Args) == Arity ->
+pending({badarg,[{erlang,Bif,BifArgs,Loc1},
+		 {?MODULE,Func,Arity,Loc2}|_]},
+	Func, Args, _Code)
+  when is_atom(Bif), is_list(BifArgs), length(Args) =:= Arity,
+       is_list(Loc1), is_list(Loc2) ->
     ok;
-pending({undef,[{non_existing_module,foo,[]}|_]}, _, _, _) ->
+pending({undef,[{non_existing_module,foo,[],Loc}|_]}, _, _, _)
+  when is_list(Loc) ->
     ok;
-pending({function_clause,[{?MODULE,Func,Args}|_]}, Func, Args, _Code) ->
+pending({function_clause,[{?MODULE,Func,Args,Loc}|_]}, Func, Args, _Code)
+  when is_list(Loc) ->
     ok;
-pending({Code,[{?MODULE,Func,Arity}|_]}, Func, Args, Code) when length(Args) == Arity ->
+pending({Code,[{?MODULE,Func,Arity,Loc}|_]}, Func, Args, Code)
+  when length(Args) =:= Arity, is_list(Loc) ->
     ok;
 pending(Reason, _Function, _Args, _Code) ->
     test_server:fail({bad_exit_reason,Reason}).
@@ -235,27 +262,36 @@ stacktrace(Conf) when is_list(Conf) ->
     ?line {_,Mref} = spawn_monitor(fun() -> exit({Tag,erlang:get_stacktrace()}) end),
     ?line {Tag,[]} = receive {'DOWN',Mref,_,_,Info} -> Info end,
     V = [make_ref()|self()],
-    ?line {value2,{caught1,badarg,[{erlang,abs,[V]}|_]=St1}} =
+    ?line {value2,{caught1,badarg,[{erlang,abs,[V],_}|_]=St1}} =
 	stacktrace_1({'abs',V}, error, {value,V}),
     ?line St1 = erase(stacktrace1),
     ?line St1 = erase(stacktrace2),
     ?line St1 = erlang:get_stacktrace(),
-    ?line {caught2,{error,badarith},[{?MODULE,my_add,2}|_]=St2} =
+    ?line {caught2,{error,badarith},[{?MODULE,my_add,2,_}|_]=St2} =
 	stacktrace_1({'div',{1,0}}, error, {'add',{0,a}}),
-    ?line [{?MODULE,my_div,2}|_] = erase(stacktrace1),
+    ?line [{?MODULE,my_div,2,_}|_] = erase(stacktrace1),
     ?line St2 = erase(stacktrace2),
     ?line St2 = erlang:get_stacktrace(),
-    ?line {caught2,{error,{try_clause,V}},[{?MODULE,stacktrace_1,3}|_]=St3} =
+    ?line {caught2,{error,{try_clause,V}},[{?MODULE,stacktrace_1,3,_}|_]=St3} =
 	stacktrace_1({value,V}, error, {value,V}),
     ?line St3 = erase(stacktrace1),
     ?line St3 = erase(stacktrace2),
     ?line St3 = erlang:get_stacktrace(),
-    ?line {caught2,{throw,V},[{?MODULE,foo,1}|_]=St4} =
+    ?line {caught2,{throw,V},[{?MODULE,foo,1,_}|_]=St4} =
 	stacktrace_1({value,V}, error, {throw,V}),
-    ?line [{?MODULE,stacktrace_1,3}|_] = erase(stacktrace1),
+    ?line [{?MODULE,stacktrace_1,3,_}|_] = erase(stacktrace1),
     ?line St4 = erase(stacktrace2),
     ?line St4 = erlang:get_stacktrace(),
-    ok.
+
+    try
+	?line stacktrace_2()
+    catch
+	error:{badmatch,_} ->
+	    [{?MODULE,stacktrace_2,0,_},
+	     {?MODULE,stacktrace,1,_}|_] =
+		erlang:get_stacktrace(),
+	    ok
+    end.
 
 stacktrace_1(X, C1, Y) ->
     erase(stacktrace1),
@@ -275,6 +311,9 @@ stacktrace_1(X, C1, Y) ->
         put(stacktrace2, erlang:get_stacktrace())
     end.
 
+stacktrace_2() ->
+    ok = erlang:process_info(self(), current_function),
+    ok.
 
 
 nested_stacktrace(Conf) when is_list(Conf) ->
@@ -283,15 +322,15 @@ nested_stacktrace(Conf) when is_list(Conf) ->
 	nested_stacktrace_1({{value,{V,x1}},void,{V,x1}},
 			    {void,void,void}),
     ?line {caught1,
-	   [{?MODULE,my_add,2}|_],
+	   [{?MODULE,my_add,2,_}|_],
 	   value2,
-	   [{?MODULE,my_add,2}|_]} =
+	   [{?MODULE,my_add,2,_}|_]} =
 	nested_stacktrace_1({{'add',{V,x1}},error,badarith},
 			    {{value,{V,x2}},void,{V,x2}}),
     ?line {caught1,
-	   [{?MODULE,my_add,2}|_],
-	   {caught2,[{erlang,abs,[V]}|_]},
-	   [{erlang,abs,[V]}|_]} =
+	   [{?MODULE,my_add,2,_}|_],
+	   {caught2,[{erlang,abs,[V],_}|_]},
+	   [{erlang,abs,[V],_}|_]} =
 	nested_stacktrace_1({{'add',{V,x1}},error,badarith},
 			    {{'abs',V},error,badarg}),
     ok.
@@ -330,14 +369,14 @@ raise(Conf) when is_list(Conf) ->
 	end,
     ?line A = erlang:get_stacktrace(),
     ?line A = get(raise),
-    ?line [{?MODULE,my_div,2}|_] = A,
+    ?line [{?MODULE,my_div,2,_}|_] = A,
     %%
     N = 8, % Must be even
     ?line N = erlang:system_flag(backtrace_depth, N),
+    ?line B = odd_even(N, []),
     ?line try even(N) 
 	  catch error:function_clause -> ok
 	  end,
-    ?line B = odd_even(N, []),
     ?line B = erlang:get_stacktrace(),
     %%
     ?line C0 = odd_even(N+1, []),
@@ -355,19 +394,12 @@ raise(Conf) when is_list(Conf) ->
 odd_even(N, R) when is_integer(N), N > 1 ->
     odd_even(N-1, 
 	     [if (N rem 2) == 0 ->
-		      {?MODULE,even,1};
+		      {?MODULE,even,1,[{file,"odd_even.erl"},{line,3}]};
 		 true ->
-		      {?MODULE,odd,1}
+		      {?MODULE,odd,1,[{file,"odd_even.erl"},{line,6}]}
 	      end|R]);
 odd_even(1, R) ->
-    [{?MODULE,odd,[1]}|R].
-
-even(N) when is_integer(N), N > 1, (N rem 2) == 0 ->
-    odd(N-1)++[N].
-
-odd(N) when is_integer(N), N > 1, (N rem 2) == 1 ->
-    even(N-1)++[N].
-    
+    [{?MODULE,odd,[1],[{file,"odd_even.erl"},{line,5}]}|R].
 
 foo({value,Value}) -> Value;
 foo({'div',{A,B}}) ->
@@ -494,4 +526,186 @@ do_exception_with_heap_frag(Bin, [Sz|Sizes]) ->
     do_exception_with_heap_frag(Bin, Sizes);
 do_exception_with_heap_frag(_, []) -> ok.
 
+line_numbers(Config) when is_list(Config) ->
+    {'EXIT',{{case_clause,bad_tag},
+	     [{?MODULE,line1,2,
+	       [{file,"fake_file.erl"},{line,3}]},
+	      {?MODULE,line_numbers,1,_}|_]}} =
+	(catch line1(bad_tag, 0)),
+    {'EXIT',{badarith,
+	     [{?MODULE,line1,2,
+	       [{file,"fake_file.erl"},{line,5}]},
+	      {?MODULE,line_numbers,1,_}|_]}} =
+	(catch line1(a, not_an_integer)),
+    {'EXIT',{{badmatch,{ok,1}},
+	     [{?MODULE,line1,2,
+	       [{file,"fake_file.erl"},{line,7}]},
+	      {?MODULE,line_numbers,1,_}|_]}} =
+	(catch line1(a, 0)),
+    {'EXIT',{crash,
+	     [{?MODULE,crash,1,
+	       [{file,"fake_file.erl"},{line,14}]},
+	      {?MODULE,line_numbers,1,_}|_]}} =
+	(catch line1(a, 41)),
+
+    ModFile = ?MODULE_STRING++".erl",
+    [{?MODULE,maybe_crash,1,[{file,"call.erl"},{line,28}]},
+     {?MODULE,call1,0,[{file,"call.erl"},{line,14}]},
+     {?MODULE,close_calls,1,[{file,"call.erl"},{line,5}]},
+     {?MODULE,line_numbers,1,[{file,ModFile},{line,_}]}|_] =
+	close_calls(call1),
+    [{?MODULE,maybe_crash,1,[{file,"call.erl"},{line,28}]},
+     {?MODULE,call2,0,[{file,"call.erl"},{line,18}]},
+     {?MODULE,close_calls,1,[{file,"call.erl"},{line,6}]},
+     {?MODULE,line_numbers,1,[{file,ModFile},{line,_}]}|_] =
+	close_calls(call2),
+    [{?MODULE,maybe_crash,1,[{file,"call.erl"},{line,28}]},
+     {?MODULE,call3,0,[{file,"call.erl"},{line,22}]},
+     {?MODULE,close_calls,1,[{file,"call.erl"},{line,7}]},
+     {?MODULE,line_numbers,1,[{file,ModFile},{line,_}]}|_] =
+	close_calls(call3),
+    no_crash = close_calls(other),
+
+    <<0,0>> = build_binary1(16),
+    {'EXIT',{badarg,
+	     [{?MODULE,build_binary1,1,
+	       [{file,"bit_syntax.erl"},{line,72503}]},
+	      {?MODULE,line_numbers,1,
+	       [{file,ModFile},{line,_}]}|_]}} =
+	(catch build_binary1(bad_size)),
+
+    <<7,1,2,3>> = build_binary2(8, <<1,2,3>>),
+    {'EXIT',{badarg,
+	     [{?MODULE,build_binary2,2,
+	       [{file,"bit_syntax.erl"},{line,72507}]},
+	      {?MODULE,line_numbers,1,
+	       [{file,ModFile},{line,_}]}|_]}} =
+	(catch build_binary2(bad_size, <<>>)),
+    {'EXIT',{badarg,
+	     [{erlang,bit_size,[bad_binary],[]},
+	      {?MODULE,build_binary2,2,
+	       [{file,"bit_syntax.erl"},{line,72507}]},
+	      {?MODULE,line_numbers,1,
+	       [{file,ModFile},{line,_}]}|_]}} =
+	(catch build_binary2(8, bad_binary)),
+
+    {'EXIT',{function_clause,
+	     [{?MODULE,do_call_abs,[y,y],
+	       [{file,"gc_bif.erl"},{line,18}]},
+	      {?MODULE,line_numbers,1,_}|_]}} =
+	(catch do_call_abs(y, y)),
+    {'EXIT',{badarg,
+	     [{erlang,abs,[[]],[]},
+	      {?MODULE,do_call_abs,2,
+	       [{file,"gc_bif.erl"},{line,19}]},
+	      {?MODULE,line_numbers,1,_}|_]}} =
+	(catch do_call_abs(x, [])),
+
+    {'EXIT',{{badmatch,"42"},
+	     [{MODULE,applied_bif_1,1,[{file,"applied_bif.erl"},{line,5}]},
+	      {?MODULE,line_numbers,1,_}|_]}} =
+	(catch applied_bif_1(42)),
+
+    {'EXIT',{{badmatch,{current_location,
+			{?MODULE,applied_bif_2,0,
+			 [{file,"applied_bif.erl"},{line,9}]}}},
+	     [{MODULE,applied_bif_2,0,[{file,"applied_bif.erl"},{line,10}]},
+	      {?MODULE,line_numbers,1,_}|_]}} =
+	(catch applied_bif_2()),
+
+    ok.
+
 id(I) -> I.
+
+-file("odd_even.erl", 1).			%Line 1
+even(N) when is_integer(N), N > 1, (N rem 2) == 0 ->
+    odd(N-1)++[N].				%Line 3
+
+odd(N) when is_integer(N), N > 1, (N rem 2) == 1 ->
+    even(N-1)++[N].				%Line 6
+
+%%
+%% If the compiler removes redundant line instructions (any
+%% line instruction with the same location as the previous),
+%% and the loader also removes line instructions before
+%% tail-recursive calls to external functions, then the
+%% badmatch exception in line 7 below will be reported as
+%% occurring in line 6.
+%%
+%% That means that any removal of redundant line instructions
+%% must all be done in the compiler OR in the loader.
+%%
+-file("fake_file.erl", 1).			%Line 1
+line1(Tag, X) ->				%Line 2
+    case Tag of					%Line 3
+	a ->
+	    Y = X + 1,				%Line 5
+	    Res = id({ok,Y}),			%Line 6
+	    ?MODULE:crash({ok,42} = Res);	%Line 7
+	b ->
+	    x = id(x),				%Line 9
+	    ok					%Line 10
+    end.					%Line 11
+
+crash(_) ->					%Line 13
+    erlang:error(crash).			%Line 14
+
+-file("call.erl", 1).				%Line 1
+close_calls(Where) ->				%Line 2
+    put(where_to_crash, Where),			%Line 3
+    try
+	call1(),				%Line 5
+	call2(),				%Line 6
+	call3(),				%Line 7
+	no_crash				%Line 8
+    catch error:crash ->
+	    erlang:get_stacktrace()		%Line 10
+    end.					%Line 11
+
+call1() ->					%Line 13
+    maybe_crash(call1),				%Line 14
+    ok.						%Line 15
+
+call2() ->					%Line 17
+    maybe_crash(call2),				%Line 18
+    ok.						%Line 19
+
+call3() ->					%Line 21
+    maybe_crash(call3),				%Line 22
+    ok.						%Line 23
+
+maybe_crash(Name) ->				%Line 25
+    case get(where_to_crash) of			%Line 26
+	Name ->
+	    erlang:error(crash);		%Line 28
+	_ ->
+	    ok					%Line 30
+    end.
+
+-file("bit_syntax.erl", 72500).			%Line 72500
+build_binary1(Size) ->				%Line 72501
+    id(42),					%Line 72502
+    <<0:Size>>.					%Line 72503
+
+build_binary2(Size, Bin) ->			%Line 72505
+    id(0),					%Line 72506
+    <<7:Size,Bin/binary>>.			%Line 72507
+
+-file("gc_bif.erl", 17).
+do_call_abs(x, Arg) ->				%Line 18
+    abs(Arg).					%Line 19
+
+%% Make sure a BIF that is applied does not leave the p->cp
+%% set (and thus generating an extra entry on the stack).
+
+-file("applied_bif.erl", 1).
+%% Explicit apply.
+applied_bif_1(I) ->				%Line 3
+    L = apply(erlang, integer_to_list, [I]),	%Line 4
+    fail = L,					%Line 5
+    ok.						%Line 6
+%% Implicit apply.
+applied_bif_2() ->				%Line 8
+    R = process_info(self(), current_location),	%Line 9
+    fail = R,					%Line 10
+    ok.						%Line 11
diff --git a/erts/emulator/test/float_SUITE.erl b/erts/emulator/test/float_SUITE.erl
index 102e472ea6..8e6923ce9f 100644
--- a/erts/emulator/test/float_SUITE.erl
+++ b/erts/emulator/test/float_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,21 +19,64 @@
 
 -module(float_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
+	 fpe/1,fp_drv/1,fp_drv_thread/1,denormalized/1,match/1,
+	 bad_float_unpack/1,cmp_zero/1, cmp_integer/1, cmp_bignum/1]).
+-export([otp_7178/1]).
+-export([hidden_inf/1]).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
-	 fpe/1,fp_drv/1,fp_drv_thread/1,denormalized/1,match/1,bad_float_unpack/1]).
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog = ?t:timetrap(?t:minutes(3)),
     [{watchdog, Dog},{testcase,Func}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
-all(suite) ->
-    [fpe,fp_drv,fp_drv_thread,denormalized,match,bad_float_unpack].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [fpe, fp_drv, fp_drv_thread, otp_7178, denormalized,
+     match, bad_float_unpack, {group, comparison}
+     ,hidden_inf
+    ].
+
+groups() -> 
+    [{comparison, [parallel], [cmp_zero, cmp_integer, cmp_bignum]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
+%%
+%% OTP-7178, list_to_float on very small numbers should give 0.0
+%% instead of exception, i.e. ignore underflow.
+%%
+otp_7178(suite) ->
+    [];
+otp_7178(doc) ->
+    ["test that list_to_float on very small numbers give 0.0"];
+otp_7178(Config) when is_list(Config) ->
+    ?line X = list_to_float("1.0e-325"),
+    ?line true = (X < 0.00000001) and (X > -0.00000001),
+    ?line Y = list_to_float("1.0e-325325325"),
+    ?line true = (Y < 0.00000001) and (Y > -0.00000001),
+    ?line {'EXIT', {badarg,_}} = (catch list_to_float("1.0e83291083210")),
+    ok.
 
 %% Forces floating point exceptions and tests that subsequent, legal,
 %% operations are calculated correctly.  Original version by Sebastian
@@ -147,6 +190,101 @@ bad_float_unpack(Config) when is_list(Config) ->
 bad_float_unpack_match(<<F:64/float>>) -> F;
 bad_float_unpack_match(<<I:64/integer-signed>>) -> I.
 
+cmp_zero(_Config) ->
+    cmp(0.5e-323,0).
+
+cmp_integer(_Config) ->
+    Axis = (1 bsl 53)-2.0, %% The point where floating points become unprecise
+    span_cmp(Axis,2,200),
+    cmp(Axis*Axis,round(Axis)).
+
+cmp_bignum(_Config) ->
+    span_cmp((1 bsl 58) - 1.0),%% Smallest bignum float
+
+    %% Test when the big num goes from I to I+1 in size
+    [span_cmp((1 bsl (32*I)) - 1.0) || I <- lists:seq(2,30)],
+
+    %% Test bignum greater then largest float
+    cmp((1 bsl (64*16)) - 1, (1 bsl (64*15)) * 1.0),
+    %% Test when num is much larger then float
+    [cmp((1 bsl (32*I)) - 1, (1 bsl (32*(I-2))) * 1.0) || I <- lists:seq(3,30)],
+    %% Test when float is much larger than num
+    [cmp((1 bsl (64*15)) * 1.0, (1 bsl (32*(I)))) || I <- lists:seq(1,29)],
+
+    %% Test that all int == float works as they should
+    [true = 1 bsl N == (1 bsl N)*1.0 || N <- lists:seq(0, 1023)],
+    [true = (1 bsl N)*-1 == (1 bsl N)*-1.0 || N <- lists:seq(0, 1023)].
+
+span_cmp(Axis) ->
+    span_cmp(Axis, 25).
+span_cmp(Axis, Length) ->
+    span_cmp(Axis, round(Axis) bsr 52, Length).
+span_cmp(Axis, Incr, Length) ->
+    [span_cmp(Axis, Incr, Length, 1 bsl (1 bsl I)) || I <- lists:seq(0,6)].
+%% This function creates tests around number axis. Both <, > and == is tested
+%% for both negative and positive numbers.
+%%
+%% Axis: The number around which to do the tests eg. (1 bsl 58) - 1.0
+%% Incr: How much to increment the test numbers inbetween each test.
+%% Length: Length/2 is the number of Incr away from Axis to test on the
+%%         negative and positive plane.
+%% Diff: How much the float and int should differ when comparing
+span_cmp(Axis, Incr, Length, Diff) ->
+    [begin
+	 cmp(round(Axis*-1.0)+Diff+I*Incr,Axis*-1.0+I*Incr),
+	 cmp(Axis*-1.0+I*Incr,round(Axis*-1.0)-Diff+I*Incr)
+     end || I <- lists:seq((Length div 2)*-1,(Length div 2))],
+    [begin
+	 cmp(round(Axis)+Diff+I*Incr,Axis+I*Incr),
+	 cmp(Axis+I*Incr,round(Axis)-Diff+I*Incr)
+     end || I <- lists:seq((Length div 2)*-1,(Length div 2))].
+
+cmp(Big,Small) when is_float(Big) ->
+    BigGtSmall = lists:flatten(
+		 io_lib:format("~f > ~p",[Big,Small])),
+    BigLtSmall = lists:flatten(
+		 io_lib:format("~f < ~p",[Big,Small])),
+    BigEqSmall = lists:flatten(
+		 io_lib:format("~f == ~p",[Big,Small])),
+    SmallGtBig = lists:flatten(
+		   io_lib:format("~p > ~f",[Small,Big])),
+    SmallLtBig = lists:flatten(
+		   io_lib:format("~p < ~f",[Small,Big])),
+    SmallEqBig = lists:flatten(
+		   io_lib:format("~p == ~f",[Small,Big])),
+    cmp(Big,Small,BigGtSmall,BigLtSmall,SmallGtBig,SmallLtBig,
+	SmallEqBig,BigEqSmall);
+cmp(Big,Small) when is_float(Small) ->
+    BigGtSmall = lists:flatten(
+		   io_lib:format("~p > ~f",[Big,Small])),
+    BigLtSmall = lists:flatten(
+		   io_lib:format("~p < ~f",[Big,Small])),
+    BigEqSmall = lists:flatten(
+		   io_lib:format("~p == ~f",[Big,Small])),
+    SmallGtBig = lists:flatten(
+		   io_lib:format("~f > ~p",[Small,Big])),
+    SmallLtBig = lists:flatten(
+		   io_lib:format("~f < ~p",[Small,Big])),
+    SmallEqBig = lists:flatten(
+		   io_lib:format("~f == ~p",[Small,Big])),
+    cmp(Big,Small,BigGtSmall,BigLtSmall,SmallGtBig,SmallLtBig,
+	SmallEqBig,BigEqSmall).
+
+cmp(Big,Small,BigGtSmall,BigLtSmall,SmallGtBig,SmallLtBig,
+    SmallEqBig,BigEqSmall) ->
+    {_,_,_,true} = {Big,Small,BigGtSmall,
+		    Big > Small},
+    {_,_,_,false} = {Big,Small,BigLtSmall,
+		     Big < Small},
+    {_,_,_,false} = {Big,Small,SmallGtBig,
+		     Small > Big},
+    {_,_,_,true} = {Big,Small,SmallLtBig,
+		    Small < Big},
+    {_,_,_,false} = {Big,Small,SmallEqBig,
+		     Small == Big},
+    {_,_,_,false} = {Big,Small,BigEqSmall,
+		     Big == Small}.
+
 id(I) -> I.
     
 start_node(Config) when is_list(Config) ->
@@ -165,3 +303,25 @@ start_node(Config) when is_list(Config) ->
 
 stop_node(Node) ->
     ?t:stop_node(Node).
+
+
+%% Test that operations that might hide infinite intermediate results
+%% do not supress the badarith.
+hidden_inf(Config) when is_list(Config) ->
+    ZeroP = 0.0,
+    ZeroN = id(ZeroP) * (-1),
+    [hidden_inf_1(A, B, Z, 9.23e307)
+     || A <- [1.0, -1.0, 3.1415, -0.00001000131, 3.57e257, ZeroP, ZeroN],
+	B <- [1.0, -1.0, 3.1415, -0.00001000131, 3.57e257, ZeroP, ZeroN],
+	Z <- [ZeroP, ZeroN]],
+    ok.
+
+hidden_inf_1(A, B, Zero, Huge) ->
+    {'EXIT',{badarith,_}} = (catch (B / (A / Zero))),
+    {'EXIT',{badarith,_}} = (catch (B * (A / Zero))),
+    {'EXIT',{badarith,_}} = (catch (B / (Huge * Huge))),
+    {'EXIT',{badarith,_}} = (catch (B * (Huge * Huge))),
+    {'EXIT',{badarith,_}} = (catch (B / (Huge + Huge))),
+    {'EXIT',{badarith,_}} = (catch (B * (Huge + Huge))),
+    {'EXIT',{badarith,_}} = (catch (B / (-Huge - Huge))),
+    {'EXIT',{badarith,_}} = (catch (B * (-Huge - Huge))).
diff --git a/erts/emulator/test/float_SUITE_data/fp_drv.c b/erts/emulator/test/float_SUITE_data/fp_drv.c
index eb453f6cd6..b80385c3f9 100644
--- a/erts/emulator/test/float_SUITE_data/fp_drv.c
+++ b/erts/emulator/test/float_SUITE_data/fp_drv.c
@@ -22,6 +22,7 @@
 #  define PRINTF(X)
 #endif
 
+#include <string.h>
 #include <math.h>
 #ifdef __WIN32__
 #include <float.h>
@@ -37,7 +38,8 @@ int _finite(double x);
 #define ERTS_FP_CONTROL_TEST 0
 #define ERTS_FP_THREAD_TEST 1
 
-static int control(ErlDrvData, unsigned int, char *, int, char **, int);
+static ErlDrvSSizeT control(ErlDrvData, unsigned int, char *,
+			    ErlDrvSizeT, char **, ErlDrvSizeT);
 
 static ErlDrvEntry fp_drv_entry = { 
     NULL /* init */,
@@ -97,10 +99,10 @@ do_test(void *unused)
     return "ok";
 }
 
-static int control(ErlDrvData drv_data,
-		   unsigned int command,
-		   char *buf, int len,
-		   char **rbuf, int rlen)
+static ErlDrvSSizeT control(ErlDrvData drv_data,
+			    unsigned int command,
+			    char *buf, ErlDrvSizeT len,
+			    char **rbuf, ErlDrvSizeT rlen)
 {
     char *res_str;
     PRINTF(("control(%p, %d, ...) called\r\n", drv_data, command));
diff --git a/erts/emulator/test/fun_SUITE.erl b/erts/emulator/test/fun_SUITE.erl
index a7889dfe90..839ad6a4f4 100644
--- a/erts/emulator/test/fun_SUITE.erl
+++ b/erts/emulator/test/fun_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1999-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -22,7 +22,9 @@
 
 -define(default_timeout, ?t:minutes(1)).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 bad_apply/1,bad_fun_call/1,badarity/1,ext_badarity/1,
 	 equality/1,ordering/1,
 	 fun_to_port/1,t_hash/1,t_phash/1,t_phash2/1,md5/1,
@@ -32,19 +34,37 @@
 
 -export([nothing/0]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [bad_apply, bad_fun_call, badarity, ext_badarity,
+     equality, ordering, fun_to_port, t_hash, t_phash,
+     t_phash2, md5, refc, refc_ets, refc_dist,
+     const_propagation, t_arity, t_is_function2, t_fun_info].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [bad_apply,bad_fun_call,badarity,ext_badarity,equality,ordering,
-     fun_to_port,t_hash,t_phash,t_phash2,md5,
-     refc,refc_ets,refc_dist,const_propagation,
-     t_arity,t_is_function2,t_fun_info].
 
 init_per_testcase(_Case, Config) ->
     ?line Dog = test_server:timetrap(?default_timeout),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog=?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
@@ -519,12 +539,6 @@ bad_md5(Bad) ->
     {'EXIT',{badarg,_}} = (catch erlang:md5(Bad)).
 
 refc(Config) when is_list(Config) ->
-    case erlang:system_info(heap_type) of
-	private -> refc_1();
-	hybrid -> {skip,"Hybrid heap"}
-    end.
-
-refc_1() ->
     ?line F1 = fun_factory(2),
     ?line {refc,2} = erlang:fun_info(F1, refc),
     ?line F2 = fun_factory(42),
@@ -550,12 +564,6 @@ fun_factory(Const) ->
     fun(X) -> X + Const end.
 
 refc_ets(Config) when is_list(Config) ->
-    case erlang:system_info(heap_type) of
-	private -> refc_ets_1();
-	hybrid -> {skip,"Hybrid heap"}
-    end.
-
-refc_ets_1() ->
     ?line F = fun(X) -> X + 33 end,
     ?line {refc,2} = erlang:fun_info(F, refc),
 
@@ -602,12 +610,6 @@ refc_ets_bag(F1, Options) ->
     ok.
 
 refc_dist(Config) when is_list(Config) ->
-    case erlang:system_info(heap_type) of
-	private -> refc_dist_1();
-	hybrid -> {skip,"Hybrid heap"}
-    end.
-
-refc_dist_1() ->
     ?line {ok,Node} = start_node(fun_SUITE_refc_dist),
     ?line process_flag(trap_exit, true),
     ?line Pid = spawn_link(Node,
@@ -627,17 +629,11 @@ refc_dist_1() ->
     %% Fun is passed in an exit signal. Wait until it is gone.
     ?line wait_until(fun () -> 4 =/= fun_refc(F2) end),
     ?line 3 = fun_refc(F2),
-    erts_debug:set_internal_state(available_internal_state, true),
-    ?line F_refc = case erts_debug:get_internal_state(force_heap_frags) of
-		       false -> 3;
-		       true -> 2 % GC after bif already decreased it
-		   end,
-    ?line F_refc = fun_refc(F),
-    erts_debug:set_internal_state(available_internal_state, false),
+    ?line true = erlang:garbage_collect(),
+    ?line 2 = fun_refc(F),
     refc_dist_send(Node, F).
 
 refc_dist_send(Node, F) ->
-    ?line true = erlang:garbage_collect(),
     ?line Pid = spawn_link(Node,
 			   fun() -> receive
 					{To,Fun} when is_function(Fun) ->
diff --git a/erts/emulator/test/fun_r11_SUITE.erl b/erts/emulator/test/fun_r13_SUITE.erl
index 61ba816cc8..76ddf9fec9 100644
--- a/erts/emulator/test/fun_r11_SUITE.erl
+++ b/erts/emulator/test/fun_r13_SUITE.erl
@@ -1,72 +1,93 @@
 %%
 %% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 2007-2009. All Rights Reserved.
-%% 
+%%
+%% Copyright Ericsson AB 2007-2011. All Rights Reserved.
+%%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
 %% compliance with the License. You should have received a copy of the
 %% Erlang Public License along with this software. If not, it can be
 %% retrieved online at http://www.erlang.org/.
-%% 
+%%
 %% Software distributed under the License is distributed on an "AS IS"
 %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 %% the License for the specific language governing rights and limitations
 %% under the License.
-%% 
+%%
 %% %CopyrightEnd%
 %%
 
--module(fun_r11_SUITE).
--compile(r11).
+-module(fun_r13_SUITE).
+-compile(r13).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,dist_old_release/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1,
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,dist_old_release/1]).
 
 -define(default_timeout, ?t:minutes(1)).
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+    [dist_old_release].
+
+groups() ->
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) -> [dist_old_release].
 
 init_per_testcase(_Case, Config) ->
     ?line Dog = test_server:timetrap(?default_timeout),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog=?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
 
 dist_old_release(Config) when is_list(Config) ->
-    case ?t:is_release_available("r11b") of
+    case ?t:is_release_available("r12b") of
 	true -> do_dist_old(Config);
-	false -> {skip,"No R11B found"}
+	false -> {skip,"No R12B found"}
     end.
 
 do_dist_old(Config) when is_list(Config) ->
     ?line Pa = filename:dirname(code:which(?MODULE)),
-    Name = fun_dist_r11,
+    Name = fun_dist_r12,
     ?line {ok,Node} = ?t:start_node(Name, peer,
 				    [{args,"-pa "++Pa},
-				     {erl,[{release,"r11b"}]}]),
+				     {erl,[{release,"r12b"}]}]),
 
     ?line Pid = spawn_link(Node,
 			   fun() ->
 				   receive
 				       Fun when is_function(Fun) ->
-					   R11BFun = fun(H) -> cons(H, [b,c]) end,
-					   Fun(Fun, R11BFun)
+					   R12BFun = fun(H) -> cons(H, [b,c]) end,
+					   Fun(Fun, R12BFun)
 				   end
 			   end),
     Self = self(),
-    Fun = fun(F, R11BFun) ->
+    Fun = fun(F, R12BFun) ->
 		  {pid,Self} = erlang:fun_info(F, pid),
 		  {module,?MODULE} = erlang:fun_info(F, module),
-		  Self ! {ok,F,R11BFun}
+		  Self ! {ok,F,R12BFun}
 	  end,
     ?line Pid ! Fun,
     ?line receive
-	      {ok,Fun,R11BFun} ->
-		  ?line [a,b,c] = R11BFun(a);
+	      {ok,Fun,R12BFun} ->
+		  ?line [a,b,c] = R12BFun(a);
 	      Other ->
 		  ?line ?t:fail({bad_message,Other})
 	  end,
diff --git a/erts/emulator/test/gc_SUITE.erl b/erts/emulator/test/gc_SUITE.erl
index 066aa215b2..771d2c9a7a 100644
--- a/erts/emulator/test/gc_SUITE.erl
+++ b/erts/emulator/test/gc_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -21,15 +21,34 @@
 
 -module(gc_SUITE).
 
--include("test_server.hrl").
--export([all/1]).
+-include_lib("test_server/include/test_server.hrl").
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 
 -define(default_timeout, ?t:minutes(10)).
 
 -export([grow_heap/1, grow_stack/1, grow_stack_heap/1]).
 
-all(suite) ->
-    [grow_heap,grow_stack, grow_stack_heap].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [grow_heap, grow_stack, grow_stack_heap].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 grow_heap(doc) -> ["Produce a growing list of elements, ",
 		   "for X calls, then drop one item per call",
diff --git a/erts/emulator/test/guard_SUITE.erl b/erts/emulator/test/guard_SUITE.erl
index 23482a20d7..a5df9b59a0 100644
--- a/erts/emulator/test/guard_SUITE.erl
+++ b/erts/emulator/test/guard_SUITE.erl
@@ -1,33 +1,55 @@
 %%
 %% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
-%% 
+%%
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
+%%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
 %% compliance with the License. You should have received a copy of the
 %% Erlang Public License along with this software. If not, it can be
 %% retrieved online at http://www.erlang.org/.
-%% 
+%%
 %% Software distributed under the License is distributed on an "AS IS"
 %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 %% the License for the specific language governing rights and limitations
 %% under the License.
-%% 
+%%
 %% %CopyrightEnd%
 %%
 
 -module(guard_SUITE).
 
--export([all/1, bad_arith/1, bad_tuple/1, test_heap_guards/1, guard_bifs/1,
-	 type_tests/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, bad_arith/1, bad_tuple/1, 
+	 test_heap_guards/1, guard_bifs/1,
+	 type_tests/1,guard_bif_binary_part/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -export([init/3]).
 -import(lists, [member/2]).
 
-all(suite) -> [bad_arith, bad_tuple, test_heap_guards, guard_bifs, type_tests].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [bad_arith, bad_tuple, test_heap_guards, guard_bifs,
+     type_tests, guard_bif_binary_part].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 bad_arith(doc) -> "Test that a bad arithmetic operation in a guard works correctly.";
 bad_arith(Config) when is_list(Config) ->
@@ -136,6 +158,170 @@ init(Fun, Args, Filler) ->
 dummy(_) ->
     ok.
 
+-define(MASK_ERROR(EXPR),mask_error((catch (EXPR)))).
+mask_error({'EXIT',{Err,_}}) ->
+    Err;
+mask_error(Else) ->
+    Else.
+
+guard_bif_binary_part(doc) ->
+    ["Test the binary_part/2,3 guard BIF's extensively"];
+guard_bif_binary_part(Config) when is_list(Config) ->
+    %% Overflow tests that need to be unoptimized
+    ?line badarg =
+	?MASK_ERROR(
+	   binary_part(<<1,2,3>>,{16#FFFFFFFFFFFFFFFF,
+				 -16#7FFFFFFFFFFFFFFF-1})),
+    ?line badarg =
+	?MASK_ERROR(
+	   binary_part(<<1,2,3>>,{16#FFFFFFFFFFFFFFFF,
+				 16#7FFFFFFFFFFFFFFF})),
+    F = fun(X) ->
+		Master = self(),
+		{Pid,Ref} = spawn_monitor( fun() ->
+					     A = lists:duplicate(X,a),
+					     B = [do_binary_part_guard() | A],
+					     Master ! {self(),hd(B)},
+					     ok
+				     end),
+		receive
+		    {Pid,ok} ->
+			erlang:demonitor(Ref,[flush]),
+			ok;
+		    Error ->
+			Error
+		end
+	end,
+    [ ok = F(N) || N <- lists:seq(1,10000) ],
+    ok.
+
+
+do_binary_part_guard() ->
+    ?line 1 = bptest(<<1,2,3>>),
+    ?line 2 = bptest(<<2,1,3>>),
+    ?line error = bptest(<<1>>),
+    ?line error = bptest(<<>>),
+    ?line error = bptest(apa),
+    ?line 3 = bptest(<<2,3,3>>),
+    % With one variable (pos)
+    ?line 1 = bptest(<<1,2,3>>,1),
+    ?line 2 = bptest(<<2,1,3>>,1),
+    ?line error = bptest(<<1>>,1),
+    ?line error = bptest(<<>>,1),
+    ?line error = bptest(apa,1),
+    ?line 3 = bptest(<<2,3,3>>,1),
+    % With one variable (length)
+    ?line 1 = bptesty(<<1,2,3>>,1),
+    ?line 2 = bptesty(<<2,1,3>>,1),
+    ?line error = bptesty(<<1>>,1),
+    ?line error = bptesty(<<>>,1),
+    ?line error = bptesty(apa,1),
+    ?line 3 = bptesty(<<2,3,3>>,2),
+    % With one variable (whole tuple)
+    ?line 1 = bptestx(<<1,2,3>>,{1,1}),
+    ?line 2 = bptestx(<<2,1,3>>,{1,1}),
+    ?line error = bptestx(<<1>>,{1,1}),
+    ?line error = bptestx(<<>>,{1,1}),
+    ?line error = bptestx(apa,{1,1}),
+    ?line 3 = bptestx(<<2,3,3>>,{1,2}),
+    % With two variables
+    ?line 1 = bptest(<<1,2,3>>,1,1),
+    ?line 2 = bptest(<<2,1,3>>,1,1),
+    ?line error = bptest(<<1>>,1,1),
+    ?line error = bptest(<<>>,1,1),
+    ?line error = bptest(apa,1,1),
+    ?line 3 = bptest(<<2,3,3>>,1,2),
+    % Direct (autoimported) call, these will be evaluated by the compiler...
+    ?line <<2>> = binary_part(<<1,2,3>>,1,1),
+    ?line <<1>> = binary_part(<<2,1,3>>,1,1),
+    % Compiler warnings due to constant evaluation expected (3)
+    ?line badarg = ?MASK_ERROR(binary_part(<<1>>,1,1)),
+    ?line badarg = ?MASK_ERROR(binary_part(<<>>,1,1)),
+    ?line badarg = ?MASK_ERROR(binary_part(apa,1,1)),
+    ?line <<3,3>> = binary_part(<<2,3,3>>,1,2),
+    % Direct call through apply
+    ?line <<2>> = apply(erlang,binary_part,[<<1,2,3>>,1,1]),
+    ?line <<1>> = apply(erlang,binary_part,[<<2,1,3>>,1,1]),
+    % Compiler warnings due to constant evaluation expected (3)
+    ?line badarg = ?MASK_ERROR(apply(erlang,binary_part,[<<1>>,1,1])),
+    ?line badarg = ?MASK_ERROR(apply(erlang,binary_part,[<<>>,1,1])),
+    ?line badarg = ?MASK_ERROR(apply(erlang,binary_part,[apa,1,1])),
+    ?line <<3,3>> = apply(erlang,binary_part,[<<2,3,3>>,1,2]),
+    % Constant propagation
+    ?line  Bin = <<1,2,3>>,
+    ?line  ok = if
+		    binary_part(Bin,1,1) =:= <<2>> ->
+			ok;
+		    %% Compiler warning, clause cannot match (expected)
+		    true ->
+			error
+		end,
+    ?line  ok = if
+		    binary_part(Bin,{1,1}) =:= <<2>> ->
+			ok;
+		    %% Compiler warning, clause cannot match (expected)
+		    true ->
+			error
+		end,
+    ok.
+
+
+bptest(B) when length(B) =:= 1337 ->
+    1;
+bptest(B) when binary_part(B,{1,1}) =:= <<2>> ->
+    1;
+bptest(B) when erlang:binary_part(B,1,1) =:= <<1>> ->
+    2;
+bptest(B)  when erlang:binary_part(B,{1,2}) =:= <<3,3>> ->
+    3;
+bptest(_) ->
+    error.
+
+bptest(B,A) when length(B) =:= A ->
+    1;
+bptest(B,A) when binary_part(B,{A,1}) =:= <<2>> ->
+    1;
+bptest(B,A) when erlang:binary_part(B,A,1) =:= <<1>> ->
+    2;
+bptest(B,A)  when erlang:binary_part(B,{A,2}) =:= <<3,3>> ->
+    3;
+bptest(_,_) ->
+    error.
+
+bptestx(B,A) when length(B) =:= A ->
+    1;
+bptestx(B,A) when binary_part(B,A) =:= <<2>> ->
+    1;
+bptestx(B,A) when erlang:binary_part(B,A) =:= <<1>> ->
+    2;
+bptestx(B,A)  when erlang:binary_part(B,A) =:= <<3,3>> ->
+    3;
+bptestx(_,_) ->
+    error.
+
+bptesty(B,A) when length(B) =:= A ->
+    1;
+bptesty(B,A) when binary_part(B,{1,A}) =:= <<2>> ->
+    1;
+bptesty(B,A) when erlang:binary_part(B,1,A) =:= <<1>> ->
+    2;
+bptesty(B,A)  when erlang:binary_part(B,{1,A}) =:= <<3,3>> ->
+    3;
+bptesty(_,_) ->
+    error.
+
+bptest(B,A,_C) when length(B) =:= A ->
+    1;
+bptest(B,A,C) when binary_part(B,{A,C}) =:= <<2>> ->
+    1;
+bptest(B,A,C) when erlang:binary_part(B,A,C) =:= <<1>> ->
+    2;
+bptest(B,A,C)  when erlang:binary_part(B,{A,C}) =:= <<3,3>> ->
+    3;
+bptest(_,_,_) ->
+    error.
+
+
 guard_bifs(doc) -> "Test all guard bifs with nasty (but legal arguments).";
 guard_bifs(Config) when is_list(Config) ->
     ?line Big = -237849247829874297658726487367328971246284736473821617265433,
@@ -235,7 +421,7 @@ try_gbif(Id, X, Y) ->
 
 try_fail_gbif(Id, X, Y) ->
     case catch guard_bif(Id, X, Y) of
-	{'EXIT', {function_clause,[{?MODULE,guard_bif,[Id,X,Y]}|_]}} ->
+	{'EXIT',{function_clause,[{?MODULE,guard_bif,[Id,X,Y],_}|_]}} ->
 	    io:format("guard_bif(~p, ~p, ~p) -- ok", [Id,X,Y]);
 	Other ->
 	    ?line ok = io:format("guard_bif(~p, ~p, ~p) -- bad result: ~p\n",
@@ -307,9 +493,9 @@ type_tests(Test, [Type|T], Allowed) ->
 	    end;
 	false ->
 	    case catch type_test(Test, Value) of
-		{'EXIT', {function_clause, {?MODULE,type_test,[Test,Value]}}} ->
-		    ok;
-		{'EXIT', {function_clause,[{?MODULE,type_test,[Test,Value]}|_]}} ->
+		{'EXIT',{function_clause,
+			 [{?MODULE,type_test,[Test,Value],Loc}|_]}}
+		when is_list(Loc) ->
 		    ok;
 		{'EXIT',Other} ->
 		    ?line test_server:fail({unexpected_error_reason,Other});
diff --git a/erts/emulator/test/hash_SUITE.erl b/erts/emulator/test/hash_SUITE.erl
index 85bdb8bff8..830ed91da9 100644
--- a/erts/emulator/test/hash_SUITE.erl
+++ b/erts/emulator/test/hash_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2000-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2000-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -49,7 +49,7 @@
 -define(config(A,B),config(A,B)).
 -export([config/2]).
 -else.
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 -endif.
 
 -ifdef(debug).
@@ -69,22 +69,40 @@ config(priv_dir,_) ->
     ".".
 -else.
 %% When run in test server.
--export([all/1,test_basic/1,test_cmp/1,test_range/1,test_spread/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 test_basic/1,test_cmp/1,test_range/1,test_spread/1,
 	 test_phash2/1,otp_5292/1,bit_level_binaries/1,otp_7127/1,
-	 fin_per_testcase/2,init_per_testcase/2]).
+	 end_per_testcase/2,init_per_testcase/2]).
 init_per_testcase(_Case, Config) ->
     ?line Dog=test_server:timetrap(test_server:minutes(10)),
     [{watchdog, Dog}|Config].
  
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog=?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
-all(doc) ->
-    ["Test erlang:phash"];
-all(suite) ->
-    [test_basic, test_cmp, test_range, test_spread, test_phash2, otp_5292,
-     bit_level_binaries, otp_7127].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [test_basic, test_cmp, test_range, test_spread,
+     test_phash2, otp_5292, bit_level_binaries, otp_7127].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 test_basic(suite) ->
     [];
@@ -480,14 +498,14 @@ otp_5292_test() ->
     S2 = md5([md5(hash_int(S, E, PH)) || {Start, N, Sz} <- d(), 
                                          {S, E} <- int(Start, N, Sz)]),
     ?line Comment = case S1 of 
-                        <<43,186,76,102,87,4,110,245,203,177,206,6,130,69,43,99>> ->
+			<<4,248,208,156,200,131,7,1,173,13,239,173,112,81,16,174>> ->
 			    ?line big = erlang:system_info(endian),
                             "Big endian machine";
-                        <<21,206,139,15,149,28,167,81,98,225,132,254,49,125,174,195>> ->
+                        <<180,28,33,231,239,184,71,125,76,47,227,241,78,184,176,233>> ->
 			    ?line little = erlang:system_info(endian),
                             "Little endian machine"
                     end,
-    ?line <<140,37,79,80,26,242,130,22,20,229,123,240,223,244,43,99>> = S2,
+    ?line <<124,81,198,121,174,233,19,137,10,83,33,80,226,111,238,99>> = S2,
     ?line 2 = erlang:hash(1, (1 bsl 27) -1),
     ?line {'EXIT', _} = (catch erlang:hash(1, (1 bsl 27))),
     {comment, Comment}.
@@ -507,7 +525,7 @@ hash_int(Start, End, F) ->
     {Start, End, md5(HL)}.
 
 md5(T) ->
-    erlang:md5(term_to_binary(T)).
+    erlang:md5(term_to_binary(T)).   
 
 bit_level_binaries() ->
     ?line [3511317,7022633,14044578,28087749,56173436,112344123,90467083|_] =
diff --git a/erts/emulator/test/hibernate_SUITE.erl b/erts/emulator/test/hibernate_SUITE.erl
index 4d36076d12..68bc3434d4 100644
--- a/erts/emulator/test/hibernate_SUITE.erl
+++ b/erts/emulator/test/hibernate_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2003-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2003-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,23 +19,44 @@
 
 -module(hibernate_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
-	 basic/1,min_heap_size/1,bad_args/1,
-	 messages_in_queue/1,undefined_mfa/1, no_heap/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
+	 basic/1,dynamic_call/1,min_heap_size/1,bad_args/1,
+	 messages_in_queue/1,undefined_mfa/1,no_heap/1,wake_up_and_bif_trap/1]).
 
 %% Used by test cases.
--export([basic_hibernator/1,messages_in_queue_restart/2, no_heap_loop/0]).
+-export([basic_hibernator/1,dynamic_call_hibernator/2,messages_in_queue_restart/2, no_heap_loop/0,characters_to_list_trap/1]).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [basic, dynamic_call, min_heap_size, bad_args, messages_in_queue,
+     undefined_mfa, no_heap, wake_up_and_bif_trap].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [basic,min_heap_size,bad_args,messages_in_queue,undefined_mfa,no_heap].
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog = ?t:timetrap(?t:minutes(3)),
     [{watchdog,Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
@@ -46,10 +67,7 @@ fin_per_testcase(_Func, Config) ->
 basic(Config) when is_list(Config) ->
     Ref = make_ref(),
     Info = {self(),Ref},
-    ExpectedHeapSz = case erlang:system_info(heap_type) of
-			 private -> erts_debug:size([Info]);
-			 hybrid -> erts_debug:size([a|b])
-		     end,
+    ExpectedHeapSz = erts_debug:size([Info]),
     ?line Child = spawn_link(fun() -> basic_hibernator(Info) end),
     ?line hibernate_wake_up(100, ExpectedHeapSz, Child),
     ?line Child ! please_quit_now,
@@ -138,10 +156,44 @@ whats_up_calc(A1, A2, A3, A4, A5, A6, A7, A8, A9, Acc) ->
     whats_up_calc(A1-1, A2+1, A3+2, A4+3, A5+4, A6+5, A7+6, A8+7, A9+8, [A1,A2|Acc]).
 
 %%%
+%%% Testing a call to erlang:hibernate/3 that the compiler and loader do not
+%%% translate to an instruction.
+%%%
+
+dynamic_call(Config) when is_list(Config) ->
+    Ref = make_ref(),
+    Info = {self(),Ref},
+    ExpectedHeapSz = erts_debug:size([Info]),
+    ?line Child = spawn_link(fun() -> ?MODULE:dynamic_call_hibernator(Info, hibernate) end),
+    ?line hibernate_wake_up(100, ExpectedHeapSz, Child),
+    ?line Child ! please_quit_now,
+    ok.
+
+dynamic_call_hibernator(Info, Function) ->
+    {catchlevel,0} = process_info(self(), catchlevel),
+    receive
+	Any ->
+	    dynamic_call_hibernator_msg(Any, Function, Info),
+	    dynamic_call_hibernator(Info, Function)
+    end.
+
+dynamic_call_hibernator_msg({hibernate,_}, Function, Info) ->
+    catch apply(erlang, Function, [?MODULE, basic_hibernator, [Info]]),
+    exit(hibernate_returned);
+dynamic_call_hibernator_msg(Msg, _Function, Info) ->
+    basic_hibernator_msg(Msg, Info).
+
+%%%
 %%% Testing setting the minimum heap size.
 %%%
 
 min_heap_size(Config) when is_list(Config) ->
+    case test_server:is_native(?MODULE) of
+	true -> {skip, "Test case relies on trace which is not available in HiPE"};
+	false -> min_heap_size_1(Config)
+    end.
+
+min_heap_size_1(Config) when is_list(Config) ->
     ?line erlang:trace(new, true, [call]),
     MFA = {?MODULE,min_hibernator,1},
     ?line 1 = erlang:trace_pattern(MFA, true, [local]),
@@ -326,6 +378,31 @@ clean_dict() ->
     lists:foreach(fun ({Key, _}) -> erase(Key) end, Dict).
 
 %%
+%% Wake up and then immediatly bif trap with a lengthy computation.
+%%
+
+wake_up_and_bif_trap(doc) -> [];
+wake_up_and_bif_trap(suite) -> [];
+wake_up_and_bif_trap(Config) when is_list(Config) ->
+    ?line Self = self(),
+    ?line Pid = spawn_link(fun() -> erlang:hibernate(?MODULE, characters_to_list_trap, [Self]) end),
+    ?line Pid ! wakeup,
+    ?line receive
+        {ok, Pid0} when Pid0 =:= Pid -> ok
+    after 5000 ->
+        ?line ?t:fail(process_blocked)
+    end,
+    ?line unlink(Pid),
+    ?line exit(Pid, bye).
+
+%% Lengthy computation that traps (in characters_to_list_trap_3).
+characters_to_list_trap(Parent) ->
+    Bin0 = <<"abcdefghijklmnopqrstuvwxz0123456789">>,
+    Bin = binary:copy(Bin0, 1500),
+    unicode:characters_to_list(Bin),
+    Parent ! {ok, self()}.
+
+%%
 %% Misc
 %%
 
diff --git a/erts/emulator/test/ignore_cores.erl b/erts/emulator/test/ignore_cores.erl
index 1d738cbafd..8b1ac0fe6c 120000..100644
--- a/erts/emulator/test/ignore_cores.erl
+++ b/erts/emulator/test/ignore_cores.erl
@@ -1 +1,158 @@
-../../test/ignore_cores.erl
\ No newline at end of file
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File    : ignore_cores.erl
+%%% Author  : Rickard Green <[email protected]>
+%%% Description : 
+%%%
+%%% Created : 11 Feb 2008 by Rickard Green <[email protected]>
+%%%-------------------------------------------------------------------
+
+-module(ignore_cores).
+
+-include_lib("test_server/include/test_server.hrl").
+
+-export([init/1, fini/1, setup/3, setup/4, restore/1, dir/1]).
+
+-record(ignore_cores, {org_cwd,
+		       org_path,
+		       org_pwd_env,
+		       ign_dir = false,
+		       cores_dir = false}).
+
+%%
+%% Takes a testcase config
+%%
+
+init(Config) ->
+    {ok, OrgCWD} = file:get_cwd(),
+    [{ignore_cores,
+      #ignore_cores{org_cwd = OrgCWD,
+		    org_path = code:get_path(),
+		    org_pwd_env = os:getenv("PWD")}}
+     | lists:keydelete(ignore_cores, 1, Config)].
+    
+fini(Config) ->
+    #ignore_cores{org_cwd = OrgCWD,
+		  org_path = OrgPath,
+		  org_pwd_env = OrgPWD} = ?config(ignore_cores, Config),
+    ok = file:set_cwd(OrgCWD),
+    true = code:set_path(OrgPath),
+    case OrgPWD of
+	false -> ok;
+	_ -> true = os:putenv("PWD", OrgPWD)
+    end,
+    lists:keydelete(ignore_cores, 1, Config).
+
+setup(Suite, Testcase, Config) ->
+    setup(Suite, Testcase, Config, false).
+
+setup(Suite, Testcase, Config, SetCwd) when is_atom(Suite), 
+                                            is_atom(Testcase),
+                                            is_list(Config) ->
+    #ignore_cores{org_cwd = OrgCWD,
+		  org_path = OrgPath,
+		  org_pwd_env = OrgPWD} = ?config(ignore_cores, Config),
+    Path = lists:map(fun (".") -> OrgCWD; (Dir) -> Dir end, OrgPath),
+    true = code:set_path(Path),
+    PrivDir = ?config(priv_dir, Config),
+    IgnDir = filename:join([PrivDir,
+			 atom_to_list(Suite)
+			 ++ "_"
+			 ++ atom_to_list(Testcase)
+			 ++ "_wd"]),
+    ok = file:make_dir(IgnDir),
+    case SetCwd of
+	false ->
+	    ok;
+	_ ->
+	    ok = file:set_cwd(IgnDir),
+	    OrgPWD = case os:getenv("PWD") of
+			 false -> false;
+			 PWD ->
+			     os:putenv("PWD", IgnDir),
+			     PWD
+		     end
+    end,
+    ok = file:write_file(filename:join([IgnDir, "ignore_core_files"]), <<>>),
+    %% cores are dumped in /cores on MacOS X
+    CoresDir = case {?t:os_type(), filelib:is_dir("/cores")} of
+		   {{unix,darwin}, true} ->
+		       filelib:fold_files("/cores",
+					  "^core.*$",
+					  false,
+					  fun (C,Cs) -> [C|Cs] end,
+					  []);
+		   _ ->
+		       false
+	       end,
+    lists:keyreplace(ignore_cores,
+		     1,
+		     Config,
+		     {ignore_cores,
+		      #ignore_cores{org_cwd = OrgCWD,
+				    org_path = OrgPath,
+				    org_pwd_env = OrgPWD,
+				    ign_dir = IgnDir,
+				    cores_dir = CoresDir}}).
+
+restore(Config) ->
+    #ignore_cores{org_cwd = OrgCWD,
+		  org_path = OrgPath,
+		  org_pwd_env = OrgPWD,
+		  ign_dir = IgnDir,
+		  cores_dir = CoresDir} = ?config(ignore_cores, Config),
+    try
+	case CoresDir of
+	    false ->
+		ok;
+	    _ ->
+		%% Move cores dumped by these testcases in /cores
+		%% to cwd.
+		lists:foreach(fun (C) ->
+				      case lists:member(C, CoresDir) of
+					  true -> ok;
+					  _ ->
+					      Dst = filename:join(
+						      [IgnDir,
+						       filename:basename(C)]),
+					      {ok, _} = file:copy(C, Dst),
+					      file:delete(C)
+				      end
+			      end,
+			      filelib:fold_files("/cores",
+						 "^core.*$",
+						 false,
+						 fun (C,Cs) -> [C|Cs] end,
+						 []))
+	end
+    after
+	catch file:set_cwd(OrgCWD),
+	catch code:set_path(OrgPath),
+	case OrgPWD of
+	    false -> ok;
+	    _ -> catch os:putenv("PWD", OrgPWD)
+	end
+    end.
+
+
+dir(Config) ->
+    #ignore_cores{ign_dir = Dir} = ?config(ignore_cores, Config),
+    Dir.
diff --git a/erts/emulator/test/list_bif_SUITE.erl b/erts/emulator/test/list_bif_SUITE.erl
index 65ea88eb2f..45a44d8b43 100644
--- a/erts/emulator/test/list_bif_SUITE.erl
+++ b/erts/emulator/test/list_bif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,21 +18,42 @@
 %%
 
 -module(list_bif_SUITE).
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2]).
 -export([hd_test/1,tl_test/1,t_length/1,t_list_to_pid/1,
 	 t_list_to_float/1,t_list_to_integer/1]).
 
 
-all(suite) ->
-    [hd_test,tl_test,t_length,t_list_to_pid,t_list_to_float,t_list_to_integer].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [hd_test, tl_test, t_length, t_list_to_pid,
+     t_list_to_float, t_list_to_integer].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(_Case, Config) ->
     ?line Dog = test_server:timetrap(test_server:seconds(60)),
     [{watchdog,Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog = ?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
diff --git a/erts/emulator/test/long_timers_test.erl b/erts/emulator/test/long_timers_test.erl
index 28626d26fb..28a4fba9f6 100644
--- a/erts/emulator/test/long_timers_test.erl
+++ b/erts/emulator/test/long_timers_test.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2010. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl
index 69c89f5d2d..461773114e 100644
--- a/erts/emulator/test/match_spec_SUITE.erl
+++ b/erts/emulator/test/match_spec_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,44 +19,66 @@
 
 -module(match_spec_SUITE).
 
--export([all/1, not_run/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, not_run/1]).
 -export([test_1/1, test_2/1, test_3/1, bad_match_spec_bin/1,
 	 trace_control_word/1, silent/1, silent_no_ms/1, 
 	 ms_trace2/1, ms_trace3/1, boxed_and_small/1,
 	 destructive_in_test_bif/1, guard_exceptions/1,
 	 unary_plus/1, unary_minus/1, moving_labels/1]).
 -export([fpe/1]).
+-export([otp_9422/1]).
 
--export([runner/2]).
+-export([runner/2, loop_runner/3]).
 -export([f1/1, f2/2, f3/2, fn/1, fn/2, fn/3]).
 -export([do_boxed_and_small/0]).
 
 % This test suite assumes that tracing in general works. What we test is
 % the match spec functionality.
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([init_per_testcase/2, fin_per_testcase/2]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?t:seconds(10)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
 
-all(suite) ->
-    case test_server:is_native(?MODULE) of
-	false -> [test_1, test_2, test_3, bad_match_spec_bin,
-		  trace_control_word, silent, silent_no_ms, 
-		  ms_trace2, ms_trace3, boxed_and_small,
-		  destructive_in_test_bif, guard_exceptions,
-		  unary_plus, unary_minus, fpe, moving_labels];
-	true  -> [not_run]
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    case test_server:is_native(match_spec_SUITE) of
+	false ->
+	    [test_1, test_2, test_3, bad_match_spec_bin,
+	     trace_control_word, silent, silent_no_ms, ms_trace2,
+	     ms_trace3, boxed_and_small, destructive_in_test_bif,
+	     guard_exceptions, unary_plus, unary_minus, fpe,
+	     moving_labels,
+	     otp_9422];
+	true -> [not_run]
     end.
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+	Config.
+
+
 not_run(Config) when is_list(Config) ->
     {skipped, "Native Code"}.
 
@@ -188,6 +210,43 @@ test_3(Config) when is_list(Config) ->
     ?line collect(P1, [{trace, P1, call, {?MODULE, f2, [a, b]}, [true]}]),
     ?line ok.
 
+otp_9422(doc) -> [];
+otp_9422(Config) when is_list(Config) ->
+    Laps = 1000,
+    ?line Fun1 = fun() -> otp_9422_tracee() end,
+    ?line P1 = spawn_link(?MODULE, loop_runner, [self(), Fun1, Laps]),
+    io:format("spawned ~p as tracee\n", [P1]),
+
+    ?line erlang:trace(P1, true, [call, silent]),
+
+    ?line Fun2 = fun() -> otp_9422_trace_changer() end,
+    ?line P2 = spawn_link(?MODULE, loop_runner, [self(), Fun2, Laps]),
+    io:format("spawned ~p as trace_changer\n", [P2]),
+
+    start_collect(P1),
+    start_collect(P2),
+
+    %%receive after 10*1000 -> ok end,
+
+    stop_collect(P1),
+    stop_collect(P2),
+    ok.
+    
+otp_9422_tracee() ->
+    ?MODULE:f1(a),
+    ?MODULE:f1(b),
+    ?MODULE:f1(c).
+
+otp_9422_trace_changer() ->
+    Pat1 = [{[a], [], [{enable_trace, arity}]}],
+    ?line erlang:trace_pattern({?MODULE, f1, 1}, Pat1),
+    Pat2 = [{[b], [], [{disable_trace, arity}]}],
+    ?line erlang:trace_pattern({?MODULE, f1, 1}, Pat2).
+
+    
+    
+
+
 bad_match_spec_bin(Config) when is_list(Config) ->
     {'EXIT',{badarg,_}} = (catch ets:match_spec_run([1], <<>>)),
     B0 = <<1,2>>,
@@ -345,15 +404,15 @@ silent_no_ms(Config) when is_list(Config) ->
 	    fun () -> 
 		    ?MODULE:f1(a),
 		    ?MODULE:f2(b, c),
-		    erlang:integer_to_list(id(1)),
+		    _ = erlang:integer_to_list(id(1)),
 		    ?MODULE:f3(d, e),
 		    ?MODULE:f1(start),
 		    ?MODULE:f2(f, g),
-		    erlang:integer_to_list(id(2)),
+		    _ = erlang:integer_to_list(id(2)),
 		    ?MODULE:f3(h, i),
 		    ?MODULE:f1(stop),
 		    ?MODULE:f2(j, k),
-		    erlang:integer_to_list(id(3)),
+		    _ = erlang:integer_to_list(id(3)),
 		    ?MODULE:f3(l, m)
 	    end,
 	    fun (Tracee) ->
@@ -393,15 +452,15 @@ silent_no_ms(Config) when is_list(Config) ->
 	    fun () -> 
 		    ?MODULE:f1(a),
 		    ?MODULE:f2(b, c),
-		    erlang:integer_to_list(id(1)),
+		    _ = erlang:integer_to_list(id(1)),
 		    ?MODULE:f3(d, e),
 		    ?MODULE:f1(start),
 		    ?MODULE:f2(f, g),
-		    erlang:integer_to_list(id(2)),
+		    _ = erlang:integer_to_list(id(2)),
 		    ?MODULE:f3(h, i),
 		    ?MODULE:f1(stop),
 		    ?MODULE:f2(j, k),
-		    erlang:integer_to_list(id(3)),
+		    _ = erlang:integer_to_list(id(3)),
 		    ?MODULE:f3(l, m)
 	    end,
 	    fun (Tracee) ->
@@ -455,18 +514,18 @@ ms_trace2(Config) when is_list(Config) ->
 	    fun () -> 
 		    ?MODULE:f1(a),
 		    ?MODULE:f2(b, c),
-		    erlang:integer_to_list(id(1)),
+		    _ = erlang:integer_to_list(id(1)),
 		    ?MODULE:f3(d, e),
 		    fn([all], [call,return_to,{tracer,Tracer}]),
 		    ?MODULE:f1(f),
 		    f2(g, h),
 		    f1(i),
-		    erlang:integer_to_list(id(2)),
+		    _ = erlang:integer_to_list(id(2)),
 		    ?MODULE:f3(j, k),
 		    fn([call,return_to], []),
 		    ?MODULE:f1(l),
 		    ?MODULE:f2(m, n),
-		    erlang:integer_to_list(id(3)),
+		    _ = erlang:integer_to_list(id(3)),
 		    ?MODULE:f3(o, p)
 	    end,
 	    fun (Tracee) ->
@@ -551,26 +610,26 @@ ms_trace3(Config) when is_list(Config) ->
 		    register(TraceeName, self()),
 		    ?MODULE:f1(a),
 		    ?MODULE:f2(b, c),
-		    erlang:integer_to_list(id(1)),
+		    _ = erlang:integer_to_list(id(1)),
 		    ?MODULE:f3(d, e),
 		    Controller ! {self(),Tag,start},
 		    receive {Controller,Tag,started} -> ok end,
 		    ?MODULE:f1(f),
 		    f2(g, h),
 		    f1(i),
-		    erlang:integer_to_list(id(2)),
+		    _ = erlang:integer_to_list(id(2)),
 		    ?MODULE:f3(j, k),
 		    Controller ! {self(),Tag,stop_1},
 		    receive {Controller,Tag,stopped_1} -> ok end,
 		    ?MODULE:f1(l),
 		    ?MODULE:f2(m, n),
-		    erlang:integer_to_list(id(3)),
+		    _ = erlang:integer_to_list(id(3)),
 		    ?MODULE:f3(o, p),
 		    Controller ! {self(),Tag,stop_2},
 		    receive {Controller,Tag,stopped_2} -> ok end,
 		    ?MODULE:f1(q),
 		    ?MODULE:f2(r, s),
-		    erlang:integer_to_list(id(4)),
+		    _ = erlang:integer_to_list(id(4)),
 		    ?MODULE:f3(t, u)
 	    end,
 	    
@@ -912,6 +971,24 @@ runner(Collector, Fun) ->
 	    Collector ! {gone, self()}
     end.
 
+loop_runner(Collector, Fun, Laps) ->
+    receive
+	{go, Collector} ->
+	    go
+    end,
+    loop_runner_cont(Collector, Fun, 0, Laps).
+
+loop_runner_cont(_Collector, _Fun, Laps, Laps) ->
+    receive
+	{done, Collector} ->
+	    io:format("loop_runner ~p exit after ~p laps\n", [self(), Laps]),
+	    Collector ! {gone, self()}
+    end;
+loop_runner_cont(Collector, Fun, N, Laps) ->
+    Fun(),
+    loop_runner_cont(Collector, Fun, N+1, Laps).
+
+
 f1(X) ->
     {X}.
 
diff --git a/erts/emulator/test/module_info_SUITE.erl b/erts/emulator/test/module_info_SUITE.erl
index f34a2b496c..8a63d9fe3e 100644
--- a/erts/emulator/test/module_info_SUITE.erl
+++ b/erts/emulator/test/module_info_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2005-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,9 +19,11 @@
 
 -module(module_info_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,init_per_testcase/2,end_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 exports/1,functions/1,native/1]).
 
 %%-compile(native).
@@ -29,8 +31,29 @@
 %% Helper.
 -export([native_proj/1,native_filter/1]).
 
-all(suite) ->
-    [exports,functions,native].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    modules().
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
+modules() -> 
+    [exports, functions, native].
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog = ?t:timetrap(?t:minutes(3)),
@@ -42,14 +65,18 @@ end_per_testcase(_Func, Config) ->
 
 %% Should return all functions exported from this module. (local)
 all_exported() ->
-    All = add_arity(all(suite)),
-    lists:sort([{all,1},{init_per_testcase,2},{end_per_testcase,2},
+    All = add_arity(modules()),
+    lists:sort([{all,0},{suite,0},{groups,0},
+		{init_per_suite,1},{end_per_suite,1},
+		{init_per_group,2},{end_per_group,2},
+		{init_per_testcase,2},{end_per_testcase,2},
 		{module_info,0},{module_info,1},{native_proj,1},
 		{native_filter,1}|All]).
 
 %% Should return all functions in this module. (local)
 all_functions() ->
-    Locals = [{add_arity,1},{add_arity,2},{all_exported,0},{all_functions,0}],
+    Locals = [{add_arity,1},{add_arity,2},{all_exported,0},{all_functions,0},
+	      {modules,0}],
     lists:sort(Locals++all_exported()).
 
 %% Test that the list of exported functions from this module is correct.
diff --git a/erts/emulator/test/monitor_SUITE.erl b/erts/emulator/test/monitor_SUITE.erl
index 68e378dfec..aec59867d8 100644
--- a/erts/emulator/test/monitor_SUITE.erl
+++ b/erts/emulator/test/monitor_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,29 +19,49 @@
 
 -module(monitor_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
 	 case_1/1, case_1a/1, case_2/1, case_2a/1, mon_e_1/1, demon_e_1/1, demon_1/1,
-	 demon_2/1, demon_3/1, demonitor_flush/1, remove_monitor/1,
+	 demon_2/1, demon_3/1, demonitor_flush/1,
 	 local_remove_monitor/1, remote_remove_monitor/1, mon_1/1, mon_2/1,
 	 large_exit/1, list_cleanup/1, mixer/1, named_down/1, otp_5827/1]).
 
--export([init_per_testcase/2, fin_per_testcase/2]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
 -export([y2/1, g/1, g0/0, g1/0, large_exit_sub/1]).
 
-all(suite) ->
-    [case_1, case_1a, case_2, case_2a, mon_e_1, demon_e_1, demon_1, mon_1,
-     mon_2, demon_2, demon_3, demonitor_flush, remove_monitor,
-     large_exit, list_cleanup, mixer, named_down,
-     otp_5827].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [case_1, case_1a, case_2, case_2a, mon_e_1, demon_e_1,
+     demon_1, mon_1, mon_2, demon_2, demon_3,
+     demonitor_flush, {group, remove_monitor}, large_exit,
+     list_cleanup, mixer, named_down, otp_5827].
+
+groups() -> 
+    [{remove_monitor, [],
+      [local_remove_monitor, remote_remove_monitor]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?t:minutes(15)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
@@ -315,8 +335,6 @@ demonitor_flush_test(Node) ->
 -define(RM_MON_GROUPS, 100).
 -define(RM_MON_GPROCS, 100).
 
-remove_monitor(suite) ->
-    [local_remove_monitor, remote_remove_monitor].
 
 local_remove_monitor(Config) when is_list(Config) ->
     Gs = generate(fun () -> start_remove_monitor_group(node()) end,
diff --git a/erts/emulator/test/mtx_SUITE.erl b/erts/emulator/test/mtx_SUITE.erl
new file mode 100644
index 0000000000..024c3456a8
--- /dev/null
+++ b/erts/emulator/test/mtx_SUITE.erl
@@ -0,0 +1,492 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%
+%% Stress tests of rwmutex implementation.
+%%
+%% Author: Rickard Green
+%%
+-module(mtx_SUITE).
+
+%%-define(line_trace,true).
+
+-include_lib("common_test/include/ct.hrl").
+
+-export([all/0,suite/0,groups/0,
+	 init_per_group/2,end_per_group/2, init_per_suite/1, 
+	 end_per_suite/1, init_per_testcase/2, end_per_testcase/2]).
+
+-export([long_rwlock/1,
+	 hammer_ets_rwlock/1,
+	 hammer_rwlock/1,
+	 hammer_rwlock_check/1,
+	 hammer_tryrwlock/1,
+	 hammer_tryrwlock_check/1,
+	 hammer_sched_long_rwlock/1,
+	 hammer_sched_long_rwlock_check/1,
+	 hammer_sched_long_freqread_rwlock/1,
+	 hammer_sched_long_freqread_rwlock_check/1,
+	 hammer_sched_long_tryrwlock/1,
+	 hammer_sched_long_tryrwlock_check/1,
+	 hammer_sched_long_freqread_tryrwlock/1,
+	 hammer_sched_long_freqread_tryrwlock_check/1,
+	 hammer_sched_rwlock/1,
+	 hammer_sched_rwlock_check/1,
+	 hammer_sched_freqread_rwlock/1,
+	 hammer_sched_freqread_rwlock_check/1,
+	 hammer_sched_tryrwlock/1,
+	 hammer_sched_tryrwlock_check/1,
+	 hammer_sched_freqread_tryrwlock/1,
+	 hammer_sched_freqread_tryrwlock_check/1]).
+
+init_per_suite(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+    Lib = filename:join([DataDir, atom_to_list(?MODULE)]),
+    ok = erlang:load_nif(Lib, none),
+    Config.
+
+end_per_suite(Config) when is_list(Config) ->
+    catch erts_debug:set_internal_state(available_internal_state, false),
+    Config.
+
+init_per_testcase(_Case, Config) ->
+    Dog = ?t:timetrap(?t:minutes(15)),
+    %% Wait for deallocations to complete since we measure
+    %% runtime in test cases.
+    wait_deallocations(),
+    [{watchdog, Dog}|Config].
+
+end_per_testcase(_Func, Config) ->
+    Dog = ?config(watchdog, Config),
+    ?t:timetrap_cancel(Dog).
+
+wait_deallocations() ->
+    try
+	erts_debug:set_internal_state(wait, deallocations)
+    catch
+	error:undef ->
+	    erts_debug:set_internal_state(available_internal_state, true),
+	    wait_deallocations()
+    end.
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [long_rwlock, hammer_rwlock_check, hammer_rwlock,
+     hammer_tryrwlock_check, hammer_tryrwlock,
+     hammer_ets_rwlock, hammer_sched_long_rwlock_check,
+     hammer_sched_long_rwlock,
+     hammer_sched_long_freqread_rwlock_check,
+     hammer_sched_long_freqread_rwlock,
+     hammer_sched_long_tryrwlock_check,
+     hammer_sched_long_tryrwlock,
+     hammer_sched_long_freqread_tryrwlock_check,
+     hammer_sched_long_freqread_tryrwlock,
+     hammer_sched_rwlock_check, hammer_sched_rwlock,
+     hammer_sched_freqread_rwlock_check,
+     hammer_sched_freqread_rwlock,
+     hammer_sched_tryrwlock_check, hammer_sched_tryrwlock,
+     hammer_sched_freqread_tryrwlock_check,
+     hammer_sched_freqread_tryrwlock].
+
+groups() -> 
+    [].
+
+init_per_group(_GroupName, Config) ->
+	Config.
+
+end_per_group(_GroupName, Config) ->
+	Config.
+
+
+long_rwlock(Config) when is_list(Config) ->
+    statistics(runtime),
+    LLRes = long_rw_test(),
+    {_, RunTime} = statistics(runtime),
+    %% A very short run time is expected, since
+    %% threads in the test mostly wait
+    ?t:format("RunTime=~p~n", [RunTime]),
+    ?line true = RunTime < 400,
+    ?line RunTimeStr = "Run-time during test was "++integer_to_list(RunTime)++" ms.",
+    case LLRes of
+	ok ->
+	    {comment, RunTimeStr};
+	{comment, Comment} ->
+	    {comment, Comment ++ " " ++ RunTimeStr}
+    end.
+
+hammer_rwlock(Config) when is_list(Config) ->
+    hammer_rw_test(false).
+
+hammer_rwlock_check(Config) when is_list(Config) ->
+    hammer_rw_test(true).
+
+hammer_tryrwlock(Config) when is_list(Config) ->
+    hammer_tryrw_test(false).
+
+hammer_tryrwlock_check(Config) when is_list(Config) ->
+    hammer_tryrw_test(true).
+
+hammer_sched_rwlock(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(false, false, true, 0, 0).
+
+hammer_sched_rwlock_check(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(false, true, true, 0, 0).
+
+hammer_sched_freqread_rwlock(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(true, false, true, 0, 0).
+
+hammer_sched_freqread_rwlock_check(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(true, true, true, 0, 0).
+
+hammer_sched_tryrwlock(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(false, false, false, 0, 100).
+
+hammer_sched_tryrwlock_check(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(false, true, false, 0, 100).
+
+hammer_sched_freqread_tryrwlock(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(true, false, false, 0, 100).
+
+hammer_sched_freqread_tryrwlock_check(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(true, true, false, 0, 100).
+
+hammer_sched_long_rwlock(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(false, false, true, 100, 0).
+
+hammer_sched_long_rwlock_check(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(false, true, true, 100, 0).
+
+hammer_sched_long_freqread_rwlock(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(true, false, true, 100, 0).
+
+hammer_sched_long_freqread_rwlock_check(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(true, true, true, 100, 0).
+
+hammer_sched_long_tryrwlock(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(false, false, false, 100, 100).
+
+hammer_sched_long_tryrwlock_check(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(false, true, false, 100, 100).
+
+hammer_sched_long_freqread_tryrwlock(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(true, false, false, 100, 100).
+
+hammer_sched_long_freqread_tryrwlock_check(Config) when is_list(Config) ->
+    hammer_sched_rwlock_test(true, true, false, 100, 100).
+
+hammer_sched_rwlock_test(FreqRead, LockCheck, Blocking, WaitLocked, WaitUnlocked) ->
+    case create_rwlock(FreqRead, LockCheck) of
+	enotsup ->
+	    {skipped, "Not supported."};
+	RWLock ->
+	    Onln = erlang:system_info(schedulers_online),
+	    NWPs = case Onln div 3 of
+		       1 -> case Onln < 4 of
+				true -> 1;
+				false -> 2
+			    end;
+		       X -> X
+		   end,
+	    NRPs = Onln - NWPs,
+	    NoLockOps = ((((50000000 div Onln)
+			       div case {Blocking, WaitLocked} of
+				       {false, 0} -> 1;
+				       _ -> 10
+				   end)
+			      div (case WaitLocked == 0 of
+				       true -> 1;
+				       false -> WaitLocked*250
+				   end))
+			     div handicap()),
+	    ?t:format("NoLockOps=~p~n", [NoLockOps]),
+	    Sleep = case Blocking of
+			true -> NoLockOps;
+			false -> NoLockOps div 10
+		    end,
+	    WPs = lists:map(
+		    fun (Sched) ->
+			    spawn_opt(
+			      fun () ->
+				      io:format("Writer on scheduler ~p.~n",
+						[Sched]),
+				      Sched = erlang:system_info(scheduler_id),
+				      receive go -> gone end,
+				      hammer_sched_rwlock_proc(RWLock,
+							       Blocking,
+							       true,
+							       WaitLocked,
+							       WaitUnlocked,
+							       NoLockOps,
+							       Sleep),
+				      Sched = erlang:system_info(scheduler_id)
+			      end,
+			      [link, {scheduler, Sched}])
+		    end,
+		    lists:seq(1, NWPs)),
+	    RPs = lists:map(
+		    fun (Sched) ->
+			    spawn_opt(
+			      fun () ->
+				      io:format("Reader on scheduler ~p.~n",
+						[Sched]),
+				      Sched = erlang:system_info(scheduler_id),
+				      receive go -> gone end,
+				      hammer_sched_rwlock_proc(RWLock,
+							       Blocking,
+							       false,
+							       WaitLocked,
+							       WaitUnlocked,
+							       NoLockOps,
+							       Sleep),
+				      Sched = erlang:system_info(scheduler_id)
+			      end,
+			      [link, {scheduler, Sched}])
+		    end,
+		    lists:seq(NWPs + 1, NWPs + NRPs)),
+	    Procs = WPs ++ RPs,
+	    case {Blocking, WaitLocked} of
+		{_, 0} -> ok;
+		{false, _} -> ok;
+		_ -> statistics(runtime)
+	    end,
+	    lists:foreach(fun (P) -> P ! go end, Procs),
+	    lists:foreach(fun (P) ->
+				  M = erlang:monitor(process, P),
+				  receive
+				      {'DOWN', M, process, P, _} ->
+					  ok
+				  end
+			  end,
+			  Procs),
+	    case {Blocking, WaitLocked} of
+		{_, 0} -> ok;
+		{false, _} -> ok;
+		_ ->
+		    {_, RunTime} = statistics(runtime),
+		    ?t:format("RunTime=~p~n", [RunTime]),
+		    ?line true = RunTime < 700,
+		    {comment,
+		     "Run-time during test was "
+		     ++ integer_to_list(RunTime)
+		     ++ " ms."}
+	    end
+    end.
+
+hammer_sched_rwlock_proc(_RWLock,
+			 _Blocking,
+			 _WriteOp,
+			 _WaitLocked,
+			 _WaitUnlocked,
+			 0,
+			 _Sleep) ->
+    ok;
+hammer_sched_rwlock_proc(RWLock,
+			 Blocking,
+			 WriteOp,
+			 WaitLocked,
+			 WaitUnlocked,
+			 Times,
+			 Sleep) when Times rem Sleep == 0 ->
+    rwlock_op(RWLock, Blocking, WriteOp, WaitLocked, WaitUnlocked),
+    hammer_sched_rwlock_proc(RWLock,
+			     Blocking,
+			     WriteOp,
+			     WaitLocked,
+			     WaitUnlocked,
+			     Times - 1,
+			     Sleep);
+hammer_sched_rwlock_proc(RWLock,
+			 Blocking,
+			 WriteOp,
+			 WaitLocked,
+			 WaitUnlocked,
+			 Times,
+			 Sleep) ->
+    rwlock_op(RWLock, Blocking, WriteOp, WaitLocked, 0),
+    hammer_sched_rwlock_proc(RWLock,
+			     Blocking,
+			     WriteOp,
+			     WaitLocked,
+			     WaitUnlocked,
+			     Times - 1,
+			     Sleep).
+
+-define(HAMMER_ETS_RWLOCK_REPEAT_TIMES, 1).
+-define(HAMMER_ETS_RWLOCK_TSIZE, 500).
+
+hammer_ets_rwlock(Config) when is_list(Config) ->
+    {Ops, Procs} = case handicap() of
+		       1 -> {20000, 500};
+		       2 -> {20000, 50};
+		       3 -> {2000, 50};
+		       _ -> {200, 50}
+		   end,
+    ?t:format("Procs=~p~nOps=~p~n", [Procs, Ops]),
+    lists:foreach(fun (XOpts) ->
+			  ?t:format("Running with extra opts: ~p", [XOpts]),
+			  hammer_ets_rwlock_test(XOpts, true, 2, Ops,
+						 Procs, false)
+		  end,
+		  [[],
+		   [{read_concurrency, true}],
+		   [{write_concurrency, true}],
+		   [{read_concurrency, true},{write_concurrency, true}]]),
+    ok.
+
+%% Aux funcs
+
+long_rw_test() ->
+    exit(no_nif_implementation).
+
+hammer_rw_test(_Arg) ->
+    exit(no_nif_implementation).
+
+hammer_tryrw_test(_Arg) ->
+    exit(no_nif_implementation).
+
+create_rwlock(_FreqRead, _LockCheck) ->
+    exit(no_nif_implementation).
+
+rwlock_op(_RWLock, _Blocking, _WriteOp, _WaitLocked, _WaitUnlocked) ->
+    exit(no_nif_implementation).
+
+hammer_ets_rwlock_put_data() ->
+    put(?MODULE, {"here are some", data, "to store", make_ref()}).
+
+hammer_ets_rwlock_get_data() ->
+    get(?MODULE).
+
+hammer_ets_rwlock_ops(_T, _UW, _N, _C, _SC, 0) ->
+    ok;
+hammer_ets_rwlock_ops(T, UW, N, C, SC, Tot) when N >= ?HAMMER_ETS_RWLOCK_TSIZE ->
+    hammer_ets_rwlock_ops(T, UW, 0, C, SC, Tot);
+hammer_ets_rwlock_ops(T, UW, N, 0, SC, Tot) ->
+    case UW of
+	true ->
+	    true = ets:insert(T, {N, Tot, hammer_ets_rwlock_get_data()});
+	false ->
+	    [{N, _, _}] = ets:lookup(T, N)
+    end,
+    hammer_ets_rwlock_ops(T, UW, N+1, SC, SC, Tot-1);
+hammer_ets_rwlock_ops(T, UW, N, C, SC, Tot) ->
+    case UW of
+	false ->
+	    true = ets:insert(T, {N, Tot, hammer_ets_rwlock_get_data()});
+	true ->
+	    [{N, _, _}] = ets:lookup(T, N)
+    end,
+    hammer_ets_rwlock_ops(T, UW, N+1, C-1, SC, Tot-1).
+
+hammer_ets_rwlock_init(T, N) when N < ?HAMMER_ETS_RWLOCK_TSIZE ->
+    ets:insert(T, {N, N, N}),
+    hammer_ets_rwlock_init(T, N+1);
+hammer_ets_rwlock_init(_T, _N) ->
+    ok.
+
+hammer_ets_rwlock_test(XOpts, UW, C, N, NP, SC) ->
+    receive after 100 -> ok end,
+    {TP, TM} = spawn_monitor(
+		 fun () ->
+			 _L = repeat_list(
+				fun () ->
+					Caller = self(),
+					T = fun () ->
+						    Parent = self(),
+						    hammer_ets_rwlock_put_data(),
+						    T=ets:new(x, [public | XOpts]),
+						    hammer_ets_rwlock_init(T, 0),
+						    Ps0 = repeat_list(
+							    fun () ->
+								    spawn_link(
+								      fun () ->
+									      hammer_ets_rwlock_put_data(),
+									      receive go -> ok end,
+									      hammer_ets_rwlock_ops(T, UW, N, C, C, N),
+									      Parent ! {done, self()},
+									      receive after infinity -> ok end
+								      end)
+							    end,
+							    NP - case SC of
+								     false -> 0;
+								     _ -> 1
+								 end),
+						    Ps = case SC of
+							     false -> Ps0;
+							     _ -> [spawn_link(fun () ->
+										      hammer_ets_rwlock_put_data(),
+										      receive go -> ok end,
+										      hammer_ets_rwlock_ops(T, UW, N, SC, SC, N),
+										      Parent ! {done, self()},
+										      receive after infinity -> ok end
+									      end) | Ps0]
+							 end,
+						    Start = now(),
+						    lists:foreach(fun (P) -> P ! go end, Ps),
+						    lists:foreach(fun (P) -> receive {done, P} -> ok end end, Ps),
+						    Stop = now(),
+						    lists:foreach(fun (P) ->
+									  unlink(P),
+									  exit(P, bang),
+									  M = erlang:monitor(process, P),
+									  receive
+									      {'DOWN', M, process, P, _} -> ok
+									  end
+								  end, Ps),
+						    Res = timer:now_diff(Stop, Start)/1000000,
+						    Caller ! {?MODULE, self(), Res}
+					    end,
+					TP = spawn_link(T),
+					receive
+					    {?MODULE, TP, Res} ->
+						Res
+					end
+				end,
+				?HAMMER_ETS_RWLOCK_REPEAT_TIMES)
+		 end),
+    receive
+	{'DOWN', TM, process, TP, _} -> ok
+    end.
+
+repeat_list(Fun, N) ->
+    repeat_list(Fun, N, []).
+
+repeat_list(_Fun, 0, Acc) ->
+    Acc;
+repeat_list(Fun, N, Acc) ->
+    repeat_list(Fun, N-1, [Fun()|Acc]).
+
+
+handicap() ->
+    X0 = case catch (erlang:system_info(logical_processors_available) >=
+			 erlang:system_info(schedulers_online)) of
+	     true -> 1;
+	     _ -> 2
+	 end,
+    case erlang:system_info(build_type) of
+	opt ->
+	    X0;
+	ReallySlow when ReallySlow == debug;
+			ReallySlow == valgrind;
+			ReallySlow == purify ->
+	    X0*3;
+	_Slow ->
+	    X0*2
+    end.
+		
diff --git a/erts/emulator/test/mtx_SUITE_data/Makefile.src b/erts/emulator/test/mtx_SUITE_data/Makefile.src
new file mode 100644
index 0000000000..b6c843269c
--- /dev/null
+++ b/erts/emulator/test/mtx_SUITE_data/Makefile.src
@@ -0,0 +1,30 @@
+# 
+# %CopyrightBegin%
+# 
+# Copyright Ericsson AB 2010. All Rights Reserved.
+# 
+# The contents of this file are subject to the Erlang Public License,
+# Version 1.1, (the "License"); you may not use this file except in
+# compliance with the License. You should have received a copy of the
+# Erlang Public License along with this software. If not, it can be
+# retrieved online at http://www.erlang.org/.
+# 
+# Software distributed under the License is distributed on an "AS IS"
+# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+# the License for the specific language governing rights and limitations
+# under the License.
+# 
+# %CopyrightEnd%
+# 
+
+include @erts_lib_include_internal_generated@@[email protected]
+include @erts_lib_include_internal_generated@@DS@erts_internal.mk
+
+NIF_LIBS = mtx_SUITE@dll@
+
+SHLIB_EXTRA_CFLAGS =  $(ETHR_DEFS) -I@erts_lib_include_internal@ -I@erts_lib_include_internal_generated@
+LIBS = @ERTS_LIBS@
+
+all: $(NIF_LIBS)
+
+@SHLIB_RULES@
diff --git a/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c b/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c
new file mode 100644
index 0000000000..7c8137dc83
--- /dev/null
+++ b/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c
@@ -0,0 +1,698 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Stress tests of rwmutex implementation.
+ *
+ * Author: Rickard Green
+ */
+
+#include "erl_nif.h"
+
+#ifdef __WIN32__
+#  ifndef WIN32_LEAN_AND_MEAN
+#    define WIN32_LEAN_AND_MEAN
+#  endif
+#  include <windows.h>
+#else
+#  include "ethread.h"
+#  include "erl_misc_utils.h"
+#  include <unistd.h>
+#endif
+
+#include <errno.h>
+#include <stdio.h>
+
+static int
+fail(const char *file, int line, const char *function, const char *assertion);
+
+#undef ASSERT
+#define ASSERT(X) ((void) ((X) ? 1 : fail(__FILE__, __LINE__, __func__, #X)))
+
+#ifdef __WIN32__
+/*
+ * We cannot access the ethread symbols directly; test
+ * what we got in the nif api instead...
+ */
+#define HAVE_FREQREAD_SUPPORT 0
+#define RWMUTEX_T ErlNifRWLock
+#define RWMUTEX_CREATE(FR) enif_rwlock_create("dummy")
+#define RWMUTEX_DESTROY enif_rwlock_destroy
+#define RWMUTEX_WLOCK enif_rwlock_rwlock
+#define RWMUTEX_TRYWLOCK enif_rwlock_tryrwlock
+#define RWMUTEX_WUNLOCK enif_rwlock_rwunlock
+#define RWMUTEX_TRYRLOCK enif_rwlock_tryrlock
+#define RWMUTEX_RLOCK enif_rwlock_rlock
+#define RWMUTEX_RUNLOCK enif_rwlock_runlock
+#define THR_ID ErlNifTid
+#define THR_CREATE(A, B, C, D) enif_thread_create("dummy", (A), (B), (C), (D))
+#define THR_JOIN enif_thread_join
+#define ATOMIC_T volatile LONG
+#define ATOMIC_INIT(VarP, Val) (*(VarP) = (Val))
+#define ATOMIC_SET(VarP, Val) (*(VarP) = (Val))
+#define ATOMIC_READ(VarP) (*(VarP))
+#define ATOMIC_INC InterlockedIncrement
+#define ATOMIC_DEC InterlockedDecrement
+
+#else
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+#  define HAVE_FREQREAD_SUPPORT 1
+#else
+#  define HAVE_FREQREAD_SUPPORT 0
+#endif
+
+#define RWMUTEX_T ethr_rwmutex
+static ethr_rwmutex *
+RWMUTEX_CREATE(int freqread)
+{
+    ethr_rwmutex *rwmtx = enif_alloc(sizeof(ethr_rwmutex));
+    ethr_rwmutex_opt rwmtx_opt = ETHR_RWMUTEX_OPT_DEFAULT_INITER;
+    if (freqread)
+	rwmtx_opt.type = ETHR_RWMUTEX_TYPE_FREQUENT_READ;
+    ASSERT(rwmtx);
+    ASSERT(ethr_rwmutex_init_opt(rwmtx, &rwmtx_opt) == 0);
+    return rwmtx;
+}
+static void
+RWMUTEX_DESTROY(ethr_rwmutex *rwmtx)
+{
+    ASSERT(ethr_rwmutex_destroy(rwmtx) == 0);
+    enif_free(rwmtx);
+}
+#define RWMUTEX_TRYWLOCK ethr_rwmutex_tryrwlock
+#define RWMUTEX_WLOCK ethr_rwmutex_rwlock
+#define RWMUTEX_WUNLOCK ethr_rwmutex_rwunlock
+#define RWMUTEX_TRYRLOCK ethr_rwmutex_tryrlock
+#define RWMUTEX_RLOCK ethr_rwmutex_rlock
+#define RWMUTEX_RUNLOCK ethr_rwmutex_runlock
+#define THR_ID ethr_tid
+#define THR_CREATE ethr_thr_create
+#define THR_JOIN ethr_thr_join
+#define ATOMIC_T ethr_atomic_t
+#define ATOMIC_INIT ethr_atomic_init
+#define ATOMIC_SET ethr_atomic_set
+#define ATOMIC_READ ethr_atomic_read
+#define ATOMIC_INC ethr_atomic_inc
+#define ATOMIC_DEC ethr_atomic_dec
+
+#endif
+
+
+#if !defined(__func__)
+#  if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L
+#    if !defined(__GNUC__) ||  __GNUC__ < 2
+#      define __func__ "[unknown_function]"
+#    else
+#      define __func__ __FUNCTION__
+#    endif
+#  endif
+#endif
+
+static void milli_sleep(int ms);
+static int get_bool(ErlNifEnv* env, ERL_NIF_TERM term);
+
+/*
+ * Long rwlock testcase
+ */
+
+#define LONG_RW_NO_W_THREADS 6
+#define LONG_RW_NO_THREADS 20
+#define LONG_RW_NO_WLOCK_COUNT 100
+
+typedef struct {
+    RWMUTEX_T *rwlock;
+    ATOMIC_T *is_wlocked;
+    ATOMIC_T *is_rlocked;
+    int *stop;
+    int *count;
+    int sleep;
+} long_rw_t;
+
+static void *
+long_rw_w(void *varg)
+{
+    long_rw_t *arg = varg;
+    int stop = 0;
+    do {
+	RWMUTEX_WLOCK(arg->rwlock);
+	ASSERT(!ATOMIC_READ(arg->is_wlocked));
+	ATOMIC_SET(arg->is_wlocked, 1);
+	ASSERT(!ATOMIC_READ(arg->is_rlocked));
+	milli_sleep(arg->sleep);
+	if (++(*arg->count) > LONG_RW_NO_WLOCK_COUNT)
+	    stop = *arg->stop = 1;
+	ATOMIC_SET(arg->is_wlocked, 0);
+	ASSERT(!ATOMIC_READ(arg->is_rlocked));
+	RWMUTEX_WUNLOCK(arg->rwlock);
+    } while (!stop);
+    return NULL;
+}
+
+static void *
+long_rw_r(void *varg)
+{
+    long_rw_t *arg = varg;
+    int stop;
+    do {
+	RWMUTEX_RLOCK(arg->rwlock);
+	ASSERT(!ATOMIC_READ(arg->is_wlocked));
+	ATOMIC_INC(arg->is_rlocked);
+	milli_sleep(arg->sleep);
+	stop = *arg->stop;
+	ATOMIC_DEC(arg->is_rlocked);
+	ASSERT(!ATOMIC_READ(arg->is_wlocked));
+	RWMUTEX_RUNLOCK(arg->rwlock);
+    } while (!stop);
+    return NULL;
+}
+
+
+static ERL_NIF_TERM long_rw_test(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
+{
+    int res, freqread, i, count, stop;
+    ATOMIC_T is_wlocked, is_rlocked;
+    THR_ID tid[LONG_RW_NO_THREADS];
+    long_rw_t arg;
+    long_rw_t targ[LONG_RW_NO_THREADS];
+
+    ATOMIC_INIT(&is_wlocked, 0);
+    ATOMIC_INIT(&is_rlocked, 0);
+
+    freqread = 0;
+
+    arg.is_wlocked = &is_wlocked;
+    arg.is_rlocked = &is_rlocked;
+    arg.count = &count;
+    arg.stop = &stop;
+
+ restart:
+
+    stop = 0;
+    count = 0;
+
+    arg.rwlock = RWMUTEX_CREATE(freqread);
+
+    ASSERT(arg.rwlock);
+
+    for (i = 0; i < LONG_RW_NO_W_THREADS; i++) {
+	targ[i] = arg;
+	targ[i].sleep = 100 + i*10;
+	ASSERT(THR_CREATE(&tid[i], long_rw_w, &targ[i], NULL) == 0);
+    }
+    for (; i < LONG_RW_NO_THREADS; i++) {
+	targ[i] = arg;
+	targ[i].sleep = 100;
+	ASSERT(THR_CREATE(&tid[i], long_rw_r, &targ[i], NULL) == 0);
+    }
+    for (i = 0; i < LONG_RW_NO_THREADS; i++)
+	ASSERT(THR_JOIN(tid[i], NULL) == 0);
+
+    ASSERT(!ATOMIC_READ(arg.is_wlocked));
+    ASSERT(!ATOMIC_READ(arg.is_rlocked));
+
+    RWMUTEX_DESTROY(arg.rwlock);
+
+    if (HAVE_FREQREAD_SUPPORT && !freqread) {
+	freqread = 1;
+	goto restart;
+    }
+
+    if (freqread)
+	return enif_make_atom(env, "ok");
+    else
+	return enif_make_tuple2(env,
+				enif_make_atom(env,
+					       "comment"),
+				enif_make_string(env,
+						 "No frequent read test made.",
+						 ERL_NIF_LATIN1));
+}
+
+/*
+ * Hammer rwlock testcase
+ */
+
+#define HAMMER_RW_NO_W_THREADS 6
+#define HAMMER_RW_NO_THREADS 20
+#define HAMMER_RW_NO_WLOCK_COUNT 1000000
+
+typedef struct {
+    RWMUTEX_T *rwlock;
+    ATOMIC_T is_locked;
+    int lock_check;
+    int stop;
+    int count;
+} hammer_rw_t;
+
+static void *
+hammer_rw_w(void *varg)
+{
+    hammer_rw_t *arg = varg;
+    int stop = 0;
+    do {
+	RWMUTEX_WLOCK(arg->rwlock);
+	if (arg->lock_check) {
+	    ASSERT(!ATOMIC_READ(&arg->is_locked));
+	    ATOMIC_SET(&arg->is_locked, -1);
+	}
+	if (++arg->count > HAMMER_RW_NO_WLOCK_COUNT)
+	    stop = arg->stop = 1;
+	if (arg->lock_check) {
+	    ASSERT(ATOMIC_READ(&arg->is_locked) == -1);
+	    ATOMIC_SET(&arg->is_locked, 0);
+	}
+	RWMUTEX_WUNLOCK(arg->rwlock);
+    } while (!stop);
+    return NULL;
+}
+
+static void *
+hammer_rw_r(void *varg)
+{
+    hammer_rw_t *arg = varg;
+    int stop;
+    do {
+	RWMUTEX_RLOCK(arg->rwlock);
+	if (arg->lock_check) {
+	    ASSERT(ATOMIC_READ(&arg->is_locked) >= 0);
+	    ATOMIC_INC(&arg->is_locked);
+	}
+	stop = arg->stop;
+	if (arg->lock_check) {
+	    ASSERT(ATOMIC_READ(&arg->is_locked) > 0);
+	    ATOMIC_DEC(&arg->is_locked);
+	}
+	RWMUTEX_RUNLOCK(arg->rwlock);
+    } while (!stop); 
+    return NULL;
+}
+
+
+static ERL_NIF_TERM hammer_rw_test(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
+{
+    hammer_rw_t arg;
+    char buf[10];
+    int res, freqread, i;
+    THR_ID tid[HAMMER_RW_NO_THREADS];
+
+    if (argc != 1)
+	goto badarg;
+
+    arg.lock_check = get_bool(env, argv[0]);
+    if (arg.lock_check < 0)
+	goto badarg;
+
+    ATOMIC_INIT(&arg.is_locked, 0);
+
+    freqread = 0;
+
+ restart:
+    arg.stop = 0;
+    arg.count = 0;
+
+    arg.rwlock = RWMUTEX_CREATE(freqread);
+
+    ASSERT(arg.rwlock);
+
+    for (i = 0; i < HAMMER_RW_NO_W_THREADS; i++)
+	ASSERT(THR_CREATE(&tid[i], hammer_rw_w, &arg, NULL) == 0);
+    for (; i < HAMMER_RW_NO_THREADS; i++)
+	ASSERT(THR_CREATE(&tid[i], hammer_rw_r, &arg, NULL) == 0);
+    for (i = 0; i < HAMMER_RW_NO_THREADS; i++)
+	ASSERT(THR_JOIN(tid[i], NULL) == 0);
+
+    ASSERT(!ATOMIC_READ(&arg.is_locked));
+
+    RWMUTEX_DESTROY(arg.rwlock);
+
+    if (HAVE_FREQREAD_SUPPORT && !freqread) {
+	freqread = 1;
+	goto restart;
+    }
+
+    if (freqread)
+	return enif_make_atom(env, "ok");
+    else
+	return enif_make_tuple2(env,
+				enif_make_atom(env,
+					       "comment"),
+				enif_make_string(env,
+						 "No frequent read test made.",
+						 ERL_NIF_LATIN1));
+ badarg:
+    return enif_make_badarg(env);
+}
+
+/*
+ * Hammer try rwlock testcase
+ */
+
+#define HAMMER_TRYRW_NO_W_THREADS 10
+#define HAMMER_TRYRW_NO_THREADS 20
+#define HAMMER_TRYRW_NO_WLOCK_COUNT 10000000
+#define HAMMER_TRYRW_NO_RLOCK_COUNT 10000000
+#define HAMMER_TRYRW_NO_WLOCK_WAIT_COUNT ((10*HAMMER_TRYRW_NO_WLOCK_COUNT)/8)
+#define HAMMER_TRYRW_NO_RLOCK_WAIT_COUNT ((10*HAMMER_TRYRW_NO_RLOCK_COUNT)/8)
+
+typedef struct {
+    RWMUTEX_T *rwlock;
+    ATOMIC_T is_locked;
+    int lock_check;
+    int w_count;
+    ATOMIC_T r_count;
+} hammer_tryrw_t;
+
+static void *
+hammer_tryrw_w(void *varg)
+{
+    hammer_tryrw_t *arg = varg;
+    int stop = 0;
+    int wait = 0;
+    do {
+	while (EBUSY == RWMUTEX_TRYWLOCK(arg->rwlock));
+	if (arg->lock_check) {
+	    ASSERT(!ATOMIC_READ(&arg->is_locked));
+	    ATOMIC_SET(&arg->is_locked, -1);
+	}
+	if (++arg->w_count > HAMMER_TRYRW_NO_WLOCK_COUNT)
+	    stop = 1;
+	else if (arg->w_count > HAMMER_TRYRW_NO_RLOCK_WAIT_COUNT)
+	    wait = 1;
+	if (arg->lock_check) {
+	    ASSERT(ATOMIC_READ(&arg->is_locked) == -1);
+	    ATOMIC_SET(&arg->is_locked, 0);
+	}
+	RWMUTEX_WUNLOCK(arg->rwlock);
+	if (wait)
+	    milli_sleep(1);
+    } while (!stop);
+    return NULL;
+}
+
+static void *
+hammer_tryrw_r(void *varg)
+{
+    hammer_tryrw_t *arg = varg;
+    long r_count;
+    int stop = 0;
+    int wait = 0;
+    do {
+	while (EBUSY == RWMUTEX_TRYRLOCK(arg->rwlock));
+	if (arg->lock_check) {
+	    ASSERT(ATOMIC_READ(&arg->is_locked) >= 0);
+	    ATOMIC_INC(&arg->is_locked);
+	}
+	ATOMIC_INC(&arg->r_count);
+	r_count = ATOMIC_READ(&arg->r_count);
+	if (r_count > HAMMER_TRYRW_NO_RLOCK_COUNT)
+	    stop = 1;
+	else if (r_count > HAMMER_TRYRW_NO_RLOCK_WAIT_COUNT)
+	    wait = 1;
+	if (arg->lock_check) {
+	    ASSERT(ATOMIC_READ(&arg->is_locked) > 0);
+	    ATOMIC_DEC(&arg->is_locked);
+	}
+	RWMUTEX_RUNLOCK(arg->rwlock);
+	if (wait)
+	    milli_sleep(1);
+    } while (!stop);
+    return NULL;
+}
+
+
+static ERL_NIF_TERM hammer_tryrw_test(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
+{
+    hammer_tryrw_t arg;
+    char buf[10];
+    int res, freqread, i;
+    THR_ID tid[HAMMER_TRYRW_NO_THREADS];
+
+    if (argc != 1)
+	goto badarg;
+
+    arg.lock_check = get_bool(env, argv[0]);
+    if (arg.lock_check < 0)
+	goto badarg;
+
+    ATOMIC_INIT(&arg.is_locked, 0);
+    freqread = 0;
+
+ restart:
+
+    arg.w_count = 0;
+    ATOMIC_INIT(&arg.r_count, 0);
+
+    arg.rwlock = RWMUTEX_CREATE(freqread);
+
+    ASSERT(arg.rwlock);
+
+    for (i = 0; i < HAMMER_TRYRW_NO_W_THREADS; i++)
+	ASSERT(THR_CREATE(&tid[i], hammer_tryrw_w, &arg, NULL) == 0);
+    for (; i < HAMMER_TRYRW_NO_THREADS; i++)
+	ASSERT(THR_CREATE(&tid[i], hammer_tryrw_r, &arg, NULL) == 0);
+    for (i = 0; i < HAMMER_TRYRW_NO_THREADS; i++)
+	ASSERT(THR_JOIN(tid[i], NULL) == 0);
+
+    ASSERT(!ATOMIC_READ(&arg.is_locked));
+
+    RWMUTEX_DESTROY(arg.rwlock);
+
+    if (HAVE_FREQREAD_SUPPORT && !freqread) {
+	freqread = 1;
+	goto restart;
+    }
+
+    if (freqread)
+	return enif_make_atom(env, "ok");
+    else
+	return enif_make_tuple2(env,
+				enif_make_atom(env,
+					       "comment"),
+				enif_make_string(env,
+						 "No frequent read test made.",
+						 ERL_NIF_LATIN1));
+ badarg:
+    return enif_make_badarg(env);
+}
+
+typedef struct {
+    int lock_check;
+    ATOMIC_T is_locked;
+    RWMUTEX_T *rwlock;
+} rwlock_resource_t;
+
+static void
+rwlock_destructor(ErlNifEnv* env, void* obj)
+{
+    rwlock_resource_t *rwlr = obj;
+    if (rwlr->lock_check)
+	ASSERT(!ATOMIC_READ(&rwlr->is_locked));
+    RWMUTEX_DESTROY(rwlr->rwlock);
+}
+
+/*
+ * create_rwlock(FreqRead, LockCheck)
+ */
+
+static ERL_NIF_TERM
+create_rwlock(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
+{
+    int lock_check, freqread;
+    ERL_NIF_TERM rwlock_term;
+    rwlock_resource_t *rwlr;
+    char buf[100];
+
+    if (argc != 2)
+	goto badarg;
+
+    freqread = get_bool(env, argv[0]);
+    if (freqread < 0)
+	goto badarg;
+
+    if (!HAVE_FREQREAD_SUPPORT && freqread)
+	return enif_make_atom(env, "enotsup");
+
+    lock_check = get_bool(env, argv[1]);
+    if (lock_check < 0)
+	goto badarg;
+
+    rwlr = enif_alloc_resource(enif_priv_data(env), sizeof(rwlock_resource_t));
+    rwlr->lock_check = lock_check;
+    ATOMIC_INIT(&rwlr->is_locked, 0);
+    rwlr->rwlock = RWMUTEX_CREATE(freqread);
+    rwlock_term = enif_make_resource(env, rwlr);
+    enif_release_resource(rwlr);
+    return rwlock_term;
+
+ badarg:
+    return enif_make_badarg(env);
+}
+
+/*
+ * rwlock_op(RWLock, Blocking, WriteOp, WaitTime) 
+ */
+
+static ERL_NIF_TERM
+rwlock_op(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
+{
+    /*
+     * Use a union for pointer type conversion to avoid compiler warnings
+     * about strict-aliasing violations with gcc-4.1. gcc >= 4.2 does not
+     * emit the warning.
+     * TODO: Reconsider use of union once gcc-4.1 is obsolete?
+     */
+    union { void* vp; rwlock_resource_t *p; } rwlr;
+    int blocking, write, wait_locked, wait_unlocked;
+
+    if (argc != 5)
+	goto badarg;
+
+    if (!enif_get_resource(env, argv[0], enif_priv_data(env), &rwlr.vp))
+	goto badarg;
+
+    blocking = get_bool(env, argv[1]);
+    if (blocking < 0)
+	goto badarg;
+
+    write = get_bool(env, argv[2]);
+    if (write < 0)
+	goto badarg;
+
+    if (!enif_get_int(env, argv[3], &wait_locked))
+	goto badarg;
+    if (wait_locked < 0)
+	goto badarg;
+
+    if (!enif_get_int(env, argv[4], &wait_unlocked))
+	goto badarg;
+    if (wait_unlocked < 0)
+	goto badarg;
+
+    if (write) {
+	if (blocking)
+	    RWMUTEX_WLOCK(rwlr.p->rwlock);
+	else
+	    while (EBUSY == RWMUTEX_TRYWLOCK(rwlr.p->rwlock));
+	if (rwlr.p->lock_check) {
+	    ASSERT(!ATOMIC_READ(&rwlr.p->is_locked));
+	    ATOMIC_SET(&rwlr.p->is_locked, -1);
+	}
+    }
+    else {
+	if (blocking)
+	    RWMUTEX_RLOCK(rwlr.p->rwlock);
+	else
+	    while (EBUSY == RWMUTEX_TRYRLOCK(rwlr.p->rwlock));
+	if (rwlr.p->lock_check) {
+	    ASSERT(ATOMIC_READ(&rwlr.p->is_locked) >= 0);
+	    ATOMIC_INC(&rwlr.p->is_locked);
+	}
+    }
+
+    if (wait_locked)
+	milli_sleep(wait_locked);
+
+    if (write) {
+	if (rwlr.p->lock_check) {
+	    ASSERT(ATOMIC_READ(&rwlr.p->is_locked) == -1);
+	    ATOMIC_SET(&rwlr.p->is_locked, 0);
+	}
+	RWMUTEX_WUNLOCK(rwlr.p->rwlock);
+    }
+    else {
+	if (rwlr.p->lock_check) {
+	    ASSERT(ATOMIC_READ(&rwlr.p->is_locked) > 0);
+	    ATOMIC_DEC(&rwlr.p->is_locked);
+	}
+	RWMUTEX_RUNLOCK(rwlr.p->rwlock);
+    }
+
+    if (wait_unlocked)
+	milli_sleep(wait_unlocked);
+
+    return enif_make_atom(env, "ok");
+ badarg:
+    return enif_make_badarg(env);
+}
+
+static int load_nif_lib(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
+{
+    *priv_data = enif_open_resource_type(env,
+					 NULL,
+					 "rwlock_resource",
+					 rwlock_destructor,
+					 ERL_NIF_RT_CREATE,
+					 NULL);
+    if (*priv_data)
+	return 0;
+    else
+	return -1;
+}
+
+/*
+ * 0 -> false
+ * >0 -> true
+ * <0 -> error
+ */
+
+static int
+get_bool(ErlNifEnv* env, ERL_NIF_TERM term)
+{
+    int res;
+    char buf[10];
+
+    res = enif_get_atom(env, term, buf, sizeof(buf), ERL_NIF_LATIN1);
+    if (res == 0)
+	return -1;
+    if (strcmp("false", buf) == 0)
+	return 0;
+    else if (strcmp("true", buf) == 0)
+	return 1;
+    else
+	return -1;
+}
+
+static int
+fail(const char *file, int line, const char *function, const char *assertion)
+{
+    fprintf(stderr, "%s:%d: Assertion failed in %s(): %s\n",
+	    file, line, function, assertion);
+    abort();
+}
+
+static void
+milli_sleep(int ms)
+{
+#ifdef __WIN32__
+    Sleep(ms);
+#else
+    while (erts_milli_sleep(ms) != 0);
+#endif
+}
+
+static ErlNifFunc nif_funcs[] = {
+    {"long_rw_test", 0, long_rw_test},
+    {"hammer_rw_test", 1, hammer_rw_test},
+    {"hammer_tryrw_test", 1, hammer_tryrw_test},
+    {"create_rwlock", 2, create_rwlock},
+    {"rwlock_op", 5, rwlock_op}
+};
+
+ERL_NIF_INIT(mtx_SUITE, nif_funcs, load_nif_lib, NULL, NULL, NULL)
diff --git a/erts/emulator/test/nested_SUITE.erl b/erts/emulator/test/nested_SUITE.erl
index 310892424e..2cd67ebaae 100644
--- a/erts/emulator/test/nested_SUITE.erl
+++ b/erts/emulator/test/nested_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,11 +19,33 @@
 
 -module(nested_SUITE).
 
--export([all/1, case_in_case/1, case_in_after/1, catch_in_catch/1, bif_in_bif/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 case_in_case/1, case_in_after/1, catch_in_catch/1, bif_in_bif/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [case_in_case, case_in_after, catch_in_catch,
+     bif_in_bif].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) -> [case_in_case, case_in_after, catch_in_catch, bif_in_bif].
 
 case_in_case(suite) -> [];
 case_in_case(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 522caec8f1..6bd7361612 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -20,29 +20,73 @@
 -module(nif_SUITE).
 
 %%-define(line_trace,true).
-%%-define(CHECK(Exp,Got), ?line check(Exp,Got,?LINE)).
--define(CHECK(Exp,Got), ?line Exp = Got).
+-define(CHECK(Exp,Got), check(Exp,Got,?LINE)).
+%%-define(CHECK(Exp,Got), ?line Exp = Got).
+
+-include_lib("test_server/include/test_server.hrl").
+
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, 
+	 end_per_testcase/2, basic/1, reload/1, upgrade/1, heap_frag/1,
+	 types/1, many_args/1, binaries/1, get_string/1, get_atom/1, 
+	 api_macros/1,
+	 from_array/1, iolist_as_binary/1, resource/1, resource_binary/1, 
+	 resource_takeover/1,
+	 threading/1, send/1, send2/1, send3/1, send_threaded/1, neg/1, 
+	 is_checks/1,
+	 get_length/1, make_atom/1, make_string/1, reverse_list_test/1,
+	 otp_9668/1
+	]).
 
--include("test_server.hrl").
+-export([many_args_100/100]).
+
+
+%% -export([lib_version/0,call_history/0,hold_nif_mod_priv_data/1,nif_mod_call_history/0,
+%% 	 list_seq/1,type_test/0,tuple_2_list/1,is_identical/2,compare/2,
+%% 	 clone_bin/1,make_sub_bin/3,string_to_bin/2,atom_to_bin/2,macros/1,
+%% 	 tuple_2_list_and_tuple/1,iolist_2_bin/1,get_resource_type/1,alloc_resource/2,
+%% 	 make_resource/1,get_resource/2,release_resource/1,last_resource_dtor_call/0, suite/0,
+%% 	 make_new_resource/2,make_new_resource_binary/1,send_list_seq/2,send_new_blob/2,
+%% 	 alloc_msgenv/0,clear_msgenv/1,grow_blob/2,send_blob/2,send_blob_thread/3,
+%% 	 join_send_thread/1]).
 
--export([all/1, fin_per_testcase/2, basic/1, reload/1, upgrade/1, heap_frag/1,
-	 types/1, many_args/1, binaries/1, get_string/1, get_atom/1, api_macros/1,
-	 from_array/1, iolist_as_binary/1, resource/1, resource_takeover/1,
-	 threading/1, neg/1]).
 
--export([many_args_100/100]).
 -define(nif_stub,nif_stub_error(?LINE)).
 
-all(suite) ->
-    [basic, reload, upgrade, heap_frag, types, many_args, binaries, get_string,
-     get_atom, api_macros, from_array, iolist_as_binary, resource,
-     resource_takeover, threading, neg].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [basic, reload, upgrade, heap_frag, types, many_args,
+     binaries, get_string, get_atom, api_macros, from_array,
+     iolist_as_binary, resource, resource_binary,
+     resource_takeover, threading, send, send2, send3,
+     send_threaded, neg, is_checks, get_length, make_atom,
+     make_string,reverse_list_test,
+     otp_9668
+    ].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
 
-%%init_per_testcase(_Case, Config) ->
-%%    ?line Dog = ?t:timetrap(?t:seconds(60*60*24)),
-%%    [{watchdog, Dog}|Config].
+end_per_group(_GroupName, Config) ->
+    Config.
 
-fin_per_testcase(_Func, _Config) ->
+
+init_per_testcase(_Case, Config) ->
+%    ?line Dog = ?t:timetrap(?t:seconds(60*60*24)),
+    Config.
+
+end_per_testcase(_Func, _Config) ->
     %%Dog = ?config(watchdog, Config),
     %%?t:timetrap_cancel(Dog),
     P1 = code:purge(nif_mod),
@@ -57,7 +101,7 @@ basic(Config) when is_list(Config) ->
     ?line true = (lib_version() =/= undefined),
     ?line [{load,1,1,101},{lib_version,1,2,102}] = call_history(),
     ?line [] = call_history(),
-    ?line [?MODULE] = erlang:system_info(taints),
+    ?line true = lists:member(?MODULE, erlang:system_info(taints)),
     ok.
 
 reload(doc) -> ["Test reload callback in nif lib"];
@@ -91,7 +135,8 @@ reload(Config) when is_list(Config) ->
     ?line true = erlang:purge_module(nif_mod),
     ?line [{unload,1,3,103}] = nif_mod_call_history(),    
 
-    ?line [?MODULE, nif_mod] = erlang:system_info(taints),
+    ?line true = lists:member(?MODULE, erlang:system_info(taints)),
+    ?line true = lists:member(nif_mod, erlang:system_info(taints)),
     ?line verify_tmpmem(TmpMem),
     ok.
 
@@ -181,7 +226,8 @@ upgrade(Config) when is_list(Config) ->
     ?line true = erlang:purge_module(nif_mod),
     ?line [{unload,2,4,204}] = nif_mod_call_history(),    
 
-    ?line [?MODULE, nif_mod] = erlang:system_info(taints),    
+    ?line true = lists:member(?MODULE, erlang:system_info(taints)),
+    ?line true = lists:member(nif_mod, erlang:system_info(taints)),
     ?line verify_tmpmem(TmpMem),
     ok.
 
@@ -215,10 +261,54 @@ types(Config) when is_list(Config) ->
                   end,
                   [{},{ok},{{}},{[],{}},{1,2,3,4,5}]),
     Stuff = [[],{},0,0.0,(1 bsl 100),(fun()-> ok end),make_ref(),self()],
-    [eq_cmp(A,clone(B)) || A<-Stuff, B<-Stuff],                   
+    [eq_cmp(A,clone(B)) || A<-Stuff, B<-Stuff],
+
+    {IntSz, LongSz} = type_sizes(),
+    UintMax = (1 bsl (IntSz*8)) - 1,
+    IntMax = UintMax bsr 1,
+    IntMin = -(IntMax+1),
+    UlongMax = (1 bsl (LongSz*8)) - 1,
+    LongMax = UlongMax bsr 1,
+    LongMin = -(LongMax+1),
+    Uint64Max = (1 bsl 64) - 1,
+    Int64Max = Uint64Max bsr 1,
+    Int64Min = -(Int64Max+1),
+    Limits = [{IntMin,IntMax},{0,UintMax},{LongMin,LongMax},{0,UlongMax},{Int64Min,Int64Max},{0,Uint64Max}],
+    io:format("Limits = ~p\n", [Limits]),
+    lists:foreach(fun(I) ->
+			  R1 = echo_int(I),
+			  %%io:format("echo_int(~p) -> ~p\n", [I, R1]),
+			  R2 = my_echo_int(I, Limits),
+			  ?line R1 = R2,
+			  ?line true = (R1 =:= R2),
+			  ?line true = (R1 == R2)
+		  end, int_list()),
+
     ?line verify_tmpmem(TmpMem),
+    ?line true = (compare(-1294536544000, -1178704800000) < 0),
+    ?line true = (compare(-1178704800000, -1294536544000) > 0),
+    ?line true = (compare(-295147905179352825856, -36893488147419103232) < 0),
+    ?line true = (compare(-36893488147419103232, -295147905179352825856) > 0),
+    ?line true = (compare(-29514790517935282585612345678, -36893488147419103232) < 0),
+    ?line true = (compare(-36893488147419103232, -29514790517935282585612345678) > 0),
     ok.
 
+int_list() ->
+    Start = 1 bsl 200,
+    int_list([Start], -Start).
+int_list([N | _]=List, End) when N<End ->
+    List;
+int_list([N | _]=List, End) ->
+    int_list([N - (1 + (abs(N) div 3)) | List], End).
+    
+my_echo_int(I, Limits) ->
+    lists:map(fun({Min,Max}) ->
+		      if I < Min -> false;
+			 I > Max -> false;
+			 true -> I
+		      end
+	      end, Limits).
+
 clone(X) ->
     binary_to_term(term_to_binary(X)).
 
@@ -473,12 +563,51 @@ resource_new_do2(Type) ->
     {{PtrA,BinA}, {ResB,PtrB,BinB}}.
 
 resource_neg(TypeA) ->
+    resource_neg_do(TypeA),
+
+    catch exit(42), % dummy exception to purge saved stacktraces from earlier exception
+    erlang:garbage_collect(),
+    ?line {_,_,2} = last_resource_dtor_call(),
+    ok.
+
+resource_neg_do(TypeA) ->
     TypeB = get_resource_type(1),
-    Aptr = alloc_resource(TypeA, <<"Arnold">>),
-    Bptr = alloc_resource(TypeB, <<"Bobo">>),
-    ?line {'EXIT',{badarg,_}} = (catch get_resource(TypeA, Bptr)),
-    ?line {'EXIT',{badarg,_}} = (catch get_resource(TypeB, Aptr)),
+    ResA = make_new_resource(TypeA, <<"Arnold">>),
+    ResB= make_new_resource(TypeB, <<"Bobo">>),
+    ?line {'EXIT',{badarg,_}} = (catch get_resource(TypeA, ResB)),
+    ?line {'EXIT',{badarg,_}} = (catch get_resource(TypeB, ResA)),
     ok.
+
+resource_binary(doc) -> ["Test enif_make_resource_binary"];
+resource_binary(suite) -> [];
+resource_binary(Config) when is_list(Config) ->
+    ?line ensure_lib_loaded(Config, 1),
+    ?line {Ptr,Bin} = resource_binary_do(),
+    erlang:garbage_collect(),
+    Last = last_resource_dtor_call(),
+    ?CHECK({Ptr,Bin,1}, Last),
+    ok.
+
+resource_binary_do() ->
+    Bin = <<"Hej Hopp i lingonskogen">>,
+    ?line {Ptr,ResBin1} = make_new_resource_binary(Bin),
+    ?line ResBin1 = Bin,          
+    ?line ResInfo = {Ptr,_} = get_resource(binary_resource_type,ResBin1),
+
+    Papa = self(),
+    Forwarder = spawn_link(fun() -> forwarder(Papa) end),
+    io:format("sending to forwarder pid=~p\n",[Forwarder]),  
+    Forwarder ! ResBin1,
+    ResBin2 = receive_any(),
+    ?line ResBin2 = ResBin1,
+    ?line ResInfo = get_resource(binary_resource_type,ResBin2),
+    Forwarder ! terminate,
+    ?line {Forwarder, 1} = receive_any(),
+    erlang:garbage_collect(),
+    ?line ResInfo = get_resource(binary_resource_type,ResBin1),
+    ?line ResInfo = get_resource(binary_resource_type,ResBin2),
+    ResInfo.
+
     
 -define(RT_CREATE,1).
 -define(RT_TAKEOVER,2).
@@ -672,7 +801,8 @@ resource_takeover(Config) when is_list(Config) ->
     ?line ok = forget_resource(AN4),
     ?line [] = nif_mod_call_history(),
 
-    ?line [?MODULE, nif_mod] = erlang:system_info(taints),
+    ?line true = lists:member(?MODULE, erlang:system_info(taints)),
+    ?line true = lists:member(nif_mod, erlang:system_info(taints)),
     ?line verify_tmpmem(TmpMem),    
     ok.
 
@@ -729,7 +859,13 @@ resource_holder(Pid,Reply,List) ->
 
 
 threading(doc) -> ["Test the threading API functions (reuse tests from driver API)"];
-threading(Config) when is_list(Config) ->    
+threading(Config) when is_list(Config) ->
+    case erlang:system_info(threads) of
+    	true -> threading_do(Config);
+	false -> {skipped,"No thread support"}
+    end.
+
+threading_do(Config) ->
     ?line Data = ?config(data_dir, Config),
     ?line File = filename:join(Data, "tester"),
     ?line {ok,tester,ModBin} = compile:file(File, [binary,return_errors]),
@@ -743,7 +879,282 @@ threading(Config) when is_list(Config) ->
 
     ?line ok = tester:load_nif_lib(Config, "tsd"),
     ?line ok = tester:run().
+
+send(doc) -> ["Test NIF message sending"];
+send(Config) when is_list(Config) ->    
+    ensure_lib_loaded(Config),
+
+    N = 1500,
+    List = lists:seq(1,N),
+    ?line {ok,1} = send_list_seq(N, self),
+    ?line {ok,1} = send_list_seq(N, self()),
+    ?line List = receive_any(),
+    ?line List = receive_any(),
+    Papa = self(),
+    spawn_link(fun() -> ?line {ok,1} = send_list_seq(N, Papa) end),
+    ?line List = receive_any(),
+
+    ?line {ok, 1, BlobS} = send_new_blob(self(), other_term()),
+    ?line BlobR = receive_any(),
+    io:format("Sent ~p\nGot ~p\n", [BlobS, BlobR]),
+    ?line BlobR = BlobS,
+
+    %% send to dead pid
+    {DeadPid, DeadMon} = spawn_monitor(fun() -> void end),
+    ?line {'DOWN', DeadMon, process, DeadPid, normal} = receive_any(),
+    {ok,0} = send_list_seq(7, DeadPid),
+    ok.
+
+send2(doc) -> ["More NIF message sending"];
+send2(Config) when is_list(Config) ->
+    ensure_lib_loaded(Config),
+
+    send2_do1(fun send_blob_dbg/2),
+    ok.
+
+send_threaded(doc) -> ["Send msg from user thread"];
+send_threaded(Config) when is_list(Config) ->
+    case erlang:system_info(smp_support) of
+	true ->
+	    send2_do1(fun(ME,To) -> send_blob_thread_dbg(ME,To,join) end),
+	    send2_do1(fun(ME,To) -> send_blob_thread_and_join(ME,To) end),
+	    ok;
+	false ->
+	    {skipped,"No threaded send on non-SMP"}
+    end.
+
+
+send2_do1(SendBlobF) ->
+    io:format("sending to self=~p\n",[self()]),
+    send2_do2(SendBlobF, self()),
+
+    Papa = self(),
+    Forwarder = spawn_link(fun() -> forwarder(Papa) end),
+    io:format("sending to forwarder pid=~p\n",[Forwarder]),
+    send2_do2(SendBlobF, Forwarder),
+    Forwarder ! terminate,
+    ?line {Forwarder, 4} = receive_any(),
+    ok.
+
+send2_do2(SendBlobF, To) ->   
+    MsgEnv = alloc_msgenv(),
+    repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []),
+    ?line {ok,1,Blob0} = SendBlobF(MsgEnv, To),
+    ?line Blob1 = receive_any(),
+    ?line Blob1 = Blob0,
     
+    clear_msgenv(MsgEnv),
+    repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []),
+    ?line {ok,1,Blob2} = SendBlobF(MsgEnv, To),
+    ?line Blob3 = receive_any(),
+    ?line Blob3 = Blob2,
+
+    clear_msgenv(MsgEnv),
+    repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []),
+
+    clear_msgenv(MsgEnv),
+    repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []),
+    ?line {ok,1,Blob4} = SendBlobF(MsgEnv, To),
+    ?line Blob5 = receive_any(),
+    ?line Blob5 = Blob4,
+
+    clear_msgenv(MsgEnv),
+    clear_msgenv(MsgEnv),
+    repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []),
+    ?line {ok,1,Blob6} = SendBlobF(MsgEnv, To),
+    ?line Blob7 = receive_any(),
+    ?line Blob7 = Blob6,
+
+    ok.
+
+
+send_blob_thread_and_join(MsgEnv, To) ->
+    ?line {ok,Blob} = send_blob_thread_dbg(MsgEnv, To, no_join),
+    ?line {ok,SendRes} = join_send_thread(MsgEnv),
+    {ok,SendRes,Blob}.
+
+send_blob_dbg(MsgEnv, To) ->
+    Ret = send_blob(MsgEnv, To),
+    %%io:format("send_blob to ~p returned ~p\n",[To,Ret]),
+    Ret.
+
+send_blob_thread_dbg(MsgEnv, To, Join) ->
+    Ret = send_blob_thread(MsgEnv, To, Join),
+    %%io:format("send_blob_thread to ~p Join=~p returned ~p\n",[To,Join,Ret]),
+    Ret.
+
+
+forwarder(To) ->
+    forwarder(To, 0).
+forwarder(To, N) ->
+    case receive_any() of
+	terminate ->
+	    To ! {self(), N};
+	Msg ->	    
+	    To ! Msg,	   
+	    forwarder(To, N+1)
+    end.
+
+other_term() ->
+    {fun(X,Y) -> X*Y end, make_ref()}.
+
+send3(doc) -> ["Message sending stress test"];
+send3(Config) when is_list(Config) ->
+    %% Let a number of processes send random message blobs between each other
+    %% using enif_send. Kill and spawn new ones randomly to keep a ~constant
+    %% number of workers running.
+    Seed = now(),
+    io:format("seed: ~p\n",[Seed]), 
+    random:seed(Seed),    
+    ets:new(nif_SUITE,[named_table,public]),
+    ?line true = ets:insert(nif_SUITE,{send3,0,0,0,0}),
+    timer:send_after(10000, timeout), % Run for 10 seconds
+    SpawnCnt = send3_controller(0, [], [], 20),
+    ?line [{_,Rcv,SndOk,SndFail,Balance}] = ets:lookup(nif_SUITE,send3),
+    io:format("spawns=~p received=~p, sent=~p send-failure=~p balance=~p\n",
+              [SpawnCnt,Rcv,SndOk,SndFail,Balance]),
+    ets:delete(nif_SUITE).
+
+send3_controller(SpawnCnt, [], _, infinity) ->
+    SpawnCnt;
+send3_controller(SpawnCnt0, Mons0, Pids0, Tick) ->
+    receive 
+        timeout ->
+            io:format("Timeout. Sending 'halt' to ~p\n",[Pids0]),            
+            lists:foreach(fun(P) -> P ! {halt,self()} end, Pids0),
+            lists:foreach(fun(P) -> receive {halted,P} -> ok end end, Pids0),
+            QTot = lists:foldl(fun(P,QSum) ->
+                                 {message_queue_len,QLen} = 
+                                    erlang:process_info(P,message_queue_len),
+                                 QSum + QLen
+                               end, 0, Pids0),
+            io:format("Total queue length ~p\n",[QTot]),            
+            lists:foreach(fun(P) -> P ! die end, Pids0),
+            send3_controller(SpawnCnt0, Mons0, [], infinity);
+        {'DOWN', MonRef, process, _Pid, _} ->
+            Mons1 = lists:delete(MonRef, Mons0),
+            %%io:format("Got DOWN from ~p. Monitors left: ~p\n",[Pid,Mons1]),            
+            send3_controller(SpawnCnt0, Mons1, Pids0, Tick)
+    after Tick -> 
+        Max = 20,
+        N = length(Pids0),
+        PidN = random:uniform(Max),
+        %%io:format("N=~p PidN=~p Pids0=~p\n", [N,PidN,Pids0]), 
+        case PidN > N of
+            true ->
+                {NewPid,Mon} = spawn_opt(fun send3_proc/0, [link,monitor]),                
+                lists:foreach(fun(P) -> P ! {is_born,NewPid} end, Pids0),
+                ?line Balance = ets:lookup_element(nif_SUITE,send3,5),
+                Inject = (Balance =< 0),
+                case Inject of
+                    true ->  ok;
+                    false -> ets:update_element(nif_SUITE,send3,{5,-1})
+                end,
+                NewPid ! {pids,Pids0,Inject},
+                send3_controller(SpawnCnt0+1, [Mon|Mons0], [NewPid|Pids0], Tick);
+            false ->
+                KillPid = lists:nth(PidN,Pids0),
+                KillPid ! die,
+                Pids1 = lists:delete(KillPid, Pids0),
+                lists:foreach(fun(P) -> P ! {is_dead,KillPid} end, Pids1),
+                send3_controller(SpawnCnt0, Mons0, Pids1, Tick)
+        end        
+   end.
+
+send3_proc() ->
+    %%io:format("Process ~p spawned\n",[self()]),
+    send3_proc([self()], {0,0,0}, {1,2,3,4,5}).
+send3_proc(Pids0, Counters={Rcv,SndOk,SndFail}, State0) ->
+    %%io:format("~p: Pids0=~p", [self(), Pids0]),
+    %%timer:sleep(10),
+    receive 
+        {pids, Pids1, Inject} ->
+            %%io:format("~p: got ~p Inject=~p\n", [self(), Pids1, Inject]), 
+            ?line Pids0 = [self()],
+            Pids2 = [self() | Pids1],
+            case Inject of
+                true -> send3_proc_send(Pids2, Counters, State0);
+                false -> send3_proc(Pids2, Counters, State0)
+            end;
+        {is_born, Pid} ->
+            %%io:format("~p: is_born ~p, got ~p\n", [self(), Pid, Pids0]), 
+            send3_proc([Pid | Pids0], Counters, State0);
+        {is_dead, Pid} ->            
+            Pids1 = lists:delete(Pid,Pids0),
+            %%io:format("~p: is_dead ~p, got ~p\n", [self(), Pid, Pids1]),
+            send3_proc(Pids1, Counters, State0);
+        {blob, Blob0} ->
+            %%io:format("~p: blob ~p\n", [self(), Blob0]),
+            State1 = send3_new_state(State0, Blob0),
+            send3_proc_send(Pids0, {Rcv+1,SndOk,SndFail}, State1);
+        die ->
+            %%io:format("Process ~p terminating, stats = ~p\n",[self(),Counters]),
+            {message_queue_len,Dropped} = erlang:process_info(self(),message_queue_len),
+            _R = ets:update_counter(nif_SUITE,send3,
+                               [{2,Rcv},{3,SndOk},{4,SndFail},{5,1-Dropped}]),
+            %%io:format("~p: dies R=~p\n", [self(), R]),
+            ok;
+        {halt,Papa} ->
+            Papa ! {halted,self()},
+            io:format("~p halted\n",[self()]),
+            receive die -> ok end,
+            io:format("~p dying\n",[self()])
+    end.
+
+send3_proc_send(Pids, {Rcv,SndOk,SndFail}, State0) ->
+    To = lists:nth(random:uniform(length(Pids)),Pids),
+    Blob = send3_make_blob(),
+    State1 = send3_new_state(State0,Blob), 
+    case send3_send(To, Blob) of
+        true ->
+            send3_proc(Pids, {Rcv,SndOk+1,SndFail}, State1);
+        false ->
+            send3_proc(Pids, {Rcv,SndOk,SndFail+1}, State1)
+    end.
+
+
+send3_make_blob() ->    
+    case random:uniform(20)-1 of
+        0 -> {term,[]};
+        N ->
+            MsgEnv = alloc_msgenv(), 
+            repeat(N bsr 1,
+                   fun(_) -> grow_blob(MsgEnv,other_term(),random:uniform(1 bsl 20))
+                   end, void),
+            case (N band 1) of
+                0 -> {term,copy_blob(MsgEnv)};
+                1 -> {msgenv,MsgEnv}
+            end
+    end.
+
+send3_send(Pid, Msg) ->
+    %% 90% enif_send and 10% normal bang
+    case random:uniform(10) of
+        1 -> send3_send_bang(Pid,Msg);
+        _ -> send3_send_nif(Pid,Msg)
+    end.
+send3_send_nif(Pid, {term,Blob}) ->
+    %%io:format("~p send term nif\n",[self()]),
+    send_term(Pid, {blob, Blob}) =:= 1;
+send3_send_nif(Pid, {msgenv,MsgEnv}) ->
+    %%io:format("~p send blob nif\n",[self()]),   
+    send3_blob(MsgEnv, Pid, blob) =:= 1.
+
+send3_send_bang(Pid, {term,Blob}) ->
+    %%io:format("~p send term bang\n",[self()]),   
+    Pid ! {blob, Blob},
+    true;
+send3_send_bang(Pid, {msgenv,MsgEnv}) ->
+    %%io:format("~p send blob bang\n",[self()]),   
+    Pid ! {blob, copy_blob(MsgEnv)},
+    true.
+
+send3_new_state(State, Blob) ->
+    case random:uniform(5+2) of
+        N when N =< 5-> setelement(N, State, Blob);
+        _ -> State  % Don't store blob
+    end.
+
 neg(doc) -> ["Negative testing of load_nif"];
 neg(Config) when is_list(Config) ->
     TmpMem = tmpmem(),
@@ -759,7 +1170,45 @@ neg(Config) when is_list(Config) ->
     ?line verify_tmpmem(TmpMem),
     ?line ok.
 
+is_checks(doc) -> ["Test all enif_is functions"];
+is_checks(Config) when is_list(Config) ->
+    ?line ensure_lib_loaded(Config, 1),
+    ?line ok = check_is(hejsan, <<19,98>>, make_ref(), ok, fun() -> ok end,
+			self(), hd(erlang:ports()), [], [1,9,9,8],
+			{hejsan, "hejsan", [$h,"ejs",<<"an">>]}, 12),
+    ?line ok = check_is(hejsan, <<19,98>>, make_ref(), ok, fun() -> ok end,
+			self(), hd(erlang:ports()), [], [1,9,9,8],
+			{hejsan, "hejsan", [$h,"ejs",<<"an">>]}, -12),
+    ?line ok = check_is(hejsan, <<19,98>>, make_ref(), ok, fun() -> ok end,
+			self(), hd(erlang:ports()), [], [1,9,9,8],
+			{hejsan, "hejsan", [$h,"ejs",<<"an">>]}, 18446744073709551617),
+    ?line ok = check_is(hejsan, <<19,98>>, make_ref(), ok, fun() -> ok end,
+			self(), hd(erlang:ports()), [], [1,9,9,8],
+			{hejsan, "hejsan", [$h,"ejs",<<"an">>]}, -18446744073709551617),
+    ?line ok = check_is(hejsan, <<19,98>>, make_ref(), ok, fun() -> ok end,
+			self(), hd(erlang:ports()), [], [1,9,9,8],
+			{hejsan, "hejsan", [$h,"ejs",<<"an">>]}, 99.146),
+    ?line ok = check_is(hejsan, <<19,98>>, make_ref(), ok, fun() -> ok end,
+			self(), hd(erlang:ports()), [], [1,9,9,8],
+			{hejsan, "hejsan", [$h,"ejs",<<"an">>]}, -99.146),
+    ?line ok = check_is(hejsan, <<19,98>>, make_ref(), ok, fun() -> ok end,
+			self(), hd(erlang:ports()), [], [1,9,9,8],
+			{hejsan, "hejsan", [$h,"ejs",<<"an">>]}, 18446744073709551616.2e2),
+    ?line ok = check_is(hejsan, <<19,98>>, make_ref(), ok, fun() -> ok end,
+			self(), hd(erlang:ports()), [], [1,9,9,8],
+			{hejsan, "hejsan", [$h,"ejs",<<"an">>]}, -18446744073709551616.2e2),
+    try
+        ?line error = check_is_exception(),
+        ?line throw(expected_badarg)
+    catch
+        error:badarg ->
+            ?line ok
+    end.
 
+get_length(doc) -> ["Test all enif_get_length functions"];
+get_length(Config) when is_list(Config) ->
+    ?line ensure_lib_loaded(Config, 1),
+    ?line ok = length_test(hejsan, "hejsan", [], [], not_a_list).
 
 ensure_lib_loaded(Config) ->
     ensure_lib_loaded(Config, 1).
@@ -773,15 +1222,54 @@ ensure_lib_loaded(Config, Ver) ->
 		  ok
 	  end.
 
+make_atom(Config) when is_list(Config) ->
+    ?line ensure_lib_loaded(Config, 1),
+    An0Atom = an0atom,
+    An0Atom0 = 'an\000atom\000',
+    ?line Atoms = make_atoms(),
+    ?line 7 = size(Atoms),
+    ?line Atoms = {An0Atom,An0Atom,An0Atom,An0Atom0,An0Atom,An0Atom,An0Atom0}.
+
+make_string(Config) when is_list(Config) ->
+    ?line ensure_lib_loaded(Config, 1),
+    ?line Strings = make_strings(),
+    ?line 5 = size(Strings),
+    A0String = "a0string",
+    A0String0 = [$a,0,$s,$t,$r,$i,$n,$g,0],
+    AStringWithAccents = [$E,$r,$l,$a,$n,$g,$ ,16#e4,$r,$ ,$e,$t,$t,$ ,$g,$e,$n,$e,$r,$e,$l,$l,$t,$ ,$p,$r,$o,$g,$r,$a,$m,$s,$p,$r,16#e5,$k],
+    ?line Strings = {A0String,A0String,A0String,A0String0, AStringWithAccents}.
+
+reverse_list_test(Config) ->
+    ?line ensure_lib_loaded(Config, 1),
+    List = lists:seq(1,100),
+    RevList = lists:reverse(List),
+    ?line RevList = reverse_list(List),
+    ?line badarg = reverse_list(foo).
+
+otp_9668(doc) -> ["Memory leak of tmp-buffer when inspecting iolist or unaligned binary in unbound environment"];
+otp_9668(Config) ->
+    ensure_lib_loaded(Config, 1),
+    TmpMem = tmpmem(),
+    IOList = ["This",' ',<<"is">>,' ',[<<"an iolist">>,'.']],    
+    otp_9668_nif(IOList),
+
+    <<_:5/bitstring,UnalignedBin:10/binary,_/bitstring>> = <<"Abuse me as unaligned">>,
+    otp_9668_nif(UnalignedBin),
+
+    ?line verify_tmpmem(TmpMem),
+    ok.
+
+
 tmpmem() ->
     case erlang:system_info({allocator,temp_alloc}) of
 	false -> undefined;
 	MemInfo ->
 	    MSBCS = lists:foldl(
 		      fun ({instance, _, L}, Acc) ->
+			      {value,{_,SBMBCS}} = lists:keysearch(sbmbcs, 1, L),
 			      {value,{_,MBCS}} = lists:keysearch(mbcs, 1, L),
 			      {value,{_,SBCS}} = lists:keysearch(sbcs, 1, L),
-			      [MBCS,SBCS | Acc]
+			      [SBMBCS,MBCS,SBCS | Acc]
 		      end,
 		      [],
 		      MemInfo),
@@ -821,13 +1309,18 @@ call(Pid,Cmd) ->
 receive_any() ->
     receive M -> M end.	     
 
-%% check(Exp,Got,Line) ->
-%%     case Got of
-%% 	Exp -> Exp;	    
-%% 	_ ->
-%% 	    io:format("CHECK at ~p: Expected ~p but got ~p\n",[Line,Exp,Got]),
-%% 	    Got
-%%     end.
+repeat(0, _, Arg) ->
+    Arg;
+repeat(N, Fun, Arg0) ->
+    repeat(N-1, Fun, Fun(Arg0)).
+
+check(Exp,Got,Line) ->
+    case Got of
+ 	Exp -> Exp;	    
+  	_ ->
+  	    io:format("CHECK at ~p: Expected ~p but got ~p\n",[Line,Exp,Got]),
+ 	    Got
+    end.
 	    
 
 %% The NIFs:
@@ -855,6 +1348,28 @@ get_resource(_,_) -> ?nif_stub.
 release_resource(_) -> ?nif_stub.
 last_resource_dtor_call() -> ?nif_stub.
 make_new_resource(_,_) -> ?nif_stub.
+check_is(_,_,_,_,_,_,_,_,_,_,_) -> ?nif_stub.
+check_is_exception() -> ?nif_stub.
+length_test(_,_,_,_,_) -> ?nif_stub.
+make_atoms() -> ?nif_stub.
+make_strings() -> ?nif_stub.
+make_new_resource_binary(_) -> ?nif_stub.
+send_list_seq(_,_) -> ?nif_stub.     
+send_new_blob(_,_) -> ?nif_stub.     
+alloc_msgenv() -> ?nif_stub.
+clear_msgenv(_) -> ?nif_stub.
+grow_blob(_,_) -> ?nif_stub.
+grow_blob(_,_,_) -> ?nif_stub.
+send_blob(_,_) -> ?nif_stub.
+send3_blob(_,_,_) -> ?nif_stub.
+send_blob_thread(_,_,_) -> ?nif_stub.
+join_send_thread(_) -> ?nif_stub.
+copy_blob(_) -> ?nif_stub.
+send_term(_,_) -> ?nif_stub.
+reverse_list(_) -> ?nif_stub.
+echo_int(_) -> ?nif_stub.
+type_sizes() -> ?nif_stub.
+otp_9668_nif(_) -> ?nif_stub.
 
 nif_stub_error(Line) ->
     exit({nif_not_loaded,module,?MODULE,line,Line}).
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 7d05a9a880..03092fef5e 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -1,3 +1,21 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2009-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
 #include "erl_nif.h"
 
 #include <stdio.h>
@@ -10,17 +28,35 @@
 static int static_cntA; /* zero by default */
 static int static_cntB = NIF_SUITE_LIB_VER * 100;
 
+static ERL_NIF_TERM atom_false;
+static ERL_NIF_TERM atom_self;
+static ERL_NIF_TERM atom_ok;
+static ERL_NIF_TERM atom_join;
+static ERL_NIF_TERM atom_binary_resource_type;
+
+
 typedef struct
 {
     int ref_cnt;
     CallInfo* call_history;
     NifModPrivData* nif_mod;
     union { ErlNifResourceType* t; long l; } rt_arr[2];
-}PrivData;
+} PrivData;
+
+/*
+ * Use a union for pointer type conversion to avoid compiler warnings
+ * about strict-aliasing violations with gcc-4.1. gcc >= 4.2 does not
+ * emit the warning.
+ * TODO: Reconsider use of union once gcc-4.1 is obsolete?
+ */
+typedef union {
+    void* vp;
+    struct make_term_info* p;
+} mti_t;
 
 void add_call(ErlNifEnv* env, PrivData* data, const char* func_name)
 {
-    CallInfo* call = enif_alloc(env, sizeof(CallInfo)+strlen(func_name));
+    CallInfo* call = enif_alloc(sizeof(CallInfo)+strlen(func_name));
     strcpy(call->func_name, func_name);
     call->lib_ver = NIF_SUITE_LIB_VER;
     call->next = data->call_history;
@@ -31,7 +67,7 @@ void add_call(ErlNifEnv* env, PrivData* data, const char* func_name)
     call->arg_sz = 0;
 }
 
-#define ADD_CALL(FUNC_NAME) add_call(env, enif_get_data(env),FUNC_NAME)
+#define ADD_CALL(FUNC_NAME) add_call(env, enif_priv_data(env),FUNC_NAME)
 
 static void*    resource_dtor_last = NULL;
 static unsigned resource_dtor_last_sz = 0;
@@ -42,15 +78,24 @@ static void resource_dtor(ErlNifEnv* env, void* obj)
 {
     resource_dtor_last = obj;
     resource_dtor_cnt++;
-    resource_dtor_last_sz = enif_sizeof_resource(env, obj);
+    resource_dtor_last_sz = enif_sizeof_resource(obj);
     assert(resource_dtor_last_sz <= sizeof(resource_dtor_last_data));
     memcpy(resource_dtor_last_data, obj, resource_dtor_last_sz);
 }
 
+static ErlNifResourceType* msgenv_resource_type;
+static void msgenv_dtor(ErlNifEnv* env, void* obj);
+
+static ErlNifResourceType* binary_resource_type;
+static void binary_resource_dtor(ErlNifEnv* env, void* obj);
+struct binary_resource {
+    unsigned char* data;
+    unsigned size;
+};
+
 static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
 {
-    /*ERL_NIF_TERM head, tail;*/
-    PrivData* data = enif_alloc(env, sizeof(PrivData));
+    PrivData* data = enif_alloc(sizeof(PrivData));
     assert(data != NULL);
     data->ref_cnt = 1;
     data->call_history = NULL;
@@ -58,41 +103,71 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
 
     add_call(env, data, "load");
 
-    /*
-    head = load_info;
-    data->rt_cnt = 0;
-    for (head=load_info; enif_get_list_cell(env,load_info,&head,&tail);
-	  head=tail) {
-	char buf[20];
-	int n = enif_get_string(env,head,buf,sizeof(buf));
-	assert(n > 0);
-	assert(i < sizeof(data->rt_arr)/sizeof(*data->rt_arr));
-	data->rt_arr[data->rt_cnt++].t = enif_create_resource_type(env,buf,resource_dtor,
-								   ERL_NIF_RT_CREATE,NULL);	   
-    }
-    assert(enif_is_empty_list(env,head)); 
-    */
-    data->rt_arr[0].t = enif_open_resource_type(env,"Gold",resource_dtor,
+    data->rt_arr[0].t = enif_open_resource_type(env,NULL,"Gold",resource_dtor,
 						ERL_NIF_RT_CREATE,NULL);
-    data->rt_arr[1].t = enif_open_resource_type(env,"Silver",resource_dtor,
+    data->rt_arr[1].t = enif_open_resource_type(env,NULL,"Silver",resource_dtor,
 						ERL_NIF_RT_CREATE,NULL);
 
+    binary_resource_type =  enif_open_resource_type(env,NULL,"nif_SUITE.binary",
+						    binary_resource_dtor,
+						    ERL_NIF_RT_CREATE, NULL);
+
+    msgenv_resource_type =  enif_open_resource_type(env,NULL,"nif_SUITE.msgenv",
+						    msgenv_dtor,
+						    ERL_NIF_RT_CREATE, NULL);
+    atom_false = enif_make_atom(env,"false");
+    atom_self = enif_make_atom(env,"self");
+    atom_ok = enif_make_atom(env,"ok");
+    atom_join = enif_make_atom(env,"join");
+    atom_binary_resource_type = enif_make_atom(env,"binary_resource_type");
+
     *priv_data = data;
     return 0;
 }
 
+static void resource_takeover(ErlNifEnv* env, PrivData* priv)
+{
+    ErlNifResourceFlags tried;
+    ErlNifResourceType* rt;
+    rt = enif_open_resource_type(env, NULL, "Gold", resource_dtor,
+				 ERL_NIF_RT_TAKEOVER, &tried);
+    assert(rt == priv->rt_arr[0].t); 
+    assert(tried == ERL_NIF_RT_TAKEOVER);
+    rt = enif_open_resource_type(env, NULL, "Silver", resource_dtor,
+				 ERL_NIF_RT_TAKEOVER, &tried);
+    assert(rt == priv->rt_arr[1].t); 
+    assert(tried == ERL_NIF_RT_TAKEOVER);
+
+    rt =  enif_open_resource_type(env, NULL, "nif_SUITE.binary", binary_resource_dtor,
+				  ERL_NIF_RT_TAKEOVER, &tried);
+    assert(rt != NULL);
+    assert(tried == ERL_NIF_RT_TAKEOVER);
+    assert(binary_resource_type==NULL || binary_resource_type == rt); 
+    binary_resource_type = rt;
+
+    rt = enif_open_resource_type(env, NULL, "nif_SUITE.msgenv", msgenv_dtor,
+				  ERL_NIF_RT_TAKEOVER, &tried);
+    assert(rt != NULL);
+    assert(tried == ERL_NIF_RT_TAKEOVER);
+    assert(msgenv_resource_type==NULL || msgenv_resource_type == rt); 
+    msgenv_resource_type = rt;
+}
+
 static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
 {
-    add_call(env, *priv_data, "reload");
+    PrivData* priv = (PrivData*) *priv_data;
+    add_call(env, priv, "reload");
+    resource_takeover(env,priv);
     return 0;
 }
 
 static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info)
 {
-    PrivData* data = *old_priv_data;
-    add_call(env, data, "upgrade");
-    data->ref_cnt++;
-    *priv_data = *old_priv_data;    
+    PrivData* priv = (PrivData*) *old_priv_data;
+    add_call(env, priv, "upgrade");
+    priv->ref_cnt++;
+    *priv_data = *old_priv_data;
+    resource_takeover(env,priv);
     return 0;
 }
 
@@ -101,7 +176,10 @@ static void unload(ErlNifEnv* env, void* priv_data)
     PrivData* data = priv_data;
     add_call(env, data, "unload");
     if (--data->ref_cnt == 0) {
-	enif_free(env, priv_data);
+	if (data->nif_mod != NULL) {
+	    NifModPrivData_release(data->nif_mod);
+	}
+	enif_free(priv_data);
     }
 }
 
@@ -120,11 +198,10 @@ static ERL_NIF_TERM make_call_history(ErlNifEnv* env, CallInfo** headp)
 	ERL_NIF_TERM func_term = enif_make_atom(env,call->func_name);
 	ERL_NIF_TERM tpl;
 	if (call->arg != NULL) {
-	    ErlNifBinary arg_bin;
-	    enif_alloc_binary(env, call->arg_sz, &arg_bin);
-	    memcpy(arg_bin.data, call->arg, call->arg_sz);
-	    func_term = enif_make_tuple2(env, func_term,
-					 enif_make_binary(env, &arg_bin));
+	    ERL_NIF_TERM arg_bin;	    
+	    memcpy(enif_make_new_binary(env, call->arg_sz, &arg_bin),
+		   call->arg, call->arg_sz);
+	    func_term = enif_make_tuple2(env, func_term, arg_bin);
 	}
 	tpl = enif_make_tuple4(env, func_term, 					    
 			       enif_make_int(env,call->lib_ver),
@@ -132,28 +209,28 @@ static ERL_NIF_TERM make_call_history(ErlNifEnv* env, CallInfo** headp)
 			       enif_make_int(env,call->static_cntB));
 	list = enif_make_list_cell(env, tpl, list);
 	*headp = call->next;
-	enif_free(env,call);
+	enif_free(call);
     }
     return list;
 }
 
 static ERL_NIF_TERM call_history(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    PrivData* data = (PrivData*) enif_get_data(env);
+    PrivData* data = (PrivData*) enif_priv_data(env);
 
     return make_call_history(env,&data->call_history);
 }
 
 static ERL_NIF_TERM hold_nif_mod_priv_data(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    PrivData* data = (PrivData*) enif_get_data(env);
+    PrivData* data = (PrivData*) enif_priv_data(env);
     unsigned long ptr_as_ulong;
     
     if (!enif_get_ulong(env,argv[0],&ptr_as_ulong)) {
 	return enif_make_badarg(env);
     }
-    if (data->nif_mod != NULL && --(data->nif_mod->ref_cnt) == 0) {
-	enif_free(env,data->nif_mod);
+    if (data->nif_mod != NULL) {
+	NifModPrivData_release(data->nif_mod);
     }
     data->nif_mod = (NifModPrivData*) ptr_as_ulong;    
     return enif_make_int(env,++(data->nif_mod->ref_cnt)); 
@@ -161,7 +238,7 @@ static ERL_NIF_TERM hold_nif_mod_priv_data(ErlNifEnv* env, int argc, const ERL_N
 
 static ERL_NIF_TERM nif_mod_call_history(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    PrivData* data = (PrivData*) enif_get_data(env);
+    PrivData* data = (PrivData*) enif_priv_data(env);
     ERL_NIF_TERM ret;
     if (data->nif_mod == NULL) {
 	return enif_make_string(env,"nif_mod pointer is NULL", ERL_NIF_LATIN1);
@@ -231,6 +308,30 @@ static int test_ulong(ErlNifEnv* env, unsigned long i1)
     return 1;
 }
 
+static int test_int64(ErlNifEnv* env, ErlNifSInt64 i1)
+{
+    ErlNifSInt64 i2 = 0;
+    ERL_NIF_TERM int_term = enif_make_int64(env, i1);
+    if (!enif_get_int64(env,int_term, &i2) || i1 != i2) {
+	fprintf(stderr, "test_int64(%ld) ...FAILED i2=%ld\r\n",
+		(long)i1, (long)i2);
+	return 0;
+    }
+    return 1;
+}
+
+static int test_uint64(ErlNifEnv* env, ErlNifUInt64 i1)
+{
+    ErlNifUInt64 i2 = 0;
+    ERL_NIF_TERM int_term = enif_make_uint64(env, i1);
+    if (!enif_get_uint64(env,int_term, &i2) || i1 != i2) {
+	fprintf(stderr, "test_ulong(%lu) ...FAILED i2=%lu\r\n",
+		(unsigned long)i1, (unsigned long)i2);
+	return 0;
+    }
+    return 1;
+}
+
 static int test_double(ErlNifEnv* env, double d1)
 {
     double d2 = 0;
@@ -244,8 +345,13 @@ static int test_double(ErlNifEnv* env, double d1)
 
 #define TAG_BITS        4
 #define SMALL_BITS	(sizeof(void*)*8 - TAG_BITS)
+#ifdef _WIN64
+#define MAX_SMALL	((1LL << (SMALL_BITS-1))-1)
+#define MIN_SMALL	(-(1LL << (SMALL_BITS-1)))
+#else
 #define MAX_SMALL	((1L << (SMALL_BITS-1))-1)
 #define MIN_SMALL	(-(1L << (SMALL_BITS-1)))
+#endif
 
 static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
@@ -254,6 +360,8 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
     unsigned uint;
     long slong;
     unsigned long ulong;
+    ErlNifSInt64 sint64;
+    ErlNifUInt64 uint64;
     double d;
     ERL_NIF_TERM atom, ref1, ref2;
 
@@ -287,11 +395,25 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
 	slong -= slong / 3 + 1;
     } while (slong >= 0);
 
+    sint64 = ((ErlNifSInt64)1 << 63); /* INT64_MIN */
+    do {
+	if (!test_int64(env,sint64)) {
+	    goto error;
+	}
+	sint64 += ~sint64 / 3 + 1;
+    } while (sint64 < 0);
+    sint64 = ((ErlNifUInt64)1 << 63) - 1; /* INT64_MAX */
+    do {
+	if (!test_int64(env,sint64)) {
+	    goto error;
+	}
+	sint64 -= sint64 / 3 + 1;
+    } while (sint64 >= 0);
 
     uint = UINT_MAX;
     for (;;) {
 	if (!test_uint(env,uint)) {
-	    
+	    goto error;
 	}
 	if (uint == 0) break;
 	uint -= uint / 3 + 1;
@@ -299,11 +421,19 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
     ulong = ULONG_MAX;
     for (;;) {
 	if (!test_ulong(env,ulong)) {
-	    
+	    goto error;
 	}
 	if (ulong == 0) break;
 	ulong -= ulong / 3 + 1;
     }    
+    uint64 = (ErlNifUInt64)-1; /* UINT64_MAX */
+    for (;;) {
+	if (!test_uint64(env,uint64)) {
+	    goto error;
+	}
+	if (uint64 == 0) break;
+	uint64 -= uint64 / 3 + 1;
+    }    
 
     if (MAX_SMALL < INT_MAX) { /* 32-bit */
 	for (i=-10 ; i <= 10; i++) {
@@ -326,24 +456,31 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
 
     for (i=-10 ; i < 10; i++) {
 	if (!test_long(env,MAX_SMALL+i) || !test_ulong(env,MAX_SMALL+i) ||
-	    !test_long(env,MIN_SMALL+i)) {
+	    !test_long(env,MIN_SMALL+i) ||
+	    !test_int64(env,MAX_SMALL+i) || !test_uint64(env,MAX_SMALL+i) ||
+	    !test_int64(env,MIN_SMALL+i)) {
 	    goto error;
 	}
+	if (MAX_SMALL < INT_MAX) {
+	    if (!test_int(env,MAX_SMALL+i) || !test_uint(env,MAX_SMALL+i) ||
+		!test_int(env,MIN_SMALL+i)) {
+		goto error;
+	    }
+	}
     }
-
     for (d=3.141592e-100 ; d < 1e100 ; d *= 9.97) {
 	if (!test_double(env,d) || !test_double(env,-d)) {
 	    goto error;
 	}	
     }
 
-    if (!enif_make_existing_atom(env,"nif_SUITE", &atom)
-	|| !enif_is_identical(env,atom,enif_make_atom(env,"nif_SUITE"))) {
+    if (!enif_make_existing_atom(env,"nif_SUITE", &atom, ERL_NIF_LATIN1)
+	|| !enif_is_identical(atom,enif_make_atom(env,"nif_SUITE"))) {
 	fprintf(stderr, "nif_SUITE not an atom?\r\n");
 	goto error;
     }
     for (i=2; i; i--) {
-	if (enif_make_existing_atom(env,"nif_SUITE_pink_unicorn", &atom)) {
+	if (enif_make_existing_atom(env,"nif_SUITE_pink_unicorn", &atom, ERL_NIF_LATIN1)) {
 	    fprintf(stderr, "pink unicorn exist?\r\n");
 	    goto error;
 	}
@@ -351,7 +488,7 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
     ref1 = enif_make_ref(env);
     ref2 = enif_make_ref(env);
     if (!enif_is_ref(env,ref1) || !enif_is_ref(env,ref2) 
-	|| enif_is_identical(env,ref1,ref2) || enif_compare(env,ref1,ref2)==0) {
+	|| enif_is_identical(ref1,ref2) || enif_compare(ref1,ref2)==0) {
 	fprintf(stderr, "strange refs?\r\n");
 	goto error;
     }
@@ -361,6 +498,45 @@ error:
     return enif_make_atom(env,"error");
 }
 
+static ERL_NIF_TERM echo_int(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    int sint;
+    unsigned uint;
+    long slong;
+    unsigned long ulong;
+    ErlNifSInt64 sint64;
+    ErlNifUInt64 uint64;    
+    ERL_NIF_TERM sint_term = atom_false,   uint_term = atom_false;
+    ERL_NIF_TERM slong_term = atom_false,  ulong_term = atom_false;
+    ERL_NIF_TERM sint64_term = atom_false, uint64_term = atom_false;
+
+    if (enif_get_int(env, argv[0], &sint)) {
+	sint_term = enif_make_int(env, sint);
+    }
+    if (enif_get_uint(env, argv[0], &uint)) {
+	uint_term = enif_make_uint(env, uint);
+    }
+    if (enif_get_long(env, argv[0], &slong)) {
+	slong_term = enif_make_long(env, slong);
+    }
+    if (enif_get_ulong(env, argv[0], &ulong)) {
+	ulong_term = enif_make_ulong(env, ulong);
+    }
+    if (enif_get_int64(env, argv[0], &sint64)) {
+	sint64_term = enif_make_int64(env, sint64);
+    }
+    if (enif_get_uint64(env, argv[0], &uint64)) {
+	uint64_term = enif_make_uint64(env, uint64);
+    }
+    return enif_make_list6(env, sint_term, uint_term, slong_term, ulong_term, sint64_term, uint64_term);
+}
+
+static ERL_NIF_TERM type_sizes(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    return enif_make_tuple2(env, enif_make_int(env, sizeof(int)),
+			    enif_make_int(env, sizeof(long)));
+}
+
 static ERL_NIF_TERM tuple_2_list(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     int arity = -1;
@@ -381,7 +557,7 @@ static ERL_NIF_TERM is_identical(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
     if (argc != 2) {
 	return enif_make_badarg(env);
     }
-    return enif_make_atom(env, (enif_is_identical(env,argv[0],argv[1]) ?
+    return enif_make_atom(env, (enif_is_identical(argv[0],argv[1]) ?
 				"true" : "false"));
 }
 
@@ -390,7 +566,7 @@ static ERL_NIF_TERM compare(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     if (argc != 2) {
 	return enif_make_badarg(env);
     }
-    return enif_make_int(env, enif_compare(env,argv[0],argv[1]));
+    return enif_make_int(env, enif_compare(argv[0],argv[1]));
 }
 
 static ERL_NIF_TERM many_args_100(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
@@ -412,11 +588,10 @@ static ERL_NIF_TERM clone_bin(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
 {
     ErlNifBinary ibin;
     if (enif_inspect_binary(env,argv[0],&ibin)) {
-	ErlNifBinary obin;
-	enif_alloc_binary(env,ibin.size,&obin);
-	memcpy(obin.data,ibin.data,ibin.size);
-	/*enif_release_binary(env,&ibin);*/
-	return enif_make_binary(env,&obin);
+	ERL_NIF_TERM obin;	
+	memcpy(enif_make_new_binary(env, ibin.size, &obin),
+	       ibin.data, ibin.size);
+	return obin;
     }
     else {
 	return enif_make_badarg(env);
@@ -438,7 +613,7 @@ static ERL_NIF_TERM string_to_bin(ErlNifEnv* env, int argc, const ERL_NIF_TERM a
     unsigned size;
     int n;
     if (!enif_get_int(env,argv[1],(int*)&size) 
-	|| !enif_alloc_binary(env,size,&obin)) {
+	|| !enif_alloc_binary(size,&obin)) {
 	return enif_make_badarg(env);
     }
     n = enif_get_string(env, argv[0], (char*)obin.data, size, ERL_NIF_LATIN1);
@@ -452,10 +627,10 @@ static ERL_NIF_TERM atom_to_bin(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg
     unsigned size;
     int n;
     if (!enif_get_int(env,argv[1],(int*)&size) 
-	|| !enif_alloc_binary(env,size,&obin)) {
+	|| !enif_alloc_binary(size,&obin)) {
 	return enif_make_badarg(env);
     }
-    n = enif_get_atom(env, argv[0], (char*)obin.data, size);
+    n = enif_get_atom(env, argv[0], (char*)obin.data, size, ERL_NIF_LATIN1);
     return enif_make_tuple(env, 2, enif_make_int(env,n),
 			   enif_make_binary(env,&obin));
 }
@@ -515,14 +690,14 @@ static ERL_NIF_TERM iolist_2_bin(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
 
 static ERL_NIF_TERM last_resource_dtor_call(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    ErlNifBinary bin;
     ERL_NIF_TERM ret;
     if (resource_dtor_last != NULL) {
-	enif_alloc_binary(env, resource_dtor_last_sz, &bin);
-	memcpy(bin.data, resource_dtor_last_data, resource_dtor_last_sz);
+	ERL_NIF_TERM bin;	
+	memcpy(enif_make_new_binary(env, resource_dtor_last_sz, &bin),
+	       resource_dtor_last_data, resource_dtor_last_sz);
 	ret = enif_make_tuple3(env,
 				enif_make_long(env, (long)resource_dtor_last),
-				enif_make_binary(env, &bin),  
+				bin,  
 				enif_make_int(env, resource_dtor_cnt));
     }
     else {
@@ -536,7 +711,7 @@ static ERL_NIF_TERM last_resource_dtor_call(ErlNifEnv* env, int argc, const ERL_
 
 static ERL_NIF_TERM get_resource_type(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    PrivData* data = (PrivData*) enif_get_data(env);
+    PrivData* data = (PrivData*) enif_priv_data(env);
     int ix;
     
     if (!enif_get_int(env, argv[0], &ix) || ix >= 2) {
@@ -548,11 +723,11 @@ static ERL_NIF_TERM get_resource_type(ErlNifEnv* env, int argc, const ERL_NIF_TE
 static ERL_NIF_TERM alloc_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     ErlNifBinary data_bin;
-    union { ErlNifResourceType* t; long l;} type;
+    union { ErlNifResourceType* t; long l; } type;
     union { void* p; long l;} data;
     if (!enif_get_long(env, argv[0], &type.l)
 	|| !enif_inspect_binary(env, argv[1], &data_bin)
-	|| (data.p = enif_alloc_resource(env, type.t, data_bin.size))==NULL) {
+	|| (data.p = enif_alloc_resource(type.t, data_bin.size))==NULL) {
 
 	return enif_make_badarg(env);
     }
@@ -572,33 +747,70 @@ static ERL_NIF_TERM make_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM a
 static ERL_NIF_TERM make_new_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     ErlNifBinary data_bin;
-    union { ErlNifResourceType* t; long l;} type;
+    union { ErlNifResourceType* t; long l; } type;
     void* data;
     ERL_NIF_TERM ret;
     if (!enif_get_long(env, argv[0], &type.l)
 	|| !enif_inspect_binary(env, argv[1], &data_bin)
-	|| (data = enif_alloc_resource(env, type.t, data_bin.size))==NULL) {
+	|| (data = enif_alloc_resource(type.t, data_bin.size))==NULL) {
 
 	return enif_make_badarg(env);
     }
     ret = enif_make_resource(env, data);
     memcpy(data, data_bin.data, data_bin.size);
-    enif_release_resource(env, data);
+    enif_release_resource(data);
     return ret;
 }
 
+static ERL_NIF_TERM make_new_resource_binary(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ErlNifBinary data_bin;
+    union { struct binary_resource* p; void* vp; long l; } br;
+    void* buf;
+    ERL_NIF_TERM ret;
+    if (!enif_inspect_binary(env, argv[0], &data_bin)
+	|| (br.vp = enif_alloc_resource(binary_resource_type,
+					sizeof(struct binary_resource)))==NULL
+	|| (buf = enif_alloc(data_bin.size)) == NULL) {
+	
+	return enif_make_badarg(env);
+    }    
+    memset(br.vp,0xba,sizeof(struct binary_resource)); /* avoid valgrind warning */
+    br.p->data = buf;
+    br.p->size = data_bin.size;
+    memcpy(br.p->data, data_bin.data, data_bin.size);    
+    ret = enif_make_resource_binary(env, br.vp, br.p->data, br.p->size);    
+    enif_release_resource(br.p);
+    return enif_make_tuple2(env, enif_make_long(env,br.l), ret);
+}
+
+static void binary_resource_dtor(ErlNifEnv* env, void* obj)
+{
+    struct binary_resource* br = (struct binary_resource*) obj;
+    resource_dtor(env,obj);
+    assert(br->data != NULL);
+    enif_free(br->data);
+    br->data = NULL;
+}
+
 static ERL_NIF_TERM get_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     ErlNifBinary data_bin;
     union { ErlNifResourceType* t; long l; } type;
     union { void* p; long l; } data;
 
-    if (!enif_get_long(env, argv[0], &type.l)
+    type.t = NULL;
+    if (enif_is_identical(argv[0], atom_binary_resource_type)) {
+	type.t = binary_resource_type;
+    }
+    else {
+	enif_get_long(env, argv[0], &type.l);
+    }
+    if (type.t == NULL 
 	|| !enif_get_resource(env, argv[1], type.t, &data.p)) {
 	return enif_make_badarg(env);
     }
-
-    enif_alloc_binary(env, enif_sizeof_resource(env,data.p), &data_bin);    
+    enif_alloc_binary(enif_sizeof_resource(data.p), &data_bin);    
     memcpy(data_bin.data, data.p, data_bin.size);
     return enif_make_tuple2(env, enif_make_long(env,data.l),
 			    enif_make_binary(env, &data_bin));
@@ -610,10 +822,639 @@ static ERL_NIF_TERM release_resource(ErlNifEnv* env, int argc, const ERL_NIF_TER
     if (!enif_get_long(env, argv[0], &data.l)) {
 	return enif_make_badarg(env);
     }
-    enif_release_resource(env, data.p);
+    enif_release_resource(data.p);
     return enif_make_atom(env,"ok");
 }
 
+/*
+ * argv[0] an atom
+ * argv[1] a binary
+ * argv[2] a ref
+ * argv[3] 'ok'
+ * argv[4] a fun
+ * argv[5] a pid
+ * argv[6] a port
+ * argv[7] an empty list
+ * argv[8] a non-empty list
+ * argv[9] a tuple
+ * argv[10] a number (small, big integer or float)
+ */
+static ERL_NIF_TERM check_is(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ERL_NIF_TERM ok_atom = enif_make_atom(env, "ok");
+
+    if (!enif_is_atom(env, argv[0])) return enif_make_badarg(env);
+    if (!enif_is_binary(env, argv[1])) return enif_make_badarg(env);
+    if (!enif_is_ref(env, argv[2])) return enif_make_badarg(env);
+    if (!enif_is_identical(argv[3], ok_atom)) return enif_make_badarg(env);
+    if (!enif_is_fun(env, argv[4])) return enif_make_badarg(env);
+    if (!enif_is_pid(env, argv[5])) return enif_make_badarg(env);
+    if (!enif_is_port(env, argv[6])) return enif_make_badarg(env);
+    if (!enif_is_empty_list(env, argv[7])) return enif_make_badarg(env);
+    if (!enif_is_list(env, argv[7])) return enif_make_badarg(env);
+    if (!enif_is_list(env, argv[8])) return enif_make_badarg(env);
+    if (!enif_is_tuple(env, argv[9])) return enif_make_badarg(env);
+    if (!enif_is_number(env, argv[10])) return enif_make_badarg(env);
+
+    return ok_atom;
+}
+
+/*
+ * no arguments
+ *
+ * This function is separate from check_is because it calls enif_make_badarg
+ * and so it must return the badarg exception as its return value. Thus, the
+ * badarg exception indicates success. Failure is indicated by returning an
+ * error atom.
+ */
+static ERL_NIF_TERM check_is_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ERL_NIF_TERM error_atom = enif_make_atom(env, "error");
+    ERL_NIF_TERM badarg = enif_make_badarg(env);
+    if (enif_is_exception(env, error_atom)) return error_atom;
+    if (!enif_is_exception(env, badarg)) return error_atom;
+    return badarg;
+}
+
+/*
+ * argv[0] atom with length of 6
+ * argv[1] list with length of 6
+ * argv[2] empty list
+ * argv[3] not an atom
+ * argv[4] not a list
+ */
+static ERL_NIF_TERM length_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    unsigned len;
+
+    if (!enif_get_atom_length(env, argv[0], &len, ERL_NIF_LATIN1) || len != 6)
+	return enif_make_badarg(env);
+
+    if (!enif_get_list_length(env, argv[1], &len) || len != 6)
+	return enif_make_badarg(env);
+
+    if (!enif_get_list_length(env, argv[2], &len) || len != 0)
+	return enif_make_badarg(env);
+
+    if (enif_get_atom_length(env, argv[3], &len, ERL_NIF_LATIN1))
+	return enif_make_badarg(env);
+
+    if (enif_get_list_length(env, argv[4], &len))
+	return enif_make_badarg(env);
+
+    return enif_make_atom(env, "ok");
+}
+
+static ERL_NIF_TERM make_atoms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ERL_NIF_TERM arr[7];
+    ERL_NIF_TERM existingatom0a, existingatom0b;
+    ERL_NIF_TERM existing0atom0;
+    const char * const an0atom = "an0atom";
+    const char an0atom0[8] = {'a','n','\0','a','t','o','m',0};
+
+    arr[0] = enif_make_atom(env, "an0atom");
+    arr[1] = enif_make_atom_len(env, "an0atom", 7);
+    arr[2] = enif_make_atom_len(env, an0atom, 7);
+    arr[3] = enif_make_atom_len(env, an0atom0, 8);
+
+    if (!enif_make_existing_atom(env, "an0atom", &existingatom0a, ERL_NIF_LATIN1))
+	return enif_make_atom(env, "error");
+    arr[4] = existingatom0a;
+
+    if (!enif_make_existing_atom_len(env, an0atom, 7, &existingatom0b, ERL_NIF_LATIN1))
+	return enif_make_atom(env, "error");
+    arr[5] = existingatom0b;
+
+    if (!enif_make_existing_atom_len(env, an0atom0, 8, &existing0atom0, ERL_NIF_LATIN1))
+	return enif_make_atom(env, "error");
+    arr[6] = existing0atom0;
+
+    return enif_make_tuple7(env,
+			    arr[0],arr[1],arr[2],arr[3],arr[4],arr[5],arr[6]);
+}
+
+static ERL_NIF_TERM make_strings(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    const char a0string[8] = {'a','0','s','t','r','i','n','g'};
+    const char a0string0[9] = {'a','\0','s','t','r','i','n','g',0};
+    const char astringwith8bits[37] = {'E','r','l','a','n','g',' ',0xE4 /* 'ä' */,'r',' ','e','t','t',' ','g','e','n','e','r','e','l','l','t',' ','p','r','o','g','r','a','m','s','p','r', 0xE5 /* 'å' */,'k',0};
+
+    return enif_make_tuple5(env,
+			    enif_make_string(env, "a0string", ERL_NIF_LATIN1),
+			    enif_make_string_len(env, "a0string", 8, ERL_NIF_LATIN1),
+			    enif_make_string_len(env, a0string, 8, ERL_NIF_LATIN1),
+			    enif_make_string_len(env, a0string0, 9, ERL_NIF_LATIN1),
+			    enif_make_string(env, astringwith8bits, ERL_NIF_LATIN1));
+}
+static ERL_NIF_TERM send_list_seq(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ErlNifPid to;
+    ERL_NIF_TERM msg;
+    ErlNifEnv* msg_env;
+    int i, res;
+    
+    if (!enif_get_int(env, argv[0], &i)) {
+	return enif_make_badarg(env);
+    }
+    if (argv[1] == atom_self) {
+	enif_self(env, &to);
+    }
+    else if (!enif_get_local_pid(env, argv[1], &to)) {
+	return enif_make_badarg(env);
+    }
+    msg_env = enif_alloc_env();
+    msg = enif_make_list(msg_env,0);
+    for ( ; i>0 ; i--) {
+	msg = enif_make_list_cell(msg_env, enif_make_int(msg_env, i), msg);
+    }
+    res = enif_send(env, &to, msg_env, msg);
+    enif_free_env(msg_env);
+    return enif_make_tuple2(env, atom_ok, enif_make_int(env,res));
+}
+
+static void fill(void* dst, unsigned bytes, int seed)
+{
+    unsigned char* ptr = dst;
+    int i;
+    for (i=bytes; i>0; i--) {
+	*ptr++ = seed;
+	seed += 7;
+    }
+}
+
+#define MAKE_TERM_REUSE_LEN 16
+struct make_term_info
+{
+    ErlNifEnv* caller_env;
+    ErlNifEnv* dst_env;
+    ERL_NIF_TERM reuse[MAKE_TERM_REUSE_LEN];
+    unsigned reuse_push;
+    unsigned reuse_pull;
+    ErlNifResourceType* resource_type;
+    ERL_NIF_TERM other_term;
+    ERL_NIF_TERM blob;
+    ErlNifPid to_pid;
+    ErlNifTid tid;
+    ErlNifCond* cond;
+    ErlNifMutex* mtx;
+    int send_it;
+    int send_res;
+    unsigned n;
+};
+
+
+static void push_term(struct make_term_info* mti, ERL_NIF_TERM term)
+{
+    unsigned ix = (mti->reuse_push++) % MAKE_TERM_REUSE_LEN; 
+    mti->reuse[ix] = term;
+    //enif_fprintf(stderr, "push at %u: %T\r\n", ix, term);
+}
+static ERL_NIF_TERM pull_term(struct make_term_info* mti)
+{
+    unsigned ix;
+    if (mti->reuse_pull >= mti->reuse_push &&
+	mti->reuse_push < MAKE_TERM_REUSE_LEN) {
+	mti->reuse_pull = 0;
+	if (mti->reuse_push == 0) {
+	    mti->reuse[0] = enif_make_list(mti->dst_env, 0);
+	}
+    }
+    ix = (mti->reuse_pull++) % MAKE_TERM_REUSE_LEN;
+    //enif_fprintf(stderr, "pull from %u: %T\r\n", ix, mti->reuse[ix]);
+    return mti->reuse[ix];
+}
+
+static int make_term_n(struct make_term_info* mti, int n, ERL_NIF_TERM* res);
+
+static ERL_NIF_TERM make_term_binary(struct make_term_info* mti, int n)
+{	
+    ErlNifBinary bin;
+    enif_alloc_binary(100, &bin);
+    fill(bin.data, bin.size, n);
+    return enif_make_binary(mti->dst_env, &bin);
+}
+
+static ERL_NIF_TERM make_term_int(struct make_term_info* mti, int n)
+{
+    int i;
+    fill(&i, sizeof(i), n);
+    return enif_make_int(mti->dst_env, i);
+}
+
+static ERL_NIF_TERM make_term_ulong(struct make_term_info* mti, int n)
+{
+    unsigned long ul;
+    fill(&ul, sizeof(ul), n);
+    return enif_make_ulong(mti->dst_env, ul);
+}
+
+static ERL_NIF_TERM make_term_double(struct make_term_info* mti, int n)
+{
+    double d = 3.141592;
+    return enif_make_double(mti->dst_env, d);    	
+}
+static ERL_NIF_TERM make_term_atom(struct make_term_info* mti, int n)
+{
+    return enif_make_atom(mti->dst_env, "make_term_n");    
+} 
+static ERL_NIF_TERM make_term_existing_atom(struct make_term_info* mti, int n)
+{	
+    ERL_NIF_TERM res;
+    int exist = enif_make_existing_atom(mti->dst_env, "nif_SUITE", &res,
+					ERL_NIF_LATIN1);
+    assert(exist);
+    return res;
+}
+static ERL_NIF_TERM make_term_string(struct make_term_info* mti, int n)
+{
+    return enif_make_string(mti->dst_env, "Hello!", ERL_NIF_LATIN1);
+}
+static ERL_NIF_TERM make_term_ref(struct make_term_info* mti, int n)
+{
+    return enif_make_ref(mti->dst_env);
+}
+static ERL_NIF_TERM make_term_sub_binary(struct make_term_info* mti, int n)
+{
+    ERL_NIF_TERM orig;
+    unsigned char* ptr = enif_make_new_binary(mti->dst_env, 10, &orig);
+    fill(ptr, 10, n);
+    return enif_make_sub_binary(mti->dst_env, orig, 3, 5);	
+}
+static ERL_NIF_TERM make_term_uint(struct make_term_info* mti, int n)
+{
+    unsigned int ui;
+    fill(&ui, sizeof(ui), n);
+    return enif_make_uint(mti->dst_env, ui);	       
+}
+static ERL_NIF_TERM make_term_long(struct make_term_info* mti, int n)
+{
+    long l;
+    fill(&l, sizeof(l), n);
+    return enif_make_long(mti->dst_env, l);
+}
+static ERL_NIF_TERM make_term_tuple0(struct make_term_info* mti, int n)
+{
+    return enif_make_tuple(mti->dst_env, 0);
+}
+static ERL_NIF_TERM make_term_list0(struct make_term_info* mti, int n)
+{
+    return enif_make_list(mti->dst_env, 0);
+}
+static ERL_NIF_TERM make_term_resource(struct make_term_info* mti, int n)
+{
+    void* resource = enif_alloc_resource(mti->resource_type, 10);
+    ERL_NIF_TERM term;
+    fill(resource, 10, n); 
+    term = enif_make_resource(mti->dst_env, resource);
+    enif_release_resource(resource);
+    return term;
+}
+static ERL_NIF_TERM make_term_new_binary(struct make_term_info* mti, int n)
+{
+    ERL_NIF_TERM res;
+    unsigned char* ptr = enif_make_new_binary(mti->dst_env,20,&res);
+    fill(ptr, 20, n);	
+    return res;
+}
+static ERL_NIF_TERM make_term_caller_pid(struct make_term_info* mti, int n)
+{
+    ErlNifPid pid;    
+    return enif_make_pid(mti->dst_env, enif_self(mti->caller_env, &pid));    		
+}	
+
+static ERL_NIF_TERM make_term_tuple(struct make_term_info* mti, int n)
+{
+    ERL_NIF_TERM t[3];
+    t[0] = pull_term(mti);
+    t[1] = pull_term(mti);
+    t[2] = pull_term(mti);
+    return enif_make_tuple3(mti->dst_env, t[0], t[1], t[2]);	      
+}
+static ERL_NIF_TERM make_term_list(struct make_term_info* mti, int n)
+{
+    ERL_NIF_TERM t[3];
+    t[0] = pull_term(mti);
+    t[1] = pull_term(mti);
+    t[2] = pull_term(mti);
+    return enif_make_list3(mti->dst_env, t[0], t[1], t[2]);    
+}
+static ERL_NIF_TERM make_term_list_cell(struct make_term_info* mti, int n)
+{
+    ERL_NIF_TERM t[2];
+    t[0] = pull_term(mti);
+    t[1] = pull_term(mti);
+    return enif_make_list_cell(mti->dst_env, t[0], t[1]);
+}
+static ERL_NIF_TERM make_term_tuple_from_array(struct make_term_info* mti, int n)
+{
+    ERL_NIF_TERM t[3];
+    t[0] = pull_term(mti);
+    t[1] = pull_term(mti);
+    t[2] = pull_term(mti);
+    return enif_make_tuple_from_array(mti->dst_env, t, 3);	
+}
+static ERL_NIF_TERM make_term_list_from_array(struct make_term_info* mti, int n)
+{
+    ERL_NIF_TERM t[3];
+    t[0] = pull_term(mti);
+    t[1] = pull_term(mti);
+    t[2] = pull_term(mti);
+    return enif_make_list_from_array(mti->dst_env, t, 3);    
+}
+static ERL_NIF_TERM make_term_garbage(struct make_term_info* mti, int n)
+{
+    (void) enif_make_string(mti->dst_env, "garbage string", ERL_NIF_LATIN1);
+    return pull_term(mti);
+}
+static ERL_NIF_TERM make_term_copy(struct make_term_info* mti, int n)
+{
+    return enif_make_copy(mti->dst_env, mti->other_term);
+}
+
+typedef ERL_NIF_TERM Make_term_Func(struct make_term_info*, int);
+static Make_term_Func* make_funcs[] = {
+    make_term_binary,
+    make_term_int,
+    make_term_ulong,
+    make_term_double,
+    make_term_atom,
+    make_term_existing_atom,
+    make_term_string,
+    //make_term_ref,
+    make_term_sub_binary,
+    make_term_uint,
+    make_term_long,
+    make_term_tuple0,
+    make_term_list0,
+    make_term_resource,
+    make_term_new_binary,
+    make_term_caller_pid,
+    make_term_tuple,
+    make_term_list,
+    make_term_list_cell,
+    make_term_tuple_from_array,
+    make_term_list_from_array,
+    make_term_garbage,
+    make_term_copy
+};
+static unsigned num_of_make_funcs()
+{
+    return sizeof(make_funcs)/sizeof(*make_funcs);
+}
+static int make_term_n(struct make_term_info* mti, int n, ERL_NIF_TERM* res)
+{
+    if (n < num_of_make_funcs()) {
+	*res = make_funcs[n](mti, n);
+	push_term(mti, *res);
+	return 1;
+    }
+    return 0;
+}
+
+static ERL_NIF_TERM make_blob(ErlNifEnv* caller_env, ErlNifEnv* dst_env,
+			      ERL_NIF_TERM other_term)
+{
+    PrivData* priv = (PrivData*) enif_priv_data(caller_env);
+    ERL_NIF_TERM term, list;
+    int n = 0;
+    struct make_term_info mti;
+    mti.caller_env = caller_env;
+    mti.dst_env = dst_env;
+    mti.reuse_push = 0;
+    mti.reuse_pull = 0;
+    mti.resource_type = priv->rt_arr[0].t;
+    mti.other_term = other_term;
+
+    list = enif_make_list(dst_env, 0); 
+    while (make_term_n(&mti, n++, &term)) {
+	list = enif_make_list_cell(dst_env, term, list);
+    }
+    return list;
+}
+
+static ERL_NIF_TERM send_new_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ErlNifPid to;
+    ERL_NIF_TERM msg, copy;
+    ErlNifEnv* msg_env;
+    int res;
+    
+    if (!enif_get_local_pid(env, argv[0], &to)) {
+	return enif_make_badarg(env);
+    }
+    msg_env = enif_alloc_env();
+    msg = make_blob(env,msg_env, argv[1]);
+    copy = make_blob(env,env, argv[1]);
+    res = enif_send(env, &to, msg_env, msg);
+    enif_free_env(msg_env);
+    return enif_make_tuple3(env, atom_ok, enif_make_int(env,res), copy);
+}
+
+static ERL_NIF_TERM alloc_msgenv(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    PrivData* priv = (PrivData*) enif_priv_data(env);
+    struct make_term_info* mti;
+    ERL_NIF_TERM ret;
+
+    mti = (struct make_term_info*) enif_alloc_resource(msgenv_resource_type,
+						       sizeof(*mti));
+    mti->caller_env = NULL;
+    mti->dst_env = enif_alloc_env();
+    mti->reuse_push = 0;
+    mti->reuse_pull = 0;
+    mti->resource_type = priv->rt_arr[0].t;
+    mti->other_term = enif_make_list(mti->dst_env, 0);
+    mti->blob = enif_make_list(mti->dst_env, 0);
+    mti->mtx = enif_mutex_create("nif_SUITE:mtx");
+    mti->cond = enif_cond_create("nif_SUITE:cond");
+    mti->send_res = 0xcafebabe;
+    mti->n = 0;
+    ret = enif_make_resource(env, mti);
+    enif_release_resource(mti);
+    return ret;
+}
+
+static void msgenv_dtor(ErlNifEnv* env, void* obj)
+{
+    struct make_term_info* mti = (struct make_term_info*) obj;
+    if (mti->dst_env != NULL) {
+	enif_free_env(mti->dst_env);
+    }
+    enif_mutex_destroy(mti->mtx);
+    enif_cond_destroy(mti->cond);
+}
+
+static ERL_NIF_TERM clear_msgenv(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    mti_t mti;
+    if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)) {
+	return enif_make_badarg(env);
+    }
+    enif_clear_env(mti.p->dst_env);
+    mti.p->reuse_pull = 0;
+    mti.p->reuse_push = 0;
+    mti.p->blob = enif_make_list(mti.p->dst_env, 0);
+    return atom_ok;
+}
+
+static ERL_NIF_TERM grow_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    mti_t mti;
+    ERL_NIF_TERM term;
+    if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)
+	|| (argc>2 && !enif_get_uint(env,argv[2], &mti.p->n))) {
+	return enif_make_badarg(env);
+    }
+    mti.p->caller_env = env;
+    mti.p->other_term = argv[1];
+    mti.p->n %= num_of_make_funcs();
+    make_term_n(mti.p, mti.p->n++, &term);
+    mti.p->blob = enif_make_list_cell(mti.p->dst_env, term, mti.p->blob);
+    return atom_ok;
+}
+
+static ERL_NIF_TERM send_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    mti_t mti;
+    ErlNifPid to;
+    ERL_NIF_TERM copy;
+    int res;
+    if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)
+	|| !enif_get_local_pid(env, argv[1], &to)) {
+	return enif_make_badarg(env);
+    }
+    copy = enif_make_copy(env, mti.p->blob);
+    res = enif_send(env, &to, mti.p->dst_env, mti.p->blob);
+    return enif_make_tuple3(env, atom_ok, enif_make_int(env,res), copy);
+}
+
+static ERL_NIF_TERM send3_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    mti_t mti;
+    ErlNifPid to;
+    ERL_NIF_TERM copy;
+    int res;
+    if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)
+	|| !enif_get_local_pid(env, argv[1], &to)) {
+	return enif_make_badarg(env);
+    }
+    mti.p->blob = enif_make_tuple2(mti.p->dst_env, 
+				   enif_make_copy(mti.p->dst_env, argv[2]),
+				   mti.p->blob);
+    res = enif_send(env, &to, mti.p->dst_env, mti.p->blob);
+    return enif_make_int(env,res);
+}
+
+void* threaded_sender(void *arg)
+{
+
+    mti_t mti;
+    mti.vp = arg;
+
+    enif_mutex_lock(mti.p->mtx);
+    while (!mti.p->send_it) {
+	enif_cond_wait(mti.p->cond, mti.p->mtx);
+    }
+    mti.p->send_it = 0;
+    enif_mutex_unlock(mti.p->mtx);
+    mti.p->send_res = enif_send(NULL, &mti.p->to_pid, mti.p->dst_env, mti.p->blob);
+    return NULL;
+}
+
+static ERL_NIF_TERM send_blob_thread(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    mti_t mti;
+    ERL_NIF_TERM copy;
+    if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)
+	|| !enif_get_local_pid(env,argv[1], &mti.p->to_pid)) {
+	return enif_make_badarg(env);
+    }
+    copy = enif_make_copy(env, mti.p->blob);
+
+    mti.p->send_it = enif_is_identical(argv[2],atom_join);
+    if (enif_thread_create("nif_SUITE:send_from_thread", &mti.p->tid,
+			   threaded_sender, mti.p, NULL) != 0) {
+	return enif_make_badarg(env);
+    }
+    if (enif_is_identical(argv[2],atom_join)) {
+	int err = enif_thread_join(mti.p->tid, NULL);
+	assert(err == 0);
+	return enif_make_tuple3(env, atom_ok, enif_make_int(env, mti.p->send_res), copy);
+    }
+    else {
+	enif_keep_resource(mti.vp);
+	return enif_make_tuple2(env, atom_ok, copy);
+    }
+}
+
+static ERL_NIF_TERM join_send_thread(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    mti_t mti;
+    int err;
+    if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)) {
+	return enif_make_badarg(env);
+    }
+    enif_mutex_lock(mti.p->mtx);
+    mti.p->send_it = 1;
+    enif_cond_signal(mti.p->cond);
+    enif_mutex_unlock(mti.p->mtx);
+    err = enif_thread_join(mti.p->tid, NULL);
+    assert(err == 0);
+    enif_release_resource(mti.vp);
+    return enif_make_tuple2(env, atom_ok, enif_make_int(env, mti.p->send_res));    
+}
+
+static ERL_NIF_TERM copy_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    mti_t mti;
+    if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)) {
+	return enif_make_badarg(env);
+    }
+    return enif_make_copy(env, mti.p->blob);
+}
+
+static ERL_NIF_TERM send_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    ErlNifEnv* menv;
+    ErlNifPid pid;
+    int ret;
+    if (!enif_get_local_pid(env, argv[0], &pid)) {
+	return enif_make_badarg(env);
+    }
+    menv = enif_alloc_env();
+    ret = enif_send(env, &pid, menv, enif_make_copy(menv, argv[1]));
+    enif_free_env(menv);
+    return enif_make_int(env, ret);
+}
+
+static ERL_NIF_TERM reverse_list(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {
+    ERL_NIF_TERM rev_list;
+
+    if(!enif_make_reverse_list(env, argv[0], &rev_list))
+	return enif_make_atom(env, "badarg");
+    return rev_list;
+}
+
+static ERL_NIF_TERM otp_9668_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    /* Inspect in process independent env */
+    ErlNifEnv* myenv = enif_alloc_env();
+    ERL_NIF_TERM mycopy = enif_make_copy(myenv, argv[0]);
+    ErlNifBinary obin, cbin;
+
+    if ((enif_inspect_binary(env, argv[0], &obin)
+	 && enif_inspect_binary(myenv, mycopy, &cbin))
+	||
+	(enif_inspect_iolist_as_binary(env, argv[0], &obin)
+	 && enif_inspect_iolist_as_binary(myenv, mycopy, &cbin)))
+    {
+	assert(obin.size == cbin.size);
+	assert(memcmp(obin.data, cbin.data, obin.size) == 0);
+    }	
+    enif_free_env(myenv);
+    return atom_ok;
+}
 
 static ErlNifFunc nif_funcs[] =
 {
@@ -640,8 +1481,30 @@ static ErlNifFunc nif_funcs[] =
     {"get_resource", 2, get_resource},
     {"release_resource", 1, release_resource},
     {"last_resource_dtor_call", 0, last_resource_dtor_call},
-    {"make_new_resource", 2, make_new_resource}
-
+    {"make_new_resource", 2, make_new_resource},
+    {"check_is", 11, check_is},
+    {"check_is_exception", 0, check_is_exception},
+    {"length_test", 5, length_test},
+    {"make_atoms", 0, make_atoms},
+    {"make_strings", 0, make_strings},
+    {"make_new_resource", 2, make_new_resource},
+    {"make_new_resource_binary", 1, make_new_resource_binary},
+    {"send_list_seq", 2, send_list_seq},
+    {"send_new_blob", 2, send_new_blob},
+    {"alloc_msgenv", 0, alloc_msgenv},
+    {"clear_msgenv", 1, clear_msgenv},
+    {"grow_blob", 2, grow_blob},
+    {"grow_blob", 3, grow_blob},
+    {"send_blob", 2, send_blob},
+    {"send3_blob", 3, send3_blob},
+    {"send_blob_thread", 3, send_blob_thread},
+    {"join_send_thread", 1, join_send_thread},
+    {"copy_blob", 1, copy_blob},
+    {"send_term", 2, send_term},
+    {"reverse_list",1, reverse_list},
+    {"echo_int", 1, echo_int},
+    {"type_sizes", 0, type_sizes},
+    {"otp_9668_nif", 1, otp_9668_nif}
 };
 
 ERL_NIF_INIT(nif_SUITE,nif_funcs,load,reload,upgrade,unload)
diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.c b/erts/emulator/test/nif_SUITE_data/nif_mod.c
index c075b74c57..e32d10057c 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_mod.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_mod.c
@@ -1,3 +1,21 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
 #include "erl_nif.h"
 #include <string.h>
 #include <stdio.h>
@@ -24,6 +42,11 @@ static ERL_NIF_TERM am_resource_type;
 static ERL_NIF_TERM am_resource_dtor_A;
 static ERL_NIF_TERM am_resource_dtor_B;
 
+static NifModPrivData* priv_data(ErlNifEnv* env)
+{
+    return (NifModPrivData*) enif_priv_data(env);
+}
+
 static void init(ErlNifEnv* env)
 {
     am_true = enif_make_atom(env, "true");
@@ -36,7 +59,7 @@ static void init(ErlNifEnv* env)
 static void add_call_with_arg(ErlNifEnv* env, NifModPrivData* data, const char* func_name,
 			      const char* arg, int arg_sz)
 {
-    CallInfo* call = enif_alloc(env, sizeof(CallInfo)+strlen(func_name) + arg_sz);
+    CallInfo* call = (CallInfo*)enif_alloc(sizeof(CallInfo)+strlen(func_name) + arg_sz);
     strcpy(call->func_name, func_name);
     call->lib_ver = NIF_LIB_VER;
     call->static_cntA = ++static_cntA;
@@ -60,7 +83,7 @@ static void add_call(ErlNifEnv* env, NifModPrivData* data,const char* func_name)
     add_call_with_arg(env, data, func_name, NULL, 0);
 }
 
-#define ADD_CALL(FUNC_NAME) add_call(env, enif_priv_data(env),FUNC_NAME)
+#define ADD_CALL(FUNC_NAME) add_call(env, priv_data(env),FUNC_NAME)
 
 #define STRINGIFY_(X) #X
 #define STRINGIFY(X) STRINGIFY_(X)
@@ -69,56 +92,56 @@ static void resource_dtor_A(ErlNifEnv* env, void* a)
 {
     const char dtor_name[] = "resource_dtor_A_v"  STRINGIFY(NIF_LIB_VER); 
 
-    add_call_with_arg(env, enif_priv_data(env), dtor_name,
-		      a, enif_sizeof_resource(env, a));
+    add_call_with_arg(env, priv_data(env), dtor_name, (const char*)a,
+		      enif_sizeof_resource(a));
 }
 
 static void resource_dtor_B(ErlNifEnv* env, void* a)
 {
-    const char dtor_name[] = "resource_dtor_B_v"  STRINGIFY(NIF_LIB_VER); 
+    const char dtor_name[] = "resource_dtor_B_v"  STRINGIFY(NIF_LIB_VER);
 
-    add_call_with_arg(env, enif_priv_data(env), dtor_name,
-		      a, enif_sizeof_resource(env, a));
+    add_call_with_arg(env, priv_data(env), dtor_name, (const char*)a,
+		      enif_sizeof_resource(a));
 }
 
 /* {resource_type, Ix|null, ErlNifResourceFlags in, "TypeName", dtor(A|B|null), ErlNifResourceFlags out}*/
 static void open_resource_type(ErlNifEnv* env, ERL_NIF_TERM op_tpl)
 {
-    NifModPrivData* data = enif_priv_data(env);
+    NifModPrivData* data = priv_data(env);
     const ERL_NIF_TERM* arr;
     int arity;
     char rt_name[30];
-    union { enum ErlNifResourceFlags e; int i; } flags, exp_res, got_res;
+    union { ErlNifResourceFlags e; int i; } flags, exp_res, got_res;
     unsigned ix;
     ErlNifResourceDtor* dtor;
     ErlNifResourceType* got_ptr;
 
     CHECK(enif_get_tuple(env, op_tpl, &arity, &arr));
     CHECK(arity == 6);
-    CHECK(enif_is_identical(env, arr[0], am_resource_type));
+    CHECK(enif_is_identical(arr[0], am_resource_type));
     CHECK(enif_get_int(env, arr[2], &flags.i));
     CHECK(enif_get_string(env, arr[3], rt_name, sizeof(rt_name), ERL_NIF_LATIN1) > 0);
     CHECK(enif_get_int(env, arr[5], &exp_res.i));
 	
-    if (enif_is_identical(env, arr[4], am_null)) {
+    if (enif_is_identical(arr[4], am_null)) {
 	dtor = NULL;
     }
-    else if (enif_is_identical(env, arr[4], am_resource_dtor_A)) {
+    else if (enif_is_identical(arr[4], am_resource_dtor_A)) {
 	dtor = resource_dtor_A;
     }
     else {
-	CHECK(enif_is_identical(env, arr[4], am_resource_dtor_B));
+	CHECK(enif_is_identical(arr[4], am_resource_dtor_B));
 	dtor = resource_dtor_B;
     }
 
-    got_ptr = enif_open_resource_type(env, rt_name, dtor,
+    got_ptr = enif_open_resource_type(env, NULL, rt_name, dtor,
 				      flags.e, &got_res.e);
 
     if (enif_get_uint(env, arr[1], &ix) && ix < RT_MAX && got_ptr != NULL) {
 	data->rt_arr[ix] = got_ptr;
     }
     else {
-	CHECK(enif_is_identical(env, arr[1], am_null));
+	CHECK(enif_is_identical(arr[1], am_null));
 	CHECK(got_ptr == NULL);
     }
     CHECK(got_res.e == exp_res.e);
@@ -126,7 +149,7 @@ static void open_resource_type(ErlNifEnv* env, ERL_NIF_TERM op_tpl)
 
 static void do_load_info(ErlNifEnv* env, ERL_NIF_TERM load_info)
 {
-    NifModPrivData* data = enif_priv_data(env);
+    NifModPrivData* data = priv_data(env);
     ERL_NIF_TERM head, tail;
     unsigned ix;
     for (ix=0; ix<RT_MAX; ix++) {
@@ -140,17 +163,18 @@ static void do_load_info(ErlNifEnv* env, ERL_NIF_TERM load_info)
     CHECK(enif_is_empty_list(env, head));
 }
 
-static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
+static int load(ErlNifEnv* env, void** priv, ERL_NIF_TERM load_info)
 {
     NifModPrivData* data;
 
     init(env);
-    data = enif_alloc(env, sizeof(NifModPrivData));
+    data = (NifModPrivData*) enif_alloc(sizeof(NifModPrivData));
     CHECK(data != NULL);
-    *priv_data = data;
+    *priv = data;
     data->mtx = enif_mutex_create("nif_mod_priv_data"); 
     data->ref_cnt = 1;
     data->call_history = NULL;
+
     add_call(env, data, "load");
 
     do_load_info(env, load_info);
@@ -158,39 +182,35 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
     return 0;
 }
 
-static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
+static int reload(ErlNifEnv* env, void** priv, ERL_NIF_TERM load_info)
 {
+    NifModPrivData* data = (NifModPrivData*) *priv;
     init(env);
-    add_call(env, *priv_data, "reload");
+    add_call(env, data, "reload");
     
     do_load_info(env, load_info);
     return 0;
 }
 
-static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info)
+static int upgrade(ErlNifEnv* env, void** priv, void** old_priv_data, ERL_NIF_TERM load_info)
 {
-    NifModPrivData* data = *old_priv_data;
+    NifModPrivData* data = (NifModPrivData*) *old_priv_data;
     init(env);
     add_call(env, data, "upgrade");
     data->ref_cnt++;
 
-    *priv_data = *old_priv_data;    
+    *priv = *old_priv_data;    
     do_load_info(env, load_info);
     
     return 0;
 }
 
-static void unload(ErlNifEnv* env, void* priv_data)
+static void unload(ErlNifEnv* env, void* priv)
 {
-    NifModPrivData* data = priv_data;
+    NifModPrivData* data = (NifModPrivData*) priv;
+    int is_last;
     add_call(env, data, "unload");
-    enif_mutex_lock(data->mtx);
-    if (--data->ref_cnt == 0) {
-	enif_mutex_unlock(data->mtx);
-	enif_mutex_destroy(data->mtx);
-	enif_free(env, data);
-    }
-    enif_mutex_unlock(data->mtx);
+    NifModPrivData_release(data);
 }
 
 static ERL_NIF_TERM lib_version(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
@@ -202,12 +222,12 @@ static ERL_NIF_TERM lib_version(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg
 static ERL_NIF_TERM get_priv_data_ptr(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     ADD_CALL("get_priv_data_ptr");
-    return enif_make_ulong(env, (unsigned long)enif_priv_data(env));
+    return enif_make_ulong(env, (unsigned long)priv_data(env));
 }
 
 static ERL_NIF_TERM make_new_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    NifModPrivData* data = (NifModPrivData*) enif_priv_data(env);
+    NifModPrivData* data = priv_data(env);
     ErlNifBinary ibin;
     char* a;
     ERL_NIF_TERM ret;
@@ -216,22 +236,22 @@ static ERL_NIF_TERM make_new_resource(ErlNifEnv* env, int argc, const ERL_NIF_TE
 	|| !enif_inspect_binary(env, argv[1], &ibin)) {
 	return enif_make_badarg(env);
     }
-    a = enif_alloc_resource(env, data->rt_arr[ix], ibin.size);
+    a = (char*) enif_alloc_resource(data->rt_arr[ix], ibin.size);
     memcpy(a, ibin.data, ibin.size);
     ret = enif_make_resource(env, a);
-    enif_release_resource(env, a);
+    enif_release_resource(a);
     return ret;
 }
 
 static ERL_NIF_TERM get_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    NifModPrivData* data = (NifModPrivData*) enif_priv_data(env);
+    NifModPrivData* data = priv_data(env);
     ErlNifBinary obin;
     unsigned ix;
     void* a;
     if (!enif_get_uint(env, argv[0], &ix) || ix >= RT_MAX 
 	|| !enif_get_resource(env, argv[1], data->rt_arr[ix], &a)
-	|| !enif_alloc_binary(env, enif_sizeof_resource(env, a), &obin)) { 
+	|| !enif_alloc_binary(enif_sizeof_resource(a), &obin)) { 
 	return enif_make_badarg(env);
     }
     memcpy(obin.data, a, obin.size);
diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.erl b/erts/emulator/test/nif_SUITE_data/nif_mod.erl
index 7888a589e7..6634624698 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_mod.erl
+++ b/erts/emulator/test/nif_SUITE_data/nif_mod.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2005-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,7 +19,7 @@
 
 -module(nif_mod).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -export([load_nif_lib/2, load_nif_lib/3, start/0, lib_version/0, call_history/0,
 	 get_priv_data_ptr/0, make_new_resource/2, get_resource/2]).
diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.h b/erts/emulator/test/nif_SUITE_data/nif_mod.h
index 0eaf91d6e1..cd0ecf4b54 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_mod.h
+++ b/erts/emulator/test/nif_SUITE_data/nif_mod.h
@@ -20,3 +20,15 @@ typedef struct
     ErlNifResourceType* rt_arr[RT_MAX];
 }NifModPrivData;
 
+#define NifModPrivData_release(NMPD) \
+    do { \
+	int is_last; \
+	enif_mutex_lock((NMPD)->mtx); \
+	is_last = (--(NMPD)->ref_cnt == 0); \
+        enif_mutex_unlock((NMPD)->mtx); \
+        if (is_last) { \
+	    enif_mutex_destroy((NMPD)->mtx); \
+	    enif_free((NMPD)); \
+        } \
+    }while (0)
+
diff --git a/erts/emulator/test/nif_SUITE_data/tester.c b/erts/emulator/test/nif_SUITE_data/tester.c
index 08466d0f18..257b116322 100644
--- a/erts/emulator/test/nif_SUITE_data/tester.c
+++ b/erts/emulator/test/nif_SUITE_data/tester.c
@@ -61,6 +61,7 @@ static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
 static ERL_NIF_TERM run(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     testcase_run(NULL);
+    testcase_cleanup(NULL);
     return enif_make_atom(env, "ok");
 }
 
diff --git a/erts/emulator/test/nif_SUITE_data/tester.erl b/erts/emulator/test/nif_SUITE_data/tester.erl
index 9df2158200..b393e29b82 100644
--- a/erts/emulator/test/nif_SUITE_data/tester.erl
+++ b/erts/emulator/test/nif_SUITE_data/tester.erl
@@ -1,6 +1,6 @@
 -module(tester).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -export([load_nif_lib/2, run/0]).
 
diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl
index f3d9eb783b..0bf2c03233 100644
--- a/erts/emulator/test/node_container_SUITE.erl
+++ b/erts/emulator/test/node_container_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2002-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2002-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -29,10 +29,12 @@
 
 %-define(line_trace, 1).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %-compile(export_all).
--export([all/1, init_per_testcase/2, fin_per_testcase/2, end_per_suite/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, init_per_testcase/2, 
+	 end_per_testcase/2,
 	 node_container_refc_check/1]).
 
 -export([term_to_binary_to_term_eq/1,
@@ -55,25 +57,30 @@
 
 -define(DEFAULT_TIMEOUT, ?t:minutes(10)).
 
-all(doc) -> [];
-all(suite) ->
-    [term_to_binary_to_term_eq,
-     round_trip_eq,
-     cmp,
-     ref_eq,
-     node_table_gc,
-     dist_link_refc,
-     dist_monitor_refc,
-     node_controller_refc,
-     ets_refc,
-     match_spec_refc,
-     timer_refc,
-     otp_4715,
-     pid_wrap,
-     port_wrap,
-     bad_nc,
-     unique_pid,
-     iter_max_procs].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [term_to_binary_to_term_eq, round_trip_eq, cmp, ref_eq,
+     node_table_gc, dist_link_refc, dist_monitor_refc,
+     node_controller_refc, ets_refc, match_spec_refc,
+     timer_refc, otp_4715, pid_wrap, port_wrap, bad_nc,
+     unique_pid, iter_max_procs].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    available_internal_state(false).
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 available_internal_state(Bool) when Bool == true; Bool == false ->
     case {Bool,
@@ -95,14 +102,11 @@ init_per_testcase(_Case, Config) when is_list(Config) ->
     available_internal_state(true),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) when is_list(Config) ->
+end_per_testcase(_Case, Config) when is_list(Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
 
-end_per_suite(_Config) ->
-    available_internal_state(false).
-
 %%%
 %%% The test cases -------------------------------------------------------------
 %%%
@@ -455,10 +459,6 @@ make_node_garbage(_, _, _, Ps) ->
 	  end,
     lists:foreach(fun (P) -> wait_until(fun () -> ProcIsCleanedUp(P) end) end,
 		  Ps),
-    ?line case erlang:system_info(heap_type) of
-	      shared -> ?line garbage_collect();
-	      _ -> ?line ok
-	  end,
     ?line ok.
 
 
@@ -601,17 +601,7 @@ node_controller_refc(Config) when is_list(Config) ->
     % Get rid of all references to Node
     ?line exec(P, fun () -> exit(normal) end),
     ?line wait_until(fun () -> not is_process_alive(P) end),
-    ?line case erlang:system_info(heap_type) of
-	      shared ->
-		  ?line garbage_collect();
-	      hybrid ->
-		  ?line lists:foreach(fun (Proc) -> garbage_collect(Proc) end,
-				      processes()),
-		  ?line erlang:garbage_collect_message_area();
-	      _ ->
-		  ?line lists:foreach(fun (Proc) -> garbage_collect(Proc) end,
-				       processes())
-	  end,
+    lists:foreach(fun (Proc) -> garbage_collect(Proc) end, processes()),
     ?line false = get_node_references({Node,Creation}),
     ?line false = get_dist_references(Node),
     ?line false = lists:member(Node, nodes(known)),
diff --git a/erts/emulator/test/nofrag_SUITE.erl b/erts/emulator/test/nofrag_SUITE.erl
index ece55f433c..6b6ac28e2e 100644
--- a/erts/emulator/test/nofrag_SUITE.erl
+++ b/erts/emulator/test/nofrag_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2007-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2007-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,9 +19,11 @@
 
 -module(nofrag_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,init_per_testcase/2,end_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 error_handler/1,error_handler_apply/1,
 	 error_handler_fixed_apply/1,error_handler_fun/1,
 	 error_handler_tuple_fun/1,
@@ -30,9 +32,28 @@
 %% Exported functions for an error_handler module.
 -export([undefined_function/3,undefined_lambda/3,breakpoint/3]).
 
-all(suite) ->
-    [error_handler,error_handler_apply,error_handler_fixed_apply,
-     error_handler_fun,error_handler_tuple_fun,debug_breakpoint].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [error_handler, error_handler_apply,
+     error_handler_fixed_apply, error_handler_fun,
+     error_handler_tuple_fun, debug_breakpoint].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog = ?t:timetrap(?t:minutes(3)),
diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl
index d009994e2d..4459732257 100644
--- a/erts/emulator/test/num_bif_SUITE.erl
+++ b/erts/emulator/test/num_bif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,7 +19,7 @@
 
 -module(num_bif_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %% Tests the BIFs:
 %% 	abs/1
@@ -31,15 +31,36 @@
 %%	round/1
 %%	trunc/1
 
--export([all/1, t_abs/1, t_float/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, t_abs/1, t_float/1,
 	 t_float_to_list/1, t_integer_to_list/1,
 	 t_list_to_integer/1,
-	 t_list_to_float/1, t_list_to_float_safe/1, t_list_to_float_risky/1,
+	 t_list_to_float_safe/1, t_list_to_float_risky/1,
 	 t_round/1, t_trunc/1]).
 
-all(suite) -> [t_abs, t_float, t_float_to_list, t_integer_to_list,
-	       t_list_to_float, t_list_to_integer,
-	       t_round, t_trunc].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [t_abs, t_float, t_float_to_list, t_integer_to_list,
+     {group, t_list_to_float}, t_list_to_integer, t_round,
+     t_trunc].
+
+groups() -> 
+    [{t_list_to_float, [],
+      [t_list_to_float_safe, t_list_to_float_risky]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 t_abs(Config) when is_list(Config) ->
     %% Floats.
@@ -140,7 +161,6 @@ t_integer_to_list(Config) when is_list(Config) ->
 
 %% Tests list_to_float/1.
 
-t_list_to_float(suite) -> [t_list_to_float_safe, t_list_to_float_risky].
 
 t_list_to_float_safe(Config) when is_list(Config) ->
     ?line 0.0 = list_to_float(id("0.0")),
diff --git a/erts/emulator/test/obsolete_SUITE.erl b/erts/emulator/test/obsolete_SUITE.erl
deleted file mode 100644
index 33c4726699..0000000000
--- a/erts/emulator/test/obsolete_SUITE.erl
+++ /dev/null
@@ -1,123 +0,0 @@
-%%
-%% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 2004-2009. All Rights Reserved.
-%% 
-%% The contents of this file are subject to the Erlang Public License,
-%% Version 1.1, (the "License"); you may not use this file except in
-%% compliance with the License. You should have received a copy of the
-%% Erlang Public License along with this software. If not, it can be
-%% retrieved online at http://www.erlang.org/.
-%% 
-%% Software distributed under the License is distributed on an "AS IS"
-%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
-%% the License for the specific language governing rights and limitations
-%% under the License.
-%% 
-%% %CopyrightEnd%
-%%
-
-
--module(obsolete_SUITE).
--author('[email protected]').
--compile(nowarn_obsolete_guard).
-
--export([all/1]).
-
--export([erl_threads/1]).
-
--include("test_server.hrl").
-
--define(DEFAULT_TIMETRAP_SECS, 240).
-
-all(doc) -> [];
-all(suite) ->
-    case catch erlang:system_info(wordsize) of
-	4 -> [erl_threads];
-	_ -> {skip, "Only expected to work on 32-bit architectures"}
-    end.
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%%                                                                        %%
-%% Testcases                                                              %%
-%%                                                                        %%
-
-erl_threads(suite) -> [];
-erl_threads(doc) ->   [];
-erl_threads(Cfg) ->
-    ?line case erlang:system_info(threads) of
-	      true ->
-		  ?line drv_case(Cfg, erl_threads);
-	      false ->
-		  ?line {skip, "Emulator not compiled with threads support"}
-	  end.
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%%                                                                        %%
-%% Internal functions                                                     %%
-%%                                                                        %%
-
-drv_case(Config, CaseName) ->
-    drv_case(Config, CaseName, "").
-
-drv_case(Config, CaseName, TimeTrap) when integer(TimeTrap) ->
-    drv_case(Config, CaseName, "", TimeTrap);
-drv_case(Config, CaseName, Command) when list(Command) ->
-    drv_case(Config, CaseName, Command, ?DEFAULT_TIMETRAP_SECS).
-
-drv_case(Config, CaseName, TimeTrap, Command) when list(Command),
-						   integer(TimeTrap) ->
-    drv_case(Config, CaseName, Command, TimeTrap);
-drv_case(Config, CaseName, Command, TimeTrap) when list(Config),
-						   atom(CaseName),
-						   list(Command),
-						   integer(TimeTrap) ->
-    case ?t:os_type() of
-	{Family, _} when Family == unix; Family == win32 ->
-	    ?line run_drv_case(Config, CaseName, Command, TimeTrap);
-	SkipOs ->
-	    ?line {skipped,
-		   lists:flatten(["Not run on "
-				  | io_lib:format("~p",[SkipOs])])}
-    end.
-
-run_drv_case(Config, CaseName, Command, TimeTrap) ->
-    ?line Dog = test_server:timetrap(test_server:seconds(TimeTrap)),
-    ?line DataDir = ?config(data_dir,Config),
-    case erl_ddll:load_driver(DataDir, CaseName) of
-	ok -> ok;
-	{error, Error} ->
-	    io:format("~s\n", [erl_ddll:format_error(Error)]),
-	    ?line ?t:fail()
-    end,
-    ?line Port = open_port({spawn, atom_to_list(CaseName)}, []),
-    ?line true = is_port(Port),
-    ?line Port ! {self(), {command, Command}},
-    ?line Result = receive_drv_result(Port, CaseName),
-    ?line Port ! {self(), close},
-    ?line receive 
-	      {Port, closed} ->
-		  ok
-	  end,
-    ?line ok = erl_ddll:unload_driver(CaseName),
-    ?line test_server:timetrap_cancel(Dog),
-    ?line Result.
-
-receive_drv_result(Port, CaseName) ->
-    ?line receive
-	      {print, Port, CaseName, Str} ->
-		  ?line ?t:format("~s", [Str]),
-		  ?line receive_drv_result(Port, CaseName);
-	      {'EXIT', Port, Error} ->
-		  ?line ?t:fail(Error);
-	      {'EXIT', error, Error} ->
-		  ?line ?t:fail(Error);
-	      {failed, Port, CaseName, Comment} ->
-		  ?line ?t:fail(Comment);
-	      {skipped, Port, CaseName, Comment} ->
-		  ?line {skipped, Comment};
-	      {succeeded, Port, CaseName, ""} ->
-		  ?line succeeded;
-	      {succeeded, Port, CaseName, Comment} ->
-		  ?line {comment, Comment}
-	  end.
diff --git a/erts/emulator/test/obsolete_SUITE_data/Makefile.src b/erts/emulator/test/obsolete_SUITE_data/Makefile.src
deleted file mode 100644
index d8e2b861c0..0000000000
--- a/erts/emulator/test/obsolete_SUITE_data/Makefile.src
+++ /dev/null
@@ -1,33 +0,0 @@
-# ``The contents of this file are subject to the Erlang Public License,
-# Version 1.1, (the "License"); you may not use this file except in
-# compliance with the License. You should have received a copy of the
-# Erlang Public License along with this software. If not, it can be
-# retrieved via the world wide web at http://www.erlang.org/.
-# 
-# Software distributed under the License is distributed on an "AS IS"
-# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
-# the License for the specific language governing rights and limitations
-# under the License.
-# 
-# The Initial Developer of the Original Code is Ericsson Utvecklings AB.
-# Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings
-# AB. All Rights Reserved.''
-# 
-#     $Id$
-#
-
-TEST_DRVS = erl_threads@dll@
-CC = @CC@
-LD = @LD@
-CFLAGS = @SHLIB_CFLAGS@ -I@erl_include@ @DEFS@
-SHLIB_EXTRA_LDLIBS = testcase_driver@obj@
-
-all: $(TEST_DRVS)
-
-@SHLIB_RULES@
-
-testcase_driver@obj@: testcase_driver.c testcase_driver.h
-$(TEST_DRVS): testcase_driver@obj@
-
-
-
diff --git a/erts/emulator/test/obsolete_SUITE_data/erl_threads.c b/erts/emulator/test/obsolete_SUITE_data/erl_threads.c
deleted file mode 100644
index 27a5163121..0000000000
--- a/erts/emulator/test/obsolete_SUITE_data/erl_threads.c
+++ /dev/null
@@ -1,302 +0,0 @@
-/* ``The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved via the world wide web at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * The Initial Developer of the Original Code is Ericsson Utvecklings AB.
- * Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings
- * AB. All Rights Reserved.''
- * 
- *     $Id$
- */
-
-#include "testcase_driver.h"
-
-#ifndef __WIN32__
-
-#define NO_OF_THREADS 2
-
-#include <unistd.h>
-#include <errno.h>
-
-static int die;
-static int cw_passed;
-static int res_tf0;
-static int res_tf1;
-static erl_mutex_t mtx;
-static erl_cond_t cnd;
-static erl_thread_t tid[NO_OF_THREADS];
-static int need_join[NO_OF_THREADS];
-
-typedef struct {
-    int n;
-} thr_arg_t;
-
-
-static void *tf0(void *vta)
-{
-    int r;
-
-    if (((thr_arg_t *) vta)->n != 0)
-	goto fail;
-
-    r = erts_mutex_lock(mtx);
-    if (r != 0) {
-	erts_mutex_unlock(mtx);
-	goto fail;
-    }
-
-    r = erts_cond_wait(cnd, mtx);
-    if (r != 0 || die) {
-	erts_mutex_unlock(mtx);
-	goto fail;
-    }
-
-    cw_passed++;
-
-    r = erts_cond_wait(cnd, mtx);
-    if (r != 0 || die) {
-	erts_mutex_unlock(mtx);
-	goto fail;
-    }
-
-    cw_passed++;
-
-    r = erts_mutex_unlock(mtx);
-    if (r != 0)
-	goto fail;
-
-    res_tf0 = 0;
-
-    return (void *) &res_tf0;
-
- fail:
-    return NULL;
-}
-
-
-static void *tf1(void *vta)
-{
-    int r;
-
-    if (((thr_arg_t *) vta)->n != 1)
-	goto fail;
-
-    r = erts_mutex_lock(mtx);
-    if (r != 0) {
-	erts_mutex_unlock(mtx);
-	goto fail;
-    }
-
-    r = erts_cond_wait(cnd, mtx);
-    if (r != 0 || die) {
-	erts_mutex_unlock(mtx);
-	goto fail;
-    }
-
-    cw_passed++;
-
-    r = erts_cond_wait(cnd, mtx);
-    if (r != 0 || die) {
-	erts_mutex_unlock(mtx);
-	goto fail;
-    }
-
-    cw_passed++;
-
-    r = erts_mutex_unlock(mtx);
-    if (r != 0)
-	goto fail;
-
-    res_tf1 = 1;
-
-    erts_thread_exit((void *) &res_tf1);
-
-    res_tf1 = 4711;
-
- fail:
-    return NULL;
-}
-
-#endif /* #ifndef __WIN32__ */
-
-void
-testcase_run(TestCaseState_t *tcs)
-{
-#ifdef __WIN32__
-    testcase_skipped(tcs, "Nothing to test; not supported on windows.");
-#else
-    int i, r;
-    void *tres[NO_OF_THREADS];
-    thr_arg_t ta[NO_OF_THREADS];
-    erl_thread_t t1;
-
-    die = 0;
-    cw_passed = 0;
-
-    for (i = 0; i < NO_OF_THREADS; i++)
-	need_join[i] = 0;
-
-    res_tf0 = 17;
-    res_tf1 = 17;
-
-    cnd = mtx = NULL;
-
-    /* Create mutex and cond */
-    mtx = erts_mutex_create();
-    ASSERT(tcs, mtx);
-    cnd = erts_cond_create();
-    ASSERT(tcs, cnd);
-
-    /* Create the threads */
-    ta[0].n = 0;
-    r = erts_thread_create(&tid[0], tf0, (void *) &ta[0], 0);
-    ASSERT(tcs, r == 0);
-    need_join[0] = 1;
-
-    ta[1].n = 1;
-    r = erts_thread_create(&tid[1], tf1, (void *) &ta[1], 0);
-    ASSERT(tcs, r == 0);
-    need_join[1] = 1;
-
-    /* Make sure the threads waits on cond wait */
-    sleep(1);
-
-    r = erts_mutex_lock(mtx);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-
-    ASSERT_CLNUP(tcs, cw_passed == 0, (void) erts_mutex_unlock(mtx));
-
-
-    /* Let one thread pass one cond wait */
-    r = erts_cond_signal(cnd);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-
-    r = erts_mutex_unlock(mtx);
-    ASSERT(tcs, r == 0);
-
-    sleep(1);
-
-    r = erts_mutex_lock(mtx);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-
-    ASSERT_CLNUP(tcs, cw_passed == 1, (void) erts_mutex_unlock(mtx));
-
-
-    /* Let both threads pass one cond wait */
-    r = erts_cond_broadcast(cnd);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-
-    r = erts_mutex_unlock(mtx);
-    ASSERT(tcs, r == 0);
-
-    sleep(1);
-
-    r = erts_mutex_lock(mtx);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-
-    ASSERT_CLNUP(tcs, cw_passed == 3, (void) erts_mutex_unlock(mtx));
-
-
-    /* Let the thread that only have passed one cond wait pass the other one */
-    r = erts_cond_signal(cnd);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-
-    r = erts_mutex_unlock(mtx);
-    ASSERT(tcs, r == 0);
-
-    sleep(1);
-
-    r = erts_mutex_lock(mtx);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-
-    ASSERT_CLNUP(tcs, cw_passed == 4, (void) erts_mutex_unlock(mtx));
-
-    /* Both threads should have passed both cond waits and exited;
-       join them and check returned values */
-
-    r = erts_thread_join(tid[0], &tres[0]);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-    need_join[0] = 0;
-
-    ASSERT_CLNUP(tcs, tres[0] == &res_tf0, (void) erts_mutex_unlock(mtx));
-    ASSERT_CLNUP(tcs, res_tf0 == 0, (void) erts_mutex_unlock(mtx));
-
-    r = erts_thread_join(tid[1], &tres[1]);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-    need_join[1] = 0;
-
-    ASSERT_CLNUP(tcs, tres[1] == &res_tf1, (void) erts_mutex_unlock(mtx));
-    ASSERT_CLNUP(tcs, res_tf1 == 1, (void) erts_mutex_unlock(mtx));
-
-    /* Test signaling when noone waits */
-
-    r = erts_cond_signal(cnd);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-
-    /* Test broadcasting when noone waits */
-
-    r = erts_cond_broadcast(cnd);
-    ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx));
-
-    /* erts_cond_timedwait() not supported anymore */
-    r = erts_cond_timedwait(cnd, mtx, 1000);
-    ASSERT_CLNUP(tcs, r != 0, (void) erts_mutex_unlock(mtx));
-    ASSERT_CLNUP(tcs,
-		 strcmp(erl_errno_id(r), "enotsup") == 0,
-		 (void) erts_mutex_unlock(mtx));
-
-    r = erts_mutex_unlock(mtx);
-    ASSERT(tcs, r == 0);
-
-    r = erts_mutex_destroy(mtx);
-    ASSERT(tcs, r == 0);
-    mtx = NULL;
-
-    r = erts_cond_destroy(cnd);
-    ASSERT(tcs, r == 0);
-    cnd = NULL;
-
-    /* ... */
-    t1 = erts_thread_self();
-
-    if (cw_passed == 4711) {
-	/* We don't want to execute this just check that the
-	   symbol/symbols is/are defined */
-	erts_thread_kill(t1);
-    }
-
-#endif /* #ifndef __WIN32__ */
-}
-
-char *
-testcase_name(void)
-{
-    return "erl_threads";
-}
-
-void
-testcase_cleanup(TestCaseState_t *tcs)
-{
-    int i;
-    for (i = 0; i < NO_OF_THREADS; i++) {
-	if (need_join[i]) {
-	    erts_mutex_lock(mtx);
-	    die = 1;
-	    erts_cond_broadcast(cnd);
-	    erts_mutex_unlock(mtx);
-	    erts_thread_join(tid[1], NULL);
-	}
-    }
-    if (mtx)
-	erts_mutex_destroy(mtx);
-    if (cnd)
-	erts_cond_destroy(cnd);
-}
-
diff --git a/erts/emulator/test/obsolete_SUITE_data/testcase_driver.c b/erts/emulator/test/obsolete_SUITE_data/testcase_driver.c
deleted file mode 100644
index 99d5adb041..0000000000
--- a/erts/emulator/test/obsolete_SUITE_data/testcase_driver.c
+++ /dev/null
@@ -1,262 +0,0 @@
-/* ``The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved via the world wide web at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * The Initial Developer of the Original Code is Ericsson Utvecklings AB.
- * Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings
- * AB. All Rights Reserved.''
- * 
- *     $Id$
- */
-
-#include "testcase_driver.h"
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdarg.h>
-#include <setjmp.h>
-#include <string.h>
-
-#ifdef __WIN32__
-#undef HAVE_VSNPRINTF
-#define HAVE_VSNPRINTF 1
-#define vsnprintf _vsnprintf
-#endif
-
-#ifndef HAVE_VSNPRINTF
-#define HAVE_VSNPRINTF 0
-#endif
-
-#define COMMENT_BUF_SZ 4096
-
-#define TESTCASE_FAILED		0
-#define TESTCASE_SKIPPED	1
-#define TESTCASE_SUCCEEDED	2
-
-typedef struct {
-    TestCaseState_t visible;
-    int port;
-    int result;
-    jmp_buf done_jmp_buf;
-    char *comment;
-    char comment_buf[COMMENT_BUF_SZ];
-} InternalTestCaseState_t;
-
-long testcase_drv_start(int port, char *command);
-int testcase_drv_stop(long drv_data);
-int testcase_drv_run(long drv_data, char *buf, int len);
-
-static DriverEntry testcase_drv_entry = { 
-    NULL,
-    testcase_drv_start,
-    testcase_drv_stop,
-    testcase_drv_run
-};
-
-
-int DRIVER_INIT(testcase_drv)(void *arg)
-{
-    testcase_drv_entry.driver_name = testcase_name();
-    return (int) &testcase_drv_entry;
-}
-
-long
-testcase_drv_start(int port, char *command)
-{
-    InternalTestCaseState_t *itcs = (InternalTestCaseState_t *)
-	driver_alloc(sizeof(InternalTestCaseState_t));
-    if (!itcs) {
-	return -1;
-    }
-
-    itcs->visible.testcase_name = testcase_name();
-    itcs->visible.extra = NULL;
-    itcs->port = port;
-    itcs->result = TESTCASE_FAILED;
-    itcs->comment = "";
-
-    return (long) itcs;
-}
-
-int
-testcase_drv_stop(long drv_data)
-{
-    testcase_cleanup((TestCaseState_t *) drv_data);
-    driver_free((void *) drv_data);
-    return 0;
-}
-
-int
-testcase_drv_run(long drv_data, char *buf, int len)
-{
-    InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) drv_data;
-    DriverTermData result_atom;
-    DriverTermData msg[12];
-
-    itcs->visible.command = buf;
-    itcs->visible.command_len = len;
-
-    if (setjmp(itcs->done_jmp_buf) == 0) {
-	testcase_run((TestCaseState_t *) itcs);
-	itcs->result = TESTCASE_SUCCEEDED;
-    }
-
-    switch (itcs->result) {
-    case TESTCASE_SUCCEEDED:
-	result_atom = driver_mk_atom("succeeded");
-	break;
-    case TESTCASE_SKIPPED:
-	result_atom = driver_mk_atom("skipped");
-	break;
-    case TESTCASE_FAILED:
-    default:
-	result_atom = driver_mk_atom("failed");
-	break;
-    }
-
-    msg[0] = ERL_DRV_ATOM;
-    msg[1] = (DriverTermData) result_atom;
-
-    msg[2] = ERL_DRV_PORT;
-    msg[3] = driver_mk_port(itcs->port);
-
-    msg[4] = ERL_DRV_ATOM;
-    msg[5] = driver_mk_atom(itcs->visible.testcase_name);
- 
-    msg[6] = ERL_DRV_STRING;
-    msg[7] = (DriverTermData) itcs->comment;
-    msg[8] = (DriverTermData) strlen(itcs->comment);
-
-    msg[9] = ERL_DRV_TUPLE;
-    msg[10] = (DriverTermData) 4;
-
-    driver_output_term(itcs->port, msg, 11);
-    return 0;
-}
-
-int
-testcase_assertion_failed(TestCaseState_t *tcs,
-			  char *file, int line, char *assertion)
-{
-    testcase_failed(tcs, "%s:%d: Assertion failed: \"%s\"",
-		    file, line, assertion);
-    return 0;
-}
-
-void
-testcase_printf(TestCaseState_t *tcs, char *frmt, ...)
-{
-    InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs;
-    DriverTermData msg[12];
-    va_list va;
-    va_start(va, frmt);
-#if HAVE_VSNPRINTF
-    vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va);
-#else
-    vsprintf(itcs->comment_buf, frmt, va);
-#endif
-    va_end(va);
-
-    msg[0] = ERL_DRV_ATOM;
-    msg[1] = (DriverTermData) driver_mk_atom("print");
-
-    msg[2] = ERL_DRV_PORT;
-    msg[3] = driver_mk_port(itcs->port);
-
-    msg[4] = ERL_DRV_ATOM;
-    msg[5] = driver_mk_atom(itcs->visible.testcase_name);
-
-    msg[6] = ERL_DRV_STRING;
-    msg[7] = (DriverTermData) itcs->comment_buf;
-    msg[8] = (DriverTermData) strlen(itcs->comment_buf);
-
-    msg[9] = ERL_DRV_TUPLE;
-    msg[10] = (DriverTermData) 4;
-
-    driver_output_term(itcs->port, msg, 11);
-}
-
-
-void testcase_succeeded(TestCaseState_t *tcs, char *frmt, ...)
-{
-    InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs;
-    va_list va;
-    va_start(va, frmt);
-#if HAVE_VSNPRINTF
-    vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va);
-#else
-    vsprintf(itcs->comment_buf, frmt, va);
-#endif
-    va_end(va);
-
-    itcs->result = TESTCASE_SUCCEEDED;
-    itcs->comment = itcs->comment_buf;
-
-    longjmp(itcs->done_jmp_buf, 1);
-}
-
-void testcase_skipped(TestCaseState_t *tcs, char *frmt, ...)
-{
-    InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs;
-    va_list va;
-    va_start(va, frmt);
-#if HAVE_VSNPRINTF
-    vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va);
-#else
-    vsprintf(itcs->comment_buf, frmt, va);
-#endif
-    va_end(va);
-
-    itcs->result = TESTCASE_SKIPPED;
-    itcs->comment = itcs->comment_buf;
-
-    longjmp(itcs->done_jmp_buf, 1);
-}
-
-void testcase_failed(TestCaseState_t *tcs, char *frmt, ...)
-{
-    InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs;
-    char buf[10];
-    size_t bufsz = sizeof(buf);
-    va_list va;
-    va_start(va, frmt);
-#if HAVE_VSNPRINTF
-    vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va);
-#else
-    vsprintf(itcs->comment_buf, frmt, va);
-#endif
-    va_end(va);
-
-    itcs->result = TESTCASE_FAILED;
-    itcs->comment = itcs->comment_buf;
-
-    if (erl_drv_getenv("ERL_ABORT_ON_FAILURE", buf, &bufsz) == 0
-	&& strcmp("true", buf) == 0) {
-	fprintf(stderr, "Testcase \"%s\" failed: %s\n",
-		itcs->visible.testcase_name, itcs->comment);
-	abort();
-    }
-
-    longjmp(itcs->done_jmp_buf, 1);
-}
-
-void *testcase_alloc(size_t size)
-{
-    return driver_alloc(size);
-}
-
-void *testcase_realloc(void *ptr, size_t size)
-{
-    return driver_realloc(ptr, size);
-}
-
-void testcase_free(void *ptr)
-{
-    driver_free(ptr);
-}
diff --git a/erts/emulator/test/obsolete_SUITE_data/testcase_driver.h b/erts/emulator/test/obsolete_SUITE_data/testcase_driver.h
deleted file mode 100644
index 3d85ca6df0..0000000000
--- a/erts/emulator/test/obsolete_SUITE_data/testcase_driver.h
+++ /dev/null
@@ -1,57 +0,0 @@
-/* ``The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved via the world wide web at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * The Initial Developer of the Original Code is Ericsson Utvecklings AB.
- * Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings
- * AB. All Rights Reserved.''
- * 
- *     $Id$
- */
-
-#ifndef TESTCASE_DRIVER_H__
-#define TESTCASE_DRIVER_H__
-
-#include "obsolete/driver.h"
-#include <stdlib.h>
-
-typedef struct {
-    char *testcase_name;
-    char *command;
-    int command_len;
-    void *extra;
-} TestCaseState_t;
-
-#define ASSERT_CLNUP(TCS, B, CLN)					\
-do {									\
-    if (!(B)) {								\
-	CLN;								\
-	testcase_assertion_failed((TCS), __FILE__, __LINE__, #B);	\
-    }									\
-} while (0)
-
-#define ASSERT(TCS, B) ASSERT_CLNUP(TCS, B, (void) 0)
-
-void testcase_printf(TestCaseState_t *tcs, char *frmt, ...);
-void testcase_succeeded(TestCaseState_t *tcs, char *frmt, ...);
-void testcase_skipped(TestCaseState_t *tcs, char *frmt, ...);
-void testcase_failed(TestCaseState_t *tcs, char *frmt, ...);
-int testcase_assertion_failed(TestCaseState_t *tcs, char *file, int line,
-			      char *assertion);
-void *testcase_alloc(size_t size);
-void *testcase_realloc(void *ptr, size_t size);
-void testcase_free(void *ptr);
-
-
-char *testcase_name(void);
-void testcase_run(TestCaseState_t *tcs);
-void testcase_cleanup(TestCaseState_t *tcs);
-
-#endif
diff --git a/erts/emulator/test/old_mod.erl b/erts/emulator/test/old_mod.erl
index 6c47ba6f8f..124842390a 100644
--- a/erts/emulator/test/old_mod.erl
+++ b/erts/emulator/test/old_mod.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2003-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2003-2010. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
diff --git a/erts/emulator/test/old_scheduler_SUITE.erl b/erts/emulator/test/old_scheduler_SUITE.erl
index 70348f64db..262536a068 100644
--- a/erts/emulator/test/old_scheduler_SUITE.erl
+++ b/erts/emulator/test/old_scheduler_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2004-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2004-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,24 +19,44 @@
 
 -module(old_scheduler_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2]).
 -export([equal/1, many_low/1, few_low/1, max/1, high/1]).
 
 -define(default_timeout, ?t:minutes(11)).
 
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     case catch erlang:system_info(modified_timing_level) of
 	Level when is_integer(Level) ->
 	    {skipped,
-	     "Modified timing (level " ++ integer_to_list(Level)
-	     ++ ") is enabled. Testcases gets messed up by modfied "
-	     "timing."};
-	_ ->
-	    [equal, many_low, few_low, max, high]
+	     "Modified timing (level " ++
+		 integer_to_list(Level) ++
+		 ") is enabled. Testcases gets messed "
+	     "up by modfied timing."};
+	_ -> [equal, many_low, few_low, max, high]
     end.
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 %%-----------------------------------------------------------------------------------
 %% TEST SUITE DESCRIPTION
 %%
@@ -63,7 +83,7 @@ init_per_testcase(_Case, Config) ->
     ?line MS = erlang:system_flag(multi_scheduling, block),
     [{prio,Prio},{watchdog,Dog},{multi_scheduling, MS}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     erlang:system_flag(multi_scheduling, unblock),
     Dog=?config(watchdog, Config),
     Prio=?config(prio, Config),
diff --git a/erts/emulator/test/op_SUITE.erl b/erts/emulator/test/op_SUITE.erl
index 55d8d9ab0f..ef4689b850 100644
--- a/erts/emulator/test/op_SUITE.erl
+++ b/erts/emulator/test/op_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,22 +19,43 @@
 
 -module(op_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 bsl_bsr/1,logical/1,t_not/1,relop_simple/1,relop/1,complex_relop/1]).
 
 -export([]).
 -import(lists, [foldl/3,flatmap/2]).
 
-all(suite) ->
-    [bsl_bsr,logical,t_not,relop_simple,relop,complex_relop].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [bsl_bsr, logical, t_not, relop_simple, relop,
+     complex_relop].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) ->
     Dog=?t:timetrap(?t:minutes(3)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl
index b9100738e4..0b99b3438a 100644
--- a/erts/emulator/test/port_SUITE.erl
+++ b/erts/emulator/test/port_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -73,22 +73,23 @@
 %%
 
 
--export([all/1, init_per_testcase/2, fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2,
 	 init_per_suite/1, end_per_suite/1,
-	 stream/1, stream_small/1, stream_big/1,
+	 stream_small/1, stream_big/1,
 	 basic_ping/1, slow_writes/1, bad_packet/1, bad_port_messages/1,
-	 multiple_packets/1, mul_basic/1, mul_slow_writes/1,
+	 mul_basic/1, mul_slow_writes/1,
 	 dying_port/1, port_program_with_path/1,
 	 open_input_file_port/1, open_output_file_port/1,
 	 iter_max_ports/1, eof/1, input_only/1, output_only/1,
 	 name1/1,
-	 t_binary/1, options/1, parallell/1, t_exit/1,
+	 t_binary/1, parallell/1, t_exit/1,
 	 env/1, bad_env/1, cd/1, exit_status/1,
-	 tps/1, tps_16_bytes/1, tps_1K/1, line/1, stderr_to_stdout/1,
+	 tps_16_bytes/1, tps_1K/1, line/1, stderr_to_stdout/1,
 	 otp_3906/1, otp_4389/1, win_massive/1, win_massive_client/1,
 	 mix_up_ports/1, otp_5112/1, otp_5119/1, otp_6224/1,
 	 exit_status_multi_scheduling_block/1, ports/1,
-	 spawn_driver/1,spawn_executable/1,
+	 spawn_driver/1, spawn_executable/1, close_deaf_port/1,
 	 unregister_name/1]).
 
 -export([]).
@@ -98,31 +99,42 @@
 -export([otp_3906_forker/5, otp_3906_start_forker_starter/4]).
 -export([env_slave_main/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 -include_lib("kernel/include/file.hrl").
 
-all(suite) ->
-    [
-     otp_6224, stream, basic_ping, slow_writes, bad_packet,
-     bad_port_messages, options, multiple_packets, parallell,
-     dying_port, port_program_with_path,
-     open_input_file_port, open_output_file_port,
-     name1,
-     env, bad_env, cd, exit_status,
-     iter_max_ports, t_exit, tps, line, stderr_to_stdout,
-     otp_3906, otp_4389, win_massive, mix_up_ports,
-     otp_5112, otp_5119,
-     exit_status_multi_scheduling_block,
-     ports, spawn_driver, spawn_executable,
-     unregister_name
-    ].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [otp_6224, {group, stream}, basic_ping, slow_writes,
+     bad_packet, bad_port_messages, {group, options},
+     {group, multiple_packets}, parallell, dying_port,
+     port_program_with_path, open_input_file_port,
+     open_output_file_port, name1, env, bad_env, cd,
+     exit_status, iter_max_ports, t_exit, {group, tps}, line,
+     stderr_to_stdout, otp_3906, otp_4389, win_massive,
+     mix_up_ports, otp_5112, otp_5119,
+     exit_status_multi_scheduling_block, ports, spawn_driver,
+     spawn_executable, close_deaf_port, unregister_name].
+
+groups() -> 
+    [{stream, [], [stream_small, stream_big]},
+     {options, [], [t_binary, eof, input_only, output_only]},
+     {multiple_packets, [], [mul_basic, mul_slow_writes]},
+     {tps, [], [tps_16_bytes, tps_1K]}].
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 -define(DEFAULT_TIMEOUT, ?t:minutes(5)).
 
 init_per_testcase(Case, Config) ->
     [{testcase, Case} |Config].
 
-fin_per_testcase(_Case, _Config) ->
+end_per_testcase(_Case, _Config) ->
     ok.
 
 init_per_suite(Config) when is_list(Config) ->
@@ -191,7 +203,6 @@ win_massive_loop(P,N) ->
     
 
 
-stream(suite) -> [stream_small, stream_big].
 
 %% Test that we can send a stream of bytes and get it back.
 %% We will send only a small amount of data, to avoid deadlock.
@@ -304,7 +315,6 @@ bad_message(PortTest, Message) ->
 %% Tests various options (stream and {packet, Number} are implicitly
 %% tested in other test cases).
 
-options(suite) -> [t_binary, eof, input_only, output_only].
 
 %% Tests the 'binary' option for a port.
 
@@ -416,7 +426,6 @@ output_and_verify(Config, Filename, Options, Data) ->
 %% Test that receiving several packages written in the same
 %% write operation works.
 
-multiple_packets(suite) -> [mul_basic, mul_slow_writes].
 
 %% Basic test of receiving multiple packages, written in
 %% one operation by the other end.
@@ -715,6 +724,8 @@ open_ports(Name, Settings) ->
 		    [];
 		system_limit ->
 		    [];
+		enomem ->
+		    [];
 		Other ->
 		    ?line test_server:fail({open_ports, Other})
 	    end;
@@ -740,7 +751,6 @@ suicide_port(Config) when is_list(Config) ->
     ?line exit(Port, die),
     ?line receive after infinity -> ok end.
 
-tps(suite) -> [tps_16_bytes, tps_1K].
 
 tps_16_bytes(doc) -> "";
 tps_16_bytes(suite) -> [];
@@ -878,12 +888,20 @@ env2(Config) ->
 			    "nisse" = os:getenv(Long)
 		    end),
 
-
+    
     ?line env_slave(Temp, [{"must_define_something","some_value"},
-			   {"certainly_not_existing",false},
+			    {"certainly_not_existing",false},
+                           {"ends_with_equal", "value="},
 			   {Long,false},
 			   {"glurf","a glorfy string"}]),
 
+    %% A lot of non existing variables (mingled with existing)
+    NotExistingList = [{lists:flatten(io_lib:format("V~p_not_existing",[X])),false} 
+                        ||  X <- lists:seq(1,150)],
+    ExistingList = [{lists:flatten(io_lib:format("V~p_existing",[X])),"a_value"} 
+                        ||  X <- lists:seq(1,150)],
+    ?line env_slave(Temp, lists:sort(ExistingList ++ NotExistingList)),
+
     ?line test_server:timetrap_cancel(Dog),
     ok.
 
@@ -1041,8 +1059,10 @@ otp_3906(Config)  when is_list(Config) ->
 -define(OTP_3906_MAX_CONC_OSP, 50).
 
 otp_3906(Config, OSName) ->
-    ?line TSDir = filename:dirname(code:which(test_server)),
-    ?line {ok, Variables} = file:consult(filename:join(TSDir, "variables")),
+    ?line DataDir = filename:dirname(proplists:get_value(data_dir,Config)),
+    ?line {ok, Variables} = file:consult(
+			      filename:join([DataDir,"..","..",
+					     "test_server","variables"])),
     case lists:keysearch('CC', 1, Variables) of
 	{value,{'CC', CC}} ->
 	    SuiteDir = filename:dirname(code:which(?MODULE)),
@@ -1224,8 +1244,8 @@ otp_3906_forker(N, Parent, Ref, Sup, Prog) ->
 otp_4389(suite) -> [];
 otp_4389(doc) -> [];
 otp_4389(Config)  when is_list(Config) ->
-    case {os:type(),erlang:system_info(heap_type)} of
-	{{unix, _},private} ->
+    case os:type() of
+	{unix, _} ->
 	    ?line Dog = test_server:timetrap(test_server:seconds(240)),
 	    ?line TCR = self(),
 	    case get_true_cmd() of
@@ -1273,7 +1293,7 @@ otp_4389(Config)  when is_list(Config) ->
 		    ?line {skipped, "\"true\" command not found"}
 	    end;
 	_ ->
-	    {skip,"Only run on Unix and private heaps"}
+	    {skip,"Only run on Unix"}
     end.
 
 get_true_cmd() ->
@@ -2292,3 +2312,37 @@ load_driver(Dir, Driver) ->
 	    io:format("~s\n", [erl_ddll:format_error(Error)]),
 	    Res
     end.
+
+
+close_deaf_port(doc) -> ["Send data to port program that does not read it, then close port."
+			 "Primary targeting Windows to test threaded_handle_closer in sys.c"];
+close_deaf_port(suite) -> [];
+close_deaf_port(Config) when is_list(Config) ->
+    ?line Dog = test_server:timetrap(test_server:seconds(100)),
+    ?line DataDir = ?config(data_dir, Config),
+    ?line DeadPort = os:find_executable("dead_port", DataDir),
+    ?line Port = open_port({spawn,DeadPort++" 60"},[]),
+    ?line erlang:port_command(Port,"Hello, can you hear me!?!?"),
+    ?line port_close(Port),
+
+    Res = close_deaf_port_1(0, DeadPort),
+    io:format("Waiting for OS procs to terminate...\n"),
+    receive after 5*1000 -> ok end,
+    ?line test_server:timetrap_cancel(Dog),
+    Res.
+
+close_deaf_port_1(1000, _) ->
+    ok;
+close_deaf_port_1(N, Cmd) ->
+    Timeout = integer_to_list(random:uniform(5*1000)),
+    ?line try open_port({spawn_executable,Cmd},[{args,[Timeout]}]) of
+    	Port ->
+	    ?line erlang:port_command(Port,"Hello, can you hear me!?!?"),
+	    ?line port_close(Port),
+	    close_deaf_port_1(N+1, Cmd)
+    catch
+    	_:eagain ->
+	    {comment, "Could not spawn more than " ++ integer_to_list(N) ++ " OS processes."}
+    end.
+    
+
diff --git a/erts/emulator/test/port_SUITE_data/Makefile.src b/erts/emulator/test/port_SUITE_data/Makefile.src
index d97b37c9ae..ff822ae720 100644
--- a/erts/emulator/test/port_SUITE_data/Makefile.src
+++ b/erts/emulator/test/port_SUITE_data/Makefile.src
@@ -3,7 +3,7 @@ LD = @LD@
 CFLAGS = @CFLAGS@ -I@erl_include@ @DEFS@
 CROSSLDFLAGS = @CROSSLDFLAGS@
 
-PROGS = port_test@exe@ echo_args@exe@
+PROGS = port_test@exe@ echo_args@exe@ dead_port@exe@
 DRIVERS = echo_drv@dll@ exit_drv@dll@ failure_drv@dll@
 
 all: $(PROGS) $(DRIVERS) port_test.@EMULATOR@
diff --git a/erts/emulator/test/port_SUITE_data/dead_port.c b/erts/emulator/test/port_SUITE_data/dead_port.c
new file mode 100644
index 0000000000..68e96fbf14
--- /dev/null
+++ b/erts/emulator/test/port_SUITE_data/dead_port.c
@@ -0,0 +1,102 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2001-2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifdef VXWORKS
+#include <vxWorks.h>
+#include <taskVarLib.h>
+#include <taskLib.h>
+#include <sysLib.h>
+#include <string.h>
+#include <ioLib.h>
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#ifndef __WIN32__
+#include <unistd.h>
+
+#ifdef VXWORKS
+#include "reclaim.h"
+#include <sys/times.h>
+#else
+#include <sys/time.h>
+#endif
+
+#define O_BINARY 0
+#define _setmode(fd, mode)
+#endif
+
+#ifdef __WIN32__
+#include "windows.h"
+#include "winbase.h"
+#endif
+
+
+#ifdef VXWORKS
+#define MAIN(argc, argv) port_test(argc, argv)
+#else
+#define MAIN(argc, argv) main(argc, argv)
+#endif
+
+
+extern int errno;
+
+static void delay(unsigned ms);
+
+
+MAIN(argc, argv)
+int argc;
+char *argv[];
+{
+    int x;
+    if (argc < 2) {
+	fprintf(stderr,"Usage %s <milliseconds>\n",argv[0]);
+	return 1;
+    }
+    if ((x = atoi(argv[1])) <= 0) {
+	fprintf(stderr,"Usage %s <milliseconds>\n",argv[0]);
+	return 1;
+    }
+    delay(x);
+    return 0;
+}
+
+static void
+delay(unsigned ms)
+{
+#ifdef VXWORKS
+  taskDelay((sysClkRateGet() * ms) / 1000);
+#else
+#ifdef __WIN32__
+  Sleep(ms);
+#else
+  struct timeval t;
+  t.tv_sec = ms/1000;
+  t.tv_usec = (ms % 1000) * 1000;
+
+  select(0, NULL, NULL, NULL, &t);
+#endif
+#endif
+}
diff --git a/erts/emulator/test/port_SUITE_data/echo_drv.c b/erts/emulator/test/port_SUITE_data/echo_drv.c
index 25eda116fe..1d39c6a00c 100644
--- a/erts/emulator/test/port_SUITE_data/echo_drv.c
+++ b/erts/emulator/test/port_SUITE_data/echo_drv.c
@@ -1,4 +1,5 @@
 #include <stdio.h>
+#include <string.h>
 #include "erl_driver.h"
 
 
@@ -17,11 +18,9 @@ typedef struct _erl_drv_data EchoDrvData;
 
 static EchoDrvData *echo_drv_start(ErlDrvPort port, char *command);
 static void         echo_drv_stop(EchoDrvData *data_p);
-static void         echo_drv_output(EchoDrvData *data_p, char *buf, int len);
+static void         echo_drv_output(ErlDrvData drv_data, char *buf,
+				    ErlDrvSizeT len);
 static void         echo_drv_finish(void);
-static int          echo_drv_control(EchoDrvData *data_p, unsigned int command,
-				     char *buf, int len,
-				     char **rbuf, int rlen);
 
 static ErlDrvEntry echo_drv_entry = { 
     NULL, /* init */
@@ -33,10 +32,20 @@ static ErlDrvEntry echo_drv_entry = {
     "echo_drv",
     echo_drv_finish,
     NULL, /* handle */
-    echo_drv_control,
+    NULL, /* control */
     NULL, /* timeout */
     NULL, /* outputv */
-    NULL  /* ready_async */
+    NULL, /* ready_async */
+    NULL,
+    NULL,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
+    NULL,
 };
 
 
@@ -68,7 +77,8 @@ static EchoDrvData *echo_drv_start(ErlDrvPort port, char *command) {
 static void echo_drv_stop(EchoDrvData *data_p) {
 }
 
-static void echo_drv_output(EchoDrvData *data_p, char *buf, int len) {
+static void echo_drv_output(ErlDrvData drv_data, char *buf, ErlDrvSizeT len) {
+    EchoDrvData *data_p = (EchoDrvData *) drv_data;
     void *void_ptr;
     ErlDrvPort port = void_ptr = data_p;
     
@@ -77,9 +87,3 @@ static void echo_drv_output(EchoDrvData *data_p, char *buf, int len) {
 
 static void echo_drv_finish() {
 }
-
-static int echo_drv_control(EchoDrvData *data_p, unsigned int command,
-			    char *buf, int len,
-			    char **rbuf, int rlen) {
-    return 0;
-}
diff --git a/erts/emulator/test/port_SUITE_data/exit_drv.c b/erts/emulator/test/port_SUITE_data/exit_drv.c
index 60f1b321bd..5f366b3545 100644
--- a/erts/emulator/test/port_SUITE_data/exit_drv.c
+++ b/erts/emulator/test/port_SUITE_data/exit_drv.c
@@ -5,11 +5,9 @@ typedef struct _erl_drv_data ExitDrvData;
 
 static ExitDrvData *exit_drv_start(ErlDrvPort port, char *command);
 static void         exit_drv_stop(ExitDrvData *data_p);
-static void         exit_drv_output(ExitDrvData *data_p, char *buf, int len);
+static void         exit_drv_output(ExitDrvData *data_p, char *buf,
+				    ErlDrvSizeT len);
 static void         exit_drv_finish(void);
-static int          exit_drv_control(ExitDrvData *data_p, unsigned int command,
-				     char *buf, int len,
-				     char **rbuf, int rlen);
 
 static ErlDrvEntry exit_drv_entry = { 
     NULL, /* init */
@@ -21,10 +19,20 @@ static ErlDrvEntry exit_drv_entry = {
     "exit_drv",
     exit_drv_finish,
     NULL, /* handle */
-    exit_drv_control,
+    NULL, /* control */
     NULL, /* timeout */
     NULL, /* outputv */
-    NULL  /* ready_async */
+    NULL, /* ready_async */
+    NULL,
+    NULL,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
+    NULL,
 };
 
 DRIVER_INIT(exit_drv)
@@ -45,7 +53,7 @@ exit_drv_stop(ExitDrvData *datap)
 }
 
 static void
-exit_drv_output(ExitDrvData *datap, char *buf, int len)
+exit_drv_output(ExitDrvData *datap, char *buf, ErlDrvSizeT len)
 {
     driver_exit((ErlDrvPort) datap, 0);
 }
@@ -55,14 +63,3 @@ exit_drv_finish(void)
 {
 
 }
-
-static int
-exit_drv_control(ExitDrvData *datap,
-		 unsigned int command,
-		 char *buf,
-		 int len,
-		 char **rbuf,
-		 int rlen)
-{
-    return 0;
-}
diff --git a/erts/emulator/test/port_SUITE_data/failure_drv.c b/erts/emulator/test/port_SUITE_data/failure_drv.c
index 34d48e00f8..5826e6d5a9 100644
--- a/erts/emulator/test/port_SUITE_data/failure_drv.c
+++ b/erts/emulator/test/port_SUITE_data/failure_drv.c
@@ -5,10 +5,8 @@ typedef struct _erl_drv_data FailureDrvData;
 
 static FailureDrvData *failure_drv_start(ErlDrvPort, char *);
 static void failure_drv_stop(FailureDrvData *);
-static void failure_drv_output(FailureDrvData *, char *, int);
+static void failure_drv_output(ErlDrvData, char *, ErlDrvSizeT);
 static void failure_drv_finish(void);
-static int failure_drv_control(FailureDrvData *, unsigned int,
-			       char *, int, char **, int);
 
 static ErlDrvEntry failure_drv_entry = { 
     NULL, /* init */
@@ -18,12 +16,22 @@ static ErlDrvEntry failure_drv_entry = {
     NULL, /* ready_input */
     NULL, /* ready_output */
     "failure_drv",
-    failure_drv_finish,
+    NULL, /* finish */
     NULL, /* handle */
-    failure_drv_control,
+    NULL, /* control */
     NULL, /* timeout */
     NULL, /* outputv */
-    NULL  /* ready_async */
+    NULL, /* ready_async */
+    NULL,
+    NULL,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
+    NULL,
 };
 
 
@@ -46,7 +54,8 @@ static FailureDrvData *failure_drv_start(ErlDrvPort port, char *command) {
 static void failure_drv_stop(FailureDrvData *data_p) {
 }
 
-static void failure_drv_output(FailureDrvData *data_p, char *buf, int len) {
+static void failure_drv_output(ErlDrvData drv_data, char *buf, ErlDrvSizeT len) {
+    FailureDrvData *data_p = (FailureDrvData *) drv_data;
     void *void_ptr;
     ErlDrvPort port = void_ptr = data_p;
     
@@ -55,9 +64,3 @@ static void failure_drv_output(FailureDrvData *data_p, char *buf, int len) {
 
 static void failure_drv_finish() {
 }
-
-static int failure_drv_control(FailureDrvData *data_p, unsigned int command,
-			       char *buf, int len,
-			       char **rbuf, int rlen) {
-    return 0;
-}
diff --git a/erts/emulator/test/port_bif_SUITE.erl b/erts/emulator/test/port_bif_SUITE.erl
index f4e0bb9fa8..8feea87d7e 100644
--- a/erts/emulator/test/port_bif_SUITE.erl
+++ b/erts/emulator/test/port_bif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -20,38 +20,61 @@
 -module(port_bif_SUITE).
 
 
--export([all/1, command/1, command_e/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, command/1,
 	 command_e_1/1, command_e_2/1, command_e_3/1, command_e_4/1,
-	 port_info/1, port_info1/1, port_info2/1,
+	 port_info1/1, port_info2/1,
+	 port_info_os_pid/1,
 	 connect/1, control/1, echo_to_busy/1]).
 
 -export([do_command_e_1/1, do_command_e_2/1, do_command_e_4/1]).
 
--export([init_per_testcase/2, fin_per_testcase/2]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [command, {group, port_info}, connect, control,
+     echo_to_busy].
+
+groups() -> 
+    [{command_e, [],
+      [command_e_1, command_e_2, command_e_3, command_e_4]},
+     {port_info, [], [port_info1, port_info2, port_info_os_pid]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) ->
-    [command, port_info, connect, control, echo_to_busy].
 
 
 init_per_testcase(_Func, Config) when is_list(Config) ->
     Dog=test_server:timetrap(test_server:minutes(10)),
     [{watchdog, Dog}|Config].
-fin_per_testcase(_Func, Config) when is_list(Config) ->
+end_per_testcase(_Func, Config) when is_list(Config) ->
     Dog=?config(watchdog, Config),
     test_server:timetrap_cancel(Dog).
 
 command(Config) when is_list(Config) ->
-    ?line load_control_drv(Config),
-
-    ?line P = open_port({spawn, control_drv}, []),
-    ?line do_command(P, "hello"),
-    ?line do_command(P, <<"hello">>),
-    ?line do_command(P, sub_bin(<<"1234kalle">>)),
-    ?line do_command(P, unaligned_sub_bin(<<"blurf">>)),
-    ?line do_command(P, ["bl"|unaligned_sub_bin(<<"urf">>)]),
-    ?line true = erlang:port_close(P),
+    load_control_drv(Config),
+
+    P = open_port({spawn, control_drv}, []),
+    do_command(P, "hello"),
+    do_command(P, <<"hello">>),
+    do_command(P, sub_bin(<<"1234kalle">>)),
+    do_command(P, unaligned_sub_bin(<<"blurf">>)),
+    do_command(P, ["bl"|unaligned_sub_bin(<<"urf">>)]),
+    true = erlang:port_close(P),
     ok.
 
 do_command(P, Data) ->
@@ -69,148 +92,166 @@ do_command(P, Data) ->
     end.
 
 
-command_e(suite) -> [command_e_1,
-		     command_e_2,
-		     command_e_3,
-		     command_e_4];
-command_e(doc) -> "Tests port_command/2 with errors".
 
 %% port_command/2: badarg 1st arg
 command_e_1(Config) when is_list(Config) ->
-    ?line DataDir = ?config(data_dir, Config),
-    ?line Program = filename:join(DataDir, "port_test"),
+    DataDir = ?config(data_dir, Config),
+    Program = filename:join(DataDir, "port_test"),
 
     process_flag(trap_exit, true),
-    ?line _ = spawn_link(?MODULE, do_command_e_1, [Program]),
-    ?line receive
-	      {'EXIT', Pid, {badarg, _}} when is_pid(Pid) ->
-		  ok;
-	      Other ->
-		  ?line test_server:fail(Other)
-	  after 10000 ->
-		  ?line test_server:fail(timeout)
-	  end,
+    _ = spawn_link(?MODULE, do_command_e_1, [Program]),
+    receive
+        {'EXIT', Pid, {badarg, _}} when is_pid(Pid) ->
+            ok;
+        Other ->
+            test_server:fail(Other)
+    after 10000 ->
+            test_server:fail(timeout)
+    end,
     ok.
 
 do_command_e_1(Program) ->
-    ?line _ = open_port({spawn, Program}, []),
-    ?line erlang:port_command(apple, "plock"),
+    _ = open_port({spawn, Program}, []),
+    erlang:port_command(apple, "plock"),
     exit(survived).
 
 %% port_command/2: badarg 2nd arg
 command_e_2(Config) when is_list(Config) ->
-    ?line DataDir = ?config(data_dir, Config),
-    ?line Program = filename:join(DataDir, "port_test"),
+    DataDir = ?config(data_dir, Config),
+    Program = filename:join(DataDir, "port_test"),
 
     process_flag(trap_exit, true),
-    ?line _ = spawn_link(?MODULE, do_command_e_2, [Program]),
-    ?line receive
-	      {'EXIT', Pid, {badarg, _}} when is_pid(Pid) ->
-		  ok;
-	      Other ->
-		  ?line test_server:fail(Other)
-	  after 10000 ->
-		  ?line test_server:fail(timeout)
-	  end,
+    _ = spawn_link(?MODULE, do_command_e_2, [Program]),
+    receive
+        {'EXIT', Pid, {badarg, _}} when is_pid(Pid) ->
+            ok;
+        Other ->
+            test_server:fail(Other)
+    after 10000 ->
+            test_server:fail(timeout)
+    end,
     ok.
 
 do_command_e_2(Program) ->
-    ?line P = open_port({spawn, Program}, []),
-    ?line erlang:port_command(P, 1),
+    P = open_port({spawn, Program}, []),
+    erlang:port_command(P, 1),
     exit(survived).
 
 %% port_command/2: Posix signals trapped
 command_e_3(Config) when is_list(Config) ->
-    ?line DataDir = ?config(data_dir, Config),
-    ?line Program = filename:join(DataDir, "port_test"),
+    DataDir = ?config(data_dir, Config),
+    Program = filename:join(DataDir, "port_test"),
 
     process_flag(trap_exit, true),
-    ?line P = open_port({spawn, Program}, [{packet, 1}]),
-    ?line Data = lists:duplicate(257, $a),
-    ?line erlang:port_command(P, Data),
-    ?line receive
-	      {'EXIT', Port, einval} when is_port(Port) ->
-		  ok;
-	      Other ->
-		  test_server:fail(Other)
-	  after 10000 ->
-		  test_server:fail(timeout)
-	  end,
+    P = open_port({spawn, Program}, [{packet, 1}]),
+    Data = lists:duplicate(257, $a),
+    erlang:port_command(P, Data),
+    receive
+        {'EXIT', Port, einval} when is_port(Port) ->
+            ok;
+        Other ->
+            test_server:fail(Other)
+    after 10000 ->
+            test_server:fail(timeout)
+    end,
     ok.
 
 %% port_command/2: Posix exit signals not trapped
 command_e_4(Config) when is_list(Config) ->
-    ?line DataDir = ?config(data_dir, Config),
-    ?line Program = filename:join(DataDir, "port_test"),
+    DataDir = ?config(data_dir, Config),
+    Program = filename:join(DataDir, "port_test"),
 
     process_flag(trap_exit, true),
-    ?line _ = spawn_link(?MODULE, do_command_e_4, [Program]),
-    ?line receive
-	      {'EXIT', Pid, {einval, _}} when is_pid(Pid) ->
-		  ok;
-	      Other ->
-		  ?line test_server:fail(Other)
-	  after 10000 ->
-		  ?line test_server:fail(timeout)
-	  end,
+    _ = spawn_link(?MODULE, do_command_e_4, [Program]),
+    receive
+        {'EXIT', Pid, {einval, _}} when is_pid(Pid) ->
+            ok;
+        Other ->
+            test_server:fail(Other)
+    after 10000 ->
+            test_server:fail(timeout)
+    end,
     ok.
 
 do_command_e_4(Program) ->
-    ?line P = open_port({spawn, Program}, [{packet, 1}]),
-    ?line Data = lists:duplicate(257, $a),
-    ?line erlang:port_command(P, Data),
+    P = open_port({spawn, Program}, [{packet, 1}]),
+    Data = lists:duplicate(257, $a),
+    erlang:port_command(P, Data),
     exit(survived).
 
-port_info(suite) -> [port_info1, port_info2].
 
 %% Tests the port_info/1 BIF
 port_info1(Config) when is_list(Config) ->
-    ?line load_control_drv(Config),
+    load_control_drv(Config),
     Me=self(),
-    ?line P = open_port({spawn, control_drv}, []),
-    ?line A1 = erlang:port_info(P),
-    ?line false = lists:keysearch(registered_name, 1, A1),
-    ?line register(myport, P),
-    ?line A = erlang:port_info(P),
-    ?line {value,{registered_name,myport}}=
-	lists:keysearch(registered_name, 1, A),
-    ?line {value,{name,"control_drv"}}=lists:keysearch(name, 1, A),
-    ?line {value,{links,[Me]}}=lists:keysearch(links, 1, A),
-    ?line {value,{id,_IdNum}}=lists:keysearch(id, 1, A),
-    ?line {value,{connected,_}}=lists:keysearch(connected, 1, A),
-    ?line {value,{input,0}}=lists:keysearch(input, 1, A),
-    ?line {value,{output,0}}=lists:keysearch(output, 1, A),
-    ?line true=erlang:port_close(P),
+    P = open_port({spawn, control_drv}, []),
+    A1 = erlang:port_info(P),
+    false = lists:keysearch(registered_name, 1, A1),
+    register(myport, P),
+    A = erlang:port_info(P),
+    {value,{registered_name,myport}}= lists:keysearch(registered_name, 1, A),
+    {value,{name,"control_drv"}}=lists:keysearch(name, 1, A),
+    {value,{links,[Me]}}=lists:keysearch(links, 1, A),
+    {value,{id,_IdNum}}=lists:keysearch(id, 1, A),
+    {value,{connected,_}}=lists:keysearch(connected, 1, A),
+    {value,{input,0}}=lists:keysearch(input, 1, A),
+    {value,{output,0}}=lists:keysearch(output, 1, A),
+    {value,{os_pid,undefined}}=lists:keysearch(os_pid, 1, A),  % linked-in driver doesn't have a OS pid
+    true=erlang:port_close(P),
     ok.
 
 %% Tests erlang:port_info/2"
 port_info2(Config) when is_list(Config) ->
-    ?line load_control_drv(Config),
+    load_control_drv(Config),
 
-    ?line P = open_port({spawn,control_drv}, [binary]),
-    ?line [] = erlang:port_info(P, registered_name),
-    ?line register(myport, P),
-    ?line {registered_name, myport} = erlang:port_info(P, registered_name),
+    P = open_port({spawn,control_drv}, [binary]),
+    [] = erlang:port_info(P, registered_name),
+    register(myport, P),
+    {registered_name, myport} = erlang:port_info(P, registered_name),
 
-    ?line {name, "control_drv"}=erlang:port_info(P, name),
-    ?line {id, _IdNum} = erlang:port_info(P, id),
+    {name, "control_drv"}=erlang:port_info(P, name),
+    {id, _IdNum} = erlang:port_info(P, id),
     Me=self(),
-    ?line {links, [Me]} = erlang:port_info(P, links),
-    ?line {connected, Me} = erlang:port_info(P, connected),
-    ?line {input, 0}=erlang:port_info(P, input),
-    ?line {output,0}=erlang:port_info(P, output),
-
-    ?line erlang:port_control(P, $i, "abc"),
-    ?line receive
-	      {P,{data,<<"abc">>}} -> ok
-	  end,
-    ?line {input,3} = erlang:port_info(P, input),
-    ?line {output,0} = erlang:port_info(P, output),
-
-    ?line Bin = list_to_binary(lists:duplicate(2047, 42)),
-    ?line output_test(P, Bin, 3, 0),
+    {links, [Me]} = erlang:port_info(P, links),
+    {connected, Me} = erlang:port_info(P, connected),
+    {input, 0}=erlang:port_info(P, input),
+    {output,0}=erlang:port_info(P, output),
+    {os_pid, undefined}=erlang:port_info(P, os_pid),  % linked-in driver doesn't have a OS pid
+
+    erlang:port_control(P, $i, "abc"),
+    receive
+        {P,{data,<<"abc">>}} -> ok
+    end,
+    {input,3} = erlang:port_info(P, input),
+    {output,0} = erlang:port_info(P, output),
+
+    Bin = list_to_binary(lists:duplicate(2047, 42)),
+    output_test(P, Bin, 3, 0),
     
-    ?line true = erlang:port_close(P),
+    true = erlang:port_close(P),
+    ok.
+
+%% Tests the port_info/1,2 os_pid option BIF
+port_info_os_pid(Config) when is_list(Config) ->
+    case os:type() of
+	{unix,_} ->
+	    do_port_info_os_pid();
+	_ ->
+	    {skip,"Only on Unix."}
+    end.
+
+do_port_info_os_pid() ->
+    P = open_port({spawn, "echo $$"}, [eof]),
+    A = erlang:port_info(P),
+    {os_pid, InfoOSPid} = erlang:port_info(P, os_pid),
+    EchoPidStr = receive
+	    {P, {data, EchoPidStr0}} -> EchoPidStr0
+	    after 10000 -> test_server:fail(timeout)
+    end,
+    {ok, [EchoPid], []} = io_lib:fread("~u\n", EchoPidStr),
+    {value,{os_pid, InfoOSPid}}=lists:keysearch(os_pid, 1, A),
+    EchoPid = InfoOSPid,
+    true = erlang:port_close(P),
     ok.
 
 output_test(_, _, Input, Output) when Output > 16#1fffffff ->
@@ -221,7 +262,7 @@ output_test(P, Bin, Input0, Output0) ->
 	{P,{data,Bin}} -> ok;
 	Other ->
 	    io:format("~p", [Other]),
-	    ?line ?t:fail()
+	    ?t:fail()
     end,
     Input = Input0 + size(Bin),
     Output = Output0 + size(Bin),
@@ -238,109 +279,106 @@ output_test(P, Bin, Input0, Output0) ->
 
 %% Tests the port_connect/2 BIF.
 connect(Config) when is_list(Config) ->
-    ?line load_control_drv(Config),
+    load_control_drv(Config),
 
-    ?line P = open_port({spawn, control_drv}, []),
+    P = open_port({spawn, control_drv}, []),
     register(myport, P),
 
-    ?line true = erlang:port_connect(myport, self()),
+    true = erlang:port_connect(myport, self()),
 
     %% Connect the port to another process.
 
     Data = "hello, world",
     Parent = self(),
-    ?line Rec =
-	fun(Me) -> receive
-		       {P,{data,Data}} ->
-			   Parent ! connect_ok,
-			   Me(Me)
-		   end
-	end,
-    ?line RecPid = spawn_link(fun() -> Rec(Rec) end),
-    ?line true = erlang:port_connect(P, RecPid),
-    ?line unlink(P),
+    Rec = fun(Me) ->
+	    receive
+		{P,{data,Data}} ->
+		    Parent ! connect_ok,
+		    Me(Me)
+	    end
+    end,
+    RecPid = spawn_link(fun() -> Rec(Rec) end),
+    true = erlang:port_connect(P, RecPid),
+    unlink(P),
 
     %% Send a command to the port and make sure that the
     %% other process receives the echo.
 
-    ?line erlang:port_command(P, Data),
-    ?line receive
-	      connect_ok -> ok
-	  end,
+    erlang:port_command(P, Data),
+    receive
+        connect_ok -> ok
+    end,
 
     %% Tests some errors.
 
-    ?line {'EXIT',{badarg, _}}=(catch erlang:port_connect(self(), self())),
-    ?line {'EXIT',{badarg, _}}=(catch erlang:port_connect(self(), P)),
-    ?line {'EXIT',{badarg, _}}=(catch erlang:port_connect(P, P)),
-    ?line {'EXIT',{badarg, _}}=(catch erlang:port_connect(P, xxxx)),
-    ?line {'EXIT',{badarg, _}}=(catch erlang:port_connect(P, [])),
+    {'EXIT',{badarg, _}}=(catch erlang:port_connect(self(), self())),
+    {'EXIT',{badarg, _}}=(catch erlang:port_connect(self(), P)),
+    {'EXIT',{badarg, _}}=(catch erlang:port_connect(P, P)),
+    {'EXIT',{badarg, _}}=(catch erlang:port_connect(P, xxxx)),
+    {'EXIT',{badarg, _}}=(catch erlang:port_connect(P, [])),
 
-    ?line process_flag(trap_exit, true),
-    ?line exit(P, you_should_die),
-    ?line receive
-	      {'EXIT',RecPid,you_should_die} -> ok;
-	      Other -> ?line ?t:fail({bad_message,Other})
-	  end,
+    process_flag(trap_exit, true),
+    exit(P, you_should_die),
+    receive
+        {'EXIT',RecPid,you_should_die} -> ok;
+        Other -> ?line ?t:fail({bad_message,Other})
+    end,
 
     %% Done.
     ok.
 
 %% Tests port_control/3
 control(Config) when is_list(Config) ->
-    ?line load_control_drv(Config),
-    ?line P = open_port({spawn, control_drv}, []),
+    load_control_drv(Config),
+    P = open_port({spawn, control_drv}, []),
 
     %% Test invalid (out-of-range) arguments.
 
-    ?line {'EXIT', {badarg, _}} = (catch erlang:port_control(self(), 1, [])),
+    {'EXIT', {badarg, _}} = (catch erlang:port_control(self(), 1, [])),
 
-    ?line {'EXIT', {badarg, _}} = (catch erlang:port_control(P, -1, [])),
-    ?line {'EXIT', {badarg, _}} = (catch erlang:port_control(P, -34887348739733833, [])),
-    ?line {'EXIT', {badarg, _}} = (catch erlang:port_control(P, 16#100000000, [])),
-    ?line {'EXIT', {badarg, _}} = (catch erlang:port_control(P, a, [])),
-    ?line {'EXIT', {badarg, _}} = (catch erlang:port_control(P, 'e', dum)),
-    ?line {'EXIT', {badarg, _}} = (catch erlang:port_control(P, $e, dum)),
-    ?line {'EXIT', {badarg, _}} = (catch erlang:port_control(P, $e, fun(X) -> X end)),
-    ?line {'EXIT', {badarg, _}} = (catch erlang:port_control(P, $e,
-							     [fun(X) -> X end])),
-    ?line {'EXIT', {badarg, _}} = (catch erlang:port_control(P, $e,
-							     [1|fun(X) -> X end])),
+    {'EXIT', {badarg, _}} = (catch erlang:port_control(P, -1, [])),
+    {'EXIT', {badarg, _}} = (catch erlang:port_control(P, -34887348739733833, [])),
+    {'EXIT', {badarg, _}} = (catch erlang:port_control(P, 16#100000000, [])),
+    {'EXIT', {badarg, _}} = (catch erlang:port_control(P, a, [])),
+    {'EXIT', {badarg, _}} = (catch erlang:port_control(P, 'e', dum)),
+    {'EXIT', {badarg, _}} = (catch erlang:port_control(P, $e, dum)),
+    {'EXIT', {badarg, _}} = (catch erlang:port_control(P, $e, fun(X) -> X end)),
+    {'EXIT', {badarg, _}} = (catch erlang:port_control(P, $e, [fun(X) -> X end])),
+    {'EXIT', {badarg, _}} = (catch erlang:port_control(P, $e, [1|fun(X) -> X end])),
 
     %% Test errors detected by the driver.
 
-    ?line {'EXIT', {badarg, _}} = (catch erlang:port_control(P, 177, [])),
-    ?line {'EXIT', {badarg, _}} = (catch erlang:port_control(P, 155,
-							     random_packet(1024))),
+    {'EXIT', {badarg, _}} = (catch erlang:port_control(P, 177, [])),
+    {'EXIT', {badarg, _}} = (catch erlang:port_control(P, 155, random_packet(1024))),
 
     %% Test big op codes.
 
     register(myport, P),
-    ?line test_op(myport, 256),
-    ?line test_op(P, 256),
-    ?line test_op(P, 16#0033A837),
-    ?line test_op(P, 16#0ab37938),
-    ?line test_op(P, 16#eab37938),
-    ?line test_op(P, 16#ffffFFFF),
+    test_op(myport, 256),
+    test_op(P, 256),
+    test_op(P, 16#0033A837),
+    test_op(P, 16#0ab37938),
+    test_op(P, 16#eab37938),
+    test_op(P, 16#ffffFFFF),
 
     %% Test the echo function of the driver.
 
-    ?line echo(P, 0),
-    ?line echo(P, 1),
-    ?line echo(P, 10),
-    ?line echo(P, 13),
-    ?line echo(P, 63),
-    ?line echo(P, 64),
-    ?line echo(P, 65),
-    ?line echo(P, 127),
-    ?line echo(P, 1023),
-    ?line echo(P, 1024),
-    ?line echo(P, 11243),
-    ?line echo(P, 70000),
+    echo(P, 0),
+    echo(P, 1),
+    echo(P, 10),
+    echo(P, 13),
+    echo(P, 63),
+    echo(P, 64),
+    echo(P, 65),
+    echo(P, 127),
+    echo(P, 1023),
+    echo(P, 1024),
+    echo(P, 11243),
+    echo(P, 70000),
 
     %% Done.
 
-    ?line true=erlang:port_close(myport),
+    true = erlang:port_close(myport),
     ok.
 
 test_op(P, Op) ->
@@ -348,23 +386,23 @@ test_op(P, Op) ->
     <<Op:32>> = list_to_binary(R).
 
 echo_to_busy(Config) when is_list(Config) ->
-    ?line Dog = test_server:timetrap(test_server:seconds(10)),
-    ?line load_control_drv(Config),
-    ?line P = open_port({spawn, control_drv}, []),
-    ?line erlang:port_control(P, $b, [1]),	% Set to busy.
+    Dog = test_server:timetrap(test_server:seconds(10)),
+    load_control_drv(Config),
+    P = open_port({spawn, control_drv}, []),
+    erlang:port_control(P, $b, [1]),	% Set to busy.
     Self = self(),
-    ?line Echoer = spawn(fun() -> echoer(P, Self) end),
-    ?line receive after 500 -> ok end,
-    ?line erlang:port_control(P, $b, [0]),	% Set to not busy.
-    ?line receive 
-	      {Echoer, done} ->
-		  ok;
-	      {Echoer, Other} ->
-		  test_server:fail(Other);
-	      Other ->
-		  test_server:fail({unexpected_message, Other})
-	  end,
-    ?line test_server:timetrap_cancel(Dog),
+    Echoer = spawn(fun() -> echoer(P, Self) end),
+    receive after 500 -> ok end,
+    erlang:port_control(P, $b, [0]),	% Set to not busy.
+    receive
+        {Echoer, done} ->
+            ok;
+        {Echoer, Other} ->
+            test_server:fail(Other);
+        Other ->
+            test_server:fail({unexpected_message, Other})
+    end,
+    test_server:timetrap_cancel(Dog),
     ok.
 
 echoer(P, ReplyTo) ->
@@ -389,9 +427,9 @@ echo(P, Size) ->
     Packet = erlang:port_control(P, $e, [unaligned_sub_bin(Bin)]).
 
 load_control_drv(Config) when is_list(Config) ->
-    ?line DataDir = ?config(data_dir, Config),
-    ?line erl_ddll:start(),
-    ?line ok = load_driver(DataDir, "control_drv").
+    DataDir = ?config(data_dir, Config),
+    erl_ddll:start(),
+    ok = load_driver(DataDir, "control_drv").
 
 load_driver(Dir, Driver) ->
     case erl_ddll:load_driver(Dir, Driver) of
@@ -443,4 +481,3 @@ sub_bin(Bin) when is_binary(Bin) ->
     B.
 
 id(I) -> I.
-    
diff --git a/erts/emulator/test/port_bif_SUITE_data/control_drv.c b/erts/emulator/test/port_bif_SUITE_data/control_drv.c
index e9f57a887a..b937a8bb15 100644
--- a/erts/emulator/test/port_bif_SUITE_data/control_drv.c
+++ b/erts/emulator/test/port_bif_SUITE_data/control_drv.c
@@ -1,13 +1,15 @@
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include "erl_driver.h"
 
 
 static ErlDrvPort erlang_port;
 static ErlDrvData control_start(ErlDrvPort, char*);
 static void control_stop(ErlDrvData);
-static void control_read(ErlDrvData, char*, int);
-static int control_control(ErlDrvData, unsigned int, char*, int, char**, int);
+static void control_read(ErlDrvData, char*, ErlDrvSizeT);
+static ErlDrvSSizeT control_control(ErlDrvData, unsigned int, char*,
+				    ErlDrvSizeT, char**, ErlDrvSizeT);
 
 static ErlDrvEntry control_driver_entry =
 {
@@ -21,9 +23,19 @@ static ErlDrvEntry control_driver_entry =
     NULL,
     NULL,
     control_control,
+    NULL, /* timeout */
+    NULL, /* outputv */
+    NULL, /* ready_async */
+    NULL,
+    NULL,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
     NULL,
     NULL,
-    NULL
 };
 
 DRIVER_INIT(control_drv)
@@ -41,7 +53,7 @@ static ErlDrvData control_start(ErlDrvPort port,char *buf)
     return (ErlDrvData)port;
 }
 
-static void control_read(ErlDrvData port, char *buf, int count)
+static void control_read(ErlDrvData port, char *buf, ErlDrvSizeT count)
 {
     driver_output(erlang_port, buf, count);
 }
@@ -51,8 +63,9 @@ static void control_stop(ErlDrvData port)
     erlang_port = (ErlDrvPort)-1;
 }
 
-static int control_control(ErlDrvData port, unsigned command, char* buf, int count,
-			   char** res, int res_size)
+static ErlDrvSSizeT control_control(ErlDrvData port, unsigned command,
+				    char* buf, ErlDrvSizeT count,
+				    char** res, ErlDrvSizeT res_size)
 {
     switch (command) {
     case 'e':
diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index 77f850d0fb..fdc55a4cc5 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -25,19 +25,21 @@
 %%	process_info/1,2
 %%	register/2 (partially)
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -define(heap_binary_size, 64).
 
--export([all/1, spawn_with_binaries/1,
-	 t_exit_1/1, t_exit_2/1, t_exit_2_other/1, t_exit_2_other_normal/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, spawn_with_binaries/1,
+	 t_exit_1/1, t_exit_2_other/1, t_exit_2_other_normal/1,
 	 self_exit/1, normal_suicide_exit/1, abnormal_suicide_exit/1,
 	 t_exit_2_catch/1, trap_exit_badarg/1, trap_exit_badarg_in_bif/1,
 	 exit_and_timeout/1, exit_twice/1,
-	 t_process_info/1, process_info_other_msg/1,
+	 t_process_info/1, process_info_other/1, process_info_other_msg/1,
 	 process_info_other_dist_msg/1,
 	 process_info_2_list/1, process_info_lock_reschedule/1,
 	 process_info_lock_reschedule2/1,
+	 process_info_lock_reschedule3/1,
 	 bump_reductions/1, low_prio/1, binary_owner/1, yield/1, yield2/1,
 	 process_status_exiting/1,
 	 otp_4725/1, bad_register/1, garbage_collect/1, otp_6237/1,
@@ -46,39 +48,69 @@
 	 processes_large_tab/1, processes_default_tab/1, processes_small_tab/1,
 	 processes_this_tab/1, processes_apply_trap/1,
 	 processes_last_call_trap/1, processes_gc_trap/1,
-	 processes_term_proc_list/1, processes_bif/1,
-	 otp_7738/1, otp_7738_waiting/1, otp_7738_suspended/1,
-	 otp_7738_resume/1]).
+	 processes_term_proc_list/1,
+	 otp_7738_waiting/1, otp_7738_suspended/1,
+	 otp_7738_resume/1,
+	 garb_other_running/1]).
 -export([prio_server/2, prio_client/2]).
 
--export([init_per_testcase/2, fin_per_testcase/2, end_per_suite/1]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
 -export([hangaround/2, processes_bif_test/0, do_processes/1,
 	 processes_term_proc_list_test/1]).
 
-all(suite) ->
-    [spawn_with_binaries, t_exit_1, t_exit_2,
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [spawn_with_binaries, t_exit_1, {group, t_exit_2},
      trap_exit_badarg, trap_exit_badarg_in_bif,
-     t_process_info, process_info_other_msg, process_info_other_dist_msg,
-     process_info_2_list,
-     process_info_lock_reschedule, process_info_lock_reschedule2,
-     process_status_exiting,
-     bump_reductions, low_prio, yield, yield2, otp_4725, bad_register,
-     garbage_collect, process_info_messages, process_flag_badarg, process_flag_heap_size,
-     spawn_opt_heap_size, otp_6237, processes_bif, otp_7738].
+     t_process_info, process_info_other, process_info_other_msg,
+     process_info_other_dist_msg, process_info_2_list,
+     process_info_lock_reschedule,
+     process_info_lock_reschedule2,
+     process_info_lock_reschedule3, process_status_exiting,
+     bump_reductions, low_prio, yield, yield2, otp_4725,
+     bad_register, garbage_collect, process_info_messages,
+     process_flag_badarg, process_flag_heap_size,
+     spawn_opt_heap_size, otp_6237, {group, processes_bif},
+     {group, otp_7738}, garb_other_running].
+
+groups() -> 
+    [{t_exit_2, [],
+      [t_exit_2_other, t_exit_2_other_normal, self_exit,
+       normal_suicide_exit, abnormal_suicide_exit,
+       t_exit_2_catch, exit_and_timeout, exit_twice]},
+     {processes_bif, [],
+      [processes_large_tab, processes_default_tab,
+       processes_small_tab, processes_this_tab,
+       processes_last_call_trap, processes_apply_trap,
+       processes_gc_trap, processes_term_proc_list]},
+     {otp_7738, [],
+      [otp_7738_waiting, otp_7738_suspended,
+       otp_7738_resume]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(Config) ->
+    catch erts_debug:set_internal_state(available_internal_state, false),
+    Config.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?t:minutes(10)),
     [{watchdog, Dog},{testcase, Func}|Config].
 
-fin_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
+end_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
-end_per_suite(Config) ->
-    catch erts_debug:set_internal_state(available_internal_state, false),
-    Config.
-
 fun_spawn(Fun) ->
     spawn_link(erlang, apply, [Fun, []]).
 
@@ -117,10 +149,6 @@ t_exit_1() ->
 	      {'EXIT', Pid, Garbage} -> ok
 	  end.
 
-t_exit_2(suite) -> [t_exit_2_other, t_exit_2_other_normal, 
-		    self_exit, normal_suicide_exit,
-		    abnormal_suicide_exit, t_exit_2_catch,
-		    exit_and_timeout, exit_twice].
 
 %% Tests exit/2 with a lot of data in the exit message.
 t_exit_2_other(Config) when is_list(Config) ->
@@ -230,7 +258,9 @@ trap_exit_badarg() ->
     ?line Pid = fun_spawn(fun() -> bad_guy(kb_128()) end),
     ?line Garbage = kb_128(),
     ?line receive
-	      {'EXIT', Pid, {badarg,[{erlang,abs,[Garbage]},{?MODULE,bad_guy,1}|_]}} ->
+	      {'EXIT',Pid,{badarg,[{erlang,abs,[Garbage],Loc1},
+				   {?MODULE,bad_guy,1,Loc2}|_]}}
+	      when is_list(Loc1), is_list(Loc2) ->
 		  ok;
 	      Other ->
 		  ?line ok = io:format("Bad EXIT message: ~P", [Other, 30]),
@@ -382,7 +412,7 @@ etwice_high(Low) ->
     exit(Low, first),
     exit(Low, second).
 
-%% Tests the process_info/1 BIF.
+%% Tests the process_info/2 BIF.
 t_process_info(Config) when is_list(Config) ->
     ?line [] = process_info(self(), registered_name),
     ?line register(my_name, self()),
@@ -390,13 +420,100 @@ t_process_info(Config) when is_list(Config) ->
     ?line {status, running} = process_info(self(), status),
     ?line {min_heap_size, 233} = process_info(self(), min_heap_size),
     ?line {min_bin_vheap_size, 46368} = process_info(self(), min_bin_vheap_size),
-    ?line {current_function, {?MODULE, t_process_info, 1}} =
+    ?line {current_function,{?MODULE,t_process_info,1}} =
 	process_info(self(), current_function),
+    ?line {current_function,{?MODULE,t_process_info,1}} =
+	apply(erlang, process_info, [self(),current_function]),
+
+    %% current_location and current_stacktrace
+    {Line1,Res1} = {?LINE,process_info(self(), current_location)},
+    verify_loc(Line1, Res1),
+    {Line2,Res2} = {?LINE,apply(erlang, process_info,
+				[self(),current_location])},
+    verify_loc(Line2, Res2),
+    pi_stacktrace([{?MODULE,t_process_info,1,?LINE}]),
+
     ?line Gleader = group_leader(),
     ?line {group_leader, Gleader} = process_info(self(), group_leader),
     ?line {'EXIT',{badarg,_Info}} = (catch process_info('not_a_pid')),
     ok.
 
+pi_stacktrace(Expected0) ->
+    {Line,Res} = {?LINE,erlang:process_info(self(), current_stacktrace)},
+    {current_stacktrace,Stack} = Res,
+    Expected = [{?MODULE,pi_stacktrace,1,Line}|Expected0],
+    pi_stacktrace_1(Stack, Expected).
+
+pi_stacktrace_1([{M,F,A,Loc}|Stk], [{M,F,A,Line}|Exp]) ->
+    case Loc of
+	[] ->
+	    %% No location info for some reason (+L, native code).
+	    io:format("Missing location information for ~w:~w/~w",
+		      [M,F,A]),
+	    ok;
+	[_|_] ->
+	    Line = proplists:get_value(line, Loc),
+	    File = proplists:get_value(file, Loc),
+	    File = ?MODULE_STRING ++ ".erl"
+    end,
+    pi_stacktrace_1(Stk, Exp);
+pi_stacktrace_1([_|_], []) -> ok.
+
+verify_loc(Line, {current_location,{?MODULE,t_process_info=F,1=A,Loc}}) ->
+    case Loc of
+	[] ->
+	    %% No location info for some reason (+L, native code).
+	    io:format("Missing location information for ~w:~w/~w",
+		      [?MODULE,F,A]),
+	    ok;
+	[_|_] ->
+	    Line = proplists:get_value(line, Loc),
+	    File = proplists:get_value(file, Loc),
+	    File = ?MODULE_STRING ++ ".erl"
+    end.
+
+process_info_other(Config) when is_list(Config) ->
+    Self = self(),
+    Pid = spawn_link(fun() -> process_info_looper(Self) end),
+    receive after 1 -> ok end,
+    pio_current_location(10000, Pid, 0, 0),
+    pio_current_stacktrace().
+
+pio_current_location(0, _, Pi, Looper) ->
+    io:format("~w call(s) to erlang:process_info/2", [Pi]),
+    io:format("~w call(s) to ~w:process_info_looper/1", [Looper,?MODULE]);
+pio_current_location(N, Pid, Pi, Looper) ->
+    erlang:yield(),
+    {current_location,Where} = process_info(Pid, current_location),
+    case Where of
+	{erlang,process_info,2,[]} ->
+	    pio_current_location(N-1, Pid, Pi+1, Looper);
+	{?MODULE,process_info_looper,1,Loc} when is_list(Loc) ->
+	    pio_current_location(N-1, Pid, Pi, Looper+1)
+    end.
+
+pio_current_stacktrace() ->
+    L = [begin
+	     {current_stacktrace,Stk} = process_info(P, current_stacktrace),
+	     {P,Stk}
+	 end || P <- processes()],
+    [erlang:garbage_collect(P) || {P,_} <- L],
+    erlang:garbage_collect(),
+    [verify_stacktrace(Stk) || {_,Stk} <- L],
+    ok.
+
+verify_stacktrace([{M,F,A,Loc}|T])
+  when is_atom(M),
+       is_atom(F),
+       is_integer(A),
+       is_list(Loc) ->
+    verify_stacktrace(T);
+verify_stacktrace([]) -> ok.
+
+process_info_looper(Parent) ->
+    process_info(Parent, current_location),
+    process_info_looper(Parent).
+
 %% Tests the process_info/1 BIF on another process with messages.
 process_info_other_msg(Config) when is_list(Config) ->
     Self = self(),
@@ -677,6 +794,52 @@ process_info_lock_reschedule2(Config) when is_list(Config) ->
     ?line unlink(P6), exit(P6, bang),
     ?line ok.
 
+many_args(0,_B,_C,_D,_E,_F,_G,_H,_I,_J) ->
+    ok;
+many_args(A,B,C,D,E,F,G,H,I,J) ->
+    many_args(A-1,B,C,D,E,F,G,H,I,J).
+
+do_pi_msg_len(PT, AT) ->
+    lists:map(fun (_) -> ok end, [a,b,c,d]),
+    {message_queue_len, _} = process_info(element(2,PT), element(2,AT)).
+    
+process_info_lock_reschedule3(doc) ->
+    [];
+process_info_lock_reschedule3(suite) ->
+    [];
+process_info_lock_reschedule3(Config) when is_list(Config) ->
+    %% We need a process that is running and an item that requires
+    %% process_info to take the main process lock.
+    ?line Target1 = spawn_link(fun tok_loop/0),
+    ?line Name1 = process_info_lock_reschedule_running,
+    ?line register(Name1, Target1),
+    ?line Target2 = spawn_link(fun () -> receive after infinity -> ok end end),
+    ?line Name2 = process_info_lock_reschedule_waiting,
+    ?line register(Name2, Target2),
+    ?line PI = fun(N) ->
+		       case N rem 10 of
+			   0 -> erlang:yield();
+			   _ -> ok
+		       end,
+		       ?line do_pi_msg_len({proc, Target1},
+					   {arg, message_queue_len})
+	       end,
+    ?line many_args(100000,1,2,3,4,5,6,7,8,9),
+    ?line lists:foreach(PI, lists:seq(1,1000000)),
+    %% Make sure Target1 still is willing to "tok loop"
+    ?line case process_info(Target1, status) of
+	      {status, OkStatus} when OkStatus == runnable;
+				      OkStatus == running;
+				      OkStatus == garbage_collecting ->
+		  ?line unlink(Target1),
+		  ?line unlink(Target2),
+		  ?line exit(Target1, bang),
+		  ?line exit(Target2, bang),
+		  ?line OkStatus;
+	      {status, BadStatus} ->
+		  ?line ?t:fail(BadStatus)
+	  end.
+
 process_status_exiting(Config) when is_list(Config) ->
     %% Make sure that erts_debug:get_internal_state({process_status,P})
     %% returns exiting if it is in status P_EXITING.
@@ -1227,17 +1390,6 @@ otp_6237_select_loop() ->
     otp_6237_select_loop().
 
 
-processes_bif(doc) ->
-    [];
-processes_bif(suite) ->
-    [processes_large_tab,
-     processes_default_tab,
-     processes_small_tab,
-     processes_this_tab,
-     processes_last_call_trap,
-     processes_apply_trap,
-     processes_gc_trap,
-     processes_term_proc_list].
 
 -define(NoTestProcs, 10000).
 -record(processes_bif_info, {min_start_reds,
@@ -1965,10 +2117,6 @@ processes_term_proc_list_test(MustChk) ->
     ?line erlang:system_flag(multi_scheduling, unblock),
     ?line as_expected.
 
-otp_7738(doc) ->
-    [];
-otp_7738(suite) ->
-    [otp_7738_waiting, otp_7738_suspended, otp_7738_resume].
 
 otp_7738_waiting(doc) ->
     [];
@@ -2058,6 +2206,41 @@ otp_7738_test(Type) ->
 	  end,
     ?line ok.
 
+gor(Reds, Stop) ->
+    receive
+	{From, reds} ->
+	    From ! {reds, Reds, self()},
+	    gor(Reds+1, Stop);
+	{From, Stop} ->
+	    From ! {stopped, Stop, Reds, self()}
+    after 0 ->
+	    gor(Reds+1, Stop)
+    end.
+
+garb_other_running(Config) when is_list(Config) ->
+    ?line Stop = make_ref(),
+    ?line {Pid, Mon} = spawn_monitor(fun () -> gor(0, Stop) end),
+    ?line Reds = lists:foldl(fun (_, OldReds) ->
+				     ?line erlang:garbage_collect(Pid),
+				     ?line receive after 1 -> ok end,
+				     ?line Pid ! {self(), reds},
+				     ?line receive
+					       {reds, NewReds, Pid} ->
+						   ?line true = (NewReds > OldReds),
+						   ?line NewReds
+					   end
+			     end,
+			     0,
+			     lists:seq(1, 10000)),
+    ?line receive after 1 -> ok end,
+    ?line Pid ! {self(), Stop},
+    ?line receive
+	      {stopped, Stop, StopReds, Pid} ->
+		  ?line true = (StopReds > Reds)
+	  end,
+    ?line receive {'DOWN', Mon, process, Pid, normal} -> ok end,
+    ?line ok.
+
 %% Internal functions
 
 wait_until(Fun) ->
diff --git a/erts/emulator/test/pseudoknot_SUITE.erl b/erts/emulator/test/pseudoknot_SUITE.erl
index 907204cf93..5a7cdcecd5 100644
--- a/erts/emulator/test/pseudoknot_SUITE.erl
+++ b/erts/emulator/test/pseudoknot_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2001-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2001-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,9 +19,29 @@
 
 -module(pseudoknot_SUITE).
 
--export([all/1,test/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,test/1]).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [test].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
-all(suite) -> [test].
     
 test(Config) when is_list(Config) ->
     statistics(runtime),
@@ -3274,13 +3294,13 @@ most_distant_atom(Sols) ->
     maximum(map(sol_most_distant_atom, Sols)).
 
 maximum([H|T]) ->
-    max(T,H).
+    max1(T,H).
 
-max([H|T],M) when is_float(H), is_float(M), H > M ->
-    max(T,H);
-max([_|T],M) ->
-    max(T,M);
-max([],M) -> M.
+max1([H|T],M) when is_float(H), is_float(M), H > M ->
+    max1(T,H);
+max1([_|T],M) ->
+    max1(T,M);
+max1([],M) -> M.
 
 map(_Func,[]) -> [];
 map(Func,[H|T]) -> 
diff --git a/erts/emulator/test/random_iolist.erl b/erts/emulator/test/random_iolist.erl
index 4bce347d9a..8f21b5a3b3 100644
--- a/erts/emulator/test/random_iolist.erl
+++ b/erts/emulator/test/random_iolist.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2008-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
diff --git a/erts/emulator/test/receive_SUITE.erl b/erts/emulator/test/receive_SUITE.erl
new file mode 100644
index 0000000000..b070e2b986
--- /dev/null
+++ b/erts/emulator/test/receive_SUITE.erl
@@ -0,0 +1,132 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(receive_SUITE).
+
+%% Tests receive after.
+
+-include_lib("test_server/include/test_server.hrl").
+
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 call_with_huge_message_queue/1,receive_in_between/1]).
+
+-export([init_per_testcase/2,end_per_testcase/2]).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [call_with_huge_message_queue, receive_in_between].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
+init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
+    Dog=?t:timetrap(?t:minutes(3)),
+    [{watchdog, Dog}|Config].
+
+end_per_testcase(_Func, Config) ->
+    Dog=?config(watchdog, Config),
+    ?t:timetrap_cancel(Dog).
+
+call_with_huge_message_queue(Config) when is_list(Config) ->
+    ?line Pid = spawn_link(fun echo_loop/0),
+
+    ?line {Time,ok} = tc(fun() -> calls(10, Pid) end),
+
+    ?line [self() ! {msg,N} || N <- lists:seq(1, 500000)],
+    erlang:garbage_collect(),
+    ?line {NewTime,ok} = tc(fun() -> calls(10, Pid) end),
+    io:format("Time for empty message queue: ~p", [Time]),
+    io:format("Time for huge message queue: ~p", [NewTime]),
+
+    case (NewTime+1) / (Time+1) of
+	Q when Q < 10 ->
+	    ok;
+	Q ->
+	    io:format("Q = ~p", [Q]),
+	    ?line ?t:fail()
+    end,
+    ok.
+
+calls(0, _) -> ok;
+calls(N, Pid) ->
+    {ok,{ultimate_answer,42}} = call(Pid, {ultimate_answer,42}),
+    calls(N-1, Pid).
+
+call(Pid, Msg) ->
+    Mref = erlang:monitor(process, Pid),
+    Pid ! {Mref,{self(),Msg}},
+    receive
+	{Mref, Reply} ->
+	    erlang:demonitor(Mref, [flush]),
+	    {ok, Reply};
+	{'DOWN', Mref, _, _, Reason} ->
+	    exit(Reason)
+    end.
+
+receive_in_between(Config) when is_list(Config) ->
+    ?line Pid = spawn_link(fun echo_loop/0),
+    ?line [{ok,{a,b}} = call2(Pid, {a,b}) || _ <- lists:seq(1, 100000)],
+    ok.
+
+call2(Pid, Msg) ->
+    self() ! dummy,
+    Mref = erlang:monitor(process, Pid),
+    Pid ! {Mref,{self(),Msg}},
+    receive_one(),
+    receive
+	{Mref,Reply} ->
+	    erlang:demonitor(Mref, [flush]),
+	    {ok,Reply};
+	{'DOWN',Mref,_,_,Reason} ->
+	    exit(Reason)
+    end.
+
+receive_one() ->
+    receive
+	dummy -> ok
+    end.
+
+%%%
+%%% Common helpers.
+%%%
+
+echo_loop() ->
+    receive
+	{Ref,{Pid,Msg}} ->
+	    Pid ! {Ref,Msg},
+	    echo_loop()
+    end.
+
+tc(Fun) ->
+    timer:tc(erlang, apply, [Fun,[]]).
diff --git a/erts/emulator/test/ref_SUITE.erl b/erts/emulator/test/ref_SUITE.erl
index fa77095efd..e13dfa1575 100644
--- a/erts/emulator/test/ref_SUITE.erl
+++ b/erts/emulator/test/ref_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,23 +19,44 @@
 
 -module(ref_SUITE).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2]).
 -export([wrap_1/1]).
 
 -export([loop_ref/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 init_per_testcase(_, Config) ->
     ?line Dog=test_server:timetrap(test_server:minutes(2)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_, Config) ->
+end_per_testcase(_, Config) ->
     Dog=?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
 
-all(suite) -> [wrap_1].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [wrap_1].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 wrap_1(doc) -> "Check that refs don't wrap around easily.";
 wrap_1(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/register_SUITE.erl b/erts/emulator/test/register_SUITE.erl
index c03ee23b2e..9953df3458 100644
--- a/erts/emulator/test/register_SUITE.erl
+++ b/erts/emulator/test/register_SUITE.erl
@@ -1,7 +1,7 @@
 %% 
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2009. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -22,24 +22,43 @@
 
 %-define(line_trace, 1).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %-compile(export_all).
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2]).
 
 -export([otp_8099/1]).
 
 -define(DEFAULT_TIMEOUT, ?t:minutes(2)).
 
-all(doc) -> [];
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     [otp_8099].
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 init_per_testcase(Case, Config) when is_list(Config) ->
     Dog = ?t:timetrap(?DEFAULT_TIMEOUT),
     [{watchdog, Dog}, {testcase, Case} | Config].
 
-fin_per_testcase(_Case, Config) when is_list(Config) ->
+end_per_testcase(_Case, Config) when is_list(Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
diff --git a/erts/emulator/test/save_calls_SUITE.erl b/erts/emulator/test/save_calls_SUITE.erl
index b56c4ad0b0..390b49b604 100644
--- a/erts/emulator/test/save_calls_SUITE.erl
+++ b/erts/emulator/test/save_calls_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,17 +19,36 @@
 
 -module(save_calls_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 
 -export([save_calls_1/1,dont_break_reductions/1]).
 
 -export([do_bopp/1, do_bipp/0, do_bepp/0]).
 
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     [save_calls_1, dont_break_reductions].
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 dont_break_reductions(suite) ->
     [];
 dont_break_reductions(doc) ->
diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl
index c9101b77c2..8931562828 100644
--- a/erts/emulator/test/scheduler_SUITE.erl
+++ b/erts/emulator/test/scheduler_SUITE.erl
@@ -1,7 +1,7 @@
 %% 
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2008-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -30,10 +30,12 @@
 
 %-define(line_trace, 1).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %-compile(export_all).
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2, end_per_suite/1]).
 
 -export([equal/1,
 	 few_low/1,
@@ -44,39 +46,65 @@
 	 equal_with_high/1,
 	 equal_with_high_max/1,
 	 bound_process/1,
-	 scheduler_bind/1,
+	
 	 scheduler_bind_types/1,
 	 cpu_topology/1,
+	 update_cpu_info/1,
 	 sct_cmd/1,
 	 sbt_cmd/1,
-	 scheduler_suspend/1]).
+	 scheduler_suspend/1,
+	 reader_groups/1]).
 
--define(DEFAULT_TIMEOUT, ?t:minutes(10)).
+-define(DEFAULT_TIMEOUT, ?t:minutes(15)).
 
 -define(MIN_SCHEDULER_TEST_TIMEOUT, ?t:minutes(1)).
 
-all(doc) -> [];
-all(suite) ->
-    [equal,
-     few_low,
-     many_low,
-     equal_with_part_time_high,
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [equal, few_low, many_low, equal_with_part_time_high,
      equal_with_part_time_max,
-     equal_and_high_with_part_time_max,
-     equal_with_high,
-     equal_with_high_max,
-     bound_process,
-     scheduler_bind,
-     scheduler_suspend].
+     equal_and_high_with_part_time_max, equal_with_high,
+     equal_with_high_max, bound_process,
+     {group, scheduler_bind}, scheduler_suspend,
+     reader_groups].
+
+groups() -> 
+    [{scheduler_bind, [],
+      [scheduler_bind_types, cpu_topology, update_cpu_info,
+       sct_cmd, sbt_cmd]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(Config) ->
+    catch erts_debug:set_internal_state(available_internal_state, false),
+    Config.
+
+init_per_group(_GroupName, Config) ->
+    Config.
 
+end_per_group(_GroupName, Config) ->
+    Config.
+
+init_per_testcase(update_cpu_info, Config) ->
+    case os:find_executable("taskset") of
+	false ->
+	    {skip,"Could not find 'taskset' in path"};
+	_ ->
+	    init_per_tc(update_cpu_info, Config)
+    end;
 init_per_testcase(Case, Config) when is_list(Config) ->
+    init_per_tc(Case, Config).
+
+init_per_tc(Case, Config) ->
     Dog = ?t:timetrap(?DEFAULT_TIMEOUT),
     process_flag(priority, max),
     erlang:display({'------------', ?MODULE, Case, '------------'}),
     OkRes = ok,
     [{watchdog, Dog}, {testcase, Case}, {ok_res, OkRes} |Config].
 
-fin_per_testcase(_Case, Config) when is_list(Config) ->
+end_per_testcase(_Case, Config) when is_list(Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
@@ -240,11 +268,6 @@ bound_loop(NS, N, M, Sched) ->
     Sched = erlang:system_info(scheduler_id),
     bound_loop(NS, N-1, M, Sched).
 
-scheduler_bind(suite) ->
-    [scheduler_bind_types,
-     cpu_topology,
-     sct_cmd,
-     sbt_cmd].
 
 -define(TOPOLOGY_A_CMD,
 	"+sct"
@@ -496,16 +519,18 @@ scheduler_bind(suite) ->
 bindings(Node, BindType) ->
     Parent = self(),
     Ref = make_ref(),
-    spawn_link(Node,
-	       fun () ->
-		       enable_internal_state(),
-		       Res = (catch erts_debug:get_internal_state(
-				      {fake_scheduler_bindings, BindType})),
-		       Parent ! {Ref, Res}
-	       end),
+    Pid = spawn_link(Node,
+		     fun () ->
+			     enable_internal_state(),
+			     Res = (catch erts_debug:get_internal_state(
+					    {fake_scheduler_bindings,
+					     BindType})),
+			     Parent ! {Ref, Res}
+		     end),
     receive
 	{Ref, Res} ->
 	    ?t:format("~p: ~p~n", [BindType, Res]),
+	    unlink(Pid),
 	    Res
     end.
 
@@ -766,6 +791,137 @@ cpu_topology_cmdline_test(Config, Topology, Cmd) ->
     ?line stop_node(Node),
     ?line ok.
 
+update_cpu_info(Config) when is_list(Config) ->
+    ?line OldOnline = erlang:system_info(schedulers_online),
+    ?line OldAff = get_affinity_mask(),
+    ?line ?t:format("START - Affinity mask: ~p - Schedulers online: ~p - Scheduler bindings: ~p~n",
+		    [OldAff, OldOnline, erlang:system_info(scheduler_bindings)]),
+    ?line case {erlang:system_info(logical_processors_available), OldAff} of
+	      {Avail, _} when Avail == unknown; OldAff == unknown ->
+		  %% Nothing much to test; just a smoke test
+		  case erlang:system_info(update_cpu_info) of
+		      unchanged -> ?line ok;
+		      changed -> ?line ok
+		  end;
+	      _ ->
+		  try
+		      ?line adjust_schedulers_online(),
+		      case erlang:system_info(schedulers_online) of
+			  1 ->
+			      %% Nothing much to test; just a smoke test
+			      ?line ok;
+			  Onln0 ->
+			      %% unset least significant bit
+			      ?line Aff = (OldAff band (OldAff - 1)),
+			      ?line set_affinity_mask(Aff),
+			      ?line Onln1 = Onln0 - 1,
+			      ?line case adjust_schedulers_online() of
+					{Onln0, Onln1} ->
+					    ?line Onln1 = erlang:system_info(schedulers_online),
+					    ?line receive after 500 -> ok end,
+					    ?line ?t:format("TEST - Affinity mask: ~p - Schedulers online: ~p - Scheduler bindings: ~p~n",
+							    [Aff, Onln1, erlang:system_info(scheduler_bindings)]),
+					    ?line unchanged = adjust_schedulers_online(),
+					    ?line ok;
+					Fail ->
+					    ?line ?t:fail(Fail)
+				    end
+		      end
+		  after
+		      set_affinity_mask(OldAff),
+		      adjust_schedulers_online(),
+		      erlang:system_flag(schedulers_online, OldOnline),
+		      receive after 500 -> ok end,
+		      ?t:format("END - Affinity mask: ~p - Schedulers online: ~p - Scheduler bindings: ~p~n",
+				[get_affinity_mask(),
+				 erlang:system_info(schedulers_online),
+				 erlang:system_info(scheduler_bindings)])
+		  end
+	  end.
+
+adjust_schedulers_online() ->
+    case erlang:system_info(update_cpu_info) of
+	unchanged ->
+	    unchanged;
+	changed ->
+	    Avail = erlang:system_info(logical_processors_available),
+	    {erlang:system_flag(schedulers_online, Avail), Avail}
+    end.
+
+read_affinity(Data) ->
+    Exp = "pid " ++ os:getpid() ++ "'s current affinity mask",
+    case string:tokens(Data, ":") of
+	[Exp, DirtyAffinityStr] ->
+	    AffinityStr = string:strip(string:strip(DirtyAffinityStr,
+						    both, $ ),
+				       both, $\n),
+	    case catch erlang:list_to_integer(AffinityStr, 16) of
+		Affinity when is_integer(Affinity) ->
+		    Affinity;
+		_ ->
+		    bad
+	    end;
+	_ ->
+	    bad
+    end.
+
+get_affinity_mask(Port, Status, Affinity) when Status == unknown;
+					       Affinity == unknown ->
+    receive
+	{Port,{data, Data}} ->
+	    get_affinity_mask(Port, Status, read_affinity(Data));
+	{Port,{exit_status,S}} ->
+	    get_affinity_mask(Port, S, Affinity)
+    end;
+get_affinity_mask(_Port, _Status, bad) ->
+    unknown;
+get_affinity_mask(_Port, _Status, Affinity) ->
+    Affinity.
+
+get_affinity_mask() ->
+    case ?t:os_type() of
+	{unix, linux} ->
+	    case catch open_port({spawn, "taskset -p " ++ os:getpid()},
+				 [exit_status]) of
+		Port when is_port(Port) ->
+		    get_affinity_mask(Port, unknown, unknown);
+		_ ->
+		    unknown
+	    end;
+	_ ->
+	    unknown
+    end.
+
+set_affinity_mask(Port, unknown) ->
+    receive
+	{Port,{data, _}} ->
+	    set_affinity_mask(Port, unknown);
+	{Port,{exit_status,Status}} ->
+	    set_affinity_mask(Port, Status)
+    end;
+set_affinity_mask(Port, Status) ->
+    receive
+	{Port,{data, _}} ->
+	    set_affinity_mask(Port, unknown)
+    after 0 ->
+	    Status
+    end.
+
+set_affinity_mask(Mask) ->
+    Cmd = lists:flatten(["taskset -p ",
+			 io_lib:format("~.16b", [Mask]),
+			 " ",
+			 os:getpid()]),
+    case catch open_port({spawn, Cmd}, [exit_status]) of
+	Port when is_port(Port) ->
+	    case set_affinity_mask(Port, unknown) of
+		0 -> ok;
+		_ -> exit(failed_to_set_affinity)
+	    end;
+	_ ->
+	    exit(failed_to_set_affinity)
+    end.
+
 sct_cmd(Config) when is_list(Config) ->
     ?line Topology = ?TOPOLOGY_A_TERM,
     ?line OldRelFlags = clear_erl_rel_flags(),
@@ -885,7 +1041,7 @@ sbt_test(Config, CpuTCmd, ClBt, Bt, LP) ->
     ?line ok.
     
 scheduler_suspend(Config) when is_list(Config) ->
-    ?line Dog = ?t:timetrap(?t:minutes(2)),
+    ?line Dog = ?t:timetrap(?t:minutes(5)),
     ?line lists:foreach(fun (S) -> scheduler_suspend_test(Config, S) end,
 			[64, 32, 16, default]),
     ?line ?t:timetrap_cancel(Dog),
@@ -902,7 +1058,8 @@ scheduler_suspend_test(Config, Schedulers) ->
     ?line [SState] = mcall(Node, [fun () ->
 					  erlang:system_info(schedulers_state)
 				  end]),
-    ?line {Sched, _, _} = SState,
+    ?line ?t:format("SState=~p~n", [SState]),
+    ?line {Sched, SchedOnln, _SchedAvail} = SState,
     ?line true = is_integer(Sched),
     ?line [ok] = mcall(Node, [fun () -> sst0_loop(300) end]),
     ?line [ok] = mcall(Node, [fun () -> sst1_loop(300) end]),
@@ -914,6 +1071,14 @@ scheduler_suspend_test(Config, Schedulers) ->
 					fun () -> sst2_loop(200) end,
 					fun () -> sst3_loop(Sched, 200) end]),
     ?line [SState] = mcall(Node, [fun () ->
+					  case Sched == SchedOnln of
+					      false ->
+						  Sched = erlang:system_flag(
+							    schedulers_online,
+							    SchedOnln);
+					      true ->
+						  ok
+					  end,
 					  erlang:system_info(schedulers_state)
 				  end]),
     ?line stop_node(Node),
@@ -956,12 +1121,300 @@ sst3_loop(S, N) ->
     erlang:system_flag(schedulers_online, 1),
     erlang:system_flag(schedulers_online, S),
     sst3_loop(S, N-1).
+
+reader_groups(Config) when is_list(Config) ->
+    %% White box testing. These results are correct, but other results
+    %% could be too...
+
+    %% The actual tilepro64 topology
+    CPUT0 = [{processor,[{node,[{core,{logical,0}},
+				{core,{logical,1}},
+				{core,{logical,2}},
+				{core,{logical,8}},
+				{core,{logical,9}},
+				{core,{logical,10}},
+				{core,{logical,11}},
+				{core,{logical,16}},
+				{core,{logical,17}},
+				{core,{logical,18}},
+				{core,{logical,19}},
+				{core,{logical,24}},
+				{core,{logical,25}},
+				{core,{logical,27}},
+				{core,{logical,29}}]},
+			 {node,[{core,{logical,3}},
+				{core,{logical,4}},
+				{core,{logical,5}},
+				{core,{logical,6}},
+				{core,{logical,7}},
+				{core,{logical,12}},
+				{core,{logical,13}},
+				{core,{logical,14}},
+				{core,{logical,15}},
+				{core,{logical,20}},
+				{core,{logical,21}},
+				{core,{logical,22}},
+				{core,{logical,23}},
+				{core,{logical,28}},
+				{core,{logical,30}}]},
+			 {node,[{core,{logical,31}},
+				{core,{logical,36}},
+				{core,{logical,37}},
+				{core,{logical,38}},
+				{core,{logical,44}},
+				{core,{logical,45}},
+				{core,{logical,46}},
+				{core,{logical,47}},
+				{core,{logical,51}},
+				{core,{logical,52}},
+				{core,{logical,53}},
+				{core,{logical,54}},
+				{core,{logical,55}},
+				{core,{logical,60}},
+				{core,{logical,61}}]},
+			 {node,[{core,{logical,26}},
+				{core,{logical,32}},
+				{core,{logical,33}},
+				{core,{logical,34}},
+				{core,{logical,35}},
+				{core,{logical,39}},
+				{core,{logical,40}},
+				{core,{logical,41}},
+				{core,{logical,42}},
+				{core,{logical,43}},
+				{core,{logical,48}},
+				{core,{logical,49}},
+				{core,{logical,50}},
+				{core,{logical,58}}]}]}],
+
+    ?line [{0,1},{1,1},{2,1},{3,3},{4,3},{5,3},{6,3},{7,3},{8,1},{9,1},{10,1},
+	   {11,1},{12,3},{13,3},{14,4},{15,4},{16,2},{17,2},{18,2},{19,2},
+	   {20,4},{21,4},{22,4},{23,4},{24,2},{25,2},{26,7},{27,2},{28,4},
+	   {29,2},{30,4},{31,5},{32,7},{33,7},{34,7},{35,7},{36,5},{37,5},
+	   {38,5},{39,7},{40,7},{41,8},{42,8},{43,8},{44,5},{45,5},{46,5},
+	   {47,6},{48,8},{49,8},{50,8},{51,6},{52,6},{53,6},{54,6},{55,6},
+	   {58,8},{60,6},{61,6}]
+	= reader_groups_map(CPUT0, 8),
+
+    CPUT1 = [n([p([c([t(l(0)),t(l(1)),t(l(2)),t(l(3))]),
+		   c([t(l(4)),t(l(5)),t(l(6)),t(l(7))]),
+		   c([t(l(8)),t(l(9)),t(l(10)),t(l(11))]),
+		   c([t(l(12)),t(l(13)),t(l(14)),t(l(15))])]),
+		p([c([t(l(16)),t(l(17)),t(l(18)),t(l(19))]),
+		   c([t(l(20)),t(l(21)),t(l(22)),t(l(23))]),
+		   c([t(l(24)),t(l(25)),t(l(26)),t(l(27))]),
+		   c([t(l(28)),t(l(29)),t(l(30)),t(l(31))])])]),
+	     n([p([c([t(l(32)),t(l(33)),t(l(34)),t(l(35))]),
+		   c([t(l(36)),t(l(37)),t(l(38)),t(l(39))]),
+		   c([t(l(40)),t(l(41)),t(l(42)),t(l(43))]),
+		   c([t(l(44)),t(l(45)),t(l(46)),t(l(47))])]),
+		p([c([t(l(48)),t(l(49)),t(l(50)),t(l(51))]),
+		   c([t(l(52)),t(l(53)),t(l(54)),t(l(55))]),
+		   c([t(l(56)),t(l(57)),t(l(58)),t(l(59))]),
+		   c([t(l(60)),t(l(61)),t(l(62)),t(l(63))])])]),
+	     n([p([c([t(l(64)),t(l(65)),t(l(66)),t(l(67))]),
+		   c([t(l(68)),t(l(69)),t(l(70)),t(l(71))]),
+		   c([t(l(72)),t(l(73)),t(l(74)),t(l(75))]),
+		   c([t(l(76)),t(l(77)),t(l(78)),t(l(79))])]),
+		p([c([t(l(80)),t(l(81)),t(l(82)),t(l(83))]),
+		   c([t(l(84)),t(l(85)),t(l(86)),t(l(87))]),
+		   c([t(l(88)),t(l(89)),t(l(90)),t(l(91))]),
+		   c([t(l(92)),t(l(93)),t(l(94)),t(l(95))])])]),
+	     n([p([c([t(l(96)),t(l(97)),t(l(98)),t(l(99))]),
+		   c([t(l(100)),t(l(101)),t(l(102)),t(l(103))]),
+		   c([t(l(104)),t(l(105)),t(l(106)),t(l(107))]),
+		   c([t(l(108)),t(l(109)),t(l(110)),t(l(111))])]),
+		p([c([t(l(112)),t(l(113)),t(l(114)),t(l(115))]),
+		   c([t(l(116)),t(l(117)),t(l(118)),t(l(119))]),
+		   c([t(l(120)),t(l(121)),t(l(122)),t(l(123))]),
+		   c([t(l(124)),t(l(125)),t(l(126)),t(l(127))])])])],
+
+    ?line [{0,1},{1,1},{2,1},{3,1},{4,2},{5,2},{6,2},{7,2},{8,3},{9,3},
+	   {10,3},{11,3},{12,4},{13,4},{14,4},{15,4},{16,5},{17,5},{18,5},
+	   {19,5},{20,6},{21,6},{22,6},{23,6},{24,7},{25,7},{26,7},{27,7},
+	   {28,8},{29,8},{30,8},{31,8},{32,9},{33,9},{34,9},{35,9},{36,10},
+	   {37,10},{38,10},{39,10},{40,11},{41,11},{42,11},{43,11},{44,12},
+	   {45,12},{46,12},{47,12},{48,13},{49,13},{50,13},{51,13},{52,14},
+	   {53,14},{54,14},{55,14},{56,15},{57,15},{58,15},{59,15},{60,16},
+	   {61,16},{62,16},{63,16},{64,17},{65,17},{66,17},{67,17},{68,18},
+	   {69,18},{70,18},{71,18},{72,19},{73,19},{74,19},{75,19},{76,20},
+	   {77,20},{78,20},{79,20},{80,21},{81,21},{82,21},{83,21},{84,22},
+	   {85,22},{86,22},{87,22},{88,23},{89,23},{90,23},{91,23},{92,24},
+	   {93,24},{94,24},{95,24},{96,25},{97,25},{98,25},{99,25},{100,26},
+	   {101,26},{102,26},{103,26},{104,27},{105,27},{106,27},{107,27},
+	   {108,28},{109,28},{110,28},{111,28},{112,29},{113,29},{114,29},
+	   {115,29},{116,30},{117,30},{118,30},{119,30},{120,31},{121,31},
+	   {122,31},{123,31},{124,32},{125,32},{126,32},{127,32}]
+	= reader_groups_map(CPUT1, 128),
+
+    ?line [{0,1},{1,1},{2,1},{3,1},{4,1},{5,1},{6,1},{7,1},{8,1},{9,1},{10,1},
+	   {11,1},{12,1},{13,1},{14,1},{15,1},{16,1},{17,1},{18,1},{19,1},
+	   {20,1},{21,1},{22,1},{23,1},{24,1},{25,1},{26,1},{27,1},{28,1},
+	   {29,1},{30,1},{31,1},{32,1},{33,1},{34,1},{35,1},{36,1},{37,1},
+	   {38,1},{39,1},{40,1},{41,1},{42,1},{43,1},{44,1},{45,1},{46,1},
+	   {47,1},{48,1},{49,1},{50,1},{51,1},{52,1},{53,1},{54,1},{55,1},
+	   {56,1},{57,1},{58,1},{59,1},{60,1},{61,1},{62,1},{63,1},{64,2},
+	   {65,2},{66,2},{67,2},{68,2},{69,2},{70,2},{71,2},{72,2},{73,2},
+	   {74,2},{75,2},{76,2},{77,2},{78,2},{79,2},{80,2},{81,2},{82,2},
+	   {83,2},{84,2},{85,2},{86,2},{87,2},{88,2},{89,2},{90,2},{91,2},
+	   {92,2},{93,2},{94,2},{95,2},{96,2},{97,2},{98,2},{99,2},{100,2},
+	   {101,2},{102,2},{103,2},{104,2},{105,2},{106,2},{107,2},{108,2},
+	   {109,2},{110,2},{111,2},{112,2},{113,2},{114,2},{115,2},{116,2},
+	   {117,2},{118,2},{119,2},{120,2},{121,2},{122,2},{123,2},{124,2},
+	   {125,2},{126,2},{127,2}]
+	= reader_groups_map(CPUT1, 2),
+
+    ?line [{0,1},{1,1},{2,1},{3,1},{4,2},{5,2},{6,2},{7,2},{8,3},{9,3},{10,3},
+	   {11,3},{12,3},{13,3},{14,3},{15,3},{16,4},{17,4},{18,4},{19,4},
+	   {20,4},{21,4},{22,4},{23,4},{24,5},{25,5},{26,5},{27,5},{28,5},
+	   {29,5},{30,5},{31,5},{32,6},{33,6},{34,6},{35,6},{36,6},{37,6},
+	   {38,6},{39,6},{40,7},{41,7},{42,7},{43,7},{44,7},{45,7},{46,7},
+	   {47,7},{48,8},{49,8},{50,8},{51,8},{52,8},{53,8},{54,8},{55,8},
+	   {56,9},{57,9},{58,9},{59,9},{60,9},{61,9},{62,9},{63,9},{64,10},
+	   {65,10},{66,10},{67,10},{68,10},{69,10},{70,10},{71,10},{72,11},
+	   {73,11},{74,11},{75,11},{76,11},{77,11},{78,11},{79,11},{80,12},
+	   {81,12},{82,12},{83,12},{84,12},{85,12},{86,12},{87,12},{88,13},
+	   {89,13},{90,13},{91,13},{92,13},{93,13},{94,13},{95,13},{96,14},
+	   {97,14},{98,14},{99,14},{100,14},{101,14},{102,14},{103,14},
+	   {104,15},{105,15},{106,15},{107,15},{108,15},{109,15},{110,15},
+	   {111,15},{112,16},{113,16},{114,16},{115,16},{116,16},{117,16},
+	   {118,16},{119,16},{120,17},{121,17},{122,17},{123,17},{124,17},
+	   {125,17},{126,17},{127,17}]
+	= reader_groups_map(CPUT1, 17),
+
+    ?line [{0,1},{1,1},{2,1},{3,1},{4,1},{5,1},{6,1},{7,1},{8,1},{9,1},{10,1},
+	   {11,1},{12,1},{13,1},{14,1},{15,1},{16,2},{17,2},{18,2},{19,2},
+	   {20,2},{21,2},{22,2},{23,2},{24,2},{25,2},{26,2},{27,2},{28,2},
+	   {29,2},{30,2},{31,2},{32,3},{33,3},{34,3},{35,3},{36,3},{37,3},
+	   {38,3},{39,3},{40,3},{41,3},{42,3},{43,3},{44,3},{45,3},{46,3},
+	   {47,3},{48,4},{49,4},{50,4},{51,4},{52,4},{53,4},{54,4},{55,4},
+	   {56,4},{57,4},{58,4},{59,4},{60,4},{61,4},{62,4},{63,4},{64,5},
+	   {65,5},{66,5},{67,5},{68,5},{69,5},{70,5},{71,5},{72,5},{73,5},
+	   {74,5},{75,5},{76,5},{77,5},{78,5},{79,5},{80,6},{81,6},{82,6},
+	   {83,6},{84,6},{85,6},{86,6},{87,6},{88,6},{89,6},{90,6},{91,6},
+	   {92,6},{93,6},{94,6},{95,6},{96,7},{97,7},{98,7},{99,7},{100,7},
+	   {101,7},{102,7},{103,7},{104,7},{105,7},{106,7},{107,7},{108,7},
+	   {109,7},{110,7},{111,7},{112,7},{113,7},{114,7},{115,7},{116,7},
+	   {117,7},{118,7},{119,7},{120,7},{121,7},{122,7},{123,7},{124,7},
+	   {125,7},{126,7},{127,7}]
+	= reader_groups_map(CPUT1, 7),
+
+    ?line CPUT2 = [p([c(l(0)),c(l(1)),c(l(2)),c(l(3)),c(l(4))]),
+		   p([t(l(5)),t(l(6)),t(l(7)),t(l(8)),t(l(9))]),
+		   p([t(l(10))]),
+		   p([c(l(11)),c(l(12)),c(l(13))]),
+		   p([c(l(14)),c(l(15))])],
+
+    ?line [{0,1},{1,1},{2,1},{3,1},{4,1},
+	   {5,2},{6,2},{7,2},{8,2},{9,2},
+	   {10,3},
+	   {11,4},{12,4},{13,4},
+	   {14,5},{15,5}] = reader_groups_map(CPUT2, 5),
+
+
+    ?line [{0,1},{1,1},{2,2},{3,2},{4,2},
+	   {5,3},{6,3},{7,3},{8,3},{9,3},
+	   {10,4},
+	   {11,5},{12,5},{13,5},
+	   {14,6},{15,6}] = reader_groups_map(CPUT2, 6),
+
+    ?line [{0,1},{1,1},{2,2},{3,2},{4,2},
+	   {5,3},{6,3},{7,3},{8,3},{9,3},
+	   {10,4},
+	   {11,5},{12,6},{13,6},
+	   {14,7},{15,7}] = reader_groups_map(CPUT2, 7),
+
+    ?line [{0,1},{1,1},{2,2},{3,2},{4,2},
+	   {5,3},{6,3},{7,3},{8,3},{9,3},
+	   {10,4},
+	   {11,5},{12,6},{13,6},
+	   {14,7},{15,8}] = reader_groups_map(CPUT2, 8),
+
+    ?line [{0,1},{1,2},{2,2},{3,3},{4,3},
+	   {5,4},{6,4},{7,4},{8,4},{9,4},
+	   {10,5},
+	   {11,6},{12,7},{13,7},
+	   {14,8},{15,9}] = reader_groups_map(CPUT2, 9),
+
+    ?line [{0,1},{1,2},{2,2},{3,3},{4,3},
+	   {5,4},{6,4},{7,4},{8,4},{9,4},
+	   {10,5},
+	   {11,6},{12,7},{13,8},
+	   {14,9},{15,10}] = reader_groups_map(CPUT2, 10),
+
+    ?line [{0,1},{1,2},{2,3},{3,4},{4,4},
+	   {5,5},{6,5},{7,5},{8,5},{9,5},
+	   {10,6},
+	   {11,7},{12,8},{13,9},
+	   {14,10},{15,11}] = reader_groups_map(CPUT2, 11),
+
+    ?line [{0,1},{1,2},{2,3},{3,4},{4,5},
+	   {5,6},{6,6},{7,6},{8,6},{9,6},
+	   {10,7},
+	   {11,8},{12,9},{13,10},
+	   {14,11},{15,12}] = reader_groups_map(CPUT2, 100),
+
+    CPUT3 = [p([t(l(5)),t(l(6)),t(l(7)),t(l(8)),t(l(9))]),
+	     p([t(l(10))]),
+	     p([c(l(11)),c(l(12)),c(l(13))]),
+	     p([c(l(14)),c(l(15))]),
+	     p([c(l(0)),c(l(1)),c(l(2)),c(l(3)),c(l(4))])],
+
+    ?line [{0,5},{1,5},{2,6},{3,6},{4,6},
+	   {5,1},{6,1},{7,1},{8,1},{9,1},
+	   {10,2},{11,3},{12,3},{13,3},
+	   {14,4},{15,4}] = reader_groups_map(CPUT3, 6),
+
+    CPUT4 = [p([t(l(0)),t(l(1)),t(l(2)),t(l(3)),t(l(4))]),
+	     p([t(l(5))]),
+	     p([c(l(6)),c(l(7)),c(l(8))]),
+	     p([c(l(9)),c(l(10))]),
+	     p([c(l(11)),c(l(12)),c(l(13)),c(l(14)),c(l(15))])],
+
+    ?line [{0,1},{1,1},{2,1},{3,1},{4,1},
+	   {5,2},
+	   {6,3},{7,3},{8,3},
+	   {9,4},{10,4},
+	   {11,5},{12,5},{13,6},{14,6},{15,6}] = reader_groups_map(CPUT4, 6),
+
+    ?line [{0,1},{1,1},{2,1},{3,1},{4,1},
+	   {5,2},
+	   {6,3},{7,4},{8,4},
+	   {9,5},{10,5},
+	   {11,6},{12,6},{13,7},{14,7},{15,7}] = reader_groups_map(CPUT4, 7),
+
+    ?line [{0,1},{65535,2}] = reader_groups_map([c(l(0)),c(l(65535))], 10),
+
+    ?line ok.
     
 
+reader_groups_map(CPUT, Groups) ->
+    Old = erlang:system_info({cpu_topology, defined}),
+    erlang:system_flag(cpu_topology, CPUT),
+    enable_internal_state(),
+    Res = erts_debug:get_internal_state({reader_groups_map, Groups}),
+    erlang:system_flag(cpu_topology, Old),
+    lists:sort(Res).
+
 %%
 %% Utils
 %%
 
+l(Id) ->
+    {logical, Id}.
+
+t(X) ->
+    {thread, X}.
+
+c(X) ->
+    {core, X}.
+
+p(X) ->
+    {processor, X}.
+
+n(X) ->
+    {node, X}.
+
 mcall(Node, Funs) ->
     Parent = self(),
     Refs = lists:map(fun (Fun) ->
@@ -1233,7 +1686,7 @@ do_it(Tracer, Low, Normal, High, Max, RedsPerSchedLimit) ->
     EndWait = now(),
     BalanceWait = timer:now_diff(EndWait,StartWait) div 1000,
     erlang:display({balance_wait, BalanceWait}),
-    Timeout = ?DEFAULT_TIMEOUT - ?t:seconds(10) - BalanceWait,
+    Timeout = ?DEFAULT_TIMEOUT - ?t:minutes(4) - BalanceWait,
     Res = case Timeout < ?MIN_SCHEDULER_TEST_TIMEOUT of
 	      true ->
 		  stop_work(Low, Normal, High, Max),
diff --git a/erts/emulator/test/send_term_SUITE.erl b/erts/emulator/test/send_term_SUITE.erl
index 489adbd660..ba0ba804ca 100644
--- a/erts/emulator/test/send_term_SUITE.erl
+++ b/erts/emulator/test/send_term_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2005-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,24 +19,43 @@
 
 -module(send_term_SUITE).
 
--export([all/1,basic/1]).
--export([init_per_testcase/2,fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,basic/1]).
+-export([init_per_testcase/2,end_per_testcase/2]).
 
 -export([generate_external_terms_files/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog=?t:timetrap(?t:minutes(3)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     [basic].
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 basic(Config) when is_list(Config) ->
     Drv = "send_term_drv",
     ?line P = start_driver(Config, Drv),
@@ -61,7 +80,7 @@ basic(Config) when is_list(Config) ->
     ?line ExpectExt2Term = term(P, 5),
 
     %% ERL_DRV_INT, ERL_DRV_UINT
-    ?line case erlang:system_info(wordsize) of
+    ?line case erlang:system_info({wordsize, external}) of
 	      4 ->
 		  ?line {-1, 4294967295} = term(P, 6);
 	      8 ->
@@ -76,40 +95,43 @@ basic(Config) when is_list(Config) ->
     ?line ExpectedBinTup = term(P, 7),
 
     %% single terms
-    ?line [] = term(P, 8), % ERL_DRV_NIL
-    ?line '' = term(P, 9), % ERL_DRV_ATOM
-    ?line an_atom = term(P, 10), % ERL_DRV_ATOM
-    ?line -4711 = term(P, 11), % ERL_DRV_INT
-    ?line 4711 = term(P, 12), % ERL_DRV_UINT
-    ?line P = term(P, 13), % ERL_DRV_PORT
-    ?line <<>> = term(P, 14), % ERL_DRV_BINARY
-    ?line <<"hejsan">> = term(P, 15), % ERL_DRV_BINARY
-    ?line <<>> = term(P, 16), % ERL_DRV_BUF2BINARY
-    ?line <<>> = term(P, 17), % ERL_DRV_BUF2BINARY
-    ?line <<"hoppsan">> = term(P, 18), % ERL_DRV_BUF2BINARY
-    ?line "" = term(P, 19), % ERL_DRV_STRING
-    ?line "" = term(P, 20), % ERL_DRV_STRING
-    ?line "hippsan" = term(P, 21), % ERL_DRV_STRING
-    ?line {} = term(P, 22), % ERL_DRV_TUPLE
-    ?line [] = term(P, 23), % ERL_DRV_LIST
-    ?line Self = term(P, 24), % ERL_DRV_PID
-    ?line [] = term(P, 25), % ERL_DRV_STRING_CONS
-    ?line AFloat = term(P, 26), % ERL_DRV_FLOAT
+    Singles = [{[], 8}, % ERL_DRV_NIL
+               {'', 9}, % ERL_DRV_ATOM
+               {an_atom, 10}, % ERL_DRV_ATOM
+               {-4711, 11}, % ERL_DRV_INT
+               {4711, 12}, % ERL_DRV_UINT
+               {P, 13}, % ERL_DRV_PORT
+               {<<>>, 14}, % ERL_DRV_BINARY
+               {<<"hejsan">>, 15}, % ERL_DRV_BINARY
+               {<<>>, 16}, % ERL_DRV_BUF2BINARY
+               {<<>>, 17}, % ERL_DRV_BUF2BINARY
+               {<<"hoppsan">>, 18}, % ERL_DRV_BUF2BINARY
+               {"", 19}, % ERL_DRV_STRING
+               {"", 20}, % ERL_DRV_STRING
+               {"hippsan", 21}, % ERL_DRV_STRING
+               {{}, 22}, % ERL_DRV_TUPLE
+               {[], 23}, % ERL_DRV_LIST
+               {Self, 24}, % ERL_DRV_PID
+               {[], 25}, % ERL_DRV_STRING_CONS
+               {[], 27}, % ERL_DRV_EXT2TERM
+               {18446744073709551615, 28}, % ERL_DRV_UINT64
+               {20233590931456, 29}, % ERL_DRV_UINT64
+               {4711, 30}, % ERL_DRV_UINT64
+               {0, 31}, % ERL_DRV_UINT64
+               {9223372036854775807, 32}, % ERL_DRV_INT64
+               {20233590931456, 33}, % ERL_DRV_INT64
+               {4711, 34}, % ERL_DRV_INT64
+               {0, 35}, % ERL_DRV_INT64
+               {-1, 36}, % ERL_DRV_INT64
+               {-4711, 37}, % ERL_DRV_INT64
+               {-20233590931456, 38}, % ERL_DRV_INT64
+               {-9223372036854775808, 39}], % ERL_DRV_INT64
+    ?line {Terms, Ops} = lists:unzip(Singles),
+    ?line Terms = term(P,Ops),
+
+    AFloat = term(P, 26), % ERL_DRV_FLOAT
     ?line true = AFloat < 0.001,
     ?line true = AFloat > -0.001,
-    ?line [] = term(P, 27), % ERL_DRV_EXT2TERM
-    ?line 18446744073709551615 = term(P, 28), % ERL_DRV_UINT64
-    ?line 20233590931456 = term(P, 29), % ERL_DRV_UINT64
-    ?line 4711 = term(P, 30), % ERL_DRV_UINT64
-    ?line 0 = term(P, 31), % ERL_DRV_UINT64
-    ?line 9223372036854775807 = term(P, 32), % ERL_DRV_INT64
-    ?line 20233590931456 = term(P, 33), % ERL_DRV_INT64
-    ?line 4711 = term(P, 34), % ERL_DRV_INT64
-    ?line 0 = term(P, 35), % ERL_DRV_INT64
-    ?line -1 = term(P, 36), % ERL_DRV_INT64
-    ?line -4711 = term(P, 37), % ERL_DRV_INT64
-    ?line -20233590931456 = term(P, 38), % ERL_DRV_INT64
-    ?line -9223372036854775808 = term(P, 39), % ERL_DRV_INT64
 
     %% Failure cases.
     ?line [] = term(P, 127),
@@ -153,6 +175,10 @@ chk_temp_alloc() ->
 	    %% Verify that we havn't got anything allocated by temp_alloc
 	    lists:foreach(
 	      fun ({instance, _, TI}) ->
+		      ?line {value, {sbmbcs, SBMBCInfo}}
+			  = lists:keysearch(sbmbcs, 1, TI),
+		      ?line {value, {blocks, 0, _, _}}
+			  = lists:keysearch(blocks, 1, SBMBCInfo),
 		      ?line {value, {mbcs, MBCInfo}}
 			  = lists:keysearch(mbcs, 1, TI),
 		      ?line {value, {blocks, 0, _, _}}
diff --git a/erts/emulator/test/send_term_SUITE_data/send_term_drv.c b/erts/emulator/test/send_term_SUITE_data/send_term_drv.c
index 6638de0560..b3feca79f0 100644
--- a/erts/emulator/test/send_term_SUITE_data/send_term_drv.c
+++ b/erts/emulator/test/send_term_SUITE_data/send_term_drv.c
@@ -17,13 +17,14 @@
  */
 
 #include "erl_driver.h"
+#include <stdio.h>
 #include <errno.h>
 #include <string.h>
 
 static ErlDrvPort erlang_port;
 static ErlDrvData send_term_drv_start(ErlDrvPort port, char *command);
 static void send_term_drv_stop(ErlDrvData drv_data);
-static void send_term_drv_run(ErlDrvData drv_data, char *buf, int len);
+static void send_term_drv_run(ErlDrvData drv_data, char *buf, ErlDrvSizeT len);
 
 
 static int make_ext_term_list(ErlDrvTermData *td, int bad);
@@ -38,6 +39,22 @@ static ErlDrvEntry send_term_drv_entry = {
     NULL,
     NULL,
     "send_term_drv",
+    NULL,
+    NULL, /* handle */
+    NULL, /* control */
+    NULL, /* timeout */
+    NULL, /* outputv */
+    NULL, /* ready_async */
+    NULL,
+    NULL,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
+    NULL,
 };
 
 DRIVER_INIT(send_term_drv)
@@ -63,14 +80,23 @@ static void send_term_drv_stop(ErlDrvData drv_data)
 static void output_term(ErlDrvTermData* msg, int len);
 static void fail_term(ErlDrvTermData* msg, int len, int line);
 
-static void send_term_drv_run(ErlDrvData port, char *buf, int count)
+static void send_term_drv_run(ErlDrvData port, char *buf, ErlDrvSizeT count)
 {
-    ErlDrvTermData msg[1024];
-
-    switch (*buf) {
+    char buf7[1024];
+    ErlDrvTermData spec[1024];
+    ErlDrvTermData* msg = spec;
+    ErlDrvBinary* bins[15];
+    int bin_ix = 0;
+    ErlDrvSInt64 s64[15];
+    int s64_ix = 0;
+    ErlDrvUInt64 u64[15];
+    int u64_ix = 0;
+    int i = 0;
+
+    for (i=0; i<count; i++) switch (buf[i]) {
     case 0:
 	msg[0] = ERL_DRV_NIL;
-	output_term(msg, 1);
+	msg += 1;
 	break;
 
     case 1: 			/* Most term types inside a tuple. */
@@ -102,7 +128,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	    msg[22] = driver_connected(erlang_port);
 	    msg[23] = ERL_DRV_TUPLE;
 	    msg[24] = (ErlDrvTermData) 7;
-	    output_term(msg, 25);
+	    msg += 25;
 	}
 	break;
 
@@ -117,7 +143,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	    msg[i] = ERL_DRV_NIL;
 	    msg[i+1] = ERL_DRV_LIST;
 	    msg[i+2] = (ErlDrvTermData) 201;
-	    output_term(msg, i+3);
+	    msg += i+3;
 	}
 	break;
 
@@ -126,7 +152,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	    ErlDrvBinary* bin;
 	    int i;
 
-	    bin = driver_alloc_binary(256);
+	    bin = bins[bin_ix++] = driver_alloc_binary(256);
 	    for (i = 0; i < 256; i++) {
 		bin->orig_bytes[i] = i;
 	    }
@@ -140,8 +166,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	    msg[7] = (ErlDrvTermData) 23;
 	    msg[8] = ERL_DRV_TUPLE;
 	    msg[9] = (ErlDrvTermData) 2;
-	    output_term(msg, 10);
-	    driver_free_binary(bin);
+	    msg += 10;
 	}
 	break;
 
@@ -152,11 +177,11 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	msg[3] = driver_caller(erlang_port);
 	msg[4] = ERL_DRV_TUPLE;
 	msg[5] = (ErlDrvTermData) 2;
-	output_term(msg, 6);
+	msg += 6;
 	break;
 
     case 5:
-	output_term(msg, make_ext_term_list(msg, 0));
+	msg += make_ext_term_list(msg, 0);
 	break;
 
     case 6:
@@ -166,94 +191,91 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	msg[3] = ~((ErlDrvTermData) 0);
 	msg[4] = ERL_DRV_TUPLE;
 	msg[5] = (ErlDrvTermData) 2;
-	output_term(msg, 6);
+	msg += 6;
 	break;
 
     case 7: {
 	int len = 0;
-	char buf[1024];
-	memset(buf, 17, sizeof(buf));
+	memset(buf7, 17, sizeof(buf7));
 	/* empty heap binary */
 	msg[len++] = ERL_DRV_BUF2BINARY;
 	msg[len++] = (ErlDrvTermData) NULL; /* NULL is ok if size == 0 */
 	msg[len++] = (ErlDrvTermData) 0;
 	/* empty heap binary again */
 	msg[len++] = ERL_DRV_BUF2BINARY;
-	msg[len++] = (ErlDrvTermData) &buf[0]; /* ptr is ok if size == 0 */
+	msg[len++] = (ErlDrvTermData) buf7; /* ptr is ok if size == 0 */
 	msg[len++] = (ErlDrvTermData) 0;
 	/* heap binary */
 	msg[len++] = ERL_DRV_BUF2BINARY;
-	msg[len++] = (ErlDrvTermData) &buf[0];
+	msg[len++] = (ErlDrvTermData) buf7;
 	msg[len++] = (ErlDrvTermData) 17;
 	/* off heap binary */
 	msg[len++] = ERL_DRV_BUF2BINARY;
-	msg[len++] = (ErlDrvTermData) &buf[0];
-	msg[len++] = (ErlDrvTermData) sizeof(buf);
+	msg[len++] = (ErlDrvTermData) buf7;
+	msg[len++] = (ErlDrvTermData) sizeof(buf7);
 
 	msg[len++] = ERL_DRV_TUPLE;
 	msg[len++] = (ErlDrvTermData) 4;
 
-	output_term(msg, len);
+	msg += len;
 	break;
     }
 
     case 8:
 	msg[0] = ERL_DRV_NIL;
-	output_term(msg, 1);
+	msg += 1;
 	break;
 
     case 9:
 	msg[0] = ERL_DRV_ATOM;
 	msg[1] = (ErlDrvTermData) driver_mk_atom("");
-	output_term(msg, 2);
+	msg += 2;
 	break;
 
     case 10:
 	msg[0] = ERL_DRV_ATOM;
 	msg[1] = (ErlDrvTermData) driver_mk_atom("an_atom");
-	output_term(msg, 2);
+	msg += 2;
 	break;
 
     case 11:
 	msg[0] = ERL_DRV_INT;
 	msg[1] = (ErlDrvTermData) -4711;
-	output_term(msg, 2);
+	msg += 2;
 	break;
 	  
     case 12:  
 	msg[0] = ERL_DRV_UINT;
 	msg[1] = (ErlDrvTermData) 4711;
-	output_term(msg, 2);
+	msg += 2;
 	  
 	break;
     case 13:  
 	msg[0] = ERL_DRV_PORT;
 	msg[1] = driver_mk_port(erlang_port);
-	output_term(msg, 2);
+	msg += 2;
 	break;
 
     case 14: {
-	ErlDrvBinary *dbin = driver_alloc_binary(0);
+	ErlDrvBinary *dbin = bins[bin_ix++] = driver_alloc_binary(0);
 	msg[0] = ERL_DRV_BINARY;
 	msg[1] = (ErlDrvTermData) dbin;
 	msg[2] = (ErlDrvTermData) 0;
 	msg[3] = (ErlDrvTermData) 0;
-	output_term(msg, 4);
-	driver_free_binary(dbin);
+	msg += 4;
 	break;
 	}
 
     case 15: {
-	char buf[] = "hejsan";
-	ErlDrvBinary *dbin = driver_alloc_binary(sizeof(buf)-1);
+	static const char buf[] = "hejsan";
+	ErlDrvBinary *dbin = bins[bin_ix++] = driver_alloc_binary(sizeof(buf)-1);
 	if (dbin)
 	    memcpy((void *) dbin->orig_bytes, (void *) buf, sizeof(buf)-1);
 	msg[0] = ERL_DRV_BINARY;
 	msg[1] = (ErlDrvTermData) dbin;
 	msg[2] = (ErlDrvTermData) (dbin ? sizeof(buf)-1 : 0);
 	msg[3] = (ErlDrvTermData) 0;
-	output_term(msg, 4);
-	driver_free_binary(dbin);
+	msg += 4;
 	break;
 	}
 
@@ -261,24 +283,24 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	msg[0] = ERL_DRV_BUF2BINARY;
 	msg[1] = (ErlDrvTermData) NULL;
 	msg[2] = (ErlDrvTermData) 0;
-	output_term(msg, 3);
+	msg += 3;
 	break;
 	
     case 17: {
-	char buf[] = "";
+	static const char buf[] = "";
 	msg[0] = ERL_DRV_BUF2BINARY;
 	msg[1] = (ErlDrvTermData) buf;
 	msg[2] = (ErlDrvTermData) sizeof(buf)-1;
-	output_term(msg, 3);
+	msg += 3;
 	break;
 	}
 
     case 18: {
-	char buf[] = "hoppsan";
+	static const char buf[] = "hoppsan";
 	msg[0] = ERL_DRV_BUF2BINARY;
 	msg[1] = (ErlDrvTermData) buf;
 	msg[2] = (ErlDrvTermData) sizeof(buf)-1;
-	output_term(msg, 3);
+	msg += 3;
 	break;
     }
 
@@ -286,44 +308,44 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	msg[0] = ERL_DRV_STRING;
 	msg[1] = (ErlDrvTermData) buf;
 	msg[2] = (ErlDrvTermData) 0;
-	output_term(msg, 3);
+	msg += 3;
 	break;
 
     case 20: {
-	char buf[] = "";
+	static const char buf[] = "";
 	msg[0] = ERL_DRV_STRING;
 	msg[1] = (ErlDrvTermData) buf;
 	msg[2] = (ErlDrvTermData) sizeof(buf)-1;
-	output_term(msg, 3);
+	msg += 3;
 	break;
     }
 	
     case 21: {
-	char buf[] = "hippsan";
+	static const char buf[] = "hippsan";
 	msg[0] = ERL_DRV_STRING;
 	msg[1] = (ErlDrvTermData) buf;
 	msg[2] = (ErlDrvTermData) sizeof(buf)-1;
-	output_term(msg, 3);
+	msg += 3;
 	break;
 	}
 
     case 22:
 	msg[0] = ERL_DRV_TUPLE;
 	msg[1] = (ErlDrvTermData) 0;
-	output_term(msg, 2);
+	msg += 2;
 	break;
 
     case 23:
 	msg[0] = ERL_DRV_NIL;
 	msg[1] = ERL_DRV_LIST;
 	msg[2] = (ErlDrvTermData) 1;
-	output_term(msg, 3);
+	msg += 3;
 	break;
 	
     case 24:
 	msg[0] = ERL_DRV_PID;
 	msg[1] = driver_connected(erlang_port);
-	output_term(msg, 2);
+	msg += 2;
 	break;
 	
     case 25:
@@ -331,132 +353,131 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	msg[1] = ERL_DRV_STRING_CONS;
 	msg[2] = (ErlDrvTermData) "";
 	msg[3] = (ErlDrvTermData) 0;
-	output_term(msg, 4);
+	msg += 4;
 	break;
 
     case 26: {
-	double my_float = 0.0;
+	static double my_float = 0.0;
 	msg[0] = ERL_DRV_FLOAT;
 	msg[1] = (ErlDrvTermData) &my_float; 
-	output_term(msg, 2);
+	msg += 2;
 	break;
     }
 
     case 27: {
-	char buf[] = {131, 106}; /* [] */
+	static char buf[] = {131, 106}; /* [] */
 	msg[0] = ERL_DRV_EXT2TERM;
 	msg[1] = (ErlDrvTermData) buf;
 	msg[2] = (ErlDrvTermData) sizeof(buf);
-	output_term(msg, 3);
+	msg += 3;
 	break;
     }
 
     case 28: {
-	ErlDrvUInt64 x = ~((ErlDrvUInt64) 0);
+	ErlDrvUInt64* x = &u64[u64_ix++];
+	*x = ~((ErlDrvUInt64) 0);
 	msg[0] = ERL_DRV_UINT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 29: {
-	ErlDrvUInt64 x = ((ErlDrvUInt64) 4711) << 32;
+	ErlDrvUInt64* x = &u64[u64_ix++];
+	*x = ((ErlDrvUInt64) 4711) << 32;
 	msg[0] = ERL_DRV_UINT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 30: {
-	ErlDrvUInt64 x = 4711;
+	ErlDrvUInt64* x = &u64[u64_ix++];
+	*x = 4711;
 	msg[0] = ERL_DRV_UINT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 31: {
-	ErlDrvUInt64 x = 0;
+	ErlDrvUInt64* x = &u64[u64_ix++];
+	*x = 0;
 	msg[0] = ERL_DRV_UINT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 32: {
-	ErlDrvSInt64 x = ((((ErlDrvUInt64) 0x7fffffff) << 32)
-			  | ((ErlDrvUInt64) 0xffffffff));
+	ErlDrvSInt64* x = &s64[s64_ix++];
+	*x = ((((ErlDrvUInt64) 0x7fffffff) << 32) | ((ErlDrvUInt64) 0xffffffff));
 	msg[0] = ERL_DRV_INT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 33: {
-	ErlDrvSInt64 x = (ErlDrvSInt64) (((ErlDrvUInt64) 4711) << 32);
+	ErlDrvSInt64* x = &s64[s64_ix++];
+	*x = (ErlDrvSInt64) (((ErlDrvUInt64) 4711) << 32);
 	msg[0] = ERL_DRV_INT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 34: {
-	ErlDrvSInt64 x = 4711;
+	ErlDrvSInt64* x = &s64[s64_ix++];
+	*x = 4711;
 	msg[0] = ERL_DRV_INT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 35: {
-	ErlDrvSInt64 x = 0;
+	ErlDrvSInt64* x = &s64[s64_ix++];
+	*x = 0;
 	msg[0] = ERL_DRV_INT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 36: {
-	ErlDrvSInt64 x = -1;
+	ErlDrvSInt64* x = &s64[s64_ix++];
+	*x = -1;
 	msg[0] = ERL_DRV_INT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 37: {
-	ErlDrvSInt64 x = -4711;
+	ErlDrvSInt64* x = &s64[s64_ix++];
+	*x = -4711;
 	msg[0] = ERL_DRV_INT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 38: {
-	ErlDrvSInt64 x = ((ErlDrvSInt64) ((ErlDrvUInt64) 4711) << 32)*-1;
+	ErlDrvSInt64* x = &s64[s64_ix++];
+	*x = ((ErlDrvSInt64) ((ErlDrvUInt64) 4711) << 32)*-1;
 	msg[0] = ERL_DRV_INT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
     case 39: {
-	ErlDrvSInt64 x = ((ErlDrvSInt64) 1) << 63;
+	ErlDrvSInt64* x = &s64[s64_ix++];
+	*x = ((ErlDrvSInt64) 1) << 63;
 	msg[0] = ERL_DRV_INT64;
-	msg[1] = (ErlDrvTermData) &x;
-	output_term(msg, 2);
-
+	msg[1] = (ErlDrvTermData) x;
+	msg += 2;
 	break;
     }
 
@@ -464,7 +485,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
     case 127:			/* Error cases */
 	{
 	    long refc;
-	    ErlDrvBinary* bin = driver_alloc_binary(256);
+	    ErlDrvBinary* bin = bins[bin_ix++] = driver_alloc_binary(256);
 
 	    FAIL_TERM(msg, 0);
 
@@ -537,7 +558,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	    refc = driver_binary_get_refc(bin);
 	    if (refc > 3) {
 		char sbuf[128];
-		sprintf(sbuf, "bad_refc:%d", refc);
+		sprintf(sbuf, "bad_refc:%ld", refc);
 		driver_failure_atom(erlang_port, sbuf);
 	    }
 	    driver_free_binary(bin);
@@ -644,6 +665,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	    /* Signal end of test case */
 	    msg[0] = ERL_DRV_NIL;
 	    driver_output_term(erlang_port, msg, 1);
+	    return;
 	}
 	break;
 
@@ -651,6 +673,16 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count)
 	driver_failure_atom(erlang_port, "bad_request");
 	break;
     }
+    if (count > 1) {
+	*msg++ = ERL_DRV_NIL;
+	*msg++ = ERL_DRV_LIST;
+	*msg++ = count + 1;
+    }
+    output_term(spec, msg-spec);
+    if ((bin_ix|s64_ix|u64_ix) > 15) abort();
+    while (bin_ix) {
+	driver_free_binary(bins[--bin_ix]);
+    }
 }
 
 static void output_term(ErlDrvTermData* msg, int len)
diff --git a/erts/emulator/test/sensitive_SUITE.erl b/erts/emulator/test/sensitive_SUITE.erl
index 458275af81..e073eab596 100644
--- a/erts/emulator/test/sensitive_SUITE.erl
+++ b/erts/emulator/test/sensitive_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2007-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2007-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,9 +19,11 @@
 
 -module(sensitive_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 stickiness/1,send_trace/1,recv_trace/1,proc_trace/1,call_trace/1,
 	 meta_trace/1,running_trace/1,gc_trace/1,seq_trace/1,
 	 t_process_info/1,t_process_display/1,save_calls/1]).
@@ -34,14 +36,33 @@ init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog = ?t:timetrap(?t:minutes(5)),
     [{watchdog,Dog}|Config].
 
-fin_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
+end_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
-all(suite) ->
-    [stickiness,send_trace,recv_trace,proc_trace,call_trace,
-     meta_trace,running_trace,gc_trace,seq_trace,
-     t_process_info,t_process_display,save_calls].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [stickiness, send_trace, recv_trace, proc_trace,
+     call_trace, meta_trace, running_trace, gc_trace,
+     seq_trace, t_process_info, t_process_display,
+     save_calls].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
     
 stickiness(Config) when is_list(Config) ->
     ?line {Tracer,Mref} = spawn_monitor(fun() ->
@@ -139,8 +160,7 @@ recv_trace(Config) when is_list(Config) ->
     ?line {messages,Messages} = process_info(Tracer, messages),
     [{trace,Parent,'receive',a},
      {trace,Parent,'receive',{trace_delivered,_,_}},
-     {trace,Parent,'receive',c},
-     {trace,Parent,'receive',{trace_delivered,_,_}}] = Messages,
+     {trace,Parent,'receive',c}|_] = Messages,
 
     ?line unlink(Tracer), exit(Tracer, kill),
     ?line unlink(Sender), exit(Sender, kill),
diff --git a/erts/emulator/test/signal_SUITE.erl b/erts/emulator/test/signal_SUITE.erl
index e9103ca3c1..736dfe5b56 100644
--- a/erts/emulator/test/signal_SUITE.erl
+++ b/erts/emulator/test/signal_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -30,8 +30,9 @@
 -define(DEFAULT_TIMEOUT_SECONDS, 120).
 
 %-define(line_trace, 1).
--include("test_server.hrl").
--export([all/1]).
+-include_lib("test_server/include/test_server.hrl").
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 
 % Test cases
 -export([xm_sig_order/1,
@@ -49,38 +50,48 @@
 	 pending_exit_group_leader/1,
 	 exit_before_pending_exit/1]).
 
--export([init_per_testcase/2, fin_per_testcase/2, end_per_suite/1]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     ?line Dog = ?t:timetrap(?t:seconds(?DEFAULT_TIMEOUT_SECONDS)),
     available_internal_state(true),
     ?line [{testcase, Func},{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     ?line Dog = ?config(watchdog, Config),
     ?line ?t:timetrap_cancel(Dog).
 
+init_per_suite(Config) ->
+    Config.
+
 end_per_suite(_Config) ->
     available_internal_state(true),
-    erts_debug:set_internal_state(not_running_optimization, true),
+    catch erts_debug:set_internal_state(not_running_optimization, true),
     available_internal_state(false).
 
-all(suite) ->
-    [xm_sig_order,
-     pending_exit_unlink_process,
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [xm_sig_order, pending_exit_unlink_process,
      pending_exit_unlink_dist_process,
-     pending_exit_unlink_port,
-     pending_exit_trap_exit,
-     pending_exit_receive,
-     pending_exit_trap_exit,
-     pending_exit_gc,
-     pending_exit_is_process_alive,
+     pending_exit_unlink_port, pending_exit_trap_exit,
+     pending_exit_receive, pending_exit_trap_exit,
+     pending_exit_gc, pending_exit_is_process_alive,
      pending_exit_process_display,
      pending_exit_process_info_1,
-     pending_exit_process_info_2,
-     pending_exit_group_leader,
+     pending_exit_process_info_2, pending_exit_group_leader,
      exit_before_pending_exit].
 
+groups() -> 
+    [].
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 xm_sig_order(doc) -> ["Test that exit signals and messages are received "
 		      "in correct order"];
 xm_sig_order(suite) -> [];
diff --git a/erts/emulator/test/smoke_test_SUITE.erl b/erts/emulator/test/smoke_test_SUITE.erl
new file mode 100644
index 0000000000..98f1cf1ad5
--- /dev/null
+++ b/erts/emulator/test/smoke_test_SUITE.erl
@@ -0,0 +1,139 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(smoke_test_SUITE).
+
+-include_lib("test_server/include/test_server.hrl").
+
+%-compile(export_all).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2]).
+
+-export([boot_combo/1]).
+
+-define(DEFAULT_TIMEOUT, ?t:minutes(2)).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [boot_combo].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
+init_per_testcase(Case, Config) when is_list(Config) ->
+    Dog = ?t:timetrap(?DEFAULT_TIMEOUT),
+    [{testcase, Case},{watchdog, Dog}|Config].
+
+end_per_testcase(_Case, Config) when is_list(Config) ->
+    Dog = ?config(watchdog, Config),
+    ?t:timetrap_cancel(Dog),
+    ok.
+
+%%%
+%%% The test cases -------------------------------------------------------------
+%%%
+
+boot_combo(Config) when is_list(Config) ->
+    ZFlags = os:getenv("ERL_ZFLAGS"),
+    NOOP = fun () -> ok end,
+    A42 = fun () ->
+		  case erlang:system_info(threads) of
+		      true ->
+			  42 = erlang:system_info(thread_pool_size);
+		      false ->
+			  ok
+		  end
+	  end,
+    SMPDisable = fun () -> false = erlang:system_info(smp_support) end,
+    try
+	chk_boot(Config, "+Ktrue", NOOP),
+	chk_boot(Config, "+A42", A42),
+	chk_boot(Config, "-smp disable", SMPDisable),
+	chk_boot(Config, "+Ktrue +A42", A42),
+	chk_boot(Config, "-smp disable +A42",
+		 fun () -> SMPDisable(), A42() end),
+	chk_boot(Config, "-smp disable +Ktrue", SMPDisable),
+	chk_boot(Config, "-smp disable +Ktrue +A42",
+		 fun () -> SMPDisable(), A42() end),
+	%% A lot more combos could be implemented...
+	ok
+    after
+	os:putenv("ERL_ZFLAGS", case ZFlags of
+				    false -> "";
+				    _ -> ZFlags
+				end)
+    end.
+
+%%%
+%%% Aux functions --------------------------------------------------------------
+%%%
+
+chk_boot(Config, Args, Fun) ->
+    true = os:putenv("ERL_ZFLAGS", Args),
+    Success = make_ref(),
+    Parent = self(),
+    ?t:format("--- Testing ~s~n", [Args]),
+    {ok, Node} = start_node(Config),
+    Pid = spawn_link(Node, fun () ->
+				   Fun(),
+				   Parent ! {self(), Success}
+			   end),
+    receive
+	{Pid, Success} ->
+	    Node = node(Pid),
+	    stop_node(Node),
+	    ?t:format("--- Success!~n", []),
+	    ok
+    end.
+
+start_node(Config) ->
+    start_node(Config, "").
+
+start_node(Config, Args) when is_list(Config) ->
+    ?line Pa = filename:dirname(code:which(?MODULE)),
+    ?line {A, B, C} = now(),
+    ?line Name = list_to_atom(atom_to_list(?MODULE)
+			      ++ "-"
+			      ++ atom_to_list(?config(testcase, Config))
+			      ++ "-"
+			      ++ integer_to_list(A)
+			      ++ "-"
+			      ++ integer_to_list(B)
+			      ++ "-"
+			      ++ integer_to_list(C)),
+    ?line ?t:start_node(Name, slave, [{args, "-pa "++Pa++" "++Args}]).
+
+stop_node(Node) ->
+    ?t:stop_node(Node).
+
diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl
index 898908c40f..a93dd309c1 100644
--- a/erts/emulator/test/statistics_SUITE.erl
+++ b/erts/emulator/test/statistics_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2012. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -21,13 +21,15 @@
 
 %% Tests the statistics/1 bif.
 
--export([all/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
 	 init_per_testcase/2,
-	 fin_per_testcase/2,
-	 wall_clock/1, wall_clock_zero_diff/1, wall_clock_update/1,
-	 runtime/1, runtime_zero_diff/1,
+	 end_per_testcase/2,
+	 wall_clock_zero_diff/1, wall_clock_update/1,
+	 runtime_zero_diff/1,
 	 runtime_update/1, runtime_diff/1,
-	 run_queue/1, run_queue_one/1,
+	 run_queue_one/1,
+	 scheduler_wall_time/1,
 	 reductions/1, reductions_big/1, garbage_collection/1, io/1,
 	 badarg/1]).
 
@@ -35,24 +37,47 @@
 
 -export([hog/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 init_per_testcase(_, Config) ->
     ?line Dog = test_server:timetrap(test_server:seconds(300)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_, Config) ->
+end_per_testcase(_, Config) ->
     Dog = ?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
 
-all(suite) -> [wall_clock, runtime, reductions, reductions_big, run_queue,
-	       garbage_collection, io, badarg].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [{group, wall_clock}, {group, runtime}, reductions,
+     reductions_big, {group, run_queue}, scheduler_wall_time,
+     garbage_collection, io, badarg].
+
+groups() -> 
+    [{wall_clock, [],
+      [wall_clock_zero_diff, wall_clock_update]},
+     {runtime, [],
+      [runtime_zero_diff, runtime_update, runtime_diff]},
+     {run_queue, [], [run_queue_one]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 
 %%% Testing statistics(wall_clock).
 
-wall_clock(suite) -> [wall_clock_zero_diff, wall_clock_update].
 
 
 wall_clock_zero_diff(doc) ->
@@ -99,7 +124,6 @@ wall_clock_update1(0) ->
 
 %%% Test statistics(runtime).
 
-runtime(suite) -> [runtime_zero_diff, runtime_update, runtime_diff].
 
 runtime_zero_diff(doc) ->
     "Tests that the difference between the times returned from two consectuitive "
@@ -225,7 +249,6 @@ reductions_big_loop() ->
 
 %%% Tests of statistics(run_queue).
 
-run_queue(suite) -> [run_queue_one].
 
 run_queue_one(doc) ->
     "Tests that statistics(run_queue) returns 1 if we start a "
@@ -244,11 +267,10 @@ run_queue_one(Config) when is_list(Config) ->
     
 
 run_queue_one_test(Config) when is_list(Config) ->
-    ?line Hog = spawn_link(?MODULE, hog, [self()]),
+    ?line _Hog = spawn_link(?MODULE, hog, [self()]),
     ?line receive
-	hog_started ->
-	    Hog ! go
-    end,
+	      hog_started -> ok
+	  end,
     ?line receive after 100 -> ok end,		% Give hog a head start.
     ?line case statistics(run_queue) of
 	      N when N >= 1 -> ok;
@@ -258,18 +280,88 @@ run_queue_one_test(Config) when is_list(Config) ->
 
 %% CPU-bound process, going at low priority.  It will always be ready
 %% to run.
-    
+
 hog(Pid) ->
     ?line process_flag(priority, low),
     ?line Pid ! hog_started,
-    ?line receive
-	go -> hog_iter(0)
+    ?line Mon = erlang:monitor(process, Pid),
+    ?line hog_iter(0, Mon).
+
+hog_iter(N, Mon) when N > 0 ->
+    receive
+	{'DOWN', Mon, _, _, _} ->  ok
+    after 0 ->
+	    ?line hog_iter(N-1, Mon)
+    end;
+hog_iter(0, Mon) ->
+    ?line hog_iter(10000, Mon).
+
+%%% Tests of statistics(scheduler_wall_time).
+
+scheduler_wall_time(doc) ->
+    "Tests that statistics(scheduler_wall_time) works as intended";
+scheduler_wall_time(Config) when is_list(Config) ->
+    %% Should return undefined if system_flag is not turned on yet
+    undefined = statistics(scheduler_wall_time),
+    %% Turn on statistics
+    false = erlang:system_flag(scheduler_wall_time, true),
+    try
+	Schedulers = erlang:system_info(schedulers_online),
+	%% Let testserver and everyone else finish their work
+	timer:sleep(500),
+	%% Empty load
+	EmptyLoad = get_load(),
+	{false, _} = {lists:any(fun(Load) -> Load > 50 end, EmptyLoad),EmptyLoad},
+	MeMySelfAndI = self(),
+	StartHog = fun() ->
+			   Pid = spawn(?MODULE, hog, [self()]),
+			   receive hog_started -> MeMySelfAndI ! go end,
+			   Pid
+		   end,
+	P1 = StartHog(),
+	%% Max on one, the other schedulers empty (hopefully)
+	%% Be generous the process can jump between schedulers
+	%% which is ok and we don't want the test to fail for wrong reasons
+	_L1 = [S1Load|EmptyScheds1] = get_load(),
+	{true,_}  = {S1Load > 50,S1Load},
+	{false,_} = {lists:any(fun(Load) -> Load > 50 end, EmptyScheds1),EmptyScheds1},
+	{true,_}  = {lists:sum(EmptyScheds1) < 60,EmptyScheds1},
+
+	%% 50% load
+	HalfHogs = [StartHog() || _ <- lists:seq(1, (Schedulers-1) div 2)],
+	HalfLoad = lists:sum(get_load()) div Schedulers,
+	if Schedulers < 2, HalfLoad > 80 -> ok; %% Ok only one scheduler online and one hog
+	   %% We want roughly 50% load
+	   HalfLoad > 40, HalfLoad < 60 -> ok;
+	   true -> exit({halfload, HalfLoad})
+	end,
+
+	%% 100% load
+	LastHogs = [StartHog() || _ <- lists:seq(1, Schedulers div 2)],
+	FullScheds = get_load(),
+	{false,_} = {lists:any(fun(Load) -> Load < 80 end, FullScheds),FullScheds},
+	FullLoad = lists:sum(FullScheds) div Schedulers,
+	if FullLoad > 90 -> ok;
+	   true -> exit({fullload, FullLoad})
+	end,
+
+	[exit(Pid, kill) || Pid <- [P1|HalfHogs++LastHogs]],
+	AfterLoad = get_load(),
+	{false,_} = {lists:any(fun(Load) -> Load > 5 end, AfterLoad),AfterLoad},
+	true = erlang:system_flag(scheduler_wall_time, false)
+    after
+	erlang:system_flag(scheduler_wall_time, false)
     end.
 
-hog_iter(N) when N > 0 ->
-    ?line hog_iter(N-1);
-hog_iter(0) ->
-    ?line hog_iter(10000).
+get_load() ->
+    Start = erlang:statistics(scheduler_wall_time),
+    timer:sleep(500),
+    End = erlang:statistics(scheduler_wall_time),
+    lists:reverse(lists:sort(load_percentage(lists:sort(Start),lists:sort(End)))).
+
+load_percentage([{Id, WN, TN}|Ss], [{Id, WP, TP}|Ps]) ->
+    [100*(WN-WP) div (TN-TP)|load_percentage(Ss, Ps)];
+load_percentage([], []) -> [].
 
 
 garbage_collection(doc) ->
diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl
index e782d2f293..0350eb671d 100644
--- a/erts/emulator/test/system_info_SUITE.erl
+++ b/erts/emulator/test/system_info_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2005-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -30,23 +30,44 @@
 
 %-define(line_trace, 1).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %-compile(export_all).
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2]).
 
--export([process_count/1, system_version/1, misc_smoke_tests/1, heap_size/1]).
+-export([process_count/1, system_version/1, misc_smoke_tests/1, heap_size/1, wordsize/1, memory/1]).
 
 -define(DEFAULT_TIMEOUT, ?t:minutes(2)).
 
-all(doc) -> [];
-all(suite) -> [process_count, system_version, misc_smoke_tests, heap_size].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [process_count, system_version, misc_smoke_tests,
+     heap_size, wordsize, memory].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(_Case, Config) when is_list(Config) ->
     Dog = ?t:timetrap(?DEFAULT_TIMEOUT),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) when is_list(Config) ->
+end_per_testcase(_Case, Config) when is_list(Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
@@ -132,6 +153,7 @@ misc_smoke_tests(Config) when is_list(Config) ->
     ?line true = is_binary(erlang:system_info(procs)),
     ?line true = is_binary(erlang:system_info(loaded)),
     ?line true = is_binary(erlang:system_info(dist)),
+    ?line ok = try erlang:system_info({cpu_topology,erts_get_cpu_topology_error_case}), fail catch error:badarg -> ok end,
     ?line ok.
     
 
@@ -145,3 +167,332 @@ heap_size(Config) when is_list(Config) ->
    ?line Hmin  = proplists:get_value(min_heap_size, GCinf),
    ok.
 
+wordsize(suite) ->
+    [];
+wordsize(doc) ->
+    ["Tests the various wordsize variants"];
+wordsize(Config) when is_list(Config) ->
+    ?line A = erlang:system_info(wordsize),
+    ?line true = is_integer(A),
+    ?line A = erlang:system_info({wordsize,internal}),
+    ?line B = erlang:system_info({wordsize,external}),
+    ?line true = A =< B,
+    case {B,A} of
+	{4,4} ->
+	    {comment, "True 32-bit emulator"};
+	{8,8} ->
+	    {comment, "True 64-bit emulator"};
+	{8,4} ->
+	    {comment, "Halfword 64-bit emulator"};
+	Other ->
+	    exit({unexpected_wordsizes,Other})
+    end.
+
+memory(doc) -> ["Verify that erlang:memory/0 and memory results in crashdump produce are similar"];
+memory(Config) when is_list(Config) ->
+    %%
+    %% Verify that erlang:memory/0 and memory results in
+    %% crashdump produce are similar.
+    %%
+    %% erlang:memory/0 requests information from each scheduler
+    %% thread and puts the information together in erlang code
+    %% (erlang.erl).
+    %%
+    %% When a crash dump is written we cannot use the
+    %% erlang:memory/0 implementation. The crashdump implementation
+    %% is a pure C implementation inspecting all allocator instances
+    %% after the system has been blocked (erts_memory() in erl_alloc.c).
+    %%
+    %% Since we got two implementations, modifications can easily
+    %% cause them to produce different results.
+    %%
+    %% erts_debug:get_internal_state(memory) blocks the system and
+    %% execute the same code as the crash dump writing uses.
+    %%
+
+    erts_debug:set_internal_state(available_internal_state, true),
+    %% Use a large heap size on the controling process in
+    %% order to avoid changes in its heap size during
+    %% comparisons.
+    MinHeapSize = process_flag(min_heap_size, 1024*1024), 
+    Prio = process_flag(priority, max),
+    try
+	erlang:memory(), %% first call will init stat atoms
+	garbage_collect(), %% blow up heap
+	memory_test(Config)
+    catch
+	error:notsup -> {skipped, "erlang:memory() not supported"}
+    after
+	process_flag(min_heap_size, MinHeapSize),
+	process_flag(priority, Prio),
+	catch erts_debug:set_internal_state(available_internal_state, false)
+    end.
+
+memory_test(_Config) ->
+
+    MWs = spawn_mem_workers(),
+
+    DPs = mem_workers_call(MWs,
+			   fun () ->
+				   mapn(fun (_) ->
+						spawn(fun () ->
+							      receive
+							      after infinity ->
+								      ok
+							      end
+						      end)
+					end,
+					1000 div erlang:system_info(schedulers_online))
+			   end,
+			   []),
+    cmp_memory(MWs, "spawn procs"),
+
+    Ps = lists:flatten(DPs),
+
+    mem_workers_call(MWs, 
+		     fun () ->
+			     lists:foreach(fun (P) -> link(P) end, Ps)
+		     end,
+		     []),
+    cmp_memory(MWs, "link procs"),
+    mem_workers_call(MWs,
+		     fun () ->
+			     lists:foreach(fun (P) -> unlink(P) end, Ps)
+		     end,
+		     []),
+    cmp_memory(MWs, "unlink procs"),
+
+    DMs = mem_workers_call(MWs,
+			   fun () ->
+				   lists:map(fun (P) ->
+						     monitor(process, P)
+					     end, Ps)
+			   end,
+			   []),
+    cmp_memory(MWs, "monitor procs"),
+    Ms = lists:flatten(DMs),
+    mem_workers_call(MWs,
+		     fun () ->
+			     lists:foreach(fun (M) ->
+						   demonitor(M)
+					   end, Ms)
+		     end,
+		     []),
+    cmp_memory(MWs, "demonitor procs"),
+
+    mem_workers_call(MWs,
+		     fun () ->
+			     lists:foreach(fun (P) ->
+						   P ! {a, "message", make_ref()}
+					   end, Ps)
+		     end,
+		     []),
+    cmp_memory(MWs, "message procs"),
+
+    mem_workers_call(MWs,
+		     fun () ->
+			     Mons = lists:map(fun (P) ->
+						      exit(P, kill),
+						      monitor(process, P)
+					      end,
+					      Ps),
+			     lists:foreach(fun (Mon) ->
+						   receive
+						       {'DOWN', Mon, _, _, _} -> ok
+						   end
+					   end,
+					   Mons)
+		     end, []),
+    cmp_memory(MWs, "kill procs"),
+
+    mem_workers_call(MWs,
+		     fun () ->
+			     put(binary_data,
+				 mapn(fun (_) -> list_to_binary(lists:duplicate(256,$?)) end, 100))
+		     end,
+		     []),
+
+    cmp_memory(MWs, "store binary data"),
+
+    mem_workers_call(MWs,
+		     fun () ->
+			     put(binary_data, false),
+			     garbage_collect()
+		     end,
+		     []),
+    cmp_memory(MWs, "release binary data"),
+
+    mem_workers_call(MWs,
+		     fun () ->
+			     list_to_atom("an ugly atom "++integer_to_list(erlang:system_info(scheduler_id))),
+			     list_to_atom("another ugly atom "++integer_to_list(erlang:system_info(scheduler_id))),
+			     list_to_atom("yet another ugly atom "++integer_to_list(erlang:system_info(scheduler_id)))
+		     end,
+		     []),
+    cmp_memory(MWs, "new atoms"),
+
+
+    mem_workers_call(MWs,
+		     fun () ->
+			     T = ets:new(?MODULE, []),
+			     ets:insert(T, {gurka, lists:seq(1,10000)}),
+			     ets:insert(T, {banan, lists:seq(1,1024)}),
+			     ets:insert(T, {appelsin, make_ref()}),
+			     put(ets_id, T)
+		     end,
+		     []),
+    cmp_memory(MWs, "store ets data"),
+
+    mem_workers_call(MWs,
+		     fun () ->
+			     ets:delete(get(ets_id)),
+			     put(ets_id, false)
+		     end,
+		     []),
+    cmp_memory(MWs, "remove ets data"),
+
+    lists:foreach(fun (MW) ->
+			  unlink(MW),
+			  Mon = monitor(process, MW),
+			  exit(MW, kill),
+			  receive
+			      {'DOWN', Mon, _, _, _} -> ok
+			  end
+		  end,
+		  MWs),
+    ok.
+
+mem_worker() ->
+    receive
+	{call, From, Fun, Args} ->
+	    From ! {reply, self(), apply(Fun, Args)},
+	    mem_worker();
+	{cast, _From, Fun, Args} ->
+	    apply(Fun, Args),
+	    mem_worker()
+    end.
+
+mem_workers_call(MWs, Fun, Args) ->
+    lists:foreach(fun (MW) ->
+			  MW ! {call, self(), Fun, Args}
+		  end,
+		  MWs),
+    lists:map(fun (MW) ->
+		      receive
+			  {reply, MW, Res} ->
+			      Res
+		      end
+	      end,
+	      MWs).
+
+mem_workers_cast(MWs, Fun, Args) ->
+    lists:foreach(fun (MW) ->
+			  MW ! {cast, self(), Fun, Args}
+		  end,
+		  MWs).
+
+spawn_mem_workers() ->
+    spawn_mem_workers(erlang:system_info(schedulers_online)).
+
+spawn_mem_workers(0) ->
+    [];
+spawn_mem_workers(N) ->
+    [spawn_opt(fun () -> mem_worker() end,
+	       [{scheduler, N rem erlang:system_info(schedulers_online) + 1},
+		link]) | spawn_mem_workers(N-1)].
+
+
+
+mem_get(X, Mem) ->
+    case lists:keyfind(X, 1, Mem) of
+	{X, Val} -> Val;
+	false -> false
+    end.
+
+cmp_memory(What, Mem1, Mem2, 1) ->
+    R1 = mem_get(What, Mem1),
+    R2 = mem_get(What, Mem2),
+    true = R1 == R2;
+cmp_memory(What, Mem1, Mem2, RelDiff) ->
+    %% We allow RealDiff diff
+    R1 = mem_get(What, Mem1),
+    R2 = mem_get(What, Mem2),
+    case R1 == R2 of
+	true ->
+	    ok;
+	false ->
+	    case R1 > R2 of
+		true ->
+		    true = R2*RelDiff > R1;
+		false ->
+		    true = R1*RelDiff > R2
+	    end
+    end.
+
+pos_int(Val) when Val >= 0 ->
+    Val;
+pos_int(Val) ->
+    exit({not_pos_int, Val}).
+
+check_sane_memory(Mem) ->
+    Tot = pos_int(mem_get(total, Mem)),
+    Proc = pos_int(mem_get(processes, Mem)),
+    ProcUsed = pos_int(mem_get(processes_used, Mem)),
+    Sys = pos_int(mem_get(system, Mem)),
+    Atom = pos_int(mem_get(atom, Mem)),
+    AtomUsed = pos_int(mem_get(atom_used, Mem)),
+    Bin = pos_int(mem_get(binary, Mem)),
+    Code = pos_int(mem_get(code, Mem)),
+    Ets = pos_int(mem_get(ets, Mem)),
+
+    Tot = Proc + Sys,
+    true = Sys > Atom + Bin + Code + Ets,
+    true = Proc >= ProcUsed,
+    true = Atom >= AtomUsed,
+
+    case mem_get(maximum, Mem) of
+	false -> ok;
+	Max -> true = pos_int(Max) >= Tot
+    end,
+    ok.
+
+cmp_memory(MWs, Str) ->
+    erlang:display(Str),
+    lists:foreach(fun (MW) -> garbage_collect(MW) end, MWs),
+    garbage_collect(),
+    erts_debug:set_internal_state(wait, deallocations),
+
+    EDM = erts_debug:get_internal_state(memory),
+    EM = erlang:memory(),
+
+    io:format("~s:~n"
+	      "erlang:memory() = ~p~n"
+	      "crash dump memory = ~p~n",
+	      [Str, EM, EDM]),
+
+    ?line check_sane_memory(EM),
+    ?line check_sane_memory(EDM),
+
+    %% We expect these to always give us exactly the same result
+
+    ?line cmp_memory(atom, EM, EDM, 1),
+    ?line cmp_memory(atom_used, EM, EDM, 1),
+    ?line cmp_memory(binary, EM, EDM, 1),
+    ?line cmp_memory(code, EM, EDM, 1),
+    ?line cmp_memory(ets, EM, EDM, 1),
+
+    %% Total, processes, processes_used, and system will seldom
+    %% give us exactly the same result since the two readings
+    %% aren't taken atomically.
+
+    ?line cmp_memory(total, EM, EDM, 1.05),
+    ?line cmp_memory(processes, EM, EDM, 1.05),
+    ?line cmp_memory(processes_used, EM, EDM, 1.05),
+    ?line cmp_memory(system, EM, EDM, 1.05),
+
+    ok.
+    
+mapn(_Fun, 0) ->
+    [];
+mapn(Fun, N) ->
+    [Fun(N) | mapn(Fun, N-1)].
diff --git a/erts/emulator/test/system_profile_SUITE.erl b/erts/emulator/test/system_profile_SUITE.erl
index 7b0d6d19fe..ba94a371be 100644
--- a/erts/emulator/test/system_profile_SUITE.erl
+++ b/erts/emulator/test/system_profile_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2007-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2007-2012. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -22,35 +22,53 @@
 
 -module(system_profile_SUITE).
 
--export([all/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
 	system_profile_on_and_off/1,
 	runnable_procs/1,
 	runnable_ports/1,
+	dont_profile_profiler/1,
 	scheduler/1
         ]).
 
--export([init_per_testcase/2, fin_per_testcase/2]).
+-export([init_per_testcase/2, end_per_testcase/2]).
 
--export([profiler_process/1, ring_loop/1, port_echo_start/0, list_load/0, run_load/2]).
+-export([profiler_process/1, ring_loop/1, port_echo_start/0, 
+	 list_load/0, run_load/2]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 -define(default_timeout, ?t:minutes(1)).
 
 init_per_testcase(_Case, Config) ->
-    ?line Dog=?t:timetrap(?default_timeout),
+    Dog=?t:timetrap(?default_timeout),
     [{watchdog, Dog}|Config].
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog=?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
 
-all(suite) -> 
-    %% Test specification on test suite level
-    [system_profile_on_and_off,
-    runnable_procs,
-    runnable_ports,
-    scheduler].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [system_profile_on_and_off, runnable_procs,
+     runnable_ports, scheduler, dont_profile_profiler].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 %% No specification clause needed for an init function in a conf case!!!
 
@@ -60,31 +78,31 @@ system_profile_on_and_off(suite) ->
 system_profile_on_and_off(doc) ->
     ["Tests switching system_profiling on and off."];
 system_profile_on_and_off(Config) when is_list(Config) ->
-    ?line Pid = start_profiler_process(),
+    Pid = start_profiler_process(),
     
     % Test runnable_ports on and off
-    ?line undefined = erlang:system_profile(Pid, [runnable_ports]),
-    ?line {Pid, [runnable_ports]} = erlang:system_profile(),
-    ?line {Pid, [runnable_ports]} = erlang:system_profile(undefined, []),
+    undefined = erlang:system_profile(Pid, [runnable_ports]),
+    {Pid, [runnable_ports]} = erlang:system_profile(),
+    {Pid, [runnable_ports]} = erlang:system_profile(undefined, []),
 
     % Test runnable_procs on and off
-    ?line undefined = erlang:system_profile(Pid, [runnable_procs]),
-    ?line {Pid, [runnable_procs]} = erlang:system_profile(),
-    ?line {Pid, [runnable_procs]} = erlang:system_profile(undefined, []),
+    undefined = erlang:system_profile(Pid, [runnable_procs]),
+    {Pid, [runnable_procs]} = erlang:system_profile(),
+    {Pid, [runnable_procs]} = erlang:system_profile(undefined, []),
 
     % Test scheduler on and off
-    ?line undefined = erlang:system_profile(Pid, [scheduler]),
-    ?line {Pid, [scheduler]} = erlang:system_profile(),
-    ?line {Pid, [scheduler]} = erlang:system_profile(undefined, []),
+    undefined = erlang:system_profile(Pid, [scheduler]),
+    {Pid, [scheduler]} = erlang:system_profile(),
+    {Pid, [scheduler]} = erlang:system_profile(undefined, []),
 
     % Test combined runnable_ports, runnable_procs, scheduler; on and off
-    ?line undefined = erlang:system_profile(Pid, [scheduler, runnable_procs, runnable_ports]),
-    ?line {Pid, [scheduler,runnable_procs,runnable_ports]} = erlang:system_profile(),
-    ?line {Pid, [scheduler,runnable_procs,runnable_ports]} = erlang:system_profile(undefined, []),
+    undefined = erlang:system_profile(Pid, [scheduler, runnable_procs, runnable_ports]),
+    {Pid, [scheduler,runnable_procs,runnable_ports]} = erlang:system_profile(),
+    {Pid, [scheduler,runnable_procs,runnable_ports]} = erlang:system_profile(undefined, []),
 
     % Test turned off and kill process
-    ?line undefined = erlang:system_profile(),
-    ?line exit(Pid,kill),
+    undefined = erlang:system_profile(),
+    exit(Pid,kill),
     ok.
 
 %% Test runnable_procs
@@ -94,25 +112,25 @@ runnable_procs(suite) ->
 runnable_procs(doc) ->
     ["Tests system_profiling with runnable_procs."];
 runnable_procs(Config) when is_list(Config) ->
-    ?line Pid = start_profiler_process(),
+    Pid = start_profiler_process(),
     % start a ring of processes
     % FIXME: Set #laps and #nodes in config file
     Nodes = 10,
     Laps = 10,
-    ?line Master = ring(Nodes),
-    ?line undefined = erlang:system_profile(Pid, [runnable_procs]),
+    Master = ring(Nodes),
+    undefined = erlang:system_profile(Pid, [runnable_procs]),
     % loop a message
-    ?line ok = ring_message(Master, message, Laps),
-    ?line Events = get_profiler_events(),
-    ?line kill_em_all = kill_ring(Master),
-    ?line erlang:system_profile(undefined, []),
+    ok = ring_message(Master, message, Laps),
+    Events = get_profiler_events(),
+    kill_em_all = kill_ring(Master),
+    erlang:system_profile(undefined, []),
     put(master, Master),
     put(laps, Laps),
-    ?line true = has_runnable_event(Events),
+    true = has_runnable_event(Events),
     Pids = sort_events_by_pid(Events),
-    ?line ok = check_events(Pids),
+    ok = check_events(Pids),
     erase(),
-    ?line exit(Pid,kill),
+    exit(Pid,kill),
     ok.
 
 runnable_ports(suite) ->
@@ -120,21 +138,21 @@ runnable_ports(suite) ->
 runnable_ports(doc) ->
     ["Tests system_profiling with runnable_port."];
 runnable_ports(Config) when is_list(Config) ->
-    ?line Pid = start_profiler_process(),
-    ?line undefined = erlang:system_profile(Pid, [runnable_ports]),
-    ?line EchoPid = echo(Config),
+    Pid = start_profiler_process(),
+    undefined = erlang:system_profile(Pid, [runnable_ports]),
+    EchoPid = echo(Config),
     % FIXME: Set config to number_of_echos
     Laps = 10,
     put(laps, Laps),
-    ?line ok = echo_message(EchoPid, Laps, message),
-    ?line Events = get_profiler_events(),
-    ?line kill_em_all = kill_echo(EchoPid),
-    ?line erlang:system_profile(undefined, []),
-    ?line true = has_runnable_event(Events),
+    ok = echo_message(EchoPid, Laps, message),
+    Events = get_profiler_events(),
+    kill_em_all = kill_echo(EchoPid),
+    erlang:system_profile(undefined, []),
+    true = has_runnable_event(Events),
     Pids = sort_events_by_pid(Events),
-    ?line ok = check_events(Pids),
+    ok = check_events(Pids),
     erase(),
-    ?line exit(Pid,kill),
+    exit(Pid,kill),
     ok.
 
 scheduler(suite) ->
@@ -143,46 +161,68 @@ scheduler(doc) ->
     ["Tests system_profiling with scheduler."];
 scheduler(Config) when is_list(Config) ->
     case {erlang:system_info(smp_support), erlang:system_info(schedulers_online)} of
-	{false,_} -> ?line {skipped, "No need for scheduler test when smp support is disabled."};
-	{_,    1} -> ?line {skipped, "No need for scheduler test when only one scheduler online."};
+	{false,_} -> {skipped, "No need for scheduler test when smp support is disabled."};
+	{_,    1} -> {skipped, "No need for scheduler test when only one scheduler online."};
 	_ ->
 	    Nodes = 10,
-	    ?line ok = check_block_system(Nodes),
-	    ?line ok = check_multi_scheduling_block(Nodes),
-	    ok
+	    ok = check_block_system(Nodes),
+	    ok = check_multi_scheduling_block(Nodes)
     end.
 
+% the profiler pid should not be profiled
+dont_profile_profiler(suite) ->
+    [];
+dont_profile_profiler(doc) ->
+    ["Ensure system profiler process is not profiled."];
+dont_profile_profiler(Config) when is_list(Config) ->
+    Pid = start_profiler_process(),
+
+    Nodes = 10,
+    Laps = 10,
+    Master = ring(Nodes),
+    undefined = erlang:system_profile(Pid, [runnable_procs]),
+    % loop a message
+    ok = ring_message(Master, message, Laps),
+    erlang:system_profile(undefined, []),
+    kill_em_all = kill_ring(Master),
+    Events = get_profiler_events(),
+    false  = has_profiler_pid_event(Events, Pid),
+
+    exit(Pid,kill),
+    ok.
+
+
 %%% Check scheduler profiling
 
 check_multi_scheduling_block(Nodes) ->
-    ?line Pid = start_profiler_process(),
-    ?line undefined = erlang:system_profile(Pid, [scheduler]),
-    ?line {ok, Supervisor} = start_load(Nodes),
-    ?line erlang:system_flag(multi_scheduling, block),
-    ?line erlang:system_flag(multi_scheduling, unblock),
-    ?line {Pid, [scheduler]} = erlang:system_profile(undefined, []),
-    ?line Events = get_profiler_events(),
-    ?line true = has_scheduler_event(Events),
+    Pid = start_profiler_process(),
+    undefined = erlang:system_profile(Pid, [scheduler]),
+    {ok, Supervisor} = start_load(Nodes),
+    erlang:system_flag(multi_scheduling, block),
+    erlang:system_flag(multi_scheduling, unblock),
+    {Pid, [scheduler]} = erlang:system_profile(undefined, []),
+    Events = get_profiler_events(),
+    true = has_scheduler_event(Events),
     stop_load(Supervisor),
-    ?line exit(Pid,kill),
+    exit(Pid,kill),
     erase(),
     ok.
 
 check_block_system(Nodes) ->
-    ?line Dummy = spawn(?MODULE, profiler_process, [[]]),
-    ?line Pid = start_profiler_process(),
-    ?line undefined = erlang:system_profile(Pid, [scheduler]),
-    ?line {ok, Supervisor} = start_load(Nodes),
+    Dummy = spawn(?MODULE, profiler_process, [[]]),
+    Pid = start_profiler_process(),
+    undefined = erlang:system_profile(Pid, [scheduler]),
+    {ok, Supervisor} = start_load(Nodes),
     % FIXME: remove wait !!
     wait(300),
-    ?line undefined = erlang:system_monitor(Dummy, [busy_port]),
-    ?line {Dummy, [busy_port]} = erlang:system_monitor(undefined, []),
-    ?line {Pid, [scheduler]} = erlang:system_profile(undefined, []),
-    ?line Events = get_profiler_events(),
-    ?line true = has_scheduler_event(Events),
+    undefined = erlang:system_monitor(Dummy, [busy_port]),
+    {Dummy, [busy_port]} = erlang:system_monitor(undefined, []),
+    {Pid, [scheduler]} = erlang:system_profile(undefined, []),
+    Events = get_profiler_events(),
+    true = has_scheduler_event(Events),
     stop_load(Supervisor),
-    ?line exit(Pid,kill),
-    ?line exit(Dummy,kill),
+    exit(Pid,kill),
+    exit(Dummy,kill),
     erase(),
     ok.
 
@@ -194,17 +234,17 @@ check_events([Pid | Pids]) ->
     Laps = get(laps),
     CheckPids = get(pids),
     {Events, N} = get_pid_events(Pid),
-    ?line ok = check_event_flow(Events),
-    ?line ok = check_event_ts(Events),
+    ok = check_event_flow(Events),
+    ok = check_event_ts(Events),
     IsMember = lists:member(Pid, CheckPids),
     case Pid of
     	Master ->
 	    io:format("Expected ~p and got ~p profile events from ~p: ok~n", [Laps*2+2, N, Pid]),
-	    ?line N = Laps*2 + 2,
+	    N = Laps*2 + 2,
     	    check_events(Pids);
 	Pid when IsMember == true ->
 	    io:format("Expected ~p and got ~p profile events from ~p: ok~n", [Laps*2, N, Pid]),
-	    ?line N = Laps*2,
+	    N = Laps*2,
     	    check_events(Pids);
 	Pid ->
 	    check_events(Pids)
@@ -431,6 +471,12 @@ has_runnable_event(Events) ->
 	    end
         end, Events).
 
+has_profiler_pid_event([], _) -> false;
+has_profiler_pid_event([{profile, Pid, _Activity, _MFA, _TS}|Events], Pid) -> true;
+has_profiler_pid_event([_|Events], Pid) ->
+    has_profiler_pid_event(Events, Pid).
+
+
 wait(Time) -> receive after Time -> ok end.
 
 %%%
diff --git a/erts/emulator/test/system_profile_SUITE_data/echo_drv.c b/erts/emulator/test/system_profile_SUITE_data/echo_drv.c
index d968ff06f9..e0b6ff804c 100644
--- a/erts/emulator/test/system_profile_SUITE_data/echo_drv.c
+++ b/erts/emulator/test/system_profile_SUITE_data/echo_drv.c
@@ -9,11 +9,9 @@ static EchoDrvData echo_drv_data, *echo_drv_data_p;
 
 static EchoDrvData *echo_drv_start(ErlDrvPort port, char *command);
 static void         echo_drv_stop(EchoDrvData *data_p);
-static void         echo_drv_output(EchoDrvData *data_p, char *buf, int len);
+static void         echo_drv_output(ErlDrvData drv_data, char *buf,
+				    ErlDrvSizeT len);
 static void         echo_drv_finish(void);
-static int          echo_drv_control(EchoDrvData *data_p, unsigned int command,
-				     char *buf, int len,
-				     char **rbuf, int rlen);
 
 static ErlDrvEntry echo_drv_entry = { 
     NULL, /* init */
@@ -25,10 +23,21 @@ static ErlDrvEntry echo_drv_entry = {
     "echo_drv",
     echo_drv_finish,
     NULL, /* handle */
-    echo_drv_control,
+    NULL, /* control */
     NULL, /* timeout */
     NULL, /* outputv */
-    NULL  /* ready_async */
+    NULL,  /* ready_async */
+    NULL,
+    NULL,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
+    NULL
+
 };
 
 DRIVER_INIT(echo_drv)
@@ -51,16 +60,11 @@ static void echo_drv_stop(EchoDrvData *data_p) {
     echo_drv_data_p = NULL;
 }
 
-static void echo_drv_output(EchoDrvData *data_p, char *buf, int len) {
+static void echo_drv_output(ErlDrvData drv_data, char *buf, ErlDrvSizeT len) {
+    EchoDrvData* data_p = (EchoDrvData *) drv_data;
     driver_output(data_p->erlang_port, buf, len);
 }
 
 static void echo_drv_finish() {
     echo_drv_data_p = NULL;
 }
-
-static int echo_drv_control(EchoDrvData *data_p, unsigned int command,
-			    char *buf, int len,
-			    char **rbuf, int rlen) {
-    return 0;
-}
diff --git a/erts/emulator/test/time_SUITE.erl b/erts/emulator/test/time_SUITE.erl
index 2ad1f0d201..4d12e3449c 100644
--- a/erts/emulator/test/time_SUITE.erl
+++ b/erts/emulator/test/time_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -29,12 +29,16 @@
 %%	now/0
 %%
 
--export([all/1, univ_to_local/1, local_to_univ/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, univ_to_local/1, local_to_univ/1,
 	 bad_univ_to_local/1, bad_local_to_univ/1,
+	 univ_to_seconds/1, seconds_to_univ/1,
 	 consistency/1,
-	 now/1, now_unique/1, now_update/1, timestamp/1]).
+	 now_unique/1, now_update/1, timestamp/1]).
 
--include("test_server.hrl").
+-export([local_to_univ_utc/1]).
+
+-include_lib("test_server/include/test_server.hrl").
 
 -export([linear_time/1]).
 
@@ -52,8 +56,61 @@
 
 -define(dst_timezone, 2).
 
-all(suite) -> [univ_to_local, local_to_univ,
-	       bad_univ_to_local, bad_local_to_univ, consistency, now, timestamp].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [univ_to_local, local_to_univ, local_to_univ_utc,
+     bad_univ_to_local, bad_local_to_univ, 
+     univ_to_seconds, seconds_to_univ,
+     consistency,
+     {group, now}, timestamp].
+
+groups() -> 
+    [{now, [], [now_unique, now_update]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
+local_to_univ_utc(suite) ->
+    [];
+local_to_univ_utc(doc) ->
+    ["Test that DST = true on timezones without DST is ignored"];
+local_to_univ_utc(Config) when is_list(Config) ->
+    case os:type() of
+	{unix,_} ->
+	    %% TZ variable has a meaning
+	    ?line {ok, Node} =
+		test_server:start_node(local_univ_utc,peer,
+				       [{args, "-env TZ UTC"}]),
+	    ?line {{2008,8,1},{0,0,0}} =
+		rpc:call(Node,
+			 erlang,localtime_to_universaltime,
+			 [{{2008, 8, 1}, {0, 0, 0}},
+			  false]),
+	    ?line {{2008,8,1},{0,0,0}} =
+		rpc:call(Node,
+			 erlang,localtime_to_universaltime,
+			 [{{2008, 8, 1}, {0, 0, 0}},
+			  true]),
+	    ?line [{{2008,8,1},{0,0,0}}] =
+		rpc:call(Node,
+			 calendar,local_time_to_universal_time_dst,
+			 [{{2008, 8, 1}, {0, 0, 0}}]),
+	    ?line test_server:stop_node(Node),
+	    ok;
+	_ ->
+	    {skip,"Only valid on Unix"}
+    end.
 
 
 %% Tests conversion from univeral to local time.
@@ -108,6 +165,30 @@ bad_test_local_to_univ([Local|Rest]) ->
 bad_test_local_to_univ([]) ->
     ok.
 
+
+%% Test universaltime to seconds conversions
+univ_to_seconds(Config) when is_list(Config) ->
+    test_univ_to_seconds(ok_utc_seconds()).
+
+test_univ_to_seconds([{Datetime, Seconds}|DSs]) ->
+    io:format("universaltime = ~p -> seconds = ~p", [Datetime, Seconds]),
+    Seconds = erlang:universaltime_to_posixtime(Datetime),
+    test_univ_to_seconds(DSs);
+test_univ_to_seconds([]) -> 
+    ok.
+
+%% Test seconds to universaltime conversions
+seconds_to_univ(Config) when is_list(Config) ->
+    test_seconds_to_univ(ok_utc_seconds()).
+
+test_seconds_to_univ([{Datetime, Seconds}|DSs]) ->
+    io:format("universaltime = ~p <- seconds = ~p", [Datetime, Seconds]),
+    Datetime = erlang:posixtime_to_universaltime(Seconds),
+    test_seconds_to_univ(DSs);
+test_seconds_to_univ([]) -> 
+    ok.
+
+
 %% Test that the the different time functions return
 %% consistent results. (See the test case for assumptions
 %% and limitations.)
@@ -248,7 +329,6 @@ repeating_timestamp_check(N) ->
 
 %% Test now/0.
 
-now(suite) -> [now_unique, now_update].
 
 %% Tests that successive calls to now/0 returns different values.
 %% Also returns a comment string with the median difference between
@@ -400,6 +480,32 @@ dst_dates() ->
      {1998, 06, 3},
      {1999, 06, 4}].
 
+%% exakt utc {date(), time()} which corresponds to the same seconds since 1 jan 1970 
+%% negative seconds are ok
+%% generated with date --date='1979-05-28 12:30:35 UTC' +%s
+ok_utc_seconds() -> [
+	{ {{1970, 1, 1},{ 0, 0, 0}},            0 },
+	{ {{1970, 1, 1},{ 0, 0, 1}},            1 },
+	{ {{1969,12,31},{23,59,59}},           -1 },
+	{ {{1920,12,31},{23,59,59}},  -1546300801 },
+	{ {{1600,02,19},{15,14,08}}, -11671807552 },
+	{ {{1979,05,28},{12,30,35}},    296742635 },
+	{ {{1999,12,31},{23,59,59}},    946684799 },
+	{ {{2000, 1, 1},{ 0, 0, 0}},    946684800 },
+	{ {{2000, 1, 1},{ 0, 0, 1}},    946684801 },
+
+	{ {{2038, 1,19},{03,14,07}},   2147483647 }, % Sint32 full - 1
+	{ {{2038, 1,19},{03,14,08}},   2147483648 }, % Sint32 full
+	{ {{2038, 1,19},{03,14,09}},   2147483649 }, % Sint32 full + 1
+
+	{ {{2106, 2, 7},{ 6,28,14}},   4294967294 }, % Uint32 full  0xFFFFFFFF - 1
+	{ {{2106, 2, 7},{ 6,28,15}},   4294967295 }, % Uint32 full  0xFFFFFFFF
+	{ {{2106, 2, 7},{ 6,28,16}},   4294967296 }, % Uint32 full  0xFFFFFFFF + 1
+	{ {{2012,12, 6},{16,28,08}},   1354811288 },
+	{ {{2412,12, 6},{16,28,08}},  13977592088 }
+    ].
+
+
 %% The following dates should not be near the end or beginning of
 %% a month, because they will be used to test when the dates are
 %% different in UTC and local time.
diff --git a/erts/emulator/test/timer_bif_SUITE.erl b/erts/emulator/test/timer_bif_SUITE.erl
index 9ac5afcc45..7ff7449ff5 100644
--- a/erts/emulator/test/timer_bif_SUITE.erl
+++ b/erts/emulator/test/timer_bif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1998-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1998-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,7 +19,9 @@
 
 -module(timer_bif_SUITE).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,end_per_suite/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2]).
 -export([start_timer_1/1, send_after_1/1, send_after_2/1, send_after_3/1,
 	 cancel_timer_1/1,
 	 start_timer_big/1, send_after_big/1,
@@ -27,7 +29,7 @@
 	 read_timer_trivial/1, read_timer/1,
 	 cleanup/1, evil_timers/1, registered_process/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 init_per_testcase(_Case, Config) ->
     ?line Dog=test_server:timetrap(test_server:seconds(30)),
@@ -37,19 +39,35 @@ init_per_testcase(_Case, Config) ->
     end,
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog = ?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
 
+init_per_suite(Config) ->
+    Config.
+
 end_per_suite(_Config) ->
     catch erts_debug:set_internal_state(available_internal_state, false).
 
-all(suite) ->
-    [start_timer_1, send_after_1, send_after_2, cancel_timer_1,
-     start_timer_e, send_after_e, cancel_timer_e,
-     start_timer_big, send_after_big, read_timer_trivial, read_timer,
-     cleanup, evil_timers, registered_process].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [start_timer_1, send_after_1, send_after_2,
+     cancel_timer_1, start_timer_e, send_after_e,
+     cancel_timer_e, start_timer_big, send_after_big,
+     read_timer_trivial, read_timer, cleanup, evil_timers,
+     registered_process].
+
+groups() -> 
+    [].
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 start_timer_1(doc) -> ["Basic start_timer/3 functionality"];
 start_timer_1(Config) when is_list(Config) ->
diff --git a/erts/emulator/test/trace_SUITE.erl b/erts/emulator/test/trace_SUITE.erl
index e9713fcf0f..221b65309a 100644
--- a/erts/emulator/test/trace_SUITE.erl
+++ b/erts/emulator/test/trace_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -23,7 +23,8 @@
 %%% Tests the trace BIF.
 %%%
 
--export([all/1, receive_trace/1, self_send/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, receive_trace/1, self_send/1,
 	 timeout_trace/1, send_trace/1,
 	 procs_trace/1, dist_procs_trace/1,
 	 suspend/1, mutual_suspend/1, suspend_exit/1, suspender_exit/1,
@@ -35,22 +36,39 @@
 	 system_monitor_large_heap_1/1, system_monitor_large_heap_2/1,
 	 bad_flag/1, trace_delivered/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %%% Internal exports
 -export([process/1]).
 
-all(suite) ->
-    [cpu_timestamp, receive_trace, self_send, timeout_trace, send_trace,
-     procs_trace, dist_procs_trace,
-     suspend, mutual_suspend, suspend_exit, suspender_exit,
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [cpu_timestamp, receive_trace, self_send, timeout_trace,
+     send_trace, procs_trace, dist_procs_trace, suspend,
+     mutual_suspend, suspend_exit, suspender_exit,
      suspend_system_limit, suspend_opts, suspend_waiting,
-     new_clear, existing_clear,
-     set_on_spawn, set_on_first_spawn,
-     system_monitor_args, more_system_monitor_args,
-     system_monitor_long_gc_1, system_monitor_long_gc_2, 
-     system_monitor_large_heap_1, system_monitor_large_heap_2,
-     bad_flag, trace_delivered].
+     new_clear, existing_clear, set_on_spawn,
+     set_on_first_spawn, system_monitor_args,
+     more_system_monitor_args, system_monitor_long_gc_1,
+     system_monitor_long_gc_2, system_monitor_large_heap_1,
+     system_monitor_large_heap_2, bad_flag, trace_delivered].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 
 %% No longer testing anything, just reporting whether cpu_timestamp
diff --git a/erts/emulator/test/trace_bif_SUITE.erl b/erts/emulator/test/trace_bif_SUITE.erl
index 3f91f8dc08..2c78aa394f 100644
--- a/erts/emulator/test/trace_bif_SUITE.erl
+++ b/erts/emulator/test/trace_bif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1998-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1998-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,24 +19,44 @@
 
 -module(trace_bif_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1]).
--export([trace_bif/1, trace_bif_timestamp/1, trace_on_and_off/1, trace_bif_local/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
+-export([trace_bif/1, trace_bif_timestamp/1, trace_on_and_off/1, 
+	 trace_bif_local/1,
 	 trace_bif_timestamp_local/1, trace_bif_return/1, not_run/1,
 	 trace_info_old_code/1]).
 
 -export([bif_process/0]).
 
-all(suite) ->
-    case test_server:is_native(?MODULE) of
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    case test_server:is_native(trace_bif_SUITE) of
 	true -> [not_run];
 	false ->
 	    [trace_bif, trace_bif_timestamp, trace_on_and_off,
-	     trace_bif_local, trace_bif_timestamp_local, 
+	     trace_bif_local, trace_bif_timestamp_local,
 	     trace_bif_return, trace_info_old_code]
     end.
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 not_run(Config) when is_list(Config) -> 
     {skipped,"Native code"}.
 
diff --git a/erts/emulator/test/trace_call_count_SUITE.erl b/erts/emulator/test/trace_call_count_SUITE.erl
index 07aa7c8d8d..2ac58493ff 100644
--- a/erts/emulator/test/trace_call_count_SUITE.erl
+++ b/erts/emulator/test/trace_call_count_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2002-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2002-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -42,7 +42,7 @@
 -define(config(A,B),config(A,B)).
 -export([config/2]).
 -else.
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 -endif.
  
 -ifdef(debug).
@@ -62,7 +62,9 @@ config(priv_dir,_) ->
     ".".
 -else.
 %% When run in test server.
--export([all/1, init_per_testcase/2, fin_per_testcase/2, not_run/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2, not_run/1]).
 -export([basic/1, on_and_off/1, info/1, 
 	 pause_and_restart/1, combo/1]).
 	 
@@ -70,7 +72,7 @@ init_per_testcase(_Case, Config) ->
     ?line Dog=test_server:timetrap(test_server:seconds(30)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     erlang:trace_pattern({'_','_','_'}, false, [local,meta,call_count]),
     erlang:trace_pattern(on_load, false, [local,meta,call_count]),
     erlang:trace(all, false, [all]),
@@ -78,15 +80,31 @@ fin_per_testcase(_Case, Config) ->
     test_server:timetrap_cancel(Dog),
     ok.
 
-all(doc) ->
-    ["Test call count tracing of local function calls."];
-all(suite) ->
-    case test_server:is_native(?MODULE) of
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    case test_server:is_native(trace_call_count_SUITE) of
 	true -> [not_run];
-	false -> [basic, on_and_off, info, 
-		  pause_and_restart, combo]
+	false ->
+	    [basic, on_and_off, info, pause_and_restart, combo]
     end.
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 not_run(Config) when is_list(Config) -> 
     {skipped,"Native code"}.
 
diff --git a/erts/emulator/test/trace_call_time_SUITE.erl b/erts/emulator/test/trace_call_time_SUITE.erl
new file mode 100644
index 0000000000..5dfa87bbee
--- /dev/null
+++ b/erts/emulator/test/trace_call_time_SUITE.erl
@@ -0,0 +1,634 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%
+%%% Define to run outside of test server
+%%%
+%%% -define(STANDALONE,1).
+%%%
+%%%
+%%% Define for debug output
+%%%
+%%% -define(debug,1).
+
+-module(trace_call_time_SUITE).
+
+%% Exported end user tests
+
+-export([seq/3, seq_r/3]).
+-export([loaded/1, a_function/1, a_called_function/1, dec/1, nif_dec/1]).
+
+-define(US_ERROR, 10000).
+-define(R_ERROR, 0.8).
+-define(SINGLE_CALL_US_TIME, 10).
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% Result examination macros
+
+-define(CT(P,MFA),{trace,P,call,MFA}).
+-define(CTT(P, MFA),{trace_ts,P,call,MFA,{_,_,_}}).
+-define(RF(P,MFA,V),{trace,P,return_from,MFA,V}).
+-define(RFT(P,MFA,V),{trace_ts,P,return_from,MFA,V,{_,_,_}}).
+-define(RT(P,MFA),{trace,P,return_to,MFA}).
+-define(RTT(P,MFA),{trace_ts,P,return_to,MFA,{_,_,_}}).
+
+-ifdef(debug).
+-define(dbgformat(A,B),io:format(A,B)).
+-else.
+-define(dbgformat(A,B),noop).
+-endif.
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+-include_lib("test_server/include/test_server.hrl").
+
+%% When run in test server.
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2, not_run/1]).
+-export([basic/1, on_and_off/1, info/1,
+	 pause_and_restart/1, scheduling/1, called_function/1, combo/1, 
+	 bif/1, nif/1]).
+
+init_per_testcase(_Case, Config) ->
+    ?line Dog=test_server:timetrap(test_server:seconds(400)),
+    erlang:trace_pattern({'_','_','_'}, false, [local,meta,call_time,call_count]),
+    erlang:trace_pattern(on_load, false, [local,meta,call_time,call_count]),
+    timer:now_diff(now(),now()),
+    [{watchdog, Dog}|Config].
+
+end_per_testcase(_Case, Config) ->
+    erlang:trace_pattern({'_','_','_'}, false, [local,meta,call_time,call_count]),
+    erlang:trace_pattern(on_load, false, [local,meta,call_time,call_count]),
+    erlang:trace(all, false, [all]),
+    Dog=?config(watchdog, Config),
+    test_server:timetrap_cancel(Dog),
+    ok.
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    case test_server:is_native(trace_call_time_SUITE) of
+	true -> [not_run];
+	false ->
+	    [basic, on_and_off, info, pause_and_restart, scheduling,
+	     combo, bif, nif, called_function]
+    end.
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
+not_run(Config) when is_list(Config) ->
+    {skipped,"Native code"}.
+
+basic(suite) ->
+    [];
+basic(doc) ->
+    ["Tests basic call count trace"];
+basic(Config) when is_list(Config) ->
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line M = 1000,
+    %%
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,  '_'}, true, [call_time]),
+    ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, true, [call_time]),
+    ?line Pid = setup(),
+    ?line {L,  T1} = execute(Pid, fun() -> seq(1, M, fun(X) -> (X+1) end) end),
+    ?line ok = check_trace_info({?MODULE, seq,   3}, [{Pid, M, 0, 0}], T1),
+    ?line ok = check_trace_info({?MODULE, seq_r, 3}, [], none),
+
+    ?line {Lr, T2} = execute(Pid, fun() -> seq_r(1, M, fun(X) -> (X+1) end) end),
+    ?line ok = check_trace_info({?MODULE, seq,   3}, [{Pid, M, 0, 0}], T1),
+    ?line ok = check_trace_info({?MODULE, seq_r, 3}, [{Pid, 1, 0, 0}], T2/M),
+    ?line ok = check_trace_info({?MODULE, seq_r, 4}, [{Pid, M, 0, 0}], T2),
+    ?line L = lists:reverse(Lr),
+
+    %%
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line Pid ! quit,
+    ok.
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+
+on_and_off(suite) ->
+    [];
+on_and_off(doc) ->
+    ["Tests turning trace parameters on and off"];
+on_and_off(Config) when is_list(Config) ->
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line M = 100,
+    %%
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, true, [call_time]),
+    ?line Pid = setup(),
+    ?line {L, T1} = execute(Pid, {?MODULE, seq, [1, M, fun(X) -> X+1 end]}),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid, M, 0, 0}], T1),
+
+    ?line N = erlang:trace_pattern({?MODULE,'_','_'}, true, [call_time]),
+    ?line {L, T2} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid, M, 0, 0}], T2),
+
+    ?line P = erlang:trace_pattern({'_','_','_'}, true, [call_time]),
+    ?line {L, T3} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid, M, 0, 0}], T3),
+
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, false, [call_time]),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, false, none),
+    ?line {L, _T4} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, false, none),
+    ?line ok = check_trace_info({?MODULE, seq_r, 4}, [], none),
+    ?line {Lr, T5} = execute(Pid, fun() -> seq_r(1, M, fun(X) -> X+1 end) end),
+    ?line ok = check_trace_info({?MODULE, seq_r, 4}, [{Pid,M,0,0}], T5),
+
+    ?line N = erlang:trace_pattern({?MODULE,'_','_'}, false, [call_time]),
+    ?line ok = check_trace_info({?MODULE, seq_r, 4}, false, none),
+    ?line {Lr, _T6} = execute(Pid, fun() -> seq_r(1, M, fun(X) -> X+1 end) end),
+    ?line ok = check_trace_info({?MODULE, seq_r, 4}, false, none),
+    ?line L = lists:reverse(Lr),
+    %%
+    ?line Pid ! quit,
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ok.
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+info(suite) ->
+    [];
+info(doc) ->
+    ["Tests the trace_info BIF"];
+info(Config) when is_list(Config) ->
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    %%
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,3}, true, [call_time]),
+    ?line {call_time,[]} = erlang:trace_info({?MODULE,seq,3}, call_time),
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, pause, [call_time]),
+    ?line {call_time,[]} = erlang:trace_info({?MODULE,seq,3}, call_time),
+    ?line {all,[_|_]=L} = erlang:trace_info({?MODULE,seq,3}, all),
+    ?line {value,{call_time,[]}} = lists:keysearch(call_time, 1, L),
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, restart, [call_time]),
+    ?line {call_time,[]} = erlang:trace_info({?MODULE,seq,3}, call_time),
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, false, [call_time]),
+    ?line {call_time,false} = erlang:trace_info({?MODULE,seq,3}, call_time),
+    ?line {all,false} = erlang:trace_info({?MODULE,seq,3}, all),
+    %%
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ok.
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+pause_and_restart(suite) ->
+    [];
+pause_and_restart(doc) ->
+    ["Tests pausing and restarting call time counters"];
+pause_and_restart(Config) when is_list(Config) ->
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line M = 100,
+    ?line Pid = setup(),
+    %%
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, true, [call_time]),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, [], none),
+    ?line {L, T1} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid,M,0,0}], T1),
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, pause, [call_time]),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid,M,0,0}], T1),
+    ?line {L, T2} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid,M,0,0}], T2),
+    ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, restart, [call_time]),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, [], none),
+    ?line {L, T3} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end),
+    ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid,M,0,0}], T3),
+    %%
+    ?line Pid ! quit,
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ok.
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+scheduling(suite) ->
+    [];
+scheduling(doc) ->
+    ["Tests in/out scheduling of call time counters"];
+scheduling(Config) when is_list(Config) ->
+    ?line P  = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line M  = 1000000,
+    ?line Np = erlang:system_info(schedulers_online),
+    ?line F  = 12,
+
+    %% setup load processes
+    %% (single, no internal calls)
+
+    ?line erlang:trace_pattern({?MODULE,loaded,1}, true, [call_time]),
+
+    ?line Pids     = [setup() || _ <- lists:seq(1, F*Np)],
+    ?line {_Ls,T1} = execute(Pids, {?MODULE,loaded,[M]}),
+    ?line [Pid ! quit || Pid <- Pids],
+
+    %% logic dictates that each process will get ~ 1/F of the schedulers time
+
+    ?line {call_time, CT} = erlang:trace_info({?MODULE,loaded,1}, call_time),
+
+    ?line lists:foreach(fun (Pid) ->
+	    ?line ok = case check_process_time(lists:keysearch(Pid, 1, CT), M, F, T1) of
+		schedule_time_error ->
+		    test_server:comment("Warning: Failed time ratio"),
+		    ok;
+		Other -> Other
+	    end
+	end, Pids),
+    ?line P  = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ok.
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+combo(suite) ->
+    [];
+combo(doc) ->
+    ["Tests combining local call trace and meta trace with call time trace"];
+combo(Config) when is_list(Config) ->
+    ?line Self = self(),
+    ?line Nbc = 3,
+    ?line MetaMs = [{'_',[],[{return_trace}]}],
+    ?line Flags = lists:sort([call, return_to]),
+    ?line LocalTracer = spawn_link(fun () -> relay_n(5 + Nbc + 3, Self) end),
+    ?line MetaTracer = spawn_link(fun () -> relay_n(9 + Nbc + 3, Self) end),
+    ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, [], [local]),
+    ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, true, [call_time]),
+    ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, MetaMs, [{meta,MetaTracer}]),
+    ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, true, [call_count]),
+
+    % bifs
+    ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, [], [local]),
+    ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, true, [call_time]),
+    ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, MetaMs, [{meta,MetaTracer}]),
+    %% not implemented
+    %?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, true, [call_count]),
+
+    ?line 1 = erlang:trace(Self, true, [{tracer,LocalTracer} | Flags]),
+    %%
+    ?line {traced,local} =
+	erlang:trace_info({?MODULE,seq_r,3}, traced),
+    ?line {match_spec,[]} =
+	erlang:trace_info({?MODULE,seq_r,3}, match_spec),
+    ?line {meta,MetaTracer} =
+	erlang:trace_info({?MODULE,seq_r,3}, meta),
+    ?line {meta_match_spec,MetaMs} =
+	erlang:trace_info({?MODULE,seq_r,3}, meta_match_spec),
+    ?line ok = check_trace_info({?MODULE, seq_r, 3}, [], none),
+
+    %% check empty trace_info for ?MODULE:seq_r/3
+    ?line {all,[_|_]=TraceInfo}     = erlang:trace_info({?MODULE,seq_r,3}, all),
+    ?line {value,{traced,local}}    = lists:keysearch(traced, 1, TraceInfo),
+    ?line {value,{match_spec,[]}}   = lists:keysearch(match_spec, 1, TraceInfo),
+    ?line {value,{meta,MetaTracer}} = lists:keysearch(meta, 1, TraceInfo),
+    ?line {value,{meta_match_spec,MetaMs}} = lists:keysearch(meta_match_spec, 1, TraceInfo),
+    ?line {value,{call_count,0}} = lists:keysearch(call_count, 1, TraceInfo),
+    ?line {value,{call_time,[]}} = lists:keysearch(call_time, 1, TraceInfo),
+
+    %% check empty trace_info for erlang:term_to_binary/1
+    ?line {all, [_|_] = TraceInfoBif} = erlang:trace_info({erlang, term_to_binary, 1}, all),
+    ?line {value,{traced,local}}     = lists:keysearch(traced, 1, TraceInfoBif),
+    ?line {value,{match_spec,[]}}    = lists:keysearch(match_spec, 1, TraceInfoBif),
+    ?line {value,{meta, MetaTracer}}  = lists:keysearch(meta, 1, TraceInfoBif),
+    ?line {value,{meta_match_spec,MetaMs}} = lists:keysearch(meta_match_spec, 1, TraceInfoBif),
+    %% not implemented
+    ?line {value,{call_count,false}} = lists:keysearch(call_count, 1, TraceInfoBif),
+    %?line {value,{call_count,0}} = lists:keysearch(call_count, 1, TraceInfoBif),
+    ?line {value,{call_time,[]}} = lists:keysearch(call_time, 1, TraceInfoBif),
+
+    %%
+    ?line [3,2,1] = seq_r(1, 3, fun(X) -> X+1 end),
+    ?line T0 = now(),
+    ?line with_bif(Nbc),
+    ?line T1 = now(),
+    ?line TimeB = timer:now_diff(T1,T0),
+    %%
+
+    ?line List = collect(100),
+    ?line {MetaR, LocalR} =
+	lists:foldl(
+	  fun ({P,X}, {M,L}) when P == MetaTracer ->
+		  {[X|M],L};
+	      ({P,X}, {M,L}) when P == LocalTracer ->
+		  {M,[X|L]}
+	  end,
+	  {[],[]},
+	  List),
+    ?line Meta = lists:reverse(MetaR),
+    ?line Local = lists:reverse(LocalR),
+
+    ?line [?CTT(Self,{?MODULE,seq_r,[1,3,_]}),
+	   ?CTT(Self,{?MODULE,seq_r,[1,3,_,[]]}),
+	   ?CTT(Self,{?MODULE,seq_r,[2,3,_,[1]]}),
+	   ?CTT(Self,{?MODULE,seq_r,[3,3,_,[2,1]]}),
+	   ?RFT(Self,{?MODULE,seq_r,4},[3,2,1]),
+	   ?RFT(Self,{?MODULE,seq_r,4},[3,2,1]),
+	   ?RFT(Self,{?MODULE,seq_r,4},[3,2,1]),
+	   ?RFT(Self,{?MODULE,seq_r,3},[3,2,1]),
+	   ?CTT(Self,{erlang,term_to_binary,[3]}), % bif
+	   ?RFT(Self,{erlang,term_to_binary,1},<<131,97,3>>),
+	   ?CTT(Self,{erlang,term_to_binary,[2]}),
+	   ?RFT(Self,{erlang,term_to_binary,1},<<131,97,2>>)
+	] = Meta,
+
+    ?line [?CT(Self,{?MODULE,seq_r,[1,3,_]}),
+	   ?CT(Self,{?MODULE,seq_r,[1,3,_,[]]}),
+	   ?CT(Self,{?MODULE,seq_r,[2,3,_,[1]]}),
+	   ?CT(Self,{?MODULE,seq_r,[3,3,_,[2,1]]}),
+	   ?RT(Self,{?MODULE,combo,1}),
+	   ?CT(Self,{erlang,term_to_binary,[3]}), % bif
+	   ?RT(Self,{?MODULE,with_bif,1}),
+	   ?CT(Self,{erlang,term_to_binary,[2]}),
+	   ?RT(Self,{?MODULE,with_bif,1})
+	] = Local,
+
+    ?line ok = check_trace_info({?MODULE, seq_r, 3}, [{Self,1,0,0}], 1),
+    ?line ok = check_trace_info({?MODULE, seq_r, 4}, [{Self,3,0,0}], 1),
+    ?line ok = check_trace_info({?MODULE, seq_r, 3}, [{Self,1,0,0}], 1),
+    ?line ok = check_trace_info({?MODULE, seq_r, 4}, [{Self,3,0,0}], 1),
+    ?line ok = check_trace_info({erlang, term_to_binary, 1}, [{self(), Nbc - 1, 0, 0}], TimeB),
+    %%
+    ?line erlang:trace_pattern({'_','_','_'}, false, [local,meta,call_time]),
+    ?line erlang:trace_pattern(on_load, false, [local,meta,call_time]),
+    ?line erlang:trace(all, false, [all]),
+    ok.
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+bif(suite) ->
+    [];
+bif(doc) ->
+    ["Tests tracing of bifs"];
+bif(Config) when is_list(Config) ->
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line M = 1000000,
+    %%
+    ?line 2 = erlang:trace_pattern({erlang, binary_to_term, '_'}, true, [call_time]),
+    ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, true, [call_time]),
+    ?line Pid = setup(),
+    ?line {L, T1} = execute(Pid, fun() -> with_bif(M) end),
+
+    ?line ok = check_trace_info({erlang, binary_to_term, 1}, [{Pid, M - 1, 0, 0}], T1/2),
+    ?line ok = check_trace_info({erlang, term_to_binary, 1}, [{Pid, M - 1, 0, 0}], T1/2),
+
+    % disable term2binary
+
+    ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, false, [call_time]),
+
+    ?line {L, T2} = execute(Pid, fun() -> with_bif(M) end),
+
+    ?line ok = check_trace_info({erlang, binary_to_term, 1}, [{Pid, M*2 - 2, 0, 0}], T1/2 + T2),
+    ?line ok = check_trace_info({erlang, term_to_binary, 1}, false, none),
+
+    %%
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line Pid ! quit,
+    ok.
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+nif(suite) ->
+    [];
+nif(doc) ->
+    ["Tests tracing of nifs"];
+nif(Config) when is_list(Config) ->
+    load_nif(Config),
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line M = 1000000,
+    %%
+    ?line 1 = erlang:trace_pattern({?MODULE, nif_dec,  '_'}, true, [call_time]),
+    ?line 1 = erlang:trace_pattern({?MODULE, with_nif, '_'}, true, [call_time]),
+    ?line Pid = setup(),
+    ?line {_, T1} = execute(Pid, fun() -> with_nif(M) end),
+
+    % the nif is called M - 1 times, the last time the function with 'with_nif'
+    % returns ok and does not call the nif.
+    ?line ok = check_trace_info({?MODULE, nif_dec,  1}, [{Pid, M-1, 0, 0}], T1/5*4),
+    ?line ok = check_trace_info({?MODULE, with_nif, 1}, [{Pid, M, 0, 0}], T1/5),
+
+    %%
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line Pid ! quit,
+    ok.
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+called_function(suite) ->
+    [];
+called_function(doc) ->
+    ["Tests combining nested function calls and that the time accumulates to the right function"];
+called_function(Config) when is_list(Config) ->
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ?line M = 2100,
+    ?line Pid = setup(),
+    %%
+    ?line 1 = erlang:trace_pattern({?MODULE,a_function,'_'}, true, [call_time]),
+    ?line {L, T1} = execute(Pid, {?MODULE, a_function, [M]}),
+    ?line ok = check_trace_info({?MODULE, a_function, 1}, [{Pid, M, 0, 0}], T1),
+
+    ?line 1 = erlang:trace_pattern({?MODULE,a_called_function,'_'}, true, [call_time]),
+    ?line {L, T2} = execute(Pid, {?MODULE, a_function, [M]}),
+    ?line ok = check_trace_info({?MODULE, a_function, 1}, [{Pid, M+M, 0, 0}], T1 + M*?SINGLE_CALL_US_TIME),
+    ?line ok = check_trace_info({?MODULE, a_called_function, 1}, [{Pid, M, 0, 0}], T2),
+
+
+    ?line 1 = erlang:trace_pattern({?MODULE,dec,'_'}, true, [call_time]),
+    ?line {L, T3} = execute(Pid, {?MODULE, a_function, [M]}),
+    ?line ok = check_trace_info({?MODULE, a_function, 1}, [{Pid, M+M+M, 0, 0}], T1 + (M+M)*?SINGLE_CALL_US_TIME),
+    ?line ok = check_trace_info({?MODULE, a_called_function, 1}, [{Pid, M+M, 0, 0}], T2 + M*?SINGLE_CALL_US_TIME ),
+    ?line ok = check_trace_info({?MODULE, dec, 1}, [{Pid, M, 0, 0}], T3),
+
+    ?line Pid ! quit,
+    ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]),
+    ok.
+
+%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%% The Tests
+%%%
+
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% Local helpers
+
+
+load_nif(Config) ->
+    ?line Path = ?config(data_dir, Config),
+    ?line ok = erlang:load_nif(filename:join(Path,"trace_nif"), 0).
+
+
+%% Stack recursive seq
+seq(Stop, Stop, Succ) when is_function(Succ) ->
+    [Stop];
+seq(Start, Stop, Succ) when is_function(Succ) ->
+    [Start | seq(Succ(Start), Stop, Succ)].
+
+
+a_function(1) -> a_called_function(1);
+a_function(N) when N > 1 -> a_function(a_called_function(N)).
+
+a_called_function(N) -> dec(N).
+
+with_bif(1) -> ok;
+with_bif(N) ->
+    with_bif(erlang:binary_to_term(erlang:term_to_binary(N)) - 1).
+
+with_nif(0) -> error;
+with_nif(1) -> ok;
+with_nif(N) ->
+    with_nif(?MODULE:nif_dec(N)).
+
+
+nif_dec(_) -> 0.
+
+dec(N) ->
+    loaded(10000),
+    N - 1.
+
+loaded(N) when N > 1 -> loaded(N - 1);
+loaded(_) -> 5.
+
+
+%% Tail recursive seq, result list is reversed
+seq_r(Start, Stop, Succ) when is_function(Succ) ->
+    seq_r(Start, Stop, Succ, []).
+
+seq_r(Stop, Stop, _, R) ->
+    [Stop | R];
+seq_r(Start, Stop, Succ, R) ->
+    seq_r(Succ(Start), Stop, Succ, [Start | R]).
+
+% Check call time tracing data and print mismatches
+check_trace_info(Mfa, [{Pid, C,_,_}] = Expect, Time) ->
+    case erlang:trace_info(Mfa, call_time) of
+	% Time tests are somewhat problematic. We want to know if Time (EXPECTED_TIME) and S*1000000 + Us (ACTUAL_TIME)
+	% is the same.
+	% If the ratio EXPECTED_TIME/ACTUAL_TIME is ~ 1 or if EXPECTED_TIME - ACTUAL_TIME is near zero, the test is ok.
+	{call_time,[{Pid,C,S,Us}]} when S >= 0, Us >= 0,  abs(1 - Time/(S*1000000 + Us)) < ?R_ERROR; abs(Time - S*1000000 - Us) < ?US_ERROR ->
+	    ok;
+	{call_time,[{Pid,C,S,Us}]} ->
+	    Sum = S*1000000 + Us,
+	    io:format("Expected ~p -> {call_time, ~p (Time ~p us)}~n - got ~w s. ~w us. = ~w us. - ~w -> delta ~w (ratio ~.2f, should be 1.0)~n",
+		[Mfa, Expect, Time, S, Us, Sum, Time, Sum - Time, Time/Sum]),
+	    time_error;
+	Other ->
+	    io:format("Expected ~p -> {call_time, ~p (Time ~p us)}~n - got ~p~n", [ Mfa, Expect, Time, Other]),
+	    time_count_error
+    end;
+check_trace_info(Mfa, Expect, _) ->
+    case erlang:trace_info(Mfa, call_time) of
+	{call_time, Expect} ->
+	    ok;
+	Other ->
+	    io:format("Expected ~p -> {call_time, ~p}~n - got ~p~n", [Mfa, Expect, Other]),
+	    result_not_expected_error
+    end.
+
+
+%check process time
+check_process_time({value,{Pid, M, S, Us}}, M, F, Time) ->
+    ?line Sum = S*1000000 + Us,
+    if
+	abs(1 - (F/(Time/Sum))) < ?R_ERROR ->
+	    ok;
+	true ->
+	    io:format("- Pid ~p, Got ratio ~.2f, expected ratio ~w~n", [Pid, Time/Sum,F]),
+	    schedule_time_error
+    end;
+check_process_time(Other, M, _, _) ->
+    io:format(" - Got ~p, expected count ~w~n", [Other, M]),
+    error.
+
+
+
+%% Message relay process
+relay_n(0, _) ->
+    ok;
+relay_n(N, Dest) ->
+    receive Msg ->
+	    Dest ! {self(), Msg},
+	    relay_n(N-1, Dest)
+    end.
+
+
+
+%% Collect received messages
+collect(Time) ->
+    Ref = erlang:start_timer(Time, self(), done),
+    L = lists:reverse(collect([], Ref)),
+    ?dbgformat("Got: ~p~n",[L]),
+    L.
+
+collect(A, 0) ->
+    receive
+	Mess ->
+	    collect([Mess | A], 0)
+    after 0 ->
+	    A
+    end;
+collect(A, Ref) ->
+    receive
+	{timeout, Ref, done} ->
+	    collect(A, 0);
+	Mess ->
+	    collect([Mess | A], Ref)
+    end.
+
+setup() ->
+    Pid = spawn_link(fun() -> loop() end),
+    ?line 1 = erlang:trace(Pid, true, [call]),
+    Pid.
+
+execute(Pids, Mfa) when is_list(Pids) ->
+    T0 = now(),
+    [P  ! {self(), execute, Mfa} || P <- Pids],
+    As = [receive {P, answer, Answer} -> Answer end || P <- Pids],
+    T1 = now(),
+    {As, timer:now_diff(T1,T0)};
+execute(P, Mfa) ->
+    T0 = now(),
+    P  ! {self(), execute, Mfa},
+    A  = receive {P, answer, Answer} -> Answer end,
+    T1 = now(),
+    {A, timer:now_diff(T1,T0)}.
+
+
+
+loop() ->
+    receive
+	quit ->
+	    ok;
+	{Pid, execute, Fun } when is_function(Fun) ->
+	    Pid ! {self(), answer, erlang:apply(Fun, [])},
+	    loop();
+	{Pid, execute, {M, F, A}} ->
+	    Pid ! {self(), answer, erlang:apply(M, F, A)},
+	    loop()
+    end.
diff --git a/erts/emulator/test/trace_call_time_SUITE_data/Makefile.src b/erts/emulator/test/trace_call_time_SUITE_data/Makefile.src
new file mode 100644
index 0000000000..2b2a35bd2c
--- /dev/null
+++ b/erts/emulator/test/trace_call_time_SUITE_data/Makefile.src
@@ -0,0 +1,6 @@
+
+NIF_LIBS = trace_nif@dll@
+
+all: $(NIF_LIBS)
+
+@SHLIB_RULES@
diff --git a/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c b/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c
new file mode 100644
index 0000000000..33b346aab7
--- /dev/null
+++ b/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c
@@ -0,0 +1,37 @@
+#include "erl_nif.h"
+
+
+static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
+{
+    return 0;
+}
+
+static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
+{
+    return 0;
+}
+
+static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info)
+{
+    return 0;
+}
+
+static void unload(ErlNifEnv* env, void* priv_data)
+{
+}
+
+static ERL_NIF_TERM nif_dec_1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    int x = 0;
+    enif_get_uint(env, argv[0], &x);
+    return enif_make_int(env, x - 1);
+}
+
+
+
+static ErlNifFunc nif_funcs[] =
+{
+    {"nif_dec", 1, nif_dec_1}
+};
+
+ERL_NIF_INIT(trace_call_time_SUITE,nif_funcs,load,reload,upgrade,unload)
diff --git a/erts/emulator/test/trace_local_SUITE.erl b/erts/emulator/test/trace_local_SUITE.erl
index 24005774ba..32e2a98e3c 100644
--- a/erts/emulator/test/trace_local_SUITE.erl
+++ b/erts/emulator/test/trace_local_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2000-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2000-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -45,7 +45,7 @@
 -export([config/2]).
 -define(DEFAULT_RECEIVE_TIMEOUT, 1000).
 -else.
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 -define(DEFAULT_RECEIVE_TIMEOUT, infinity).
 -endif.
  
@@ -68,7 +68,8 @@ config(priv_dir,_) ->
 
 %%% When run in test server %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
--export([all/1, basic/1, bit_syntax/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, basic/1, bit_syntax/1,
 	 return/1, on_and_off/1, stack_grow/1,info/1, delete/1,
 	 exception/1, exception_apply/1,
 	 exception_function/1, exception_apply_function/1,
@@ -79,34 +80,51 @@ config(priv_dir,_) ->
 	 exception_meta_nocatch/1, exception_meta_nocatch_apply/1,
 	 exception_meta_nocatch_function/1,
 	 exception_meta_nocatch_apply_function/1,
-	 init_per_testcase/2, fin_per_testcase/2]).
+	 init_per_testcase/2, end_per_testcase/2]).
 init_per_testcase(_Case, Config) ->
     ?line Dog=test_server:timetrap(test_server:minutes(2)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     shutdown(),
     Dog=?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
-all(doc) ->
-    ["Test tracing of local function calls and return traces."];
-all(suite) ->
-    case test_server:is_native(?MODULE) of
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    case test_server:is_native(trace_local_SUITE) of
 	true -> [not_run];
-	false -> [basic, bit_syntax, return, on_and_off, stack_grow, info, delete,
-		  exception, exception_apply,
-		  exception_function, exception_apply_function,
-		  exception_nocatch, exception_nocatch_apply,
-		  exception_nocatch_function, 
-		  exception_nocatch_apply_function,
-		  exception_meta, exception_meta_apply,
-		  exception_meta_function, exception_meta_apply_function,
-		  exception_meta_nocatch, exception_meta_nocatch_apply,
-		  exception_meta_nocatch_function,
-		  exception_meta_nocatch_apply_function]
+	false ->
+	    [basic, bit_syntax, return, on_and_off, stack_grow,
+	     info, delete, exception, exception_apply,
+	     exception_function, exception_apply_function,
+	     exception_nocatch, exception_nocatch_apply,
+	     exception_nocatch_function,
+	     exception_nocatch_apply_function, exception_meta,
+	     exception_meta_apply, exception_meta_function,
+	     exception_meta_apply_function, exception_meta_nocatch,
+	     exception_meta_nocatch_apply,
+	     exception_meta_nocatch_function,
+	     exception_meta_nocatch_apply_function]
     end.
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 not_run(Config) when is_list(Config) -> 
     {skipped,"Native code"}.
 
@@ -749,8 +767,8 @@ exception_test(Opts, Func0, Args0) ->
 	end,
     
     ?line R1 = exc_slave(ExcOpts, Func, Args),
-    ?line Stack2 = [{?MODULE,exc_top,3},{?MODULE,slave,2}],
-    ?line Stack3 = [{?MODULE,exc,2}|Stack2],
+    ?line Stack2 = [{?MODULE,exc_top,3,[]},{?MODULE,slave,2,[]}],
+    ?line Stack3 = [{?MODULE,exc,2,[]}|Stack2],
     ?line Rs = 
 	case x_exc_top(ExcOpts, Func, Args) of % Emulation
 	    {crash,{Reason,Stack}}=R when is_list(Stack) ->
@@ -771,21 +789,29 @@ exception_test(Opts, Func0, Args0) ->
 	  end,
     ?line expect({nm}).
 
-exception_validate(R1, [R2|Rs]) ->
+exception_validate(R0, Rs0) ->
+    R = clean_location(R0),
+    Rs = [clean_location(E) || E <- Rs0],
+    exception_validate_1(R, Rs).
+
+exception_validate_1(R1, [R2|Rs]) ->
     case [R1|R2] of
 	[R|R] ->
 	    ok;
-	[{crash,{badarg,[{lists,reverse,[L1a,L1b]}|T]}}|
-	 {crash,{badarg,[{lists,reverse,[L2a,L2b]}|T]}}] ->
+	[{crash,{badarg,[{lists,reverse,[L1a,L1b],_}|T]}}|
+	 {crash,{badarg,[{lists,reverse,[L2a,L2b],_}|T]}}] ->
 	    same({crash,{badarg,[{lists,reverse,
-				  [lists:reverse(L1b, L1a),[]]}|T]}},
+				  [lists:reverse(L1b, L1a),[]],[]}|T]}},
 		 {crash,{badarg,[{lists,reverse,
-				  [lists:reverse(L2b, L2a),[]]}|T]}});
+				  [lists:reverse(L2b, L2a),[]],[]}|T]}});
 	_ when is_list(Rs), Rs =/= [] ->
 	    exception_validate(R1, Rs)
     end.
 
-
+clean_location({crash,{Reason,Stk0}}) ->
+    Stk = [{M,F,A,[]} || {M,F,A,_} <- Stk0],
+    {crash,{Reason,Stk}};
+clean_location(Term) -> Term.
 
 %%% Tracee target functions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%
@@ -796,9 +822,6 @@ loop(D1,D2,D3,0) ->
 loop(D1,D2,D3,N) ->
     max(N,loop(D1,D2,D3,N-1)).
 
-max(A, B) when A > B -> A;
-max(_, B) -> B.
-
 exported_wrap(Val) ->
     exported(Val).
 
@@ -1042,10 +1065,10 @@ x_exc_exception(_Rtt, M, F, _, Arity, CR) ->
 x_exc_stacktrace() ->
     x_exc_stacktrace(erlang:get_stacktrace()).
 %% Truncate stacktrace to below exc/2
-x_exc_stacktrace([{?MODULE,x_exc,4}|_]) -> [];
-x_exc_stacktrace([{?MODULE,x_exc_func,4}|_]) -> [];
-x_exc_stacktrace([{?MODULE,x_exc_body,4}|_]) -> [];
-x_exc_stacktrace([{?MODULE,exc,2}|_]) -> [];
+x_exc_stacktrace([{?MODULE,x_exc,4,_}|_]) -> [];
+x_exc_stacktrace([{?MODULE,x_exc_func,4,_}|_]) -> [];
+x_exc_stacktrace([{?MODULE,x_exc_body,4,_}|_]) -> [];
+x_exc_stacktrace([{?MODULE,exc,2,_}|_]) -> [];
 x_exc_stacktrace([H|T]) ->
     [H|x_exc_stacktrace(T)].
 
diff --git a/erts/emulator/test/trace_meta_SUITE.erl b/erts/emulator/test/trace_meta_SUITE.erl
index d84cb3cdf2..45987cc319 100644
--- a/erts/emulator/test/trace_meta_SUITE.erl
+++ b/erts/emulator/test/trace_meta_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2002-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2002-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -45,7 +45,7 @@
 -define(config(A,B),config(A,B)).
 -export([config/2]).
 -else.
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 -endif.
  
 -ifdef(debug).
@@ -65,7 +65,9 @@ config(priv_dir,_) ->
     ".".
 -else.
 %% When run in test server.
--export([all/1, init_per_testcase/2, fin_per_testcase/2, not_run/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2, not_run/1]).
 -export([basic/1, return/1, on_and_off/1, stack_grow/1, 
 	 info/1, tracer/1, combo/1, nosilent/1]).
 	 
@@ -73,19 +75,36 @@ init_per_testcase(_Case, Config) ->
     ?line Dog=test_server:timetrap(test_server:minutes(5)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     shutdown(),
     Dog=?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     ok.
-all(doc) ->
-    ["Test meta tracing of local function calls and return trace."];
-all(suite) ->
-    case test_server:is_native(?MODULE) of
-	true -> [not_run];
-	false -> [basic, return, on_and_off, stack_grow, 
-		  info, tracer, combo, nosilent]
-    end.
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+case test_server:is_native(trace_meta_SUITE) of
+  true -> [not_run];
+  false ->
+      [basic, return, on_and_off, stack_grow, info, tracer,
+       combo, nosilent]
+end.
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+	Config.
+
+end_per_group(_GroupName, Config) ->
+	Config.
+
 
 not_run(Config) when is_list(Config) -> 
     {skipped,"Native code"}.
@@ -594,11 +613,6 @@ loop(D1,D2,D3,0) ->
 loop(D1,D2,D3,N) ->
     max(N,loop(D1,D2,D3,N-1)).
 
-max(A,B) when A > B ->
-    A;
-max(_A,B) ->
-    B.
-
 id(X) ->
     X.
 
diff --git a/erts/emulator/test/trace_nif_SUITE.erl b/erts/emulator/test/trace_nif_SUITE.erl
index 587cc08979..a7484a22fd 100644
--- a/erts/emulator/test/trace_nif_SUITE.erl
+++ b/erts/emulator/test/trace_nif_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2009. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,9 +19,10 @@
 
 -module(trace_nif_SUITE).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
--export([all/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2]).
 -export([trace_nif/1,
 	 trace_nif_timestamp/1,
 	 trace_nif_local/1,
@@ -32,19 +33,33 @@
 
 -export([nif_process/0, nif/0, nif/1]).
 
-all(suite) ->
-    case test_server:is_native(?MODULE) of
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    case test_server:is_native(trace_nif_SUITE) of
 	true -> [not_run];
 	false ->
-	    [trace_nif,
- 	     trace_nif_timestamp,
-	     trace_nif_local,
-	     trace_nif_meta,
-	     trace_nif_timestamp_local,
- 	     trace_nif_return
-	    ]
+	    [trace_nif, trace_nif_timestamp, trace_nif_local,
+	     trace_nif_meta, trace_nif_timestamp_local,
+	     trace_nif_return]
     end.
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 not_run(Config) when is_list(Config) -> 
     {skipped,"Native code"}.
 
diff --git a/erts/emulator/test/trace_port_SUITE.erl b/erts/emulator/test/trace_port_SUITE.erl
index 5febe177f9..f81cab3114 100644
--- a/erts/emulator/test/trace_port_SUITE.erl
+++ b/erts/emulator/test/trace_port_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1999-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1999-2012. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -20,7 +20,9 @@
 
 -module(trace_port_SUITE).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 call_trace/1,
 	 return_trace/1,
 	 send/1,
@@ -34,35 +36,49 @@
 	 gc/1,
 	 default_tracer/1]).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
-test_cases() ->
-    [call_trace,
-     return_trace,
-     send,
-     receive_trace,
-     process_events,
-     schedule,
-     fake_schedule,
+test_cases() -> 
+    [call_trace, return_trace, send, receive_trace,
+     process_events, schedule, fake_schedule,
      fake_schedule_after_register,
      fake_schedule_after_getting_linked,
-     fake_schedule_after_getting_unlinked,
-     gc,
+     fake_schedule_after_getting_unlinked, gc,
      default_tracer].
 
-all(suite) -> test_cases().
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    test_cases().
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) ->
     Dog = ?t:timetrap(?t:seconds(30)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog).
 
 call_trace(doc) -> "Test sending call trace messages to a port.";
 call_trace(Config) when is_list(Config) ->
-    case test_server:is_native(?MODULE) of
+    case test_server:is_native(?MODULE) orelse
+	test_server:is_native(lists) of
 	true -> 
 	    {skip,"Native code"};
 	false ->
@@ -113,7 +129,8 @@ bs_sum_c(<<>>, Acc) -> Acc.
 
 return_trace(doc) -> "Test the new return trace.";
 return_trace(Config) when is_list(Config) ->
-    case test_server:is_native(?MODULE) of
+    case test_server:is_native(?MODULE) orelse
+	test_server:is_native(lists) of
 	true -> 
 	    {skip,"Native code"};
 	false ->
diff --git a/erts/emulator/test/trace_port_SUITE_data/echo_drv.c b/erts/emulator/test/trace_port_SUITE_data/echo_drv.c
index 15c4ca11fe..a8d4ede4fe 100644
--- a/erts/emulator/test/trace_port_SUITE_data/echo_drv.c
+++ b/erts/emulator/test/trace_port_SUITE_data/echo_drv.c
@@ -25,12 +25,14 @@ static EchoDrvData echo_drv_data, *echo_drv_data_p;
 **/
 
 static EchoDrvData *echo_drv_start(ErlDrvPort port, char *command);
-static void         echo_drv_stop(EchoDrvData *data_p);
-static void         echo_drv_output(EchoDrvData *data_p, char *buf, int len);
+static void         echo_drv_stop(ErlDrvData drv_data);
+static void         echo_drv_output(ErlDrvData drv_data, char *buf,
+				    ErlDrvSizeT len);
 static void         echo_drv_finish(void);
-static int          echo_drv_control(EchoDrvData *data_p, unsigned int command,
-				     char *buf, int len,
-				     char **rbuf, int rlen);
+static ErlDrvSSizeT echo_drv_control(ErlDrvData drv_data,
+				     unsigned int command,
+				     char *buf,  ErlDrvSizeT len,
+				     char **rbuf, ErlDrvSizeT rlen);
 
 static ErlDrvEntry echo_drv_entry = { 
     NULL, /* init */
@@ -45,11 +47,19 @@ static ErlDrvEntry echo_drv_entry = {
     echo_drv_control,
     NULL, /* timeout */
     NULL, /* outputv */
-    NULL  /* ready_async */
+    NULL, /* ready_async */
+    NULL,
+    NULL,
+    NULL,
+    ERL_DRV_EXTENDED_MARKER,
+    ERL_DRV_EXTENDED_MAJOR_VERSION,
+    ERL_DRV_EXTENDED_MINOR_VERSION,
+    0,
+    NULL,
+    NULL,
+    NULL
 };
 
-
-
 /* -------------------------------------------------------------------------
 ** Entry functions
 **/
@@ -75,7 +85,8 @@ static void echo_drv_stop(EchoDrvData *data_p) {
     echo_drv_data_p = NULL;
 }
 
-static void echo_drv_output(EchoDrvData *data_p, char *buf, int len) {
+static void echo_drv_output(ErlDrvData drv_data, char *buf, ErlDrvSizeT len) {
+    EchoDrvData* data_p = (EchoDrvData *) drv_data;
     driver_output(data_p->erlang_port, buf, len);
     switch (data_p->heavy) {
     case heavy_off:
@@ -95,9 +106,11 @@ static void echo_drv_finish() {
     echo_drv_data_p = NULL;
 }
 
-static int echo_drv_control(EchoDrvData *data_p, unsigned int command,
-			    char *buf, int len,
-			    char **rbuf, int rlen) {
+static ErlDrvSSizeT echo_drv_control(ErlDrvData drv_data,
+				     unsigned int command,
+				     char *buf, ErlDrvSizeT len,
+				     char **rbuf, ErlDrvSizeT rlen) {
+    EchoDrvData* data_p = (EchoDrvData *) drv_data;
     switch (command) {
     case 'h':
 	data_p->heavy = heavy_set;
diff --git a/erts/emulator/test/tuple_SUITE.erl b/erts/emulator/test/tuple_SUITE.erl
index c4edb16d68..bfc3910742 100644
--- a/erts/emulator/test/tuple_SUITE.erl
+++ b/erts/emulator/test/tuple_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -17,11 +17,13 @@
 %% %CopyrightEnd%
 %%
 -module(tuple_SUITE).
--export([all/1, t_size/1, t_tuple_size/1, t_element/1, t_setelement/1,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 t_size/1, t_tuple_size/1, t_element/1, t_setelement/1,
 	 t_list_to_tuple/1, t_tuple_to_list/1,
 	 t_make_tuple_2/1, t_make_tuple_3/1, t_append_element/1,
 	 build_and_match/1, tuple_with_case/1, tuple_in_guard/1]).
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %% Tests tuples and the BIFs:
 %%
@@ -33,13 +35,30 @@
 %% make_tuple/2
 %%
 
-all(suite) ->
-    [build_and_match, t_size, t_tuple_size,
-     t_list_to_tuple, t_tuple_to_list,
-     t_element, t_setelement, t_make_tuple_2,
-     t_make_tuple_3, t_append_element,
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [build_and_match, t_size, t_tuple_size, t_list_to_tuple,
+     t_tuple_to_list, t_element, t_setelement,
+     t_make_tuple_2, t_make_tuple_3, t_append_element,
      tuple_with_case, tuple_in_guard].
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 build_and_match(Config) when is_list(Config) ->
     ?line {} = id({}),
     ?line {1} = id({1}),
@@ -80,7 +99,7 @@ t_tuple_size(Config) when is_list(Config) ->
 
 ludicrous_tuple_size(T)
   when tuple_size(T) =:= 16#7777777777777777777777777777777777 -> ok;
-ludicrous_tuple_size(T) -> error.
+ludicrous_tuple_size(_) -> error.
 
 %% Tests element/2.
 
diff --git a/erts/emulator/test/z_SUITE.erl b/erts/emulator/test/z_SUITE.erl
index 67d2b288a2..4b3075a164 100644
--- a/erts/emulator/test/z_SUITE.erl
+++ b/erts/emulator/test/z_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2011. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -29,10 +29,12 @@
 
 %-define(line_trace, 1).
 
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 
 %-compile(export_all).
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, init_per_testcase/2, 
+	 end_per_testcase/2]).
 
 -export([schedulers_alive/1, node_container_refc_check/1,
 	 long_timers/1, pollset_size/1,
@@ -40,19 +42,33 @@
 
 -define(DEFAULT_TIMEOUT, ?t:minutes(5)).
 
-all(doc) -> [];
-all(suite) ->
-    [schedulers_alive,
-     node_container_refc_check,
-     long_timers,
-     pollset_size,
-     check_io_debug].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [schedulers_alive, node_container_refc_check,
+     long_timers, pollset_size, check_io_debug].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_testcase(_Case, Config) when is_list(Config) ->
     Dog = ?t:timetrap(?DEFAULT_TIMEOUT),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) when is_list(Config) ->
+end_per_testcase(_Case, Config) when is_list(Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 4a859c3094..8fe2402ca8 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -1,20 +1,20 @@
 #!/usr/bin/env perl
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 1998-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 1998-2012. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
 use strict;
@@ -27,6 +27,8 @@ my $outdir = ".";		# Directory for output files.
 my $verbose = 0;
 my $hot = 1;
 my $num_file_opcodes = 0;
+my $wordsize = 32;
+my %defs;			# Defines (from command line).
 
 # This is shift counts and mask for the packer.
 my $WHOLE_WORD = '';
@@ -36,12 +38,20 @@ my @pack_mask;
 
 $pack_instr[2] = ['6', 'i'];
 $pack_instr[3] = ['0', '0', 'i'];
+$pack_instr[4] = ['6', '6', '6', 'i']; # Only for 64 bit wordsize
 
 $pack_shift[2] = ['0', 'BEAM_LOOSE_SHIFT'];
 $pack_shift[3] = ['0', 'BEAM_TIGHT_SHIFT', '(2*BEAM_TIGHT_SHIFT)'];
+$pack_shift[4] = ['0', 'BEAM_LOOSE_SHIFT', # Only for 64 bit wordsize
+		  '(2*BEAM_LOOSE_SHIFT)',
+		  '(3*BEAM_LOOSE_SHIFT)'];
 
 $pack_mask[2]  = ['BEAM_LOOSE_MASK', $WHOLE_WORD];
 $pack_mask[3]  = ['BEAM_TIGHT_MASK', 'BEAM_TIGHT_MASK', 'BEAM_TIGHT_MASK'];
+$pack_mask[4]  = ['BEAM_LOOSE_MASK', # Only for 64 bit wordsize
+		  'BEAM_LOOSE_MASK',
+		  'BEAM_LOOSE_MASK',
+		  $WHOLE_WORD];
 
 # There are two types of instructions: generic and specific.
 # The generic instructions are those generated by the Beam compiler.
@@ -58,6 +68,10 @@ my $max_gen_operands = 8;
 # Must be even. The beam_load.c file must be updated, too.
 my $max_spec_operands = 6;
 
+# The maximum number of primitive genop_types.
+
+my $max_genop_types = 16;
+
 my %gen_opnum;
 my %num_specific;
 my %gen_to_spec;
@@ -80,6 +94,14 @@ my %cold_code;
 my @unnumbered_generic;
 my %unnumbered;
 
+my %is_transformed;
+
+#
+# Pre-processor.
+#
+my @if_val;
+my @if_line;
+
 #
 # Code transformations.
 #
@@ -90,12 +112,14 @@ my %match_engine_ops;		# All opcodes for the match engine.
 my %gen_transform_offset;
 my @transformations;
 my @call_table;
+my %call_table;
 my @pred_table;
+my %pred_table;
 
 # Operand types for generic instructions.
 
 my $compiler_types = "uiaxyfhz";
-my $loader_types = "nprvlq";
+my $loader_types = "nprvlqo";
 my $genop_types = $compiler_types . $loader_types;
 
 #
@@ -118,7 +142,8 @@ my %arg_size = ('r' => 0,	# x(0) - x register zero
 		't' => 1,	# untagged integer -- can be packed
 		'b' => 1,	# pointer to bif
 		'A' => 1,	# arity value
-		'P' => 1,	# byte offset into tuple
+		'P' => 1,	# byte offset into tuple or stack
+		'Q' => 1,	# like 'P', but packable
 		'h' => 1,	# character
 		'l' => 1,	# float reg
 		'q' => 1,	# literal term
@@ -130,33 +155,67 @@ my %arg_size = ('r' => 0,	# x(0) - x register zero
 my %type_bit;
 my @tag_type;
 
+sub define_type_bit {
+    my($tag,$val) = @_;
+    defined $type_bit{$tag} and
+	sanity("the tag '$tag' has already been defined with the value ",
+	       $type_bit{$tag});
+    $type_bit{$tag} = $val;
+}
+
 {
     my($bit) = 1;
     my(%bit);
 
     foreach (split('', $genop_types)) {
 	push(@tag_type, $_);
-	$type_bit{$_} = $bit;
+	define_type_bit($_, $bit);
 	$bit{$_} = $bit;
 	$bit *= 2;
     }
 
     # Composed types.
-    $type_bit{'d'} = $type_bit{'x'} | $type_bit{'y'} | $type_bit{'r'};
-    $type_bit{'c'} = $type_bit{'i'} | $type_bit{'a'} | $type_bit{'n'} | $type_bit{'q'};
-    $type_bit{'s'} = $type_bit{'d'} | $type_bit{'i'} | $type_bit{'a'} |	$type_bit{'n'};
-    $type_bit{'j'} = $type_bit{'f'} | $type_bit{'p'};
+    define_type_bit('d', $type_bit{'x'} | $type_bit{'y'} | $type_bit{'r'});
+    define_type_bit('c', $type_bit{'i'} | $type_bit{'a'} |
+		    $type_bit{'n'} | $type_bit{'q'});
+    define_type_bit('s', $type_bit{'d'} | $type_bit{'i'} |
+		    $type_bit{'a'} | $type_bit{'n'});
+    define_type_bit('j', $type_bit{'f'} | $type_bit{'p'});
 
     # Aliases (for matching purposes).
-    $type_bit{'I'} = $type_bit{'u'};
-    $type_bit{'t'} = $type_bit{'u'};
-    $type_bit{'A'} = $type_bit{'u'};
-    $type_bit{'L'} = $type_bit{'u'};
-    $type_bit{'b'} = $type_bit{'u'};
-    $type_bit{'N'} = $type_bit{'u'};
-    $type_bit{'U'} = $type_bit{'u'};
-    $type_bit{'e'} = $type_bit{'u'};
-    $type_bit{'P'} = $type_bit{'u'};
+    define_type_bit('I', $type_bit{'u'});
+    define_type_bit('t', $type_bit{'u'});
+    define_type_bit('A', $type_bit{'u'});
+    define_type_bit('L', $type_bit{'u'});
+    define_type_bit('b', $type_bit{'u'});
+    define_type_bit('N', $type_bit{'u'});
+    define_type_bit('U', $type_bit{'u'});
+    define_type_bit('e', $type_bit{'u'});
+    define_type_bit('P', $type_bit{'u'});
+    define_type_bit('Q', $type_bit{'u'});
+}
+
+#
+# Pre-define the 'fail' instruction. It is used internally
+# by the 'try_me_else_fail' instruction.
+#
+$match_engine_ops{'TOP_fail'} = 1;
+
+#
+# Sanity checks.
+#
+
+{
+    if (@tag_type > $max_genop_types) {
+	sanity("\$max_genop_types is $max_genop_types, ",
+	       "but there are ", scalar(@tag_type),
+	       " primitive tags defined\n");
+    }
+
+    foreach my $tag (@tag_type) {
+	sanity("tag '$tag': primitive tags must be named with lowercase letters")
+	    unless $tag =~ /^[a-z]$/;
+    }
 }
 
 #
@@ -169,7 +228,9 @@ while (@ARGV && $ARGV[0] =~ /^-(.*)/) {
     ($target = \&emulator_output), next if /^emulator/;
     ($target = \&compiler_output), next if /^compiler/;
     ($outdir = shift), next if /^outdir/;
+    ($wordsize = shift), next if /^wordsize/;
     ($verbose = 1), next if /^v/;
+    ($defs{$1} = $2), next if /^D(\w+)=(\w+)/;
     die "$0: Bad option: -$_\n";
 }
 
@@ -186,7 +247,43 @@ while (<>) {
     }
     next if /^\s*$/;
     next if /^\#/;
-    
+
+    #
+    # Handle %if.
+    #
+    if (/^\%if (\w+)/) {
+	my $name = $1;
+	my $val = $defs{$name};
+	defined $val or error("'$name' is undefined");
+	push @if_val, $val;
+	push @if_line, $.;
+	next;
+    } elsif (/^\%unless (\w+)/) {
+	my $name = $1;
+	my $val = $defs{$name};
+	defined $val or error("'$name' is undefined");
+	push @if_val, !$val;
+	push @if_line, $.;
+	next;
+    } elsif (/^\%else$/) {
+	unless (@if_line) {
+	    error("%else without a preceding %if/%unless");
+	}
+	$if_line[$#if_line] = $.;
+	$if_val[$#if_val] = !$if_val[$#if_val];
+	next;
+    } elsif (/^\%endif$/) {
+	unless (@if_line) {
+	    error("%endif without a preceding %if/%unless/%else");
+	}
+	pop @if_val;
+	pop @if_line;
+	next;
+    }
+    if (@if_val and not $if_val[$#if_val]) {
+	next;
+    }
+
     #
     # Handle assignments.
     #
@@ -296,7 +393,13 @@ while (<>) {
 	$unnumbered{$name,$arity} = 1;
     }
 } continue {
-    close(ARGV) if eof(ARGV);
+    if (eof(ARGV)) {
+	close(ARGV);
+	if (@if_line) {
+	    error("Unterminated %if/%unless/%else at " .
+		  "line $if_line[$#if_line]\n");
+	}
+    }
 }
 
 $num_file_opcodes = @gen_opname;
@@ -422,12 +525,12 @@ sub emulator_output {
 	    #
 
 	    my(@bits) = (0) x ($max_spec_operands/2);
-	    my($shift) = 16;
 	    my($i);
 	    for ($i = 0; $i < $max_spec_operands && defined $args[$i]; $i++) {
 		my $t = $args[$i];
 		if (defined $type_bit{$t}) {
-		    $bits[int($i/2)] |= $type_bit{$t} << (16*($i%2));
+		    my $shift = $max_genop_types * ($i % 2);
+		    $bits[int($i/2)] |= $type_bit{$t} << $shift;
 		}
 	    }
 
@@ -474,8 +577,9 @@ sub emulator_output {
 		$gen_transform_offset{$key} : -1;
 	    my($spec_op) = $gen_to_spec{$key};
 	    my($num_specific) = $num_specific{$key};
-	    defined $spec_op or $tr != -1 or
+	    defined $spec_op or
 		$obsolete[$gen_opnum{$name,$arity}] or
+		$is_transformed{$name,$arity} or
 		error("instruction $key has no specific instruction");
 	    $spec_op = -1 unless defined $spec_op;
 	    &init_item($name, $arity, $spec_op, $num_specific, $tr, $min_window{$key});
@@ -498,8 +602,14 @@ sub emulator_output {
     print "#define NUM_SPECIFIC_OPS ", scalar(@op_to_name), "\n";
     print "\n";
     print "#ifdef ARCH_64\n";
+    print "#  define BEAM_WIDE_MASK 0xFFFFUL\n";
     print "#  define BEAM_LOOSE_MASK 0x1FFFUL\n";
+    print "#if HALFWORD_HEAP\n";
+    print "#  define BEAM_TIGHT_MASK 0x1FFCUL\n";
+    print "#else\n";
     print "#  define BEAM_TIGHT_MASK 0x1FF8UL\n";
+    print "#endif\n";
+    print "#  define BEAM_WIDE_SHIFT 32\n";
     print "#  define BEAM_LOOSE_SHIFT 16\n";
     print "#  define BEAM_TIGHT_SHIFT 16\n";
     print "#else\n";
@@ -732,6 +842,10 @@ sub error {
     die $where, @message, "\n";
 }
 
+sub sanity {
+    die "internal error: ", @_, "\n";
+}
+
 sub comment {
     my($lang, @comments) = @_;
     my($prefix);
@@ -792,6 +906,7 @@ sub basic_generator {
 		     'I' => 1,
 		     't' => 1,
 		     'P' => 1,
+		     'Q' => 1,
 		     );
 
     # Pick up the macro to use and its flags (if any).
@@ -813,7 +928,7 @@ sub basic_generator {
     #
 
     if ($flags =~ /-pack/ && $hot) {
-	($prefix, $pack_spec, @args) = &do_pack(@args);
+        ($prefix, $pack_spec, @args) = &do_pack(@args);
     }
 
     #
@@ -907,16 +1022,27 @@ sub basic_generator {
     my($code);
     if (defined $macro{$name}) {
 	my($macro_code) = "$prefix$macro(" . join(', ', @f) . ");";
-	$var_decls .= "Uint tmp_packed1;"
+	$var_decls .= "BeamInstr tmp_packed1;"
 	    if $macro_code =~ /tmp_packed1/;
-	$var_decls .= "Uint tmp_packed2;"
+	$var_decls .= "BeamInstr tmp_packed2;"
 	    if $macro_code =~ /tmp_packed2/;
 	if ($flags =~ /-nonext/) {
-	    $code = "$macro_code\n";
+	    $code = join("\n",
+			 "{ $var_decls",
+			 $macro_code,
+			 "}");
+	} elsif ($flags =~ /-goto:(\S*)/) {
+	    my $goto = $1;
+	    $code = join("\n",
+			 "{ $var_decls",
+			 $macro_code,
+			 "I += $size + 1;",
+			 "goto $goto;",
+			 "}");
 	} else {
 	    $code = join("\n",
 			 "{ $var_decls",
-			 "Eterm* next;",
+			 "BeamInstr* next;",
 			 "PreFetch($size, next);",
 			 "$macro_code",
 			 "NextPF($size, next);",
@@ -931,18 +1057,31 @@ sub basic_generator {
 
 sub do_pack {
     my(@args) = @_;
-    my($i);
     my($packable_args) = 0;
+    my @is_packable;		# Packability (boolean) for each argument.
+    my $wide_packing = 0;
 
     #
     # Count the number of packable arguments.  If we encounter any 's' or 'd'
     # arguments, packing is not possible.
     #
-    for ($i = 0; $i < @args; $i++) {
-	if ($args[$i] =~ /[xyt]/) {
+    my $packable_types = "xytQ";
+    foreach my $arg (@args) {
+	if ($arg =~ /^[$packable_types]/) {
 	    $packable_args++;
-	} elsif ($args[$i] =~ /[sd]/) {
+	    push @is_packable, 1;
+	} elsif ($arg =~ /^I/ and $wordsize == 64 and $packable_args < 2) {
+	    $wide_packing = 1;
+	    push @is_packable, 1;
+	    if (++$packable_args == 2) {
+		# We can only pack two arguments. Turn off packing
+		# for the rest of the arguments.
+		$packable_types = "\xFF";
+	    }
+	} elsif ($arg =~ /^[sd]/) {
 	    return ('', '', @args);
+	} else {
+	    push @is_packable, 0;
 	}
     }
 
@@ -958,10 +1097,27 @@ sub do_pack {
 				# beginning).
     my($up) = '';		# Pack commands (storing back while
 				# moving forward).
-    my($args_per_word) = $packable_args < 4 ? $packable_args : 2;
-    my(@shift) = @{$pack_shift[$args_per_word]};
-    my(@mask) = @{$pack_mask[$args_per_word]};
-    my(@pack_instr) = @{$pack_instr[$args_per_word]};
+    my $args_per_word;
+    if ($packable_args < 4 or $wordsize == 64) {
+	$args_per_word = $packable_args;
+    } else {
+	# 4 packable argument, 32 bit wordsize. Need 2 words.
+	$args_per_word = 2;
+    }
+
+    my @shift;
+    my @mask;
+    my @instr;
+
+    if ($wide_packing) {
+	@shift = ('0', 'BEAM_WIDE_SHIFT');
+	@mask = ('BEAM_WIDE_MASK', $WHOLE_WORD);
+	@instr = ('w', 'i');
+    } else {
+	@shift = @{$pack_shift[$args_per_word]};
+	@mask = @{$pack_mask[$args_per_word]};
+	@instr = @{$pack_instr[$args_per_word]};
+    }
 
     #
     # Now generate the packing instructions.  One complication is that
@@ -975,10 +1131,10 @@ sub do_pack {
     my($ap) = 0;		# Argument number within word.
     my($tmpnum) = 1;		# Number of temporary variable.
     my($expr) = '';
-    for ($i = 0; $i < @args; $i++) {
+    for (my $i = 0; $i < @args; $i++) {
 	my($reg) = $args[$i];
 	my($this_size) = $arg_size{$reg};
-	if ($reg =~ /[xyt]/) {
+	if ($is_packable[$i]) {
 	    $this_size = 0;
 	    $did_some_packing = 1;
 
@@ -989,7 +1145,7 @@ sub do_pack {
 		$this_size = 1;
 	    }
 
-	    $down = "$pack_instr[$ap]$down";
+	    $down = "$instr[$ap]$down";
 	    my($unpack) = &make_unpack($tmpnum, $shift[$ap], $mask[$ap]);
 	    $args[$i] = "pack:$this_size:$reg" . "b($unpack)";
 
@@ -1099,6 +1255,10 @@ sub compile_transform {
     if ($obsolete[$gen_opnum{$name,$arity}]) {
 	error("obsolete function must not be used in transformations");
     }
+
+    if ($src) {
+	$is_transformed{$name,$arity} = 1;
+    }
     
     [$name,$arity,@ops];
 }
@@ -1202,7 +1362,8 @@ sub tr_gen {
 
     foreach $ref (@g) {
 	my($line, $orig_transform, $from_ref, $to_ref) = @$ref;
-	my $so_far = tr_gen_from($line, @$from_ref);
+	my $used_ref = used_vars($from_ref, $to_ref);
+	my $so_far = tr_gen_from($line, $used_ref, @$from_ref);
 	tr_gen_to($line, $orig_transform, $so_far, @$to_ref);
     }
 
@@ -1211,9 +1372,22 @@ sub tr_gen {
     #
     my($offset) = 0;
     print "Uint op_transform[] = {\n";
-    foreach $key (keys %gen_transform) {
+    foreach $key (sort keys %gen_transform) {
 	$gen_transform_offset{$key} = $offset;
-	foreach $instr (@{$gen_transform{$key}}) {
+	my @instr = @{$gen_transform{$key}};
+
+	#
+	# If the last instruction is 'fail', remove it and
+	# convert the previous 'try_me_else' to 'try_me_else_fail'.
+	#
+	if (is_instr($instr[$#instr], 'fail')) {
+	    pop(@instr);
+	    my $i = $#instr;
+	    $i-- while !is_instr($instr[$i], 'try_me_else');
+	    $instr[$i] = make_op('', 'try_me_else_fail');
+	}
+
+	foreach $instr (@instr) {
 	    my($size, $instr_ref, $comment) = @$instr;
 	    my($op, @args) = @$instr_ref;
 	    print "    ";
@@ -1240,8 +1414,48 @@ sub tr_gen {
     print "};\n\n";
 }
 
+sub used_vars {
+    my($from_ref,$to_ref) = @_;
+    my %used;
+    my %seen;
+
+    foreach my $ref (@$from_ref) {
+	my($name,$arity,@ops) = @$ref;
+	if ($name =~ /^[.]/) {
+	    foreach my $var (@ops) {
+		$used{$var} = 1;
+	    }
+	} else {
+	    # Any variable that is used at least twice on the
+	    # left-hand side is used. (E.g. "move R R".)
+	    foreach my $op (@ops) {
+		my($var, $type, $type_val) = @$op;
+		next if $var eq '';
+		$used{$var} = 1 if $seen{$var};
+		$seen{$var} = 1;
+	    }
+	}
+    }
+
+    foreach my $ref (@$to_ref) {
+	my($name, $arity, @ops) = @$ref;
+	if ($name =~ /^[.]/) {
+	    foreach my $var (@ops) {
+		$used{$var} = 1;
+	    }
+	} else {
+	    foreach my $op (@ops) {
+		my($var, $type, $type_val) = @$op;
+		next if $var eq '';
+		$used{$var} = 1;
+	    }
+	}
+    }
+    \%used;
+}
+
 sub tr_gen_from {
-    my($line, @tr) = @_;
+    my($line,$used_ref,@tr) = @_;
     my(%var) = ();
     my(%var_type);
     my($var_num) = 0;
@@ -1251,25 +1465,30 @@ sub tr_gen_from {
     my(@fix_pred_funcs);
     my($op, $ref);		# Loop variables.
     my $where = "left side of transformation in line $line: ";
+    my %var_used = %$used_ref;
+    my $may_fail = 0;
+    my $is_first = 1;
 
     foreach $ref (@tr) {
 	my($name, $arity, @ops) = @$ref;
 	my($key) = "$name/$arity";
 	my($opnum);
 
+	$may_fail = 1 unless $is_first;
+	$is_first = 0;
+
 	#
 	# A name starting with a period is a C pred function to be called.
 	#
 
 	if ($name =~ /^\.(\w+)/) {
 	    $name = $1;
+	    $may_fail = 1;
 	    my $var;
 	    my(@args);
 
-	    my $next_instr = pop(@code); # Get rid of 'next_instr'
 	    push(@fix_pred_funcs, scalar(@code));
 	    push(@code, [$name, @ops]);
-	    push(@code, $next_instr);
 	    next;
 	}
 
@@ -1281,63 +1500,91 @@ sub tr_gen_from {
 	    unless defined $gen_opnum{$name,$arity};
 	$opnum = $gen_opnum{$name,$arity};
 
-	push(@code, &make_op("$name/$arity", 'is_op', $opnum));
+	push(@code, make_op("$name/$arity", 'next_instr', $opnum));
 	$min_window++;
 	foreach $op (@ops) {
 	    my($var, $type, $type_val, $cond, $val) = @$op;
+	    my $ignored_var = "$var (ignored)";
 
 	    if ($type ne '' && $type ne '*') {
-		my($types) = '';
-		my($type_mask) = 0;
-		foreach (split('', $type)) {
-		    $types .= "$_ ";
-		    $type_mask |= $type_bit{$_};
+		$may_fail = 1;
+
+		#
+		# The is_bif, is_not_bif, and is_func instructions have
+		# their own built-in type test and don't need to
+		# be guarded with a type test instruction.
+		#
+		$ignored_var = '';
+		unless ($cond eq 'is_bif' or
+			$cond eq 'is_not_bif' or
+			$cond eq 'is_func') {
+		    my($types) = '';
+		    my($type_mask) = 0;
+		    foreach (split('', $type)) {
+			$types .= "$_ ";
+			$type_mask |= $type_bit{$_};
+		    }
+		    if ($cond ne 'is_eq') {
+			push(@code, &make_op($types, 'is_type', $type_mask));
+		    } else {
+			$cond = '';
+			push(@code, &make_op("$types== $val", 'is_type_eq',
+					     $type_mask, $val));
+		    }
 		}
-		push(@code, &make_op($types, 'is_type', $type_mask));
 	    }
 
 	    if ($cond eq 'is_func') {
 		my($m, $f, $a) = split(/:/, $val);
+		$ignored_var = '';
+		$may_fail = 1;
 		push(@code, &make_op('', "$cond", "am_$m",
 				     "am_$f", $a));
 	    } elsif ($cond ne '') {
+		$ignored_var = '';
+		$may_fail = 1;
 		push(@code, &make_op('', "$cond", $val));
 	    }
 
 	    if ($var ne '') {
 		if (defined $var{$var}) {
+		    $ignored_var = '';
+		    $may_fail = 1;
 		    push(@code, &make_op($var, 'is_same_var', $var{$var}));
 		} elsif ($type eq '*') {
 		    #
 		    # Reserve a hole for a 'rest_args' instruction.
 		    #
+		    $ignored_var = '';
 		    push(@fix_rest_args, scalar(@code));
 		    push(@code, $var);
-		} else {
+		} elsif ($var_used{$var}) {
+		    $ignored_var = '';
 		    $var_type{$var} = 'scalar';
 		    $var{$var} = $var_num;
 		    $var_num++;
 		    push(@code, &make_op($var, 'set_var', $var{$var}));
 		}
 	    }
-	    if (is_set_var_instr($code[$#code])) {
+	    if (is_instr($code[$#code], 'set_var')) {
 		my $ref = pop @code;
 		my $comment = $ref->[2];
 		my $var = $ref->[1][1];
 		push(@code, make_op($comment, 'set_var_next_arg', $var));
 	    } else {
-		push(@code, &make_op('', 'next_arg'));
+		push(@code, &make_op($ignored_var, 'next_arg'));
 	    }
 	}
-	push(@code, &make_op('', 'next_instr'));
-	pop(@code) if $code[$#code]->[1][0] eq 'next_arg';
+
+	# Remove redundant 'next_arg' instructions before the end
+	# of the instruction.
+	pop(@code) while is_instr($code[$#code], 'next_arg');
     }
 
     #
     # Insert the commit operation.
     #
-    pop(@code);		# Get rid of 'next_instr'
-    push(@code, &make_op('', 'commit'));
+    push(@code, make_op($may_fail ? '' : 'always reached', 'commit'));
 
     #
     # If there is an rest_args instruction, we must insert its correct
@@ -1367,9 +1614,8 @@ sub tr_gen_from {
 		push(@args, "var+$var{$var}");
 	    }
 	}
-	splice(@code, $index, 1, &make_op("$name()",
-					  'pred', scalar(@pred_table)));
-	push(@pred_table, [$name, @args]);
+	my $pi = tr_next_index(\@pred_table, \%pred_table, $name, @args);
+	splice(@code, $index, 1, make_op("$name()", 'pred', $pi));
     }
 
     $te_max_vars = $var_num
@@ -1386,6 +1632,10 @@ sub tr_gen_to {
     my($op, $ref);		# Loop variables.
     my($where) = "right side of transformation in line $line: ";
 
+    my $last_instr = $code[$#code];
+    my $cannot_fail = is_instr($last_instr, 'commit') &&
+	(get_comment($last_instr) =~ /^always/);
+
     foreach $ref (@tr) {
 	my($name, $arity, @ops) = @$ref;
 
@@ -1407,9 +1657,10 @@ sub tr_gen_to {
 		    push(@args, "var+$var{$var}");
 		}
 	    }
-	    pop(@code);	# Get rid of 'next_instr'
-	    push(@code, &make_op("$name()", 'call', scalar(@call_table)));
-	    push(@call_table, [$name, @args]);
+	    pop(@code);	# Get rid of 'commit' instruction
+	    my $index = tr_next_index(\@call_table, \%call_table,
+				      $name, @args);
+	    push(@code, make_op("$name()", 'call_end', $index));
 	    last;
 	}
 
@@ -1426,27 +1677,27 @@ sub tr_gen_to {
 	# Create code to build the generic instruction.
 	#
 
-	push(@code, &make_op('', 'new_instr'));
-	push(@code, &make_op("$name/$arity", 'store_op', $opnum, $arity));
+	push(@code, make_op("$name/$arity", 'new_instr', $opnum));
 	foreach $op (@ops) {
 	    my($var, $type, $type_val) = @$op;
 
 	    if ($var ne '') {
 		&error($where, "variable '$var' unbound")
 		    unless defined $var{$var};
-		push(@code, &make_op($var, 'store_var', $var{$var}));
+		push(@code, &make_op($var, 'store_var_next_arg', $var{$var}));
 	    } elsif ($type ne '') {
 		push(@code, &make_op('', 'store_type', "TAG_$type"));
 		if ($type_val) {
 		    push(@code, &make_op('', 'store_val', $type_val));
 		}
+		push(@code, make_op('', 'next_arg'));
 	    }
-	    push(@code, &make_op('', 'next_arg'));
 	}
-	pop(@code) if $code[$#code]->[1][0] eq 'next_arg';
+	pop(@code) if is_instr($code[$#code], 'next_arg');
     }
 
-    push(@code, &make_op('', 'end'));
+    push(@code, make_op('', 'end'))
+	unless is_instr($code[$#code], 'call_end');
 
     #
     # Chain together all codes segments having the same first operation.
@@ -1458,11 +1709,20 @@ sub tr_gen_to {
     $min_window{$key} = $min_window
 	if $min_window{$key} > $min_window;
 
-    pop(@{$gen_transform{$key}})
+    my $prev_last;
+    $prev_last = pop(@{$gen_transform{$key}})
 	if defined @{$gen_transform{$key}}; # Fail
-    my(@prefix) = (&make_op($comment), &make_op('', 'try_me_else', &tr_code_len(@code)));
-    unshift(@code, @prefix);
-    push(@{$gen_transform{$key}}, @code, &make_op('', 'fail'));
+
+    if ($prev_last && !is_instr($prev_last, 'fail')) {
+	error("Line $line: A previous transformation shadows '$orig_transform'");
+    }
+    unless ($cannot_fail) {
+	unshift(@code, make_op('', 'try_me_else',
+			       tr_code_len(@code)));
+	push(@code, make_op(""), make_op("$key", 'fail'));
+    }
+    unshift(@code, make_op($comment));
+    push(@{$gen_transform{$key}}, @code),
 }
 
 sub tr_code_len {
@@ -1480,21 +1740,38 @@ sub make_op {
     [scalar(@op), [@op], $comment];
 }
 
-sub is_set_var_instr {
-    my($ref) = @_;
+sub is_instr {
+    my($ref,$op) = @_;
     return 0 unless ref($ref) eq 'ARRAY';
-    $ref->[1][0] eq 'set_var';
+    $ref->[1][0] eq $op;
+}
+
+sub get_comment {
+    my($ref,$op) = @_;
+    return '' unless ref($ref) eq 'ARRAY';
+    $ref->[2];
+}
+
+sub tr_next_index {
+    my($lref,$href,$name,@args) = @_;
+    my $code = "RVAL = $name(" . join(', ', 'st', @args) . "); break;\n";
+    my $index;
+
+    if (defined $$href{$code}) {
+	$index = $$href{$code};
+    } else {
+	$index = scalar(@$lref);
+	push(@$lref, $code);
+	$$href{$code} = $index;
+    }
+    $index;
 }
 
 sub tr_gen_call {
     my(@call_table) = @_;
     my($i);
 
-    print "\n";
     for ($i = 0; $i < @call_table; $i++) {
-	my $ref = $call_table[$i];
-	my($name, @args) = @$ref;
-	print "case $i: RVAL = $name(", join(', ', 'st', @args), "); break;\n";
+	print "case $i: $call_table[$i]";
     }
-    print "\n";
 }
diff --git a/erts/emulator/utils/count b/erts/emulator/utils/count
new file mode 100755
index 0000000000..617f5c25e8
--- /dev/null
+++ b/erts/emulator/utils/count
@@ -0,0 +1,127 @@
+%% -*- erlang -*-
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 1998-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-mode(compile).
+
+main(_) ->
+    DisDir = "./dis",
+    ok = filelib:ensure_dir(filename:join(DisDir, "dummy")),
+    io:format("Dissambling to ~s\n", [DisDir]),
+    ok = file:set_cwd(DisDir),
+    Path = code:get_path() -- ["."],
+    Beams0 = [filelib:wildcard(filename:join(Dir, "*.beam")) ||
+		 Dir <- Path],
+    Beams = lists:append(Beams0),
+    Mods0 = [list_to_atom(filename:rootname(filename:basename(F))) ||
+	       F <- Beams],
+    Mods = lists:usort(Mods0),
+    start_sem(),
+    Ps = [begin
+	      {_,Ref} = spawn_monitor(fun() -> count(M) end),
+	      Ref
+	  end || M <- Mods],
+    [put(list_to_atom(I), 0) || I <- erts_debug:instructions()],
+    Res = wait_for_all(Ps, 1),
+    OutFile = "count",
+    {ok,Out} = file:open(OutFile, [write]),
+    [io:format(Out, "~s ~p\n", [I,C]) || {I,C} <- Res],
+    ok = file:close(Out),
+    io:format("\nResult written to ~s\n",
+	      [filename:join(DisDir, OutFile)]),
+    ok.
+
+wait_for_all([], _) ->
+    lists:reverse(lists:keysort(2, get()));
+wait_for_all([_|_]=Ps, I) ->
+    receive
+	{'DOWN',Ref,process,_,Result} ->
+	    io:format("\r~p", [I]),
+	    [increment(Key, Count) || {Key,Count} <- Result],
+	    wait_for_all(Ps -- [Ref], I+1)
+    end.
+
+count(M) ->
+    down(),
+    erts_debug:df(M),
+    {ok,Fd} = file:open(atom_to_list(M) ++ ".dis", [read,raw]),
+    count_is(Fd),
+    ok = file:close(Fd),
+    exit(get()).
+
+count_is(Fd) ->
+    case file:read_line(Fd) of
+	{ok,Line} ->
+	    count_instr(Line),
+	    count_is(Fd);
+	eof ->
+	    ok
+    end.
+
+count_instr([$\s|T]) ->
+    count_instr_1(T, []);
+count_instr([_|T]) ->
+    count_instr(T);
+count_instr([]) ->
+    %% Empty line.
+    ok.
+
+count_instr_1([$\s|_], Acc) ->
+    Instr = list_to_atom(lists:reverse(Acc)),
+    increment(Instr, 1);
+count_instr_1([H|T], Acc) ->
+    count_instr_1(T, [H|Acc]).
+
+increment(Key, Inc) -> 
+    case get(Key) of
+	undefined ->
+	    put(Key, Inc);
+	Count ->
+	    put(Key, Count+Inc)
+    end.
+
+%%%
+%%% Counting sempahore to limit the number of processes that
+%%% can run concurrently.
+%%%
+
+down() ->
+    sem ! {down,self()},
+    receive
+	sem_taken -> ok
+    end.
+
+start_sem() ->	    
+    spawn(fun() ->
+		  register(sem, self()),
+		  process_flag(trap_exit, true),
+		  do_sem(erlang:system_info(schedulers)+1) end).
+
+do_sem(0) ->
+    receive
+	{'EXIT',_,_} ->
+	    do_sem(1)
+    end;
+do_sem(C) ->
+    receive
+	{down,Pid} ->
+	    link(Pid),
+	    Pid ! sem_taken,
+	    do_sem(C-1)
+    end.
diff --git a/erts/emulator/utils/loaded b/erts/emulator/utils/loaded
new file mode 100644
index 0000000000..d124a64a78
--- /dev/null
+++ b/erts/emulator/utils/loaded
@@ -0,0 +1,44 @@
+%% -*- erlang -*-
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 1998-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%% Run like:
+%%   $ERL_TOP/bin/escript erts/emulator/utils/loaded
+
+-mode(compile).
+
+main(_) ->
+    LibDir = code:lib_dir(),
+    io:format("Library root is ~s\n", [LibDir]),
+    Wc = filename:join(LibDir, "*/ebin/*.beam"),
+    Beams = filelib:wildcard(Wc),
+    BeamFileSize = lists:sum([filelib:file_size(Beam) || Beam <- Beams]),
+    io:format("~w BEAM files containing ~w bytes\n",
+	      [length(Beams),BeamFileSize]),
+    Ms = [list_to_atom(filename:rootname(filename:basename(Beam))) ||
+	     Beam <- Beams],
+    [{module,_} = code:ensure_loaded(M) || M <- Ms],
+    <<"Current code: ",T/binary>> = erlang:system_info(loaded),
+    Digits = grab_digits(T),
+    io:format("~w modules comprising ~s words when loaded\n",
+	      [length(Ms),Digits]).
+
+grab_digits(<<H,T/binary>>) when $0 =< H, H =< $9 ->
+    [H|grab_digits(T)];
+grab_digits(<<$\n,_/binary>>) -> [].
diff --git a/erts/emulator/utils/make_preload b/erts/emulator/utils/make_preload
index d0671e998d..13019d4062 100755
--- a/erts/emulator/utils/make_preload
+++ b/erts/emulator/utils/make_preload
@@ -2,7 +2,7 @@
 #
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 1999-2009. All Rights Reserved.
+# Copyright Ericsson AB 1999-2011. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -39,6 +39,7 @@ use File::Basename;
 my $gen_rc = 0;
 my $gen_old = 0;
 my $windres = 0;
+my $msys = 0;
 my $file;
 
 my $progname = basename($0);
@@ -49,6 +50,8 @@ while (@ARGV && $ARGV[0] =~ /^-(\w+)/) {
 	$gen_rc = 1;
     } elsif ($opt eq '-windres') {
 	$windres = 1;
+    } elsif ($opt eq '-msys') {
+	$msys = 1;
     } elsif ($opt eq '-old') {
 	$gen_old = 1;
     } else {
@@ -68,7 +71,12 @@ foreach $file (@ARGV) {
 	unless $file =~ /\.beam$/;
     my $module = basename($file, ".beam");
     if ($gen_rc) {
-	my ($win_file) = split("\n", `(cygpath -d $file 2>/dev/null || cygpath -w $file)`);
+	my $win_file;
+	if ($msys) {
+	    ($win_file) = split("\n", `(msys2win_path.sh $file)`);
+	} else {
+	    ($win_file) = split("\n", `(cygpath -d $file 2>/dev/null || cygpath -w $file)`);
+	}
 	$win_file =~ s&\\&\\\\&g;
 	print "$num ERLANG_CODE \"$win_file\"\n";
 	push(@modules, "   ", -s $file, "L, $num, ",
@@ -88,6 +96,7 @@ foreach $file (@ARGV) {
 	print "unsigned char preloaded_$module", "[] = {\n";
 	for ($i = 0; $i < length($_); $i++) {
 	    if ($i % 8 == 0 && $comment ne '') {
+		$comment =~ s@/\*@..@g; # Comment start -- avoid warning.
 		$comment =~ s@\*/@..@g; # Comment terminator.
 		print " /* $comment */\n  ";
 		$comment = '';
diff --git a/erts/emulator/utils/make_tables b/erts/emulator/utils/make_tables
index b5391234cf..91efb4c023 100755
--- a/erts/emulator/utils/make_tables
+++ b/erts/emulator/utils/make_tables
@@ -1,20 +1,20 @@
 #!/usr/bin/env perl 
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 1999-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 1999-2010. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
 use strict;
@@ -181,10 +181,9 @@ for ($i = 0; $i < @bif; $i++) {
 print "\n";
 
 for ($i = 0; $i < @bif; $i++) {
-    my $arity = $bif[$i]->[2];
-    my $args = join(', ', 'Process*', ('Eterm') x $arity);
+    my $args = join(', ', 'Process*', 'Eterm*');
     print "Eterm $bif[$i]->[3]($args);\n";
-    print "Eterm wrap_$bif[$i]->[3]($args, Uint *I);\n";
+    print "Eterm wrap_$bif[$i]->[3]($args, UWord *I);\n";
 }
 print "#endif\n";
 
@@ -219,28 +218,10 @@ for ($i = 0; $i < @bif; $i++) {
     next if $bif[$i]->[3] eq $bif[$i]->[4]; # Skip unwrapped bifs
     my $arity = $bif[$i]->[2];
     my $func = $bif[$i]->[3];
-    my $arg;
     print "Eterm\n";
-    print "wrap_$func(Process* p";
-    for ($arg = 1; $arg <= $arity; $arg++) {
-	print ", Eterm arg$arg";
-    }
-    print ", Uint *I)\n";
+    print "wrap_$func(Process* p, Eterm* args, UWord* I)\n";
     print "{\n";
-    print "    return erts_bif_trace($i, p";
-    for ($arg = 1; $arg <= 3; $arg++) {
-	if ($arg <= $arity) {
-	    print ", arg$arg";
-	} elsif ($arg == ($arity + 1)) { 
-	    # Place I in correct position
-	    print ", (Eterm) I";
-	} else {
-	    print ", 0";
-	}
-    }
-    # I is always last, as well as in the correct position
-    # Note that "last" and "correct position" may be the same...
-    print ", I);\n";
+    print "    return erts_bif_trace($i, p, args, I);\n";
     print "}\n\n";
 }
 
@@ -261,19 +242,9 @@ for ($i = 0; $i < @bif; $i++) {
     my $orig_func = $1;
     $orig_func = $implementation[$i] if $implementation[$i];
     print "Eterm\n";
-    print "$func(Process* p";
-    for ($arg = 1; $arg <= $arity; $arg++) {
-	print ", Eterm arg$arg";
-    }
-    print ")\n";
+    print "$func(Process* p, Eterm* BIF__ARGS)\n";
     print "{\n";
-    print "    return $orig_func(p";
-    for ($arg = 1; $arg <= 3; $arg++) {
-	if ($arg <= $arity) {
-	    print ", arg$arg";
-        }
-    }
-    print ");\n";
+    print "    return $orig_func(p, BIF__ARGS);\n";
     print "}\n\n";
 }
 
diff --git a/erts/emulator/valgrind/suppress.patched.3.6.0 b/erts/emulator/valgrind/suppress.patched.3.6.0
new file mode 100644
index 0000000000..62ba032520
--- /dev/null
+++ b/erts/emulator/valgrind/suppress.patched.3.6.0
@@ -0,0 +1,359 @@
+# Valgrind suppression file updated to support the patched
+# Valgrind used in daily builds on ahmed.
+
+{
+   libc internal error
+   Memcheck:Addr8
+   obj:/lib64/ld-2.3.5.so
+}
+{
+   libc internal error
+   Memcheck:Addr8
+   fun:_dl_start
+}
+{
+   libc internal error
+   Memcheck:Addr8
+   fun:__libc_start_main
+   obj:*
+}
+{
+   libc internal error
+   Memcheck:Addr4
+   fun:__sigjmp_save
+   fun:__libc_start_main
+   obj:*
+}
+{
+   libc internal error
+   Memcheck:Addr8
+   fun:__sigsetjmp
+   fun:__libc_start_main
+   obj:*
+}
+{
+   Intentional error in testcase
+   Memcheck:Param
+   pipe(filedes)
+   fun:pipe
+   fun:chkio_drv_timeout
+}
+{
+   Intentional error in testcase
+   Memcheck:Param
+   pipe(filedes)
+   fun:pipe
+   fun:io_ready_exit_drv_control
+   fun:erts_port_control
+   fun:port_control_3
+   fun:process_main
+}
+{
+   Leak in libc putenv
+   Memcheck:Leak
+   fun:malloc
+   fun:realloc
+   fun:__add_to_environ
+   fun:putenv
+   fun:erts_sys_putenv
+   fun:os_putenv_2
+   ...
+}
+{
+   Leak in libc putenv
+   Memcheck:Leak
+   ...
+   fun:erts_alloc
+   fun:erts_sys_putenv
+   fun:os_putenv_2
+   ...
+}
+{
+   erronous warning
+   Memcheck:Leak
+   fun:malloc
+   fun:erts_sys_alloc
+   ...
+   fun:fix_core_alloc
+   fun:erts_init_fix_alloc
+   fun:erts_alloc_init
+   fun:early_init
+   fun:erl_start
+}
+{
+   pthread internal error
+   Memcheck:Param
+   futex(utime)
+   fun:__lll_mutex_unlock_wake
+}
+{
+   libc internal error
+   Memcheck:Param
+   socketcall.sendto(msg)
+   ...
+   fun:getifaddrs
+}
+{
+inet_drv; pointer inside allocated block
+Memcheck:Leak
+PossiblyLost
+fun:realloc
+fun:erts_sys_realloc
+...
+fun:erts_realloc_fnf
+fun:erts_bin_realloc_fnf
+fun:driver_realloc_binary
+}
+{
+inet_drv; pointer inside allocated block
+Memcheck:Leak
+PossiblyLost
+fun:malloc
+fun:erts_sys_alloc
+...
+fun:erts_alloc_fnf
+fun:erts_bin_drv_alloc_fnf
+fun:driver_alloc_binary
+}
+{
+pthread leak or erroneous valgrind warning
+Memcheck:Leak
+fun:calloc
+fun:allocate_dtv
+fun:_dl_allocate_tls
+fun:pthread_create@@GLIBC_2.2.5
+}
+{
+pthread leak or erroneous valgrind warning
+Memcheck:Leak
+fun:calloc
+fun:_dl_allocate_tls
+fun:pthread_create@@GLIBC_2.2.5
+}
+{
+zlib; ok according to zlib developers
+Memcheck:Cond
+fun:longest_match
+fun:deflate_slow
+fun:deflate
+}
+{
+zlib; ok according to zlib developers
+Memcheck:Cond
+fun:longest_match
+fun:deflate_fast
+fun:deflate
+}
+{
+zlib; ok accordnig to zlib (this one popped up with valgrind-3.6.0)
+Memcheck:Cond
+fun:deflate_slow
+fun:deflate
+fun:zlib_deflate
+fun:zlib_ctl
+}
+{
+No leak; pointer into block
+Memcheck:Leak
+fun:malloc
+fun:erts_sys_alloc
+...
+fun:erts_alloc
+fun:erts_init_scheduling
+fun:erl_init
+fun:erl_start
+fun:main
+}
+{
+No leak; pointer into block
+Memcheck:Leak
+PossiblyLost
+fun:malloc
+fun:erts_sys_alloc
+...
+fun:erts_alloc
+fun:init_db
+fun:erl_init
+fun:erl_start
+fun:main
+}
+{
+No leak; sometimes pointer into block
+Memcheck:Leak
+PossiblyLost
+fun:malloc
+fun:erts_sys_alloc
+...
+fun:erts_alloc_fnf
+fun:driver_alloc
+fun:get_bufstk
+fun:alloc_buffer
+}
+{
+   Crypto internal...
+Memcheck:Cond
+obj:*/crypto.valgrind.*
+}
+{
+   Crypto internal...
+Memcheck:Cond
+obj:*/libcrypto.*
+}
+{
+   Crypto internal...
+Memcheck:Cond
+obj:*/openssl.*
+}
+{
+   Crypto internal...
+Memcheck:Cond
+obj:*/ssleay.*
+}
+{
+   Crypto internal...
+Memcheck:Value8
+obj:*/crypto.valgrind.*
+}
+{
+   Crypto internal...
+Memcheck:Value8
+obj:*/libcrypto.*
+}
+{
+   Crypto internal...
+Memcheck:Value8
+obj:*/openssl.*
+}
+{
+   Crypto internal...
+Memcheck:Value8
+obj:*/ssleay.*
+}
+{
+   Crypto internal...
+   Memcheck:Cond
+   fun:memset
+   fun:BN_lshift
+   fun:BN_div
+   fun:BN_MONT_CTX_set
+   fun:BN_is_prime_fasttest_ex
+   fun:BN_generate_prime_ex
+   fun:DH_generate_parameters_ex
+   fun:DH_generate_parameters
+   fun:dh_generate_parameters_nif
+   fun:process_main
+   fun:sched_thread_func
+   fun:thr_wrapper
+}
+{
+   Crypto internal...
+   Memcheck:Cond
+   fun:memset
+   fun:BN_lshift
+   fun:BN_div
+   fun:BN_nnmod
+   fun:BN_mod_inverse
+   fun:BN_MONT_CTX_set
+   fun:BN_is_prime_fasttest_ex
+   fun:BN_generate_prime_ex
+   fun:DH_generate_parameters_ex
+   fun:DH_generate_parameters
+   fun:dh_generate_parameters_nif
+   fun:process_main
+}
+{
+   Crypto internal...
+   Memcheck:Value8
+   fun:BN_mod_exp_mont_consttime
+   fun:generate_key
+   fun:dh_generate_key_nif
+   fun:process_main
+   fun:sched_thread_func
+   fun:thr_wrapper
+   fun:start_thread
+   fun:clone
+}
+
+{
+erts_bits_init_state; Why is this needed?
+Memcheck:Leak
+PossiblyLost
+fun:malloc
+fun:erts_sys_alloc
+...
+fun:erts_alloc
+fun:erts_bits_init_state
+fun:erts_init_scheduling
+fun:erl_init
+fun:erl_start
+fun:main
+}
+
+{
+Prebuilt constant terms in os_info_init
+Memcheck:Leak
+PossiblyLost
+fun:malloc
+fun:erts_sys_alloc
+...
+fun:erts_alloc
+fun:os_info_init
+fun:erts_bif_info_init
+fun:erl_init
+fun:erl_start
+fun:main
+}
+
+{
+Permanent cache aligned malloc for array of mseg allocators
+Memcheck:Leak
+PossiblyLost
+fun:malloc
+fun:erts_mseg_init
+fun:erts_alloc_init
+fun:early_init
+fun:erl_start
+fun:main
+}
+
+{
+Early permanent cache aligned erl_process:aux_work_tmo
+Memcheck:Leak
+PossiblyLost
+fun:malloc
+fun:aux_work_timeout_early_init
+fun:erts_early_init_scheduling
+fun:early_init
+fun:erl_start
+fun:main
+}
+
+{
+Early permanent cache aligned ts_event_pool
+Memcheck:Leak
+PossiblyLost
+fun:malloc
+fun:erts_sys_alloc
+fun:erts_alloc_fnf
+fun:ethr_std_alloc
+fun:ts_event_pool
+fun:init_ts_event_alloc
+fun:ethr_late_init_common__
+fun:ethr_late_init
+fun:erts_thr_late_init
+fun:early_init
+fun:erl_start
+fun:main
+}
+
+{
+Harmless leak of ErtsThrPrgrData from async threads in exiting emulator
+Memcheck:Leak
+...
+fun:erts_alloc
+fun:erts_thr_progress_register_unmanaged_thread
+fun:async_thread_init
+fun:async_main
+...
+}
+
diff --git a/erts/emulator/valgrind/suppress.standard b/erts/emulator/valgrind/suppress.standard
new file mode 100644
index 0000000000..5a129bfd10
--- /dev/null
+++ b/erts/emulator/valgrind/suppress.standard
@@ -0,0 +1,317 @@
+{
+   libc internal error
+   Memcheck:Addr8
+   obj:/lib64/ld-2.3.5.so
+}
+{
+   libc internal error
+   Memcheck:Addr8
+   fun:_dl_start
+}
+{
+   libc internal error
+   Memcheck:Addr8
+   fun:__libc_start_main
+   obj:*
+}
+{
+   libc internal error
+   Memcheck:Addr4
+   fun:__sigjmp_save
+   fun:__libc_start_main
+   obj:*
+}
+{
+   libc internal error
+   Memcheck:Addr8
+   fun:__sigsetjmp
+   fun:__libc_start_main
+   obj:*
+}
+{
+   Intentional error in testcase
+   Memcheck:Param
+   pipe(filedes)
+   fun:pipe
+   fun:chkio_drv_timeout
+}
+{
+   Intentional error in testcase
+   Memcheck:Param
+   pipe(filedes)
+   fun:pipe
+   fun:io_ready_exit_drv_control
+   fun:erts_port_control
+   fun:port_control_3
+   fun:process_main
+}
+{
+   Leak in libc putenv
+   Memcheck:Leak
+   fun:malloc
+   fun:realloc
+   fun:__add_to_environ
+   fun:putenv
+   fun:erts_sys_putenv
+   fun:os_putenv_2
+   ...
+}
+{
+   Leak in libc putenv
+   Memcheck:Leak
+   ...
+   fun:erts_alloc
+   fun:erts_sys_putenv
+   fun:os_putenv_2
+   ...
+}
+{
+   erronous warning
+   Memcheck:Leak
+   fun:malloc
+   fun:erts_sys_alloc
+   fun:fix_core_alloc
+   fun:erts_init_fix_alloc
+   fun:erts_alloc_init
+   fun:early_init
+   fun:erl_start
+}
+{
+   pthread internal error
+   Memcheck:Param
+   futex(utime)
+   fun:__lll_mutex_unlock_wake
+}
+{
+   libc internal error
+   Memcheck:Param
+   socketcall.sendto(msg)
+   ...
+   fun:getifaddrs
+}
+{
+inet_drv; pointer inside allocated block
+Memcheck:Leak
+fun:realloc
+fun:erts_sys_realloc
+...
+fun:erts_realloc_fnf
+fun:erts_bin_realloc_fnf
+fun:driver_realloc_binary
+}
+{
+inet_drv; pointer inside allocated block
+Memcheck:Leak
+fun:malloc
+fun:erts_sys_alloc
+...
+fun:erts_alloc_fnf
+fun:erts_bin_drv_alloc_fnf
+fun:driver_alloc_binary
+}
+{
+pthread leak or erroneous valgrind warning
+Memcheck:Leak
+fun:calloc
+fun:allocate_dtv
+fun:_dl_allocate_tls
+fun:pthread_create@@GLIBC_2.2.5
+}
+{
+zlib; ok according to zlib developers
+Memcheck:Cond
+fun:longest_match
+fun:deflate_slow
+fun:deflate
+}
+{
+zlib; ok according to zlib developers
+Memcheck:Cond
+fun:longest_match
+fun:deflate_fast
+fun:deflate
+}
+{
+No leak; pointer into block
+Memcheck:Leak
+fun:malloc
+fun:erts_sys_alloc
+...
+fun:erts_alloc
+fun:erts_init_scheduling
+fun:erl_init
+fun:erl_start
+fun:main
+}
+{
+No leak; pointer into block
+Memcheck:Leak
+fun:malloc
+fun:erts_sys_alloc
+...
+fun:erts_alloc
+fun:init_db
+fun:erl_init
+fun:erl_start
+fun:main
+}
+{
+No leak; sometimes pointer into block
+Memcheck:Leak
+fun:malloc
+fun:erts_sys_alloc
+...
+fun:erts_alloc_fnf
+fun:driver_alloc
+fun:get_bufstk
+fun:alloc_buffer
+}
+{
+   Crypto internal...
+Memcheck:Cond
+obj:*/crypto.valgrind.*
+}
+{
+   Crypto internal...
+Memcheck:Cond
+obj:*/libcrypto.*
+}
+{
+   Crypto internal...
+Memcheck:Cond
+obj:*/openssl.*
+}
+{
+   Crypto internal...
+Memcheck:Cond
+obj:*/ssleay.*
+}
+{
+   Crypto internal...
+Memcheck:Value8
+obj:*/crypto.valgrind.*
+}
+{
+   Crypto internal...
+Memcheck:Value8
+obj:*/libcrypto.*
+}
+{
+   Crypto internal...
+Memcheck:Value8
+obj:*/openssl.*
+}
+{
+   Crypto internal...
+Memcheck:Value8
+obj:*/ssleay.*
+}
+{
+   Crypto internal...
+   Memcheck:Cond
+   fun:memset
+   fun:BN_lshift
+   fun:BN_div
+   fun:BN_MONT_CTX_set
+   fun:BN_is_prime_fasttest_ex
+   fun:BN_generate_prime_ex
+   fun:DH_generate_parameters_ex
+   fun:DH_generate_parameters
+   fun:dh_generate_parameters_nif
+   fun:process_main
+   fun:sched_thread_func
+   fun:thr_wrapper
+}
+{
+   Crypto internal...
+   Memcheck:Cond
+   fun:memset
+   fun:BN_lshift
+   fun:BN_div
+   fun:BN_nnmod
+   fun:BN_mod_inverse
+   fun:BN_MONT_CTX_set
+   fun:BN_is_prime_fasttest_ex
+   fun:BN_generate_prime_ex
+   fun:DH_generate_parameters_ex
+   fun:DH_generate_parameters
+   fun:dh_generate_parameters_nif
+   fun:process_main
+}
+{
+   Crypto internal...
+   Memcheck:Value8
+   fun:BN_mod_exp_mont_consttime
+   fun:generate_key
+   fun:dh_generate_key_nif
+   fun:process_main
+   fun:sched_thread_func
+   fun:thr_wrapper
+   fun:start_thread
+   fun:clone
+}
+
+{
+Prebuilt constant terms in os_info_init (PossiblyLost)
+Memcheck:Leak
+fun:malloc
+fun:erts_sys_alloc
+...
+fun:erts_alloc
+fun:os_info_init
+fun:erts_bif_info_init
+fun:erl_init
+fun:erl_start
+fun:main
+}
+
+{
+Permanent cache aligned malloc for array of mseg allocators
+Memcheck:Leak
+fun:malloc
+fun:erts_mseg_init
+fun:erts_alloc_init
+fun:early_init
+fun:erl_start
+fun:main
+}
+
+{
+Early permanent cache aligned erl_process:aux_work_tmo
+Memcheck:Leak
+fun:malloc
+fun:aux_work_timeout_early_init
+fun:erts_early_init_scheduling
+fun:early_init
+fun:erl_start
+fun:main
+}
+
+{
+Early permanent cache aligned ts_event_pool
+Memcheck:Leak
+fun:malloc
+fun:erts_sys_alloc
+fun:erts_alloc_fnf
+fun:ethr_std_alloc
+fun:ts_event_pool
+fun:init_ts_event_alloc
+fun:ethr_late_init_common__
+fun:ethr_late_init
+fun:erts_thr_late_init
+fun:early_init
+fun:erl_start
+fun:main
+}
+
+{
+Harmless leak of ErtsThrPrgrData from async threads in exiting emulator
+Memcheck:Leak
+...
+fun:erts_alloc
+fun:erts_thr_progress_register_unmanaged_thread
+fun:async_thread_init
+fun:async_main
+...
+}
+
diff --git a/erts/emulator/zlib/Makefile.in b/erts/emulator/zlib/Makefile.in
deleted file mode 100644
index b44a87551d..0000000000
--- a/erts/emulator/zlib/Makefile.in
+++ /dev/null
@@ -1,102 +0,0 @@
-# Makefile for zlib
-# Copyright (C) 1995-1996 Jean-loup Gailly.
-# For conditions of distribution and use, see copyright notice in zlib.h 
-
-# %ExternalCopyright%
-
-# To compile and test, type:
-#   ./configure; make test
-# The call of configure is optional if you don't have special requirements
-
-# To install /usr/local/lib/libz.* and /usr/local/include/zlib.h, type:
-#    make install
-# To install in $HOME instead of /usr/local, use:
-#    make install prefix=$HOME
-
-ARFLAGS = rc
-CFLAGS = $(subst -O2, -O3, @CFLAGS@ @DEFS@ @EMU_THR_DEFS@)
-#CFLAGS=-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7
-#CFLAGS=-g -DDEBUG
-#CFLAGS=-O3 -Wall -Wwrite-strings -Wpointer-arith -Wconversion \
-#           -Wstrict-prototypes -Wmissing-prototypes
-
-VER=1.0.4
-
-O = adler32.o compress.o crc32.o uncompr.o deflate.o trees.o \
-       zutil.o inflate.o inftrees.o inffast.o
-OBJS = $(O:%=$(OBJDIR)/%)
-
-
-#### Begin OTP targets
-
-include $(ERL_TOP)/make/target.mk
-
-ifeq ($(TYPE),gcov)
-CFLAGS = -O0 -fprofile-arcs -ftest-coverage @DEBUG_CFLAGS@ @DEFS@ @EMU_THR_DEFS@
-else  # gcov
-ifeq ($(TYPE),debug)
-CFLAGS = @DEBUG_CFLAGS@ @DEFS@ @EMU_THR_DEFS@
-endif # debug
-endif # gcov
-
-# On windows we *need* a separate zlib during debug build
-OBJDIR= $(ERL_TOP)/erts/emulator/zlib/obj/$(TARGET)/$(TYPE)
-
-include $(ERL_TOP)/make/$(TARGET)/otp.mk
-
-ifeq ($(TARGET), win32)
-LIBRARY=$(OBJDIR)/z.lib
-else
-LIBRARY=$(OBJDIR)/libz.a
-endif
-
-all: $(LIBRARY)
-
-# ----------------------------------------------------
-# Release Target
-# ---------------------------------------------------- 
-include $(ERL_TOP)/make/otp_release_targets.mk
-
-release_spec: opt
-
-tests release_tests:
-
-docs release_docs release_docs_spec:
-
-clean:
-	rm -f $(OBJS) $(OBJDIR)/libz.a
-
-#### end OTP targets
-
-ifeq ($(TARGET), win32)
-$(LIBRARY): $(OBJS)
-	$(AR) -out:$@ $(OBJS)
-else
-$(LIBRARY): $(OBJS)
-	$(AR) $(ARFLAGS) $@ $(OBJS)
-	-@ ($(RANLIB) $@ || true) 2>/dev/null
-endif
-
-$(OBJDIR)/%.o: %.c
-	$(CC) -c $(CFLAGS) -o $@ $<
-
-# DO NOT DELETE THIS LINE -- make depend depends on it.
-
-adler32.o: zlib.h zconf.h
-compress.o: zlib.h zconf.h
-crc32.o: zlib.h zconf.h
-deflate.o: deflate.h zutil.h zlib.h zconf.h
-example.o: zlib.h zconf.h
-gzio.o: zutil.h zlib.h zconf.h
-infblock.o: infblock.h inftrees.h infcodes.h infutil.h zutil.h zlib.h zconf.h
-infcodes.o: zutil.h zlib.h zconf.h
-infcodes.o: inftrees.h infblock.h infcodes.h infutil.h inffast.h
-inffast.o: zutil.h zlib.h zconf.h inftrees.h
-inffast.o: infblock.h infcodes.h infutil.h inffast.h
-inflate.o: zutil.h zlib.h zconf.h infblock.h
-inftrees.o: zutil.h zlib.h zconf.h inftrees.h
-infutil.o: zutil.h zlib.h zconf.h infblock.h inftrees.h infcodes.h infutil.h
-minigzip.o:  zlib.h zconf.h 
-trees.o: deflate.h zutil.h zlib.h zconf.h 
-uncompr.o: zlib.h zconf.h
-zutil.o: zutil.h zlib.h zconf.h  
diff --git a/erts/emulator/zlib/zlib.mk b/erts/emulator/zlib/zlib.mk
new file mode 100644
index 0000000000..fa1f159fae
--- /dev/null
+++ b/erts/emulator/zlib/zlib.mk
@@ -0,0 +1,74 @@
+#-*-makefile-*-   ; force emacs to enter makefile-mode
+# ----------------------------------------------------
+# Make include file for zlib
+#
+# %CopyrightBegin%
+#
+# Copyright Ericsson AB 2011-2012. All Rights Reserved.
+#
+# The contents of this file are subject to the Erlang Public License,
+# Version 1.1, (the "License"); you may not use this file except in
+# compliance with the License. You should have received a copy of the
+# Erlang Public License along with this software. If not, it can be
+# retrieved online at http://www.erlang.org/.
+#
+# Software distributed under the License is distributed on an "AS IS"
+# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+# the License for the specific language governing rights and limitations
+# under the License.
+#
+# %CopyrightEnd%
+#
+# ----------------------------------------------------
+# Copyright for zlib itself see copyright notice in zlib.h
+
+ZLIB_FILES = \
+	adler32 \
+	compress \
+	crc32 \
+	uncompr \
+	deflate \
+	trees \
+	zutil \
+	inflate \
+	inftrees \
+	inffast
+
+# On windows we *need* a separate zlib during debug build
+ZLIB_OBJDIR = $(ERL_TOP)/erts/emulator/zlib/obj/$(TARGET)/$(TYPE)
+
+ZLIB_OBJS = $(ZLIB_FILES:%=$(ZLIB_OBJDIR)/%.o)
+ZLIB_SRC = $(ZLIB_FILES:%=zlib/%.c)
+
+ifeq ($(TARGET), win32)
+ZLIB_LIBRARY = $(ZLIB_OBJDIR)/z.lib
+else
+ZLIB_LIBRARY = $(ZLIB_OBJDIR)/libz.a
+endif
+
+
+ifeq ($(TYPE),gcov)
+ZLIB_CFLAGS = -O0 -fprofile-arcs -ftest-coverage $(DEBUG_CFLAGS) $(DEFS) $(THR_DEFS)
+else  # gcov
+ifeq ($(TYPE),debug)
+ZLIB_CFLAGS = $(DEBUG_CFLAGS) $(DEFS) $(THR_DEFS)
+else # debug
+ZLIB_CFLAGS = $(subst -O2, -O3, $(CONFIGURE_CFLAGS) $(DEFS) $(THR_DEFS))
+#ZLIB_CFLAGS=-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7
+#ZLIB_CFLAGS=-g -DDEBUG
+#ZLIB_CFLAGS=-O3 -Wall -Wwrite-strings -Wpointer-arith -Wconversion \
+#	-Wstrict-prototypes -Wmissing-prototypes
+endif # debug
+endif # gcov
+
+ifeq ($(TARGET), win32)
+$(ZLIB_LIBRARY): $(ZLIB_OBJS)
+	$(AR) -out:$@ $(ZLIB_OBJS)
+else
+$(ZLIB_LIBRARY): $(ZLIB_OBJS)
+	$(AR) $(ARFLAGS) $@ $(ZLIB_OBJS)
+	-@ ($(RANLIB) $@ || true) 2>/dev/null
+endif
+
+$(ZLIB_OBJDIR)/%.o: zlib/%.c
+	$(CC) -c $(ZLIB_CFLAGS) -o $@ $<
diff --git a/erts/emulator/zlib/zutil.h b/erts/emulator/zlib/zutil.h
index d560382691..a8872e1c88 100644
--- a/erts/emulator/zlib/zutil.h
+++ b/erts/emulator/zlib/zutil.h
@@ -142,6 +142,7 @@ extern const char * const z_errmsg[10]; /* indexed by 2-zlib_error */
 #ifdef WIN32
 #  ifndef __CYGWIN__  /* Cygwin is Unix, not Win32 */
 #    define OS_CODE  0x0b
+#    define F_OPEN(name, mode) _wfopen((WCHAR *)(name), (WCHAR *)(mode)) /* Unicode */
 #  endif
 #endif
 
diff --git a/erts/epmd/src/Makefile.in b/erts/epmd/src/Makefile.in
index 498756b468..577fc77c13 100644
--- a/erts/epmd/src/Makefile.in
+++ b/erts/epmd/src/Makefile.in
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 1998-2009. All Rights Reserved.
+# Copyright Ericsson AB 1998-2012. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -24,6 +24,7 @@ PURIFY     =
 TYPEMARKER = .debug
 TYPE_FLAGS = -DDEBUG @DEBUG_FLAGS@
 else
+
 ifeq ($(TYPE),purify)
 PURIFY     = purify
 TYPEMARKER =
@@ -33,6 +34,8 @@ else
   TYPE_FLAGS = -O2 -DPURIFY
 endif
 else
+
+override TYPE = opt
 PURIFY     =
 TYPEMARKER =
 ifeq ($(findstring ose,$(TARGET)),ose)
@@ -49,15 +52,33 @@ include ../epmd.mk
 
 BINDIR  = $(ERL_TOP)/bin/$(TARGET)
 OBJDIR  = $(ERL_TOP)/erts/obj$(TYPEMARKER)/$(TARGET)
+ERTS_INCL = -I$(ERL_TOP)/erts/include \
+            -I$(ERL_TOP)/erts/include/$(TARGET) \
+            -I$(ERL_TOP)/erts/include/internal \
+            -I$(ERL_TOP)/erts/include/internal/$(TARGET)
+
+# On windows we always need reentrant libraries.
+ifeq ($(TARGET),win32)
+ERTS_INTERNAL_LIBS=-L../../lib/internal/$(TARGET) -lerts_internal_r$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@
+else
+ifeq ($(findstring vxworks,$(TARGET)),vxworks)
+ERTS_INTERNAL_LIBS=-L../../lib/internal/$(TARGET) -lerts_internal$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@
+else
+ERTS_INTERNAL_LIBS=-L../../lib/internal/$(TARGET) -lerts_internal$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@ -lm
+endif
+endif
+
+ERTS_LIB = $(ERL_TOP)/erts/lib_src/obj/$(TARGET)/$(TYPE)/MADE
 
 CC      = @CC@
 WFLAGS  = @WFLAGS@
-CFLAGS  = @CFLAGS@ @DEFS@ $(TYPE_FLAGS) $(WFLAGS)
+CFLAGS  = @CFLAGS@ @DEFS@ $(TYPE_FLAGS) $(WFLAGS) $(ERTS_INCL)
 LD      = @LD@
-LIBS    = @LIBS@
+LIBS    = @LIBS@ $(ERTS_INTERNAL_LIBS)
 LDFLAGS = @LDFLAGS@
 
 
+
 # ----------------------------------------------------
 # Release directory specification
 # ----------------------------------------------------
@@ -106,18 +127,20 @@ clean:
 # Objects & executables
 #
 
-$(BINDIR)/$(EPMD): $(EPMD_OBJS)
+$(BINDIR)/$(EPMD): $(EPMD_OBJS) $(ERTS_LIB)
 	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(EPMD_OBJS) $(LIBS)
 
-$(OBJDIR)/%.o: %.c
+$(OBJDIR)/%.o: %.c epmd.h epmd_int.h
 	$(CC) $(CFLAGS) $(EPMD_FLAGS) -o $@ -c $<
 
+$(ERTS_LIB):
+	cd $(ERL_TOP)/erts/lib_src && $(MAKE) $(TYPE)
+
 include $(ERL_TOP)/make/otp_release_targets.mk
 
 release_spec: all
-	$(INSTALL_DIR)     $(RELEASE_PATH)/erts-$(VSN)/bin
-	$(INSTALL_PROGRAM) $(INSTALL_PROGS) $(RELEASE_PATH)/erts-$(VSN)/bin
+	$(INSTALL_DIR)     "$(RELEASE_PATH)/erts-$(VSN)/bin"
+	$(INSTALL_PROGRAM) $(INSTALL_PROGS) "$(RELEASE_PATH)/erts-$(VSN)/bin"
 
 
 release_docs_spec:
-
diff --git a/erts/epmd/src/epmd.c b/erts/epmd/src/epmd.c
index 23ac421446..2267f9b12b 100644
--- a/erts/epmd/src/epmd.c
+++ b/erts/epmd/src/epmd.c
@@ -1,20 +1,20 @@
 /* -*- c-indent-level: 2; c-continued-statement-offset: 2 -*- */ 
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -23,11 +23,7 @@
 #endif
 #include "epmd.h"     /* Renamed from 'epmd_r4.h' */
 #include "epmd_int.h"
-
-#ifdef _OSE_
-#  include "ose.h"
-#  include "efs.h"
-#endif
+#include "erl_printf.h"
 
 #ifdef HAVE_STDLIB_H
 #  include <stdlib.h>
@@ -37,7 +33,9 @@
 
 static void usage(EpmdVars *);
 static void run_daemon(EpmdVars*);
+static char* get_addresses(void);
 static int get_port_no(void);
+static int check_relaxed(void);
 #ifdef __WIN32__
 static int has_console(void);
 #endif
@@ -82,86 +80,7 @@ static char *mystrdup(char *s)
     return r;
 }
 
-#ifdef _OSE_
-
-struct args_sig {
-  SIGSELECT sig_no;
-  int argc ;
-  char argv[20][20];
-};
-
-union SIGNAL {
-  SIGSELECT sig_no;
-  struct args_sig args;
-};
-
-/* Start function. It may be called from the start script as well as from
-   the OSE shell directly (using late start hooks). It spawns epmd as an
-   OSE process which calls the epmd main function. */
-int start_ose_epmd(int argc, char **argv) {
-  union SIGNAL *sig;
-  PROCESS epmd_;
-  OSENTRYPOINT ose_epmd;
-  int i;
-
-  if(hunt("epmd", 0, &epmd_, NULL)) {
-    fprintf(stderr, "Warning! EPMD already exists (%u).\n", epmd_);
-    return 0;
-  }
-  else {
-    /* copy start args to signal */
-    sig = alloc(sizeof(struct args_sig), 0);
-    sig->args.argc = argc;
-    for(i=0; i<argc; i++) {
-      strcpy((sig->args.argv)[i], argv[i]);
-    }    
-    /* start epmd and send signal */
-    epmd_ = create_process(OS_BG_PROC, /* processtype */
-			   "epmd",      /* name        */
-			   ose_epmd,    /* entrypoint  */
-			   16383,	/* stacksize   */
-			   20,	        /* priority    */
-			   0,	        /* timeslice   */
-			   0,	        /* block       */
-			   NULL,0,0);   /* not used    */
-    efs_clone(epmd_);
-    start(epmd_);
-    send(&sig, epmd_);	
-#ifdef DEBUG
-    printf("EPMD ID: %li\n", epmd_);
-#endif
-  }
-  return 0;
-}
-
-OS_PROCESS(ose_epmd) {
-  union SIGNAL *sig;
-  static const SIGSELECT rec_any_sig[] = { 0 };
-  int i, argc;
-  char **argv;
-
-  sig = receive((SIGSELECT*)rec_any_sig);
-
-  argc = sig->args.argc;
-  argv = (char **)malloc((argc+1)*sizeof(char *));
-  for(i=0; i<argc; i++) {
-    argv[i] = (char *)malloc(strlen((sig->args.argv)[i])+1);
-    strcpy(argv[i], (sig->args.argv)[i]);
-  }
-  argv[argc] = NULL;
-  free_buf(&sig);
-
-  epmd(argc, argv);
-
-  for(i=0; i<argc; i++) {
-    free(argv[i]);
-  }
-  free(argv);
-}
-
-#else  /* ifdef _OSE_ */
-
-/* VxWorks start function */
+#ifdef VXWORKS
 int start_epmd(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9)
 char *a0, *a1, *a2, *a3, *a4, *a5, *a6, *a7, *a8, *a9;     
 {
@@ -200,10 +119,7 @@ char *a0, *a1, *a2, *a3, *a4, *a5, *a6, *a7, *a8, *a9;
 		   argc,(int) argv,1,
 		   0,0,0,0,0,0,0);
 }
-    
-#endif    /* _OSE_ */
-    
-    
+#endif    /* WxWorks */
 
 
 int epmd(int argc, char **argv)
@@ -218,6 +134,7 @@ int main(int argc, char** argv)
 {
     EpmdVars g_empd_vars;
     EpmdVars *g = &g_empd_vars;
+    int i;
 #ifdef __WIN32__
     WORD wVersionRequested;
     WSADATA wsaData;
@@ -243,21 +160,25 @@ int main(int argc, char** argv)
     g->argv = NULL;
 #endif
 
-    g->port  = get_port_no();
-    g->debug = 0;
+    g->addresses      = get_addresses();
+    g->port           = get_port_no();
+    g->debug          = 0;
 
     g->silent         = 0; 
     g->is_daemon      = 0;
+    g->brutal_kill    = check_relaxed();
     g->packet_timeout = CLOSE_TIMEOUT; /* Default timeout */
     g->delay_accept   = 0;
     g->delay_write    = 0;
     g->progname       = argv[0];
-    g->listenfd       = -1;
     g->conn           = NULL;
     g->nodes.reg = g->nodes.unreg = g->nodes.unreg_tail = NULL;
     g->nodes.unreg_count = 0;
     g->active_conn    = 0;
 
+    for (i = 0; i < MAX_LISTEN_SOCKETS; i++)
+	g->listenfd[i] = -1;
+
     argc--;
     argv++;
     while (argc > 0) {
@@ -283,12 +204,20 @@ int main(int argc, char** argv)
 	} else if (strcmp(argv[0], "-daemon") == 0) {
 	    g->is_daemon = 1;
 	    argv++; argc--;
+	} else if (strcmp(argv[0], "-relaxed_command_check") == 0) {
+	    g->brutal_kill = 1;
+	    argv++; argc--;
 	} else if (strcmp(argv[0], "-kill") == 0) {
 	    if (argc == 1)
 		kill_epmd(g);
 	    else
 		usage(g);
 	    epmd_cleanup_exit(g,0);
+	} else if (strcmp(argv[0], "-address") == 0) {
+	    if (argc == 1)
+	      usage(g);
+	    g->addresses = argv[1];
+	    argv += 2; argc -= 2;
 	} else if (strcmp(argv[0], "-port") == 0) {
 	    if ((argc == 1) ||
 		((g->port = atoi(argv[1])) == 0))
@@ -313,11 +242,17 @@ int main(int argc, char** argv)
 	    else
 		usage(g);
 	    epmd_cleanup_exit(g,0);
+	} else if (strcmp(argv[0], "-stop") == 0) {
+	    if (argc == 2)
+		stop_cli(g, argv[1]);
+	    else
+		usage(g);
+	    epmd_cleanup_exit(g,0);
 	}
 	else
 	    usage(g);
     }
-    dbg_printf(g,0,"epmd running - daemon = %d",g->is_daemon);
+    dbg_printf(g,1,"epmd running - daemon = %d",g->is_daemon);
 
 #ifndef NO_SYSCONF
     if ((g->max_conn = sysconf(_SC_OPEN_MAX)) <= 0)
@@ -327,13 +262,10 @@ int main(int argc, char** argv)
     /*
      * max_conn must not be greater than FD_SETSIZE.
      * (at least QNX crashes)
-     *
-     * More correctly, it must be FD_SETSIZE - 1, beacuse the
-     * listen FD is stored outside the connection array.
      */
   
     if (g->max_conn > FD_SETSIZE) {
-      g->max_conn = FD_SETSIZE - 1;
+      g->max_conn = FD_SETSIZE;
     }
 
     if (g->is_daemon)  {
@@ -392,7 +324,11 @@ static void run_daemon(EpmdVars *g)
       }
 
     /* move cwd to root to make sure we are not on a mounted filesystem  */
-    chdir("/");
+    if (chdir("/") < 0)
+      {
+	dbg_perror(g,"epmd: chdir() failed");
+	epmd_cleanup_exit(g,1);
+      }
     
     umask(0);
 
@@ -453,7 +389,7 @@ static void run_daemon(EpmdVars *g)
 }
 #endif
 
-#if (defined(VXWORKS) || defined(_OSE_))
+#if defined(VXWORKS)
 static void run_daemon(EpmdVars *g)
 {
     run(g);
@@ -468,10 +404,14 @@ static void run_daemon(EpmdVars *g)
 
 static void usage(EpmdVars *g)
 {
-    fprintf(stderr, "usage: epmd [-d|-debug] [DbgExtra...] [-port No] [-daemon]\n");
-    fprintf(stderr, "            [-d|-debug] [-port No] [-names|-kill]\n\n");
-    fprintf(stderr, "See the Erlang epmd manual page for info about the usage.\n");
-    fprintf(stderr, "The -port and DbgExtra options are\n\n");
+    fprintf(stderr, "usage: epmd [-d|-debug] [DbgExtra...] [-address List]\n");
+    fprintf(stderr, "            [-port No] [-daemon] [-relaxed_command_check]\n");
+    fprintf(stderr, "       epmd [-d|-debug] [-port No] [-names|-kill|-stop name]\n\n");
+    fprintf(stderr, "See the Erlang epmd manual page for info about the usage.\n\n");
+    fprintf(stderr, "Regular options\n");
+    fprintf(stderr, "    -address List\n");
+    fprintf(stderr, "        Let epmd listen only on the comma-separated list of IP\n");
+    fprintf(stderr, "        addresses (and on the loopback interface).\n");
     fprintf(stderr, "    -port No\n");
     fprintf(stderr, "        Let epmd listen to another port than default %d\n",
 	    EPMD_PORT_NO);
@@ -481,8 +421,16 @@ static void usage(EpmdVars *g)
     fprintf(stderr, "        the standard error stream. It will shorten\n");
     fprintf(stderr, "        the number of saved used node names to 5.\n\n");
     fprintf(stderr, "        If you give more than one debug flag you may\n");
-    fprintf(stderr, "        get more debugging information.\n\n");
-    fprintf(stderr, "    -packet_timout Seconds\n");
+    fprintf(stderr, "        get more debugging information.\n");
+    fprintf(stderr, "    -daemon\n");
+    fprintf(stderr, "        Start epmd detached (as a daemon)\n");
+    fprintf(stderr, "    -relaxed_command_check\n");
+    fprintf(stderr, "        Allow this instance of epmd to be killed with\n");
+    fprintf(stderr, "        epmd -kill even if there "
+	    "are registered nodes.\n");
+    fprintf(stderr, "        Also allows forced unregister (epmd -stop).\n");
+    fprintf(stderr, "\nDbgExtra options\n");
+    fprintf(stderr, "    -packet_timeout Seconds\n");
     fprintf(stderr, "        Set the number of seconds a connection can be\n");
     fprintf(stderr, "        inactive before epmd times out and closes the\n");
     fprintf(stderr, "        connection (default 60).\n\n");
@@ -494,6 +442,18 @@ static void usage(EpmdVars *g)
     fprintf(stderr, "    -delay_write Seconds\n");
     fprintf(stderr, "        Also a simulation of a busy server. Inserts\n");
     fprintf(stderr, "        a delay before a reply is sent.\n");
+    fprintf(stderr, "\nInteractive options\n");
+    fprintf(stderr, "    -names\n");
+    fprintf(stderr, "        List names registered with the currently "
+	    "running epmd\n");
+    fprintf(stderr, "    -kill\n");
+    fprintf(stderr, "        Kill the currently running epmd\n");
+    fprintf(stderr, "        (only allowed if -names show empty database or\n");
+    fprintf(stderr, "        -relaxed_command_check was given when epmd was started).\n");
+    fprintf(stderr, "    -stop Name\n");
+    fprintf(stderr, "        Forcibly unregisters a name with epmd\n");
+    fprintf(stderr, "        (only allowed if -relaxed_command_check was given when \n");
+    fprintf(stderr, "        epmd was started).\n");
     epmd_cleanup_exit(g,1);
 }
 
@@ -513,20 +473,20 @@ static void usage(EpmdVars *g)
  *      args...      Arguments to print out according to the format
  *      
  */
-
+#define DEBUG_BUFFER_SIZE 2048
 static void dbg_gen_printf(int onsyslog,int perr,int from_level,
 			   EpmdVars *g,const char *format, va_list args)
 {
   time_t now;
   char *timestr;
-  char buf[2048];
+  char buf[DEBUG_BUFFER_SIZE];
 
   if (g->is_daemon)
     {
 #ifndef NO_SYSLOG
       if (onsyslog)
 	{
-	  vsprintf(buf, format, args);
+	  erts_vsnprintf(buf, DEBUG_BUFFER_SIZE, format, args);
 	  syslog(LOG_ERR,"epmd: %s",buf);
 	}
 #endif
@@ -537,11 +497,12 @@ static void dbg_gen_printf(int onsyslog,int perr,int from_level,
 
       time(&now);
       timestr = (char *)ctime(&now);
-      sprintf(buf, "epmd: %.*s: ", (int) strlen(timestr)-1, timestr);
+      erts_snprintf(buf, DEBUG_BUFFER_SIZE, "epmd: %.*s: ",
+		    (int) strlen(timestr)-1, timestr);
       len = strlen(buf);
-      vsprintf(buf + len, format, args);
-      if (perr == 1)
-	perror(buf);
+      erts_vsnprintf(buf + len, DEBUG_BUFFER_SIZE - len, format, args);
+      if (perr != 0)
+	fprintf(stderr,"%s: %s\r\n",buf,strerror(perr));
       else
 	fprintf(stderr,"%s\r\n",buf);
     }
@@ -552,7 +513,7 @@ void dbg_perror(EpmdVars *g,const char *format,...)
 {
   va_list args;
   va_start(args, format);
-  dbg_gen_printf(1,1,0,g,format,args);
+  dbg_gen_printf(1,errno,0,g,format,args);
   va_end(args);
 }
 
@@ -608,8 +569,9 @@ void epmd_cleanup_exit(EpmdVars *g, int exitval)
 	      epmd_conn_close(g,&g->conn[i]);
       free(g->conn);
   }
-  if(g->listenfd >= 0)
-      close(g->listenfd);
+  for(i=0; i < MAX_LISTEN_SOCKETS; i++)
+      if(g->listenfd[i] >= 0)
+          close(g->listenfd[i]);
   free_all_nodes(g);
   if(g->argv){
       for(i=0; g->argv[i] != NULL; ++i)
@@ -621,9 +583,18 @@ void epmd_cleanup_exit(EpmdVars *g, int exitval)
   exit(exitval);
 }
 
+static char* get_addresses(void)
+{
+    return getenv("ERL_EPMD_ADDRESS");
+}
 static int get_port_no(void)
 {
     char* port_str = getenv("ERL_EPMD_PORT");
     return (port_str != NULL) ? atoi(port_str) : EPMD_PORT_NO;
 }
+static int check_relaxed(void)
+{
+    char* port_str = getenv("ERL_EPMD_RELAXED_COMMAND_CHECK");
+    return (port_str != NULL) ? 1 : 0;
+}
 
diff --git a/erts/epmd/src/epmd.h b/erts/epmd/src/epmd.h
index 9e939ee38e..5d6e9ac165 100644
--- a/erts/epmd/src/epmd.h
+++ b/erts/epmd/src/epmd.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2010. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -17,21 +17,18 @@
  * %CopyrightEnd%
  */
 
-/* The port number is now defined in a makefile */
+/* The port number is defined in a makefile */
 
 /* Definitions of message codes */
 
-#define EPMD_ALIVE_REQ 'a'
-#define EPMD_ALIVE_OK_RESP 'Y'
-#define EPMD_PORT_REQ 'p'
+/* Registration and queries */
+#define EPMD_ALIVE2_REQ 'x'
+#define EPMD_PORT2_REQ 'z'
+#define EPMD_ALIVE2_RESP 'y'
+#define EPMD_PORT2_RESP 'w'
 #define EPMD_NAMES_REQ 'n'
+
+/* Interactive client command codes */
 #define EPMD_DUMP_REQ 'd'
 #define EPMD_KILL_REQ 'k'
 #define EPMD_STOP_REQ 's'
-
-/* New epmd messages */
-
-#define EPMD_ALIVE2_REQ 'x' /* 120 */
-#define EPMD_PORT2_REQ 'z' /* 122 */
-#define EPMD_ALIVE2_RESP 'y' /* 121 */
-#define EPMD_PORT2_RESP 'w' /* 119 */
diff --git a/erts/epmd/src/epmd_cli.c b/erts/epmd/src/epmd_cli.c
index c12f711bc5..74408e3ebe 100644
--- a/erts/epmd/src/epmd_cli.c
+++ b/erts/epmd/src/epmd_cli.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -42,7 +42,47 @@ void kill_epmd(EpmdVars *g)
 	epmd_cleanup_exit(g,1);
     }
     if ((rval = read_fill(fd,buf,2)) == 2) {
-	printf("Killed\n");
+	if (buf[0] == 'O' && buf[1] == 'K') {
+	    printf("Killed\n");
+	} else {
+	    printf("Killing not allowed - living nodes in database.\n");
+	}
+	epmd_cleanup_exit(g,0);
+    } else if (rval < 0) {
+	printf("epmd: failed to read answer from local epmd\n");
+	epmd_cleanup_exit(g,1);
+    } else { 			/* rval is now 0 or 1 */
+	buf[rval] = '\0';
+	printf("epmd: local epmd responded with <%s>\n", buf);
+	epmd_cleanup_exit(g,1);
+    }
+}
+
+void stop_cli(EpmdVars *g, char *name)
+{
+    char buf[1024];
+    int fd, rval, bsize;
+
+    bsize = strlen(name);
+    if (bsize > 1000) {
+	printf("epmd: Name too long!");
+	epmd_cleanup_exit(g, 1);
+    }
+
+    fd = conn_to_epmd(g);
+    bsize++;
+    put_int16(bsize, buf);
+    buf[2] = EPMD_STOP_REQ;
+    bsize += 2;
+    strcpy(buf+3, name);
+
+    if (write(fd, buf, bsize) != bsize) {
+	printf("epmd: Can't write to epmd\n");
+	epmd_cleanup_exit(g,1);
+    }
+    if ((rval = read_fill(fd,buf,7)) == 7) {
+	buf[7] = '\000';
+	printf("%s\n", buf);
 	epmd_cleanup_exit(g,0);
     } else if (rval < 0) {
 	printf("epmd: failed to read answer from local epmd\n");
@@ -64,7 +104,10 @@ void epmd_call(EpmdVars *g,int what)
     fd = conn_to_epmd(g);
     put_int16(1,buf);
     buf[2] = what;
-    write(fd,buf,3);
+    if (write(fd, buf, 3) != 3) {
+	printf("epmd: Can't write to epmd\n");
+	epmd_cleanup_exit(g,1);
+    }
     if (read(fd,(char *)&i,4) != 4) {
 	if (!g->silent)
 	    printf("epmd: no response from local epmd\n");
@@ -97,7 +140,7 @@ static int conn_to_epmd(EpmdVars *g)
 
     { /* store port number in unsigned short */
       unsigned short sport = g->port;
-      SET_ADDR_LOOPBACK(address, FAMILY, sport);
+      SET_ADDR(address, EPMD_ADDR_LOOPBACK, sport);
     }
 
     if (connect(connect_sock, (struct sockaddr*)&address, sizeof address) < 0) 
diff --git a/erts/epmd/src/epmd_int.h b/erts/epmd/src/epmd_int.h
index 65fcf9bacb..14d05c3f19 100644
--- a/erts/epmd/src/epmd_int.h
+++ b/erts/epmd/src/epmd_int.h
@@ -1,7 +1,8 @@
+/* -*- c-indent-level: 2; c-continued-statement-offset: 2 -*- */
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -37,19 +38,6 @@
 #define DONT_USE_MAIN
 #endif
 
-#ifdef _OSE_
-#define NO_SYSLOG
-#define NO_SYSCONF
-#define NO_DAEMON
-#define DONT_USE_MAIN
-#ifndef HAVE_SYS_TIME_H
-#define HAVE_SYS_TIME_H
-#endif
-#ifndef HAVE_UNISTD_H
-#define HAVE_UNISTD_H
-#endif
-#endif
-
 /* ************************************************************************ */
 /* Standard includes                                                        */
 
@@ -92,7 +80,7 @@
 #endif
 #endif /* ! VXWORKS */
 
-#if (!defined(__WIN32__) && !defined(_OSE_))
+#if !defined(__WIN32__)
 #  include <netinet/in.h>
 #  include <sys/socket.h>
 #  include <sys/stat.h>
@@ -105,10 +93,8 @@
 #  include <netinet/tcp.h>
 #endif /* ! WIN32 */
 
-#ifndef _OSE_
 #include <ctype.h>
 #include <signal.h>
-#endif
 
 #include <errno.h>
 
@@ -126,13 +112,6 @@
 
 #include <stdarg.h>
 
-#ifdef _OSE_
-#  include "ose.h"
-#  include "inet.h"
-#  include "sys/stat.h"
-#endif
-
-
 /* ************************************************************************ */
 /* Replace some functions by others by making the function name a macro */
 
@@ -148,10 +127,6 @@
 #define sleep(n) taskDelay((n) * sysClkRateGet())
 #endif /* VXWORKS */
 
-#ifdef _OSE_
-#define sleep(n) delay((n))
-#endif
-
 #ifdef USE_BCOPY
 #  define memcpy(a, b, c) bcopy((b), (a), (c))
 #  define memcmp(a, b, c) bcmp((a), (b), (c))
@@ -167,6 +142,10 @@
 #  define EADDRINUSE WSAEADDRINUSE
 #endif
 
+#if defined(__WIN32__) && !defined(ECONNABORTED)
+#  define ECONNABORTED WSAECONNABORTED
+#endif
+
 #ifndef SOMAXCONN
 #  define SOMAXCONN 128
 #endif
@@ -189,50 +168,40 @@
 #if defined(HAVE_IN6) && defined(AF_INET6) && defined(EPMD6)
 
 #define EPMD_SOCKADDR_IN sockaddr_in6
-#define FAMILY      AF_INET6
-
-#define SET_ADDR_LOOPBACK(addr, af, port) do { \
-    static u_int32_t __addr[4] = IN6ADDR_LOOPBACK_INIT; \
-    memset((char*)&(addr), 0, sizeof(addr)); \
-    (addr).sin6_family = (af); \
-    (addr).sin6_flowinfo = 0; \
-    (addr).sin6_addr.s6_addr32[0] = __addr[0]; \
-    (addr).sin6_addr.s6_addr32[1] = __addr[1]; \
-    (addr).sin6_addr.s6_addr32[2] = __addr[2]; \
-    (addr).sin6_addr.s6_addr32[3] = __addr[3]; \
-    (addr).sin6_port = htons(port); \
+#define EPMD_IN_ADDR in6_addr
+#define EPMD_S_ADDR s6_addr
+#define EPMD_ADDR_LOOPBACK in6addr_loopback.s6_addr
+#define EPMD_ADDR_ANY in6addr_any.s6_addr
+#define FAMILY AF_INET6
+
+#define SET_ADDR(dst, addr, port) do { \
+    memset((char*)&(dst), 0, sizeof(dst)); \
+    memcpy((char*)&(dst).sin6_addr.s6_addr, (char*)&(addr), 16); \
+    (dst).sin6_family = AF_INET6; \
+    (dst).sin6_flowinfo = 0; \
+    (dst).sin6_port = htons(port); \
  } while(0)
 
-#define SET_ADDR_ANY(addr, af, port) do { \
-    static u_int32_t __addr[4] = IN6ADDR_ANY_INIT; \
-    memset((char*)&(addr), 0, sizeof(addr)); \
-    (addr).sin6_family = (af); \
-    (addr).sin6_flowinfo = 0; \
-    (addr).sin6_addr.s6_addr32[0] = __addr[0]; \
-    (addr).sin6_addr.s6_addr32[1] = __addr[1]; \
-    (addr).sin6_addr.s6_addr32[2] = __addr[2]; \
-    (addr).sin6_addr.s6_addr32[3] = __addr[3]; \
-    (addr).sin6_port = htons(port); \
- } while(0)
+#define IS_ADDR_LOOPBACK(addr) \
+    (memcmp((addr).s6_addr, in6addr_loopback.s6_addr, 16) == 0)
 
 #else /* Not IP v6 */
 
 #define EPMD_SOCKADDR_IN sockaddr_in
-#define FAMILY      AF_INET
-
-#define SET_ADDR_LOOPBACK(addr, af, port) do { \
-    memset((char*)&(addr), 0, sizeof(addr)); \
-    (addr).sin_family = (af); \
-    (addr).sin_addr.s_addr = htonl(INADDR_LOOPBACK); \
-    (addr).sin_port = htons(port); \
+#define EPMD_IN_ADDR in_addr
+#define EPMD_S_ADDR s_addr
+#define EPMD_ADDR_LOOPBACK htonl(INADDR_LOOPBACK)
+#define EPMD_ADDR_ANY htonl(INADDR_ANY)
+#define FAMILY AF_INET
+
+#define SET_ADDR(dst, addr, port) do { \
+    memset((char*)&(dst), 0, sizeof(dst)); \
+    (dst).sin_family = AF_INET; \
+    (dst).sin_addr.s_addr = (addr); \
+    (dst).sin_port = htons(port); \
  } while(0)
 
-#define SET_ADDR_ANY(addr, af, port) do { \
-    memset((char*)&(addr), 0, sizeof(addr)); \
-    (addr).sin_family = (af); \
-    (addr).sin_addr.s_addr = htonl(INADDR_ANY); \
-    (addr).sin_port = htons(port); \
- } while(0)
+#define IS_ADDR_LOOPBACK(addr) ((addr).s_addr == htonl(INADDR_LOOPBACK))
 
 #endif /* Not IP v6 */
 
@@ -260,6 +229,8 @@
 /* Maximum length of a node name == atom name */
 #define MAXSYMLEN 255
 
+#define MAX_LISTEN_SOCKETS 16
+
 #define INBUF_SIZE 1024
 #define OUTBUF_SIZE 1024
 
@@ -269,14 +240,24 @@
 #define put_int16(i, s) {((unsigned char*)(s))[0] = ((i) >> 8) & 0xff; \
                         ((unsigned char*)(s))[1] = (i)         & 0xff;}
 
+#if defined(__GNUC__)
+#  define EPMD_INLINE __inline__
+#elif defined(__WIN32__)
+#  define EPMD_INLINE __inline
+#else
+#  define EPMD_INLINE
+#endif
+
 /* ************************************************************************ */
 
 /* Stuctures used by server */
 
 typedef struct {
   int fd;			/* File descriptor */
-  unsigned open:1;		/* TRUE if open */
-  unsigned keep:1;		/* Don't close when sent reply */
+  unsigned char open;		/* TRUE if open */
+  unsigned char keep;		/* Don't close when sent reply */
+  unsigned char local_peer;     /* The peer of this connection is via
+                                   loopback interface */
   unsigned got;			/* # of bytes we have got */
   unsigned want;		/* Number of bytes we want */
   char *buf;			/* The remaining buffer */
@@ -316,16 +297,19 @@ typedef struct {
   int debug;
   int silent; 
   int is_daemon;
+  int brutal_kill;
   unsigned packet_timeout;
   unsigned delay_accept;
   unsigned delay_write;
   int max_conn;
   int active_conn;
+  int select_fd_top;
   char *progname;
   Connection *conn;
   Nodes nodes;
   fd_set orig_read_mask;
-  int listenfd;
+  int listenfd[MAX_LISTEN_SOCKETS];
+  char *addresses;
   char **argv;
 } EpmdVars;
 
@@ -337,6 +321,7 @@ void epmd_call(EpmdVars*,int);
 void run(EpmdVars*);
 void epmd_cleanup_exit(EpmdVars*, int);
 int epmd_conn_close(EpmdVars*,Connection*);
+void stop_cli(EpmdVars *g, char *name);
 
 #ifdef DONT_USE_MAIN
 int  start_epmd(char *,char *,char *,char *,char *,char *,char *,char *,char *,char *);
diff --git a/erts/epmd/src/epmd_srv.c b/erts/epmd/src/epmd_srv.c
index a033fab244..da575affa1 100644
--- a/erts/epmd/src/epmd_srv.c
+++ b/erts/epmd/src/epmd_srv.c
@@ -2,7 +2,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -24,6 +24,10 @@
 #include "epmd.h"     /* Renamed from 'epmd_r4.h' */
 #include "epmd_int.h"
 
+#ifndef INADDR_NONE
+#  define INADDR_NONE 0xffffffff
+#endif
+
 /*
  *  
  *  This server is a local name server for Erlang nodes. Erlang nodes can
@@ -39,46 +43,16 @@
  *  server keeps the socket open where the request for registration was
  *  made.
  *
- *  The protocol is briefly documented in "erl_ext_dist.txt". All requests
- *  to this server are done with a packet
+ *  The protocol is briefly documented in the ERTS User's Guide, see
+ *  http://www.erlang.org/doc/apps/erts/erl_dist_protocol.html
+ *
+ *  All requests to this server are done with a packet
  *
  *      2        n
  *  +--------+---------+
  *  | Length | Request |
  *  +--------+---------+
  *
- *  In all but one case there is only one request for each connection made
- *  to this server so we can safely close the socket after sending the
- *  reply.  The exception is ALIVE_REQ where we keep the connection
- *  open without sending any data. When we receive a "close" this is
- *  an indication that the Erlang node was terminated. The termination
- *  may have been "normal" or caused by a crash. The operating system
- *  ensure that the connection is closed either way.
- *  
- *  Reading is done non-blocking, i.e. we call a "read" only if we are
- *  told by the "select" function that there are data to read.
- *  
- *  Two databases are used: One node database where the registered names
- *  of the nodes are stored, and one connection database where the state
- *  of sockets and the data buffers is stored.
- *  
- *  Incomplete packets are thrown away after a timout. The Erlang node
- *  doing the request is responsible for completing in it in a reasonable time.
- *
- *  Note that if the server gets busy it may not have time to
- *  process all requests for connection. The "accept()" function
- *  will on most operating systems silently refuse to accept more
- *  than 5 outstanding requests. It is the client's responsibility
- *  to retry the request a number of times with random time interval.
- *  The "-debug" flag will insert a delay so you can test this 
- *  behaviour.
- *
- *  FIXME: In this code we assume that the packets we send on each
- *  socket is so small that a "write()" never block
- *
- *  FIXME: We never restarts a read or write that was terminated
- *  by an interrupt. Do we need to?
- *
  */
 
 /* We use separate data structures for node names and connections
@@ -98,7 +72,6 @@ static int conn_open(EpmdVars*,int);
 static int conn_close_fd(EpmdVars*,int);
 
 static void node_init(EpmdVars*);
-static Node *node_reg(EpmdVars*,char*,int,int);
 static Node *node_reg2(EpmdVars*,char*, int, int, unsigned char, unsigned char, int, int, int, char*);
 static int node_unreg(EpmdVars*,char*);
 static int node_unreg_sock(EpmdVars*,int);
@@ -107,115 +80,175 @@ static int reply(EpmdVars*,int,char *,int);
 static void dbg_print_buf(EpmdVars*,char *,int);
 static void print_names(EpmdVars*);
 
+static EPMD_INLINE void select_fd_set(EpmdVars* g, int fd)
+{
+    FD_SET(fd, &g->orig_read_mask);
+    if (fd >= g->select_fd_top) {
+	g->select_fd_top = fd + 1;
+    }
+}
 
 void run(EpmdVars *g)
 {
-  int listensock;
+  struct EPMD_SOCKADDR_IN iserv_addr[MAX_LISTEN_SOCKETS];
+  int listensock[MAX_LISTEN_SOCKETS];
+  int num_sockets;
   int i;
   int opt;
-  struct EPMD_SOCKADDR_IN iserv_addr;
+  unsigned short sport = g->port;
 
   node_init(g);
   g->conn = conn_init(g);
 
   dbg_printf(g,2,"try to initiate listening port %d", g->port);
-  
-  if ((listensock = socket(FAMILY,SOCK_STREAM,0)) < 0) {
-    dbg_perror(g,"error opening stream socket");
-    epmd_cleanup_exit(g,1);
-  }
-  g->listenfd = listensock;
+
+  if (g->addresses != NULL && /* String contains non-separator characters if: */
+      g->addresses[strspn(g->addresses," ,")] != '\000')
+    {
+      char *tmp;
+      char *token;
+      int loopback_ok = 0;
+
+      if ((tmp = (char *)malloc(strlen(g->addresses) + 1)) == NULL)
+	{
+	  dbg_perror(g,"cannot allocate memory");
+	  epmd_cleanup_exit(g,1);
+	}
+      strcpy(tmp,g->addresses);
+
+      for(token = strtok(tmp,", "), num_sockets = 0;
+	  token != NULL;
+	  token = strtok(NULL,", "), num_sockets++)
+	{
+	  struct EPMD_IN_ADDR addr;
+#ifdef HAVE_INET_PTON
+	  int ret;
+
+	  if ((ret = inet_pton(FAMILY,token,&addr)) == -1)
+	    {
+	      dbg_perror(g,"cannot convert IP address to network format");
+	      epmd_cleanup_exit(g,1);
+	    }
+	  else if (ret == 0)
+#elif !defined(EPMD6)
+	  if ((addr.EPMD_S_ADDR = inet_addr(token)) == INADDR_NONE)
+#endif
+	    {
+	      dbg_tty_printf(g,0,"cannot parse IP address \"%s\"",token);
+	      epmd_cleanup_exit(g,1);
+	    }
+
+	  if (IS_ADDR_LOOPBACK(addr))
+	    loopback_ok = 1;
+
+	  if (num_sockets - loopback_ok == MAX_LISTEN_SOCKETS - 1)
+	    {
+	      dbg_tty_printf(g,0,"cannot listen on more than %d IP addresses",
+			     MAX_LISTEN_SOCKETS);
+	      epmd_cleanup_exit(g,1);
+	    }
+
+	  SET_ADDR(iserv_addr[num_sockets],addr.EPMD_S_ADDR,sport);
+	}
+
+      free(tmp);
+
+      if (!loopback_ok)
+	{
+	  SET_ADDR(iserv_addr[num_sockets],EPMD_ADDR_LOOPBACK,sport);
+	  num_sockets++;
+	}
+    }
+  else
+    {
+      SET_ADDR(iserv_addr[0],EPMD_ADDR_ANY,sport);
+      num_sockets = 1;
+    }
+
+#if !defined(__WIN32__)
+  /* We ignore the SIGPIPE signal that is raised when we call write
+     twice on a socket closed by the other end. */
+  signal(SIGPIPE, SIG_IGN);
+#endif
 
   /*
    * Initialize number of active file descriptors.
    * Stdin, stdout, and stderr are still open.
-   * One for the listen socket.
    */
-  g->active_conn = 3+1;
+  g->active_conn = 3 + num_sockets;
+  g->max_conn -= num_sockets;
+
+  FD_ZERO(&g->orig_read_mask);
+  g->select_fd_top = 0;
+
+  for (i = 0; i < num_sockets; i++)
+    {
+      if ((listensock[i] = socket(FAMILY,SOCK_STREAM,0)) < 0)
+	{
+	  dbg_perror(g,"error opening stream socket");
+	  epmd_cleanup_exit(g,1);
+	}
+      g->listenfd[i] = listensock[i];
   
-  /*
-   * Note that we must not enable the SO_REUSEADDR on Windows,
-   * because addresses will be reused even if they are still in use.
-   */
+      /*
+       * Note that we must not enable the SO_REUSEADDR on Windows,
+       * because addresses will be reused even if they are still in use.
+       */
   
-#if (!defined(__WIN32__) && !defined(_OSE_))
-  /* We ignore the SIGPIPE signal that is raised when we call write
-     twice on a socket closed by the other end. */
-  signal(SIGPIPE, SIG_IGN);
-
-  opt = 1;			/* Set this option */
-  if (setsockopt(listensock,SOL_SOCKET,SO_REUSEADDR,(char* ) &opt,
-		 sizeof(opt)) <0) {
-    dbg_perror(g,"can't set sockopt");
-    epmd_cleanup_exit(g,1);
-  }
+#if !defined(__WIN32__)
+      opt = 1;
+      if (setsockopt(listensock[i],SOL_SOCKET,SO_REUSEADDR,(char* ) &opt,
+		     sizeof(opt)) <0)
+	{
+	  dbg_perror(g,"can't set sockopt");
+	  epmd_cleanup_exit(g,1);
+	}
 #endif
   
-  /* In rare cases select returns because there is someone
-     to accept but the request is withdrawn before the
-     accept function is called. We set the listen socket
-     to be non blocking to prevent us from being hanging
-     in accept() waiting for the next request. */
-#ifdef _OSE_  
-  opt = 1;
-  if (ioctl(listensock, FIONBIO, (char*)&opt) != 0)
-#else
+      /* In rare cases select returns because there is someone
+	 to accept but the request is withdrawn before the
+	 accept function is called. We set the listen socket
+	 to be non blocking to prevent us from being hanging
+	 in accept() waiting for the next request. */
 #if (defined(__WIN32__) || defined(NO_FCNTL))
-  opt = 1;
-  if (ioctl(listensock, FIONBIO, &opt) != 0) /* Gives warning in VxWorks */
+      opt = 1;
+      /* Gives warning in VxWorks */
+      if (ioctl(listensock[i], FIONBIO, &opt) != 0)
 #else
-  opt = fcntl(listensock, F_GETFL, 0);
-  if (fcntl(listensock, F_SETFL, opt | O_NONBLOCK) == -1)
+      opt = fcntl(listensock[i], F_GETFL, 0);
+      if (fcntl(listensock[i], F_SETFL, opt | O_NONBLOCK) == -1)
 #endif /* __WIN32__ || VXWORKS */
-#endif /* _OSE_ */
-    dbg_perror(g,"failed to set non-blocking mode of listening socket %d",
-	       listensock);
-
-  { /* store port number in unsigned short */
-    unsigned short sport = g->port;
-    SET_ADDR_ANY(iserv_addr, FAMILY, sport);
-  }
-  
-#ifdef _OSE_
-  {
-    int optlen = sizeof(opt);
-    opt = 1;
-    if(getsockopt(listensock, SOL_SOCKET, SO_REUSEADDR,
-		  (void*)&opt, &optlen) < 0)
-      fprintf(stderr, "\n\nGETSOCKOPT FAILS! %d\n\n", errno);
-    else if(opt == 1)
-      fprintf(stderr, "SO_REUSEADDR is set!\n");
-  }
-#endif
+	dbg_perror(g,"failed to set non-blocking mode of listening socket %d",
+		   listensock[i]);
 
-  if(bind(listensock,(struct sockaddr*) &iserv_addr, sizeof(iserv_addr)) < 0 )
-    {
-      if (errno == EADDRINUSE)
-	{
-	  dbg_tty_printf(g,1,"there is already a epmd running at port %d",
-			 g->port);
-	  epmd_cleanup_exit(g,0);
-	}
-      else
+      if (bind(listensock[i], (struct sockaddr*) &iserv_addr[i],
+	  sizeof(iserv_addr[i])) < 0)
 	{
-	  dbg_perror(g,"failed to bind socket");
-	  epmd_cleanup_exit(g,1);
+	  if (errno == EADDRINUSE)
+	    {
+	      dbg_tty_printf(g,1,"there is already a epmd running at port %d",
+			     g->port);
+	      epmd_cleanup_exit(g,0);
+	    }
+	  else
+	    {
+	      dbg_perror(g,"failed to bind socket");
+	      epmd_cleanup_exit(g,1);
+	    }
 	}
-    }
-
-  dbg_printf(g,2,"starting");
-
-  listen(listensock, SOMAXCONN);
-
 
-  FD_ZERO(&g->orig_read_mask);
-  FD_SET(listensock,&g->orig_read_mask);
+      if(listen(listensock[i], SOMAXCONN) < 0) {
+          dbg_perror(g,"failed to listen on socket");
+          epmd_cleanup_exit(g,1);
+      }
+      select_fd_set(g, listensock[i]);
+    }
 
   dbg_tty_printf(g,2,"entering the main select() loop");
 
  select_again:
   while(1)
-    {	
+    {
       fd_set read_mask = g->orig_read_mask;
       struct timeval timeout;
       int ret;
@@ -227,8 +260,17 @@ void run(EpmdVars *g)
       timeout.tv_sec = (g->packet_timeout < IDLE_TIMEOUT) ? 1 : IDLE_TIMEOUT;
       timeout.tv_usec = 0;
 
-      if ((ret = select(g->max_conn,&read_mask,(fd_set *)0,(fd_set *)0,&timeout)) < 0)
+      if ((ret = select(g->select_fd_top,
+			&read_mask, (fd_set *)0,(fd_set *)0,&timeout)) < 0) {
 	dbg_perror(g,"error in select ");
+        switch (errno) {
+          case EAGAIN:
+          case EINTR:
+            break;
+          default:
+            epmd_cleanup_exit(g,1);
+        }
+      }
       else {
 	time_t now;
 	if (ret == 0) {
@@ -238,17 +280,18 @@ void run(EpmdVars *g)
 	  sleep(g->delay_accept);
 	}
 
-	if (FD_ISSET(listensock,&read_mask)) {
-	  if (do_accept(g, listensock) && g->active_conn < g->max_conn) {
-	    /*
-	     * The accept() succeeded, and we have at least one file
-	     * descriptor still free, which means that another accept()
-	     * could succeed. Go do do another select(), in case there
-	     * are more incoming connections waiting to be accepted.
-	     */
-	    goto select_again;
+	for (i = 0; i < num_sockets; i++)
+	  if (FD_ISSET(listensock[i],&read_mask)) {
+	    if (do_accept(g, listensock[i]) && g->active_conn < g->max_conn) {
+	      /*
+	       * The accept() succeeded, and we have at least one file
+	       * descriptor still free, which means that another accept()
+	       * could succeed. Go do do another select(), in case there
+	       * are more incoming connections waiting to be accepted.
+	       */
+	      goto select_again;
+	    }
 	  }
-	}
 	  
 	/* Check all open streams marked by select for data or a
 	   close.  We also close all open sockets except ALIVE
@@ -312,11 +355,9 @@ static void do_read(EpmdVars *g,Connection *s)
 		 s->fd,val);
 	  dbg_print_buf(g,s->buf,val);
 
-	  /* FIXME: Shouldn't be needed to close down.... */
 	  node_unreg_sock(g,s->fd);
 	  epmd_conn_close(g,s);
 	}
-      /* FIXME: We always close, probably the right thing to do */
       return;
     }
 
@@ -401,12 +442,21 @@ static int do_accept(EpmdVars *g,int listensock)
 
     if (msgsock < 0) {
         dbg_perror(g,"error in accept");
-	return EPMD_FALSE;
+        switch (errno) {
+            case EAGAIN:
+            case ECONNABORTED:
+            case EINTR:
+	            return EPMD_FALSE;
+            default:
+	            epmd_cleanup_exit(g,1);
+        }
     }
 
     return conn_open(g,msgsock);
 }
 
+/* buf is actually one byte larger than bsize,
+   giving place for null termination */
 static void do_request(g, fd, s, buf, bsize)
      EpmdVars *g;
      int fd;
@@ -417,117 +467,23 @@ static void do_request(g, fd, s, buf, bsize)
   char wbuf[OUTBUF_SIZE];	/* Buffer for writing */
   int i;
 
-  /*
-   * Terminate packet as a C string.  Needed for requests received from Erlang
-   * nodes with lower version than R3A.
-   */
-
-  buf[bsize] = '\0';
+  buf[bsize] = '\0'; /* Needed for strcmp in PORT2 and STOP requests
+			buf is always large enough */
 
   switch (*buf)
     {
-    case EPMD_ALIVE_REQ:
-      dbg_printf(g,1,"** got ALIVE_REQ");
-
-      /* The packet has the format "axxyyyyyy" where xx is port, given
-	 in network byte order, and yyyyyy is symname, possibly null
-	 terminated. */
-
-      if (buf[bsize - 1] == '\000') /* Skip null termination */
-	bsize--;
-	
-      if (bsize <= 3)
-	{
-	  dbg_printf(g,0,"packet to small for request ALIVE_REQ (%d)", bsize);
-	  return;
-	}
-
-      for (i = 3; i < bsize; i++)
-	if (buf[i] == '\000')
-	  {
-	    dbg_printf(g,0,"node name contains ascii 0 in ALIVE_REQ");
-	    return;
-	  }
-
-      {
-	Node *node;
-	int eport;
-	char *name  = &buf[3];	/* points to node name */
-
-	eport = get_int16(&buf[1]);
-
-	if ((node = node_reg(g, name, fd, eport)) == NULL)
-	  return;
-
-	wbuf[0] = EPMD_ALIVE_OK_RESP;
-	put_int16(node->creation, wbuf+1);
-  
-	if (g->delay_write)		/* Test of busy server */
-	  sleep(g->delay_write);
-
-	if (reply(g, fd, wbuf, 3) != 3)
-	  {
-	    dbg_tty_printf(g,1,"failed to send ALIVE_OK_RESP for \"%s\"",name);
-	    return;
-	  }
-
-	dbg_tty_printf(g,1,"** sent ALIVE_OK_RESP for \"%s\"",name);
-	s->keep = EPMD_TRUE;		/* Don't close on inactivity */
-      }
-      break;
-
-    case EPMD_PORT_REQ:
-      dbg_printf(g,1,"** got PORT_REQ");
-
-      if (buf[bsize - 1] == '\000') /* Skip null termination */
-	bsize--;
-	
-      if (bsize <= 1)
-	{
-	  dbg_printf(g,0,"packet to small for request PORT_REQ (%d)", bsize);
-	  return;
-	}
-
-      for (i = 1; i < bsize; i++)
-	if (buf[i] == '\000')
-	  {
-	    dbg_printf(g,0,"node name contains ascii 0 in PORT_REQ");
-	    return;
-	  }
-
-      {
-	char *name = &buf[1]; /* Points to node name */
-	Node *node;
-
-	for (node = g->nodes.reg; node; node = node->next)
-	  {
-	    if (strcmp(node->symname, name) == 0)
-	      {
-		put_int16(node->port,wbuf);
-		if (reply(g, fd, wbuf, 2) != 2)
-		  {
-		    dbg_tty_printf(g,1,"failed to send PORT_RESP for %s: %d",
-				   name,node->port);
-		    return;
-		  }
-		dbg_tty_printf(g,1,"** sent PORT_RESP for %s: %d",
-			       name,node->port);
-		return;
-	      }
-	  }
-	dbg_tty_printf(g,1,"Closed on PORT_REQ for %s",name);
-      }
-      /* FIXME: How about an answer if no port? Is a close enough? */
-      break;
-
     case EPMD_ALIVE2_REQ:
       dbg_printf(g,1,"** got ALIVE2_REQ");
+      if (!s->local_peer) {
+	   dbg_printf(g,0,"ALIVE2_REQ from non local address");
+	   return;
+      }
 
       /* The packet has the format "axxyyyyyy" where xx is port, given
 	 in network byte order, and yyyyyy is symname, possibly null
 	 terminated. */
 
-      if (bsize <= 13)
+      if (bsize <= 13) /* at least one character for the node name */
 	{
 	  dbg_printf(g,0,"packet to small for request ALIVE2_REQ (%d)",bsize);
 	  return;
@@ -550,7 +506,17 @@ static void do_request(g, fd, s, buf, bsize)
 	highvsn = get_int16(&buf[5]);
 	lowvsn = get_int16(&buf[7]);
 	namelen = get_int16(&buf[9]);
+	if (namelen + 13 > bsize) {
+	    dbg_printf(g,0,"Node name size error in ALIVE2_REQ");
+	    return;
+	}
 	extralen = get_int16(&buf[11+namelen]);
+
+	if (extralen + namelen + 13 > bsize) {
+	    dbg_printf(g,0,"Extra info size error in ALIVE2_REQ");
+	    return;
+	}
+
 	for (i = 11 ; i < 11 + namelen; i ++)
 	    if (buf[i] == '\000')  {
 		dbg_printf(g,0,"node name contains ascii 0 in ALIVE2_REQ");
@@ -593,7 +559,7 @@ static void do_request(g, fd, s, buf, bsize)
 	
       if (bsize <= 1)
 	{
-	  dbg_printf(g,0,"packet to small for request PORT2_REQ (%d)", bsize);
+	  dbg_printf(g,0,"packet too small for request PORT2_REQ (%d)", bsize);
 	  return;
 	}
 
@@ -681,6 +647,10 @@ static void do_request(g, fd, s, buf, bsize)
 
     case EPMD_DUMP_REQ:
       dbg_printf(g,1,"** got DUMP_REQ");
+      if (!s->local_peer) {
+	   dbg_printf(g,0,"DUMP_REQ from non local address");
+	   return;
+      }
       {
 	Node *node;
 
@@ -730,7 +700,19 @@ static void do_request(g, fd, s, buf, bsize)
       break;
 
     case EPMD_KILL_REQ:
+      if (!s->local_peer) {
+	   dbg_printf(g,0,"KILL_REQ from non local address");
+	   return;
+      }
       dbg_printf(g,1,"** got KILL_REQ");
+
+      if (!g->brutal_kill && (g->nodes.reg != NULL)) {
+	  dbg_printf(g,0,"Disallowed KILL_REQ, live nodes");
+	  if (reply(g, fd,"NO",2) != 2)
+	      dbg_printf(g,0,"failed to send reply to KILL_REQ");
+	  return;
+      }
+
       if (reply(g, fd,"OK",2) != 2)
 	dbg_printf(g,0,"failed to send reply to KILL_REQ");
       dbg_tty_printf(g,1,"epmd killed");
@@ -740,9 +722,18 @@ static void do_request(g, fd, s, buf, bsize)
 
     case EPMD_STOP_REQ:
       dbg_printf(g,1,"** got STOP_REQ");
+      if (!s->local_peer) {
+	   dbg_printf(g,0,"STOP_REQ from non local address");
+	   return;
+      }
+      if (!g->brutal_kill) {
+	  dbg_printf(g,0,"Disallowed STOP_REQ, no relaxed_command_check");
+	  return;
+      }
+
       if (bsize <= 1 )
 	{
-	  dbg_printf(g,0,"packet to small for request STOP_REQ (%d)",bsize);
+	  dbg_printf(g,0,"packet too small for request STOP_REQ (%d)",bsize);
 	  return;
 	}
 
@@ -827,15 +818,42 @@ static int conn_open(EpmdVars *g,int fd)
 
   for (i = 0; i < g->max_conn; i++) {
     if (g->conn[i].open == EPMD_FALSE) {
+      struct sockaddr_in si;
+      struct sockaddr_in di;
+#ifdef HAVE_SOCKLEN_T
+      socklen_t st;
+#else
+      int st;
+#endif
+      st = sizeof(si);
+
       g->active_conn++;
       s = &g->conn[i];
      
       /* From now on we want to know if there are data to be read */
-      FD_SET(fd, &g->orig_read_mask);
+      select_fd_set(g, fd);
 
       s->fd   = fd;
       s->open = EPMD_TRUE;
       s->keep = EPMD_FALSE;
+
+      /* Determine if connection is from localhost */
+      if (getpeername(s->fd,(struct sockaddr*) &si,&st) ||
+	  st < sizeof(si)) {
+	  /* Failure to get peername is regarded as non local host */
+	  s->local_peer = EPMD_FALSE;
+      } else {
+	  /* Only 127.x.x.x and connections from the host's IP address
+	     allowed, no false positives */
+	  s->local_peer =
+	      (((((unsigned) ntohl(si.sin_addr.s_addr)) & 0xFF000000U) ==
+	       0x7F000000U) ||
+	       (getsockname(s->fd,(struct sockaddr*) &di,&st) ?
+	       EPMD_FALSE : si.sin_addr.s_addr == di.sin_addr.s_addr));
+      }
+      dbg_tty_printf(g,2,(s->local_peer) ? "Local peer connected" :
+		     "Non-local peer connected");
+
       s->want = 0;		/* Currently unknown */
       s->got  = 0;
       s->mod_time = current_time(g); /* Note activity */
@@ -878,6 +896,7 @@ int epmd_conn_close(EpmdVars *g,Connection *s)
   dbg_tty_printf(g,2,"closing connection on file descriptor %d",s->fd);
 
   FD_CLR(s->fd,&g->orig_read_mask);
+  /* we don't bother lowering g->select_fd_top */
   close(s->fd);			/* Sometimes already closed but close anyway */
   s->open = EPMD_FALSE;
   if (s->buf != NULL) {		/* Should never be NULL but test anyway */
@@ -904,7 +923,7 @@ static void node_init(EpmdVars *g)
 
 
 /* We have got a close on a connection and it may be a
-   EPMD_ALIVE_CLOSE_REQ. Note that this call shouild be called
+   EPMD_ALIVE_CLOSE_REQ. Note that this call should be called
    *before* calling conn_close() */
 
 static int node_unreg(EpmdVars *g,char *name)
@@ -992,11 +1011,6 @@ static int node_unreg_sock(EpmdVars *g,int fd)
  *     Perhaps use the oldest or something.
  */
 
-static Node *node_reg(EpmdVars *g,char *name,int fd, int port)
-{
-    return node_reg2(g, name, fd, port, 0, 0, 0, 0, 0, NULL);
-}
-
 static Node *node_reg2(EpmdVars *g,
 		       char* name,
 		       int fd,
@@ -1022,6 +1036,11 @@ static Node *node_reg2(EpmdVars *g,
       dbg_printf(g,0,"node name is too long (%d) %s", strlen(name), name);
       return NULL;
     }
+  if (extralen > MAXSYMLEN)
+    {
+      dbg_printf(g,0,"extra data is too long (%d) %s", strlen(name), name);
+      return NULL;
+    }
 
   /* Fail if it is already registered */
 
@@ -1107,7 +1126,7 @@ static Node *node_reg2(EpmdVars *g,
   node->extralen = extralen;
   memcpy(node->extra,extra,extralen);
   strcpy(node->symname,name);
-  FD_SET(fd,&g->orig_read_mask);
+  select_fd_set(g, fd);
 
   if (highvsn == 0) {
     dbg_tty_printf(g,1,"registering '%s:%d', port %d",
diff --git a/erts/epmd/test/Makefile b/erts/epmd/test/Makefile
index 13dad09ae3..868b1c023b 100644
--- a/erts/epmd/test/Makefile
+++ b/erts/epmd/test/Makefile
@@ -71,10 +71,10 @@ include $(ERL_TOP)/make/otp_release_targets.mk
 release_spec:
 
 release_tests_spec: opt
-	$(INSTALL_DIR) $(RELEPMDDIR)
+	$(INSTALL_DIR) "$(RELEPMDDIR)"
 	$(INSTALL_DATA) epmd.spec epmd.spec.vxworks $(ERL_FILES) \
-		$(EMAKEFILE) $(RELEPMDDIR)
-	chmod -f -R u+w $(RELEPMDDIR)
+		$(EMAKEFILE) "$(RELEPMDDIR)"
+	chmod -R u+w "$(RELEPMDDIR)"
 
 release_docs_spec:
 
diff --git a/erts/epmd/test/epmd.spec b/erts/epmd/test/epmd.spec
index 0e2496bc72..e72272cf94 100644
--- a/erts/epmd/test/epmd.spec
+++ b/erts/epmd/test/epmd.spec
@@ -1 +1 @@
-{topcase, {dir, "../epmd_test"}}.
+{suites,"../epmd_test",all}.
diff --git a/erts/epmd/test/epmd_SUITE.erl b/erts/epmd/test/epmd_SUITE.erl
index 91e09faf75..3f31cd979c 100644
--- a/erts/epmd/test/epmd_SUITE.erl
+++ b/erts/epmd/test/epmd_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1998-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1998-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -17,7 +17,7 @@
 %% %CopyrightEnd%
 %%
 -module(epmd_SUITE).
--include("test_server.hrl").
+-include_lib("test_server/include/test_server.hrl").
 -include_lib("kernel/include/file.hrl").
 
 
@@ -35,7 +35,9 @@
 -record(node_info, {port, node_type, prot, lvsn, hvsn, node_name, extra}).
 
 % Test server specific exports
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2]).
 
 -export(
    [
@@ -63,7 +65,13 @@
     alive_req_too_large/1,
 
     returns_valid_empty_extra/1,
-    returns_valid_populated_extra_with_nulls/1
+    returns_valid_populated_extra_with_nulls/1,
+
+    buffer_overrun_1/1,
+    buffer_overrun_2/1,
+    no_nonlocal_register/1,
+    no_nonlocal_kill/1,
+    no_live_killing/1
    ]).
 
 
@@ -82,11 +90,8 @@
 -define(REG_REPEAT_LIM,1000).
 
 % Message codes in epmd protocol
--define(EPMD_ALIVE_REQ,		$a).
 -define(EPMD_ALIVE2_REQ,	$x).
--define(EPMD_ALIVE_OK_RESP,	$Y).
 -define(EPMD_ALIVE2_RESP,	$y).
--define(EPMD_PORT_REQ,		$p).
 -define(EPMD_PORT_PLEASE2_REQ,	$z).
 -define(EPMD_PORT2_RESP,	$w).
 -define(EPMD_NAMES_REQ,	$n).
@@ -98,45 +103,48 @@
 %% all/1
 %%
 
-all(suite) ->
-    [
-     register_name,
-     register_names_1,
-     register_names_2,
-     register_duplicate_name,
-     get_port_nr,
-     slow_get_port_nr,
-     unregister_others_name_1,
-     unregister_others_name_2,
-     register_overflow,
-     name_with_null_inside,
-     name_null_terminated,
-     stupid_names_req,
-
-     no_data,
-     one_byte,
-     two_bytes,
-     partial_packet,
-     zero_length,
-     too_large,
-     alive_req_too_small_1,
-     alive_req_too_small_2,
-     alive_req_too_large,
-
-     returns_valid_empty_extra,
-     returns_valid_populated_extra_with_nulls
-    ].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [register_name, register_names_1, register_names_2,
+     register_duplicate_name, get_port_nr, slow_get_port_nr,
+     unregister_others_name_1, unregister_others_name_2,
+     register_overflow, name_with_null_inside,
+     name_null_terminated, stupid_names_req, no_data,
+     one_byte, two_bytes, partial_packet, zero_length,
+     too_large, alive_req_too_small_1, alive_req_too_small_2,
+     alive_req_too_large, returns_valid_empty_extra,
+     returns_valid_populated_extra_with_nulls,
+     {group, buffer_overrun}, no_nonlocal_register,
+     no_nonlocal_kill, no_live_killing].
+
+groups() -> 
+    [{buffer_overrun, [],
+      [buffer_overrun_1, buffer_overrun_2]}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 %%
 %% Run before and after each test case
 %%
 
 init_per_testcase(_Func, Config) ->
-    Dog = test_server:timetrap(?SHORT_TEST_TIMEOUT),
+    Dog = test_server:timetrap(?MEDIUM_TEST_TIMEOUT),
     cleanup(),
     [{watchdog, Dog} | Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     cleanup(),
     Dog = ?config(watchdog, Config),
     catch test_server:timetrap_cancel(Dog),	% We may have canceled already
@@ -148,7 +156,7 @@ register_name(doc) ->
     ["Register a name"];
 register_name(suite) ->
     [];
-register_name(Config) when list(Config) ->
+register_name(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = register_node("foobar"),
     ?line ok = close(Sock),			% Unregister
@@ -158,7 +166,7 @@ register_names_1(doc) ->
     ["Register and unregister two nodes"];
 register_names_1(suite) ->
     [];
-register_names_1(Config) when list(Config) ->
+register_names_1(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock1} = register_node("foobar"),
     ?line {ok,Sock2} = register_node("foozap"),
@@ -170,7 +178,7 @@ register_names_2(doc) ->
     ["Register and unregister two nodes"];
 register_names_2(suite) ->
     [];
-register_names_2(Config) when list(Config) ->
+register_names_2(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock1} = register_node("foobar"),
     ?line {ok,Sock2} = register_node("foozap"),
@@ -182,7 +190,7 @@ register_duplicate_name(doc) ->
     ["Two nodes with the same name"];
 register_duplicate_name(suite) ->
     [];
-register_duplicate_name(Config) when list(Config) ->
+register_duplicate_name(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = register_node("foobar"),
     ?line error = register_node("foobar"),
@@ -192,22 +200,9 @@ register_duplicate_name(Config) when list(Config) ->
 % Internal function to register a node name, no close, i.e. unregister
 
 register_node(Name) ->
-    register_node(Name,?DUMMY_PORT).
-
-register_node(Name, Port) ->
-    case send_req([?EPMD_ALIVE_REQ, put16(Port), Name]) of
-	{ok,Sock} ->
-	    case recv(Sock,3) of
-		{ok, [?EPMD_ALIVE_OK_RESP,_D1,_D0]} ->
-		    {ok,Sock};
-		Other ->
-		    test_server:format("recv on sock ~w: ~p~n",
-				       [Sock,Other]),
-		    error
-	    end;
-	error ->
-	    error
-    end.
+    register_node_v2(?DUMMY_PORT,$M,0,5,5,Name,"").
+register_node(Name,Port) ->
+    register_node_v2(Port,$M,0,5,5,Name,"").
 
 register_node_v2(Port, NodeType, Prot, HVsn, LVsn, Name, Extra) ->
     Req = [?EPMD_ALIVE2_REQ, put16(Port), NodeType, Prot,
@@ -254,7 +249,7 @@ parse_port2_resp(Resp) ->
 			    hvsn=HVsn,lvsn=LVsn,
 			    node_name=binary_to_list(NodeName),
 			    extra=binary_to_list(Extra)}};
-	Other ->
+	_Other ->
 	    test_server:format("invalid port2 resp: ~p~n",
 			       [Resp]),
 	    error
@@ -266,7 +261,7 @@ name_with_null_inside(doc) ->
     ["Register a name with a null char in it"];
 name_with_null_inside(suite) ->
     [];
-name_with_null_inside(Config) when list(Config) ->
+name_with_null_inside(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line error = register_node("foo\000bar"),
     ok.
@@ -277,11 +272,9 @@ name_null_terminated(doc) ->
     ["Register a name with terminating null byte"];
 name_null_terminated(suite) ->
     [];
-name_null_terminated(Config) when list(Config) ->
+name_null_terminated(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
-    ?line {ok,Sock} = register_node("foobar\000"),
-    ?line error = register_node("foobar"),
-    ?line ok = close(Sock),			% Unregister
+    ?line error = register_node("foobar\000"),
     ok.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -290,7 +283,7 @@ stupid_names_req(doc) ->
     ["Read names from epmd in a stupid way"];
 stupid_names_req(suite) ->
     [];
-stupid_names_req(Config) when list(Config) ->
+stupid_names_req(Config) when is_list(Config) ->
     Dog = ?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     LongDog = test_server:timetrap(?MEDIUM_TEST_TIMEOUT),
@@ -394,15 +387,15 @@ get_port_nr(doc) ->
     ["Register a name on a port and ask about port nr"];
 get_port_nr(suite) ->
     [];
-get_port_nr(Config) when list(Config) ->
-    port_request([?EPMD_PORT_REQ,"foo"]).
+get_port_nr(Config) when is_list(Config) ->
+    port_request([?EPMD_PORT_PLEASE2_REQ,"foo"]).
 
 slow_get_port_nr(doc) ->
     ["Register with slow write and ask about port nr"];
 slow_get_port_nr(suite) ->
     [];
-slow_get_port_nr(Config) when list(Config) ->
-    port_request([?EPMD_PORT_REQ,d,$f,d,$o,d,$o]).
+slow_get_port_nr(Config) when is_list(Config) ->
+    port_request([?EPMD_PORT_PLEASE2_REQ,d,$f,d,$o,d,$o]).
 
 
 % Internal function used above
@@ -413,9 +406,18 @@ port_request(M) ->
     ?line {ok,RSock} = register_node("foo", Port),
     ?line {ok,Sock} = connect(),
     ?line ok = send(Sock,[size16(M),M]),
-    R = put16(Port),
-    ?line {ok,R} = recv(Sock, length(R)),
-    ?line ok = close(RSock),
+    ?line case recv_until_sock_closes(Sock) of
+	      {ok, Resp} ->
+		  ?line close(RSock),
+		  ?line {ok,Rec} = parse_port2_resp(Resp),
+		  ?line Port = Rec#node_info.port,
+		  ok;
+	      Other ->
+		  ?line close(RSock),
+		  ?line test_server:format("recv on sock ~w: ~p~n",
+					   [Sock,Other]),
+		  ?line throw({error,Other})
+	  end,
     ok.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -424,8 +426,8 @@ unregister_others_name_1(doc) ->
     ["Unregister name of other node"];
 unregister_others_name_1(suite) ->
     [];
-unregister_others_name_1(Config) when list(Config) ->
-    ?line ok = epmdrun(),
+unregister_others_name_1(Config) when is_list(Config) ->
+    ?line ok = epmdrun("-relaxed_command_check"),
     ?line {ok,RSock} = register_node("foo"),
     ?line {ok,Sock} = connect(),
     M = [?EPMD_STOP_REQ,"foo"],
@@ -441,8 +443,8 @@ unregister_others_name_2(doc) ->
     ["Unregister name of other node"];
 unregister_others_name_2(suite) ->
     [];
-unregister_others_name_2(Config) when list(Config) ->
-    ?line ok = epmdrun(),
+unregister_others_name_2(Config) when is_list(Config) ->
+    ?line ok = epmdrun("-relaxed_command_check"),
     ?line {ok,Sock} = connect(),
     M = [?EPMD_STOP_REQ,"xxx42"],
     ?line ok = send(Sock,[size16(M),M]),
@@ -456,7 +458,7 @@ register_overflow(doc) ->
     ["Register too many, clean and redo 10 times"];
 register_overflow(suite) ->
     [];
-register_overflow(Config) when list(Config) ->
+register_overflow(Config) when is_list(Config) ->
     Dog = ?config(watchdog, Config),
     test_server:timetrap_cancel(Dog),
     LongDog = test_server:timetrap(?LONG_TEST_TIMEOUT),
@@ -546,7 +548,7 @@ no_data(doc) ->
     ["Open but send no data"];
 no_data(suite) ->
     [];
-no_data(Config) when list(Config) ->
+no_data(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = connect(),
     sleep(?LONG_PAUSE),
@@ -559,7 +561,7 @@ one_byte(doc) ->
     ["Send one byte only"];
 one_byte(suite) ->
     [];
-one_byte(Config) when list(Config) ->
+one_byte(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = connect(),
     ?line ok = send(Sock,[0]),
@@ -573,7 +575,7 @@ two_bytes(doc) ->
     ["Send packet size only"];
 two_bytes(suite) ->
     [];
-two_bytes(Config) when list(Config) ->
+two_bytes(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = connect(),
     ?line ok = send(Sock,[put16(3)]),
@@ -587,7 +589,7 @@ partial_packet(doc) ->
     ["Got only part of a packet"];
 partial_packet(suite) ->
     [];
-partial_packet(Config) when list(Config) ->
+partial_packet(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = connect(),
     ?line ok = send(Sock,[put16(100),"only a few bytes"]),
@@ -601,7 +603,7 @@ zero_length(doc) ->
     ["Invalid zero packet size"];
 zero_length(suite) ->
     [];
-zero_length(Config) when list(Config) ->
+zero_length(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = connect(),
     ?line ok = send(Sock,[0,0,0,0,0,0,0,0,0,0]),
@@ -615,15 +617,20 @@ too_large(doc) ->
     ["Invalid large packet"];
 too_large(suite) ->
     [];
-too_large(Config) when list(Config) ->
+too_large(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = connect(),
     Size = 63000,
     M = lists:duplicate(Size, $z),
     ?line ok = send(Sock,[put16(Size),M]),
     sleep(?MEDIUM_PAUSE),
-    ?line closed = recv(Sock,1),
-    ok.
+    % With such a large packet, even the writes can fail as the
+    % daemon closes before everything is delivered -> econnaborted
+    case recv(Sock,1) of
+	closed -> ok;
+	{error,econnaborted} -> ok;
+	Other -> exit({unexpected,Other})
+    end.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
@@ -631,10 +638,11 @@ alive_req_too_small_1(doc) ->
     ["Try to register but not enough data"];
 alive_req_too_small_1(suite) ->
     [];
-alive_req_too_small_1(Config) when list(Config) ->
+alive_req_too_small_1(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = connect(),
-    M = [?EPMD_ALIVE_REQ, 42],
+    M = [?EPMD_ALIVE2_REQ, put16(?DUMMY_PORT),$M,0, put16(5),
+	 put16(5),put16(0)],
     ?line ok = send(Sock, [size16(M), M]),
     sleep(?MEDIUM_PAUSE),
     ?line closed = recv(Sock,1),
@@ -646,10 +654,11 @@ alive_req_too_small_2(doc) ->
     ["Try to register but not enough data"];
 alive_req_too_small_2(suite) ->
     [];
-alive_req_too_small_2(Config) when list(Config) ->
+alive_req_too_small_2(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = connect(),
-    M = [?EPMD_ALIVE_REQ, put16(?DUMMY_PORT)],
+    M =  [?EPMD_ALIVE2_REQ, put16(?DUMMY_PORT),$M,0, put16(5),
+	  put16(5)],
     ?line ok = send(Sock, [size16(M), M]),
     sleep(?MEDIUM_PAUSE),
     ?line closed = recv(Sock,1),
@@ -661,7 +670,7 @@ alive_req_too_large(doc) ->
     ["Try to register but node name too large"];
 alive_req_too_large(suite) ->
     [];
-alive_req_too_large(Config) when list(Config) ->
+alive_req_too_large(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = connect(),
     L = [
@@ -678,10 +687,12 @@ alive_req_too_large(Config) when list(Config) ->
 	 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
 	 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
 	 ],
-    M = [?EPMD_ALIVE_REQ, put16(?DUMMY_PORT), L],
+    S = length(lists:flatten(L)),
+    M = [?EPMD_ALIVE2_REQ, put16(?DUMMY_PORT),$M,0, put16(5),
+	 put16(5), put16(S),L,put16(0)],
     ?line ok = send(Sock, [size16(M), M]),
     sleep(?MEDIUM_PAUSE),
-    ?line closed = recv(Sock,1),
+    ?line {ok,[?EPMD_ALIVE2_RESP,1]} = recv(Sock,2),
     ok.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -690,7 +701,7 @@ returns_valid_empty_extra(doc) ->
     ["Check that an empty extra is prefixed by a two byte length"];
 returns_valid_empty_extra(suite) ->
     [];
-returns_valid_empty_extra(Config) when list(Config) ->
+returns_valid_empty_extra(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = register_node_v2(4711, 72, 0, 5, 5, "foo", []),
     ?line {ok,#node_info{extra=[]}} = port_please_v2("foo"),
@@ -703,7 +714,7 @@ returns_valid_populated_extra_with_nulls(doc) ->
     ["Check a populated extra with embedded null characters"];
 returns_valid_populated_extra_with_nulls(suite) ->
     [];
-returns_valid_populated_extra_with_nulls(Config) when list(Config) ->
+returns_valid_populated_extra_with_nulls(Config) when is_list(Config) ->
     ?line ok = epmdrun(),
     ?line {ok,Sock} = register_node_v2(4711, 72, 0, 5, 5, "foo", "ABC\000\000"),
     ?line {ok,#node_info{extra="ABC\000\000"}} = port_please_v2("foo"),
@@ -711,6 +722,165 @@ returns_valid_populated_extra_with_nulls(Config) when list(Config) ->
     ok.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+buffer_overrun_1(suite) ->
+    [];
+buffer_overrun_1(doc) ->
+    ["Test security vulnerability in fake extra lengths in alive2_req"];
+buffer_overrun_1(Config) when is_list(Config) ->
+    ?line ok = epmdrun(),
+    ?line true = alltrue([hostile(N) || N <- lists:seq(1,10000)]),
+    ok.
+buffer_overrun_2(suite) ->
+    [];
+buffer_overrun_2(doc) ->
+    ["Test security vulnerability in fake extra lengths in alive2_req"];
+buffer_overrun_2(Config) when is_list(Config) ->
+    ?line ok = epmdrun(),
+    ?line [false | Rest] = [hostile2(N) || N <- lists:seq(255,10000)],
+    ?line true = alltrue(Rest),
+    ok.
+hostile(N) ->
+    try
+	Bin= <<$x:8,4747:16,$M:8,0:8,5:16,5:16,5:16,"gurka",N:16>>,
+	S = size(Bin),
+	{ok,E}=connect_sturdy(),
+	gen_tcp:send(E,[<<S:16>>,Bin]),
+	closed = recv(E,1),
+	gen_tcp:close(E),
+	true
+    catch
+	_:_ ->
+	    false
+    end.
+hostile2(N) ->
+    try
+	B2 = list_to_binary(lists:duplicate(N,255)),
+	Bin= <<$x:8,4747:16,$M:8,0:8,5:16,5:16,5:16,"gurka",N:16,B2/binary>>,
+	S = size(Bin),
+	{ok,E}=connect_sturdy(),
+	gen_tcp:send(E,[<<S:16>>,Bin]),
+	Z = recv(E,2),
+	gen_tcp:close(E),
+	(Z =:= closed) or (Z =:= {ok, [$y,1]})
+    catch
+	_A:_B ->
+	    false
+    end.
+
+alltrue([]) ->
+    true;
+alltrue([true|T]) ->
+    alltrue(T);
+alltrue([_|_]) ->
+    false.
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+no_nonlocal_register(suite) ->
+    [];
+no_nonlocal_register(doc) ->
+    ["Ensure that we cannot register throug a nonlocal connection"];
+no_nonlocal_register(Config) when is_list(Config) ->
+    ?line case {os:find_executable("ssh"),ct:get_config(ssh_proxy_host)} of
+	      {SSH,Name} when is_list(Name), is_list(SSH) ->
+		  do_no_nonlocal_register(Config,Name);
+	      {false,_} ->
+		  {skip, "No ssh command found to create proxy"};
+	      _ ->
+		  {skip, "No ssh_proxy_host configured in ts.config"}
+	  end.
+do_no_nonlocal_register(Config,SSHHost) when is_list(Config) ->
+    ?line ok = epmdrun(),
+    ?line ProxyPort = proxy_port(),
+    ?line ok = ssh_proxy(SSHHost,ProxyPort),
+    Res = try
+	      ?line Name = "gurka_"
+	      %++
+	      %integer_to_list(A1)++"_"++
+	      %integer_to_list(A2)++"_"++
+	      %integer_to_list(A3)++"_"++
+	      %integer_to_list(A4)
+	      ,
+	      ?line Bname = list_to_binary(Name),
+	      ?line NameS = byte_size(Bname),
+	      ?line Bin= <<$x:8,4747:16,$M:8,0:8,5:16,
+			  5:16,NameS:16,Bname/binary,
+			  0:16>>,
+	      ?line S = size(Bin),
+	      ?line {ok, E} = connect("localhost",ProxyPort,passive),
+	      ?line gen_tcp:send(E,[<<S:16>>,Bin]),
+	      ?line closed = recv(E,1),
+	      ?line gen_tcp:close(E),
+	      true
+	  catch
+	      _:_ ->
+		  false
+	  end,
+    %erlang:display(Res),
+    true = Res,
+    ok.
+
+no_nonlocal_kill(suite) ->
+    [];
+no_nonlocal_kill(doc) ->
+    ["Ensure that we cannot kill through nonlocal connection"];
+no_nonlocal_kill(Config) when is_list(Config) ->
+    ?line case {os:find_executable("ssh"),ct:get_config(ssh_proxy_host)} of
+	      {SSH,Name} when is_list(Name), is_list(SSH) ->
+		  do_no_nonlocal_kill(Config,Name);
+	      {false,_} ->
+		  {skip, "No ssh command found to create proxy"};
+	      _ ->
+		  {skip, "No ssh_proxy_host configured in ts.config"}
+	  end.
+do_no_nonlocal_kill(Config,SSHHost) when is_list(Config) ->
+    ?line ok = epmdrun(),
+    ?line ProxyPort = proxy_port(),
+    ?line ok = ssh_proxy(SSHHost,ProxyPort),
+    Res = try
+	      {ok, E} = connect("localhost",ProxyPort,passive),
+	      M = [?EPMD_KILL_REQ],
+	      send(E, [size16(M), M]),
+	      closed = recv(E,2),
+	      gen_tcp:close(E),
+	      sleep(?MEDIUM_PAUSE),
+	      {ok, E2} = connect("localhost",ProxyPort,passive),
+	      gen_tcp:close(E2),
+	      true
+	  catch
+	      _:_ ->
+		  false
+	  end,
+    %erlang:display(Res),
+    true = Res,
+    ok.
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+no_live_killing(doc) ->
+    ["Dont allow killing with live nodes or any unregistering w/o -relaxed_command_check"];
+no_live_killing(suite) ->
+    [];
+no_live_killing(Config) when is_list(Config) ->
+    ?line ok = epmdrun(),
+    ?line {ok,RSock} = register_node("foo"),
+    ?line {ok,Sock} = connect(),
+    ?line M = [?EPMD_KILL_REQ],
+    ?line ok = send(Sock,[size16(M),M]),
+    ?line {ok,"NO"} = recv(Sock,2),
+    ?line close(Sock),
+    ?line {ok,Sock2} = connect(),
+    ?line M2 = [?EPMD_STOP_REQ,"foo"],
+    ?line ok = send(Sock2,[size16(M2),M2]),
+    ?line closed = recv(Sock2,1),
+    ?line close(Sock2),
+    ?line close(RSock),
+    ?line sleep(?MEDIUM_PAUSE),
+    ?line {ok,Sock3} = connect(),
+    ?line M3 = [?EPMD_KILL_REQ],
+    ?line ok = send(Sock3,[size16(M3),M3]),
+    ?line {ok,"OK"} = recv(Sock3,2),
+    ?line close(Sock3),
+    ok.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 % Terminate all tests with killing epmd.
 
 cleanup() ->
@@ -726,21 +896,42 @@ cleanup() ->
 	    true
     end.
 
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% Start an ssh channel to simulate remote access
+
+proxy_port() ->
+    ?PORT+1.
+
+ssh_proxy(SSHHost,ProxyPort) ->
+    ?line Host = lists:nth(2,string:tokens(atom_to_list(node()),"@")),
+    % Requires proxy to be a unix host with the command 'read' accessible
+    ?line osrun("ssh -L "++integer_to_list(ProxyPort)++":"++Host++":"
+		++integer_to_list(?PORT)++" "++SSHHost++" read").
+    
+
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 % Normal debug start of epmd
 
 epmdrun() ->
+    epmdrun([]).
+epmdrun(Args) ->
     case os:find_executable(epmd) of
 	false ->
 	    {error, {could_not_find_epmd_in_path}};
 	Path ->
-	    epmdrun(Path)
+	    epmdrun(Path,Args)
     end.
 
-epmdrun(Epmd) ->
+epmdrun(Epmd,Args0) ->
   %% test_server:format("epmdrun() => Epmd = ~p",[Epmd]),
-  osrun(Epmd ++ " " ?EPMDARGS " -port " ++ integer_to_list(?PORT)).
+    Args = case Args0 of
+	       [] ->
+		   [];
+	       O ->
+		   " "++O
+	   end,
+  osrun("\"" ++ Epmd ++ "\"" ++ Args ++ " " ?EPMDARGS " -port " ++ integer_to_list(?PORT)).
 
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -753,20 +944,27 @@ osrun(Cmd) ->
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 % Wrappers of TCP functions
 
-% These two functions is the interface for connect.
+% These functions is the interface for connect.
 % Passive mode is the default
 
 connect() ->
-    connect(?PORT, passive).
+    connect("localhost",?PORT, passive).
+
+connect(Addr) ->
+    connect(Addr,?PORT, passive).
 
 connect_active() ->
-    connect(?PORT, active).
+    connect("localhost",?PORT, active).
 
+%% Retry after 15 seconds, to avoid TIME_WAIT socket exhaust.
+connect_sturdy() ->
+    connect("localhost",?PORT, passive, 15000, 3).
 
 % Try a few times before giving up
-
-connect(Port, Mode) ->
-    case connect_repeat(?CONN_RETRY, Port, Mode) of
+connect(Addr, Port, Mode) ->
+    connect(Addr, Port, Mode, ?CONN_SLEEP, ?CONN_RETRY).
+connect(Addr, Port, Mode, Sleep, Retry) ->
+    case connect_repeat(Addr, Retry, Port, Mode, Sleep) of
 	{ok,Sock} ->
 	    {ok,Sock};
 	{error,timeout} ->
@@ -783,25 +981,25 @@ connect(Port, Mode) ->
 % Try a few times before giving up. Pause a small time between
 % each try.
 
-connect_repeat(1, Port, Mode) ->
-    connect_mode(Port, Mode);
-connect_repeat(Retry, Port, Mode) ->
-    case connect_mode(Port, Mode) of
+connect_repeat(Addr, 1, Port, Mode, _Sleep) ->
+    connect_mode(Addr,Port, Mode);
+connect_repeat(Addr,Retry, Port, Mode, Sleep) ->
+    case connect_mode(Addr,Port, Mode) of
 	{ok,Sock} ->
 	    {ok,Sock};
 	{error,Reason} ->
 	    test_server:format("connect: error: ~w~n",[Reason]),
-	    timer:sleep(?CONN_SLEEP),
-	    connect_repeat(Retry - 1, Port, Mode);
+	    timer:sleep(Sleep),
+	    connect_repeat(Addr, Retry - 1, Port, Mode, Sleep);
 	Any ->
 	    test_server:format("connect: unknown message: ~w~n",[Any]),
 	    exit(1)
     end.
 
-connect_mode(Port, active) ->
-    gen_tcp:connect("localhost", Port, [{packet, 0}], ?CONN_TIMEOUT);
-connect_mode(Port, passive) ->
-    gen_tcp:connect("localhost", Port, [{packet, 0}, {active, false}],
+connect_mode(Addr,Port, active) ->
+    gen_tcp:connect(Addr, Port, [{packet, 0}], ?CONN_TIMEOUT);
+connect_mode(Addr, Port, passive) ->
+    gen_tcp:connect(Addr, Port, [{packet, 0}, {active, false}],
 		    ?CONN_TIMEOUT).
 
 
@@ -858,9 +1056,9 @@ send(Sock, SendSpec) ->
 
 send([], RevBytes, _Sock) ->
     {ok,RevBytes};
-send([Byte | Spec], RevBytes, Sock) when integer(Byte) ->
+send([Byte | Spec], RevBytes, Sock) when is_integer(Byte) ->
     send(Spec, [Byte | RevBytes], Sock);
-send([List | Spec], RevBytes, Sock) when list(List) ->
+send([List | Spec], RevBytes, Sock) when is_list(List) ->
     case send(List, RevBytes, Sock) of
 	{ok,Left} ->
 	    send(Spec, Left, Sock);
diff --git a/erts/etc/common/Makefile.in b/erts/etc/common/Makefile.in
index 20c4c3c2b1..83fe97df8e 100644
--- a/erts/etc/common/Makefile.in
+++ b/erts/etc/common/Makefile.in
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1996-2010. All Rights Reserved.
+# Copyright Ericsson AB 1996-2012. All Rights Reserved.
 #
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -22,13 +22,14 @@ include $(ERL_TOP)/make/target.mk
 ERTS_LIB_TYPEMARKER=.$(TYPE)
 
 USING_MINGW=@MIXED_CYGWIN_MINGW@
-USING_VC=@MIXED_CYGWIN_VC@
+USING_VC=@MIXED_VC@
 
 ifeq ($(TYPE),debug)
 PURIFY =
 TYPEMARKER = .debug
 TYPE_FLAGS = -DDEBUG @DEBUG_FLAGS@
 else
+
 ifeq ($(TYPE),purify)
 PURIFY = purify
 TYPEMARKER =
@@ -38,6 +39,8 @@ else
 TYPE_FLAGS = -g -O2 -DPURIFY
 endif
 else
+
+override TYPE=opt
 PURIFY =
 TYPEMARKER =
 ERTS_LIB_TYPEMARKER=
@@ -99,11 +102,13 @@ endif
 
 # On windows we always need reentrant libraries.
 ifeq ($(TARGET),win32)
-ERLEXEC_XLIBS=-L../../lib/internal/$(TARGET) -lerts_internal_r$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@
+ERTS_INTERNAL_LIBS=-L../../lib/internal/$(TARGET) -lerts_internal_r$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@
 else
-ERLEXEC_XLIBS=-L../../lib/internal/$(TARGET) -lerts_internal$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@
+ERTS_INTERNAL_LIBS=-L../../lib/internal/$(TARGET) -lerts_internal$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@ -lm
 endif
 
+ERTS_LIB = $(ERL_TOP)/erts/lib_src/obj/$(TARGET)/$(TYPE)/MADE
+
 # ----------------------------------------------------
 # Release directory specification
 # ----------------------------------------------------
@@ -164,9 +169,8 @@ ERLSRV_OBJECTS= \
 	$(OBJDIR)/erlsrv_main.o \
 	$(OBJDIR)/erlsrv_util.o \
 	$(OBJDIR)/erlsrv_logmess.res
-MC_OUTPUTS= \
-	$(OBJDIR)/erlsrv_logmess.h $(OBJDIR)/erlsrv_logmess.rc
-	MT_FLAG="-MT"
+MC_OUTPUTS=$(OBJDIR)/erlsrv_logmess.h $(OBJDIR)/erlsrv_logmess.rc
+MT_FLAG="-MT"
 else
 ERLRES_OBJ=erl_res.o
 ERLSRV_OBJECTS= \
@@ -176,12 +180,11 @@ ERLSRV_OBJECTS= \
 	$(OBJDIR)/erlsrv_main.o \
 	$(OBJDIR)/erlsrv_util.o \
 	$(OBJDIR)/erlsrv_logmess.o
-MC_OUTPUTS= \
-	$(OBJDIR)/erlsrv_logmess.h $(OBJDIR)/erlsrv_logmess.res
-	MT_FLAG="-MD"
+MC_OUTPUTS=$(OBJDIR)/erlsrv_logmess.h $(OBJDIR)/erlsrv_logmess.res
+MT_FLAG="-MD"
 endif
 INET_GETHOST = $(BINDIR)/inet_gethost.exe
-INSTALL_EMBEDDED_PROGS += $(BINDIR)/typer.exe $(BINDIR)/dialyzer.exe $(BINDIR)/erlc.exe $(BINDIR)/start_erl.exe $(BINDIR)/escript.exe
+INSTALL_EMBEDDED_PROGS += $(BINDIR)/typer.exe $(BINDIR)/dialyzer.exe $(BINDIR)/erlc.exe $(BINDIR)/start_erl.exe $(BINDIR)/escript.exe $(BINDIR)/ct_run.exe
 INSTALL_SRC = $(WINETC)/start_erl.c $(WINETC)/Nmakefile.start_erl
 ERLEXECDIR=.
 INSTALL_LIBS =
@@ -214,7 +217,7 @@ ERLSRV_OBJECTS=
 MC_OUTPUTS=
 INET_GETHOST = $(BINDIR)/inet_gethost@EXEEXT@
 INSTALL_EMBEDDED_PROGS += $(BINDIR)/typer@EXEEXT@ $(BINDIR)/dialyzer@EXEEXT@ \
- $(BINDIR)/erlc@EXEEXT@ $(BINDIR)/escript@EXEEXT@ \
+ $(BINDIR)/erlc@EXEEXT@ $(BINDIR)/escript@EXEEXT@ $(BINDIR)/ct_run@EXEEXT@ \
  $(BINDIR)/run_erl $(BINDIR)/to_erl $(BINDIR)/dyn_erl
 INSTALL_EMBEDDED_DATA = ../unix/start.src ../unix/start_erl.src
 INSTALL_TOP = Install
@@ -235,14 +238,17 @@ endif
 endif
 endif
 
-etc:	erts_lib $(ENTRY_OBJ) $(INSTALL_PROGS) $(INSTALL_LIBS) $(TEXTFILES) $(INSTALL_TOP_BIN)
+.PHONY: etc
+etc:	$(ENTRY_OBJ) $(INSTALL_PROGS) $(INSTALL_LIBS) $(TEXTFILES) $(INSTALL_TOP_BIN)
 
 # erlexec needs the erts_internal library...
-erts_lib:
+$(ERTS_LIB):
 	cd $(ERL_TOP)/erts/lib_src && $(MAKE) $(TYPE)
 
+.PHONY: docs
 docs:
 
+.PHONY: clean
 clean:
 ifneq ($(INSTALL_PROGS),)
 	rm -f $(INSTALL_PROGS)
@@ -277,81 +283,13 @@ endif
 	rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/dyn_erl.o
 	rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/safe_string.o	
 	rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/typer.o
+	rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/ct_run.o
 	rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/vxcall.o
 	rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/erl.o
 	rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/werl.o
+	rm -f $(TEXTFILES)
 	rm -f *~ core
 
-#
-# Objects & executables
-#
-#$(OBJDIR)/%.o: %.c
-#	$(CC) $(CFLAGS) -o $@ -c $<
-#
-#$(OBJDIR)/%.o: ../unix/%.c
-#	$(CC) $(CFLAGS) -o $@ -c $<
-#
-#$(BINDIR)/%: $(OBJDIR)/%.o
-#	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $< $(LIBS)
-
-$(OBJDIR)/inet_gethost.o: inet_gethost.c
-	$(CC) $(CFLAGS) -o $@ -c inet_gethost.c
-
-$(BINDIR)/inet_gethost@EXEEXT@: $(OBJDIR)/inet_gethost.o $(ENTRY_OBJ)
-	$(PURIFY) $(LD) $(LDFLAGS) $(ENTRY_LDFLAGS) -o $@ $(OBJDIR)/inet_gethost.o $(ENTRY_OBJ) $(LIBS) 
-
-$(BINDIR)/run_erl: $(OBJDIR)/safe_string.o $(OBJDIR)/run_erl.o
-	$(LD) $(LDFLAGS) -o $@ $(OBJDIR)/safe_string.o $(OBJDIR)/run_erl.o $(LIBS)
-
-$(OBJDIR)/run_erl.o: ../unix/run_erl.c
-	$(CC) $(CFLAGS) -o $@ -c ../unix/run_erl.c
-
-$(BINDIR)/to_erl: $(OBJDIR)/safe_string.o $(OBJDIR)/to_erl.o
-	$(LD) $(LDFLAGS) -o $@  $(OBJDIR)/safe_string.o $(OBJDIR)/to_erl.o
-
-$(OBJDIR)/to_erl.o: ../unix/to_erl.c
-	$(CC) $(CFLAGS) -o $@ -c ../unix/to_erl.c
-
-$(BINDIR)/dyn_erl: $(OBJDIR)/safe_string.o $(OBJDIR)/dyn_erl.o
-	$(LD) $(LDFLAGS) -o $@  $(OBJDIR)/safe_string.o $(OBJDIR)/dyn_erl.o
-
-$(OBJDIR)/dyn_erl.o: ../unix/dyn_erl.c
-	$(CC) $(CFLAGS) -o $@ -c ../unix/dyn_erl.c
-
-$(OBJDIR)/safe_string.o: ../unix/safe_string.c
-	$(CC) $(CFLAGS) -o $@ -c ../unix/safe_string.c
-
-ifneq ($(TARGET),win32)
-$(BINDIR)/$(ERLEXEC): $(OBJDIR)/$(ERLEXEC).o
-	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/$(ERLEXEC).o $(ERLEXEC_XLIBS)
-
-$(OBJDIR)/$(ERLEXEC).o: $(ERLEXECDIR)/$(ERLEXEC).c
-	$(CC) -I$(EMUDIR) $(CFLAGS) -o $@ -c $(ERLEXECDIR)/$(ERLEXEC).c
-endif
-$(BINDIR)/erlc@EXEEXT@: $(OBJDIR)/erlc.o
-	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/erlc.o -L$(OBJDIR) $(LIBS)
-
-$(OBJDIR)/erlc.o: erlc.c
-	$(CC) $(CFLAGS) -o $@ -c erlc.c
-
-$(BINDIR)/dialyzer@EXEEXT@: $(OBJDIR)/dialyzer.o
-	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/dialyzer.o -L$(OBJDIR) $(LIBS)
-
-$(OBJDIR)/dialyzer.o: dialyzer.c
-	$(CC) $(CFLAGS) -o $@ -c dialyzer.c
-
-$(BINDIR)/typer@EXEEXT@: $(OBJDIR)/typer.o
-	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/typer.o -L$(OBJDIR) $(LIBS)
-
-$(OBJDIR)/typer.o: typer.c
-	$(CC) $(CFLAGS) -o $@ -c typer.c
-
-$(BINDIR)/escript@EXEEXT@: $(OBJDIR)/escript.o
-	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/escript.o -L$(OBJDIR) $(LIBS)
-
-$(OBJDIR)/escript.o: escript.c
-	$(CC) $(CFLAGS) -o $@ -c escript.c
-
 #------------------------------------------------------------------------
 # Windows specific targets
 # The windows platform is quite different from the others. erl/werl are small C programs
@@ -362,8 +300,8 @@ $(OBJDIR)/escript.o: escript.c
 
 ifeq ($(TARGET),win32)
 
-$(BINDIR)/$(ERLEXEC): $(OBJDIR)/erlexec.o $(OBJDIR)/win_erlexec.o $(OBJDIR)/init_file.o  $(OBJDIR)/$(ERLRES_OBJ)
-	$(LD) -dll $(LDFLAGS) -o $@ $(OBJDIR)/erlexec.o $(OBJDIR)/win_erlexec.o $(OBJDIR)/init_file.o $(OBJDIR)/$(ERLRES_OBJ) $(ERLEXEC_XLIBS)
+$(BINDIR)/$(ERLEXEC): $(OBJDIR)/erlexec.o $(OBJDIR)/win_erlexec.o $(OBJDIR)/init_file.o  $(OBJDIR)/$(ERLRES_OBJ) $(ERTS_LIB)
+	$(LD) -dll $(LDFLAGS) -o $@ $(OBJDIR)/erlexec.o $(OBJDIR)/win_erlexec.o $(OBJDIR)/init_file.o $(OBJDIR)/$(ERLRES_OBJ) $(ERTS_INTERNAL_LIBS)
 
 $(BINDIR)/erl@EXEEXT@: $(OBJDIR)/erl.o $(OBJDIR)/init_file.o $(OBJDIR)/$(ERLRES_OBJ)
 	$(LD) $(LDFLAGS) -o $@ $(OBJDIR)/erl.o $(OBJDIR)/init_file.o $(OBJDIR)/$(ERLRES_OBJ) 
@@ -377,58 +315,75 @@ $(BINDIR)/start_erl@EXEEXT@: $(OBJDIR)/start_erl.o
 $(BINDIR)/Install@EXEEXT@: $(OBJDIR)/Install.o  $(OBJDIR)/init_file.o 
 	$(LD) $(LDFLAGS) -o $@  $(OBJDIR)/Install.o  $(OBJDIR)/init_file.o
 
+# The service expects to be compiled with $(MT_FLAG) flag.
 $(BINDIR)/erlsrv@EXEEXT@: $(ERLSRV_OBJECTS)
 	$(LD) $(LDFLAGS) $(MT_FLAG) -o $@ $(ERLSRV_OBJECTS)
 
-# The service expects to be compiled with $(MT_FLAG) flag.
-
-$(OBJDIR)/%.o: $(WINETC)/erlsrv/%.c $(ERLSRV_HEADERS)
-	$(CC) $(CFLAGS) $(MT_FLAG) -o $@ -c $<
-
-$(OBJDIR)/erlsrv_util.o: $(WINETC)/erlsrv/erlsrv_util.c $(ERLSRV_HEADERS) \
-$(OBJDIR)/erlsrv_logmess.h
-	$(CC) $(CFLAGS) -I$(OBJDIR) $(MT_FLAG) -o $@ -c $<
+# To fix a spooky parallel make build problem on Windows there are some
+# false dependencies on the $(MC), $(RC) and .o rules. The theory behind
+# is that the $(MC) and/or $(RC) compilers seems to temporarily create and
+# destroy a toplevel file vc100.pdb that simultaneous ordinary compilations
+# (that have been by flags explicitly ordered to not use .pdb files)
+# gets upset when that file vanishes from under their feet.
+#
+# The false dependencies are targeted to make make (pun intended) run
+# the $(MC) and $(RC) compilations to completion before all others.
+
+LOGMESS_GENERATED = $(OBJDIR)/LOGMESS-GENERATED
+$(MC_OUTPUTS): $(LOGMESS_GENERATED)
+$(LOGMESS_GENERATED): $(WINETC)/erlsrv/erlsrv_logmess.mc
+	$(MC) -o $(OBJDIR) $(WINETC)/erlsrv/erlsrv_logmess.mc && \
+		echo $? >$(LOGMESS_GENERATED)
+
+$(OBJDIR)/$(ERLRES_OBJ): $(WINETC)/erl.rc $(WINETC)/erlang.ico \
+		$(WINETC)/erl_icon.ico $(WINETC)/hrl_icon.ico \
+		$(WINETC)/beam_icon.ico $(LOGMESS_GENERATED)
+	$(RC) -o $@ -I$(WINETC) $(WINETC)/erl.rc
 
 ifeq ($(USING_VC), yes)
-$(OBJDIR)/erlsrv_logmess.res: $(OBJDIR)/erlsrv_logmess.rc
+RC_GENERATED = $(OBJDIR)/erlsrv_logmess.res
+$(RC_GENERATED): $(OBJDIR)/erlsrv_logmess.rc $(OBJDIR)/$(ERLRES_OBJ)
 	$(RC) -o $(OBJDIR)/erlsrv_logmess.res -I$(OBJDIR) $(OBJDIR)/erlsrv_logmess.rc
 else
-$(OBJDIR)/erlsrv_logmess.o: $(OBJDIR)/erlsrv_logmess.res
+RC_GENERATED = $(OBJDIR)/erlsrv_logmess.o
+$(RC_GENERATED): $(OBJDIR)/erlsrv_logmess.res $(OBJDIR)/$(ERLRES_OBJ)
 	$(RC) -o $(OBJDIR)/erlsrv_logmess.o -I$(OBJDIR) $(OBJDIR)/erlsrv_logmess.res
 endif
 
-$(MC_OUTPUTS): $(WINETC)/erlsrv/erlsrv_logmess.mc
-	$(MC) -o $(OBJDIR) $(WINETC)/erlsrv/erlsrv_logmess.mc
+# The service expects to be compiled with $(MT_FLAG) flag.
+$(OBJDIR)/%.o: $(WINETC)/erlsrv/%.c $(ERLSRV_HEADERS) $(RC_GENERATED)
+	$(CC) $(CFLAGS) $(MT_FLAG) -o $@ -c $<
+
+$(OBJDIR)/erlsrv_util.o: $(WINETC)/erlsrv/erlsrv_util.c $(ERLSRV_HEADERS) \
+		$(OBJDIR)/erlsrv_logmess.h $(RC_GENERATED)
+	$(CC) $(CFLAGS) -I$(OBJDIR) $(MT_FLAG) -o $@ -c $<
 
-$(OBJDIR)/werl.o: $(WINETC)/erl.c
+$(OBJDIR)/werl.o: $(WINETC)/erl.c $(WINETC)/init_file.h $(RC_GENERATED)
 	$(CC) $(CFLAGS) -DBUILD_TYPE=\"-$(TYPE)\" -DERL_RUN_SHARED_LIB=1 \
 	-DWIN32_WERL -o $@ -c $(WINETC)/erl.c
 
-$(OBJDIR)/erl.o: $(WINETC)/erl.c
+$(OBJDIR)/erl.o: $(WINETC)/erl.c $(WINETC)/init_file.h $(RC_GENERATED)
 	$(CC) $(CFLAGS) -DBUILD_TYPE=\"-$(TYPE)\" -DERL_RUN_SHARED_LIB=1 \
 	-o $@ -c $(WINETC)/erl.c
 
-$(OBJDIR)/erlexec.o: $(ERLEXECDIR)/erlexec.c
+$(OBJDIR)/erlexec.o: $(ERLEXECDIR)/erlexec.c $(RC_GENERATED)
 	$(CC) $(CFLAGS) -DBUILD_TYPE=\"-$(TYPE)\" -DERL_RUN_SHARED_LIB=1 \
 	-o $@ -c $(ERLEXECDIR)/erlexec.c
 
-$(OBJDIR)/win_erlexec.o: $(WINETC)/win_erlexec.c
+$(OBJDIR)/win_erlexec.o: $(WINETC)/win_erlexec.c $(RC_GENERATED)
 	$(CC) $(CFLAGS) -DBUILD_TYPE=\"-$(TYPE)\" -DERL_RUN_SHARED_LIB=1 \
 	-o $@ -c $(WINETC)/win_erlexec.c
 
-$(OBJDIR)/init_file.o: $(WINETC)/init_file.c $(WINETC)/init_file.h
+$(OBJDIR)/init_file.o: $(WINETC)/init_file.c $(WINETC)/init_file.h $(RC_GENERATED)
 	$(CC) $(CFLAGS) -o $@ -c $(WINETC)/init_file.c
 
-$(OBJDIR)/Install.o: $(WINETC)/Install.c $(WINETC)/init_file.h
+$(OBJDIR)/Install.o: $(WINETC)/Install.c $(WINETC)/init_file.h $(RC_GENERATED)
 	$(CC) $(CFLAGS) -o $@ -c $(WINETC)/Install.c
 
-$(OBJDIR)/$(ERLRES_OBJ): $(WINETC)/erl.rc $(WINETC)/erlang.ico $(WINETC)/erl_icon.ico $(WINETC)/hrl_icon.ico $(WINETC)/beam_icon.ico
-	$(RC) -o $@ -I$(WINETC) $(WINETC)/erl.rc
-
-$(OBJDIR)/start_erl.o: $(WINETC)/start_erl.c
+$(OBJDIR)/start_erl.o: $(WINETC)/start_erl.c $(RC_GENERATED)
 	$(CC) $(CFLAGS) -o $@ -c $(WINETC)/start_erl.c
 
-$(ENTRY_OBJ): $(ENTRY_SRC)
+$(ENTRY_OBJ): $(ENTRY_SRC) $(RC_GENERATED)
 	$(CC) $(CFLAGS) -o $@ -c $(ENTRY_SRC)
 
 Install.ini:	../$(TARGET)/Install.src ../../vsn.mk $(TARGET)/Makefile
@@ -437,6 +392,8 @@ Install.ini:	../$(TARGET)/Install.src ../../vsn.mk $(TARGET)/Makefile
           ../$(TARGET)/Install.src > Install.ini
 
 
+else
+RC_GENERATED =
 endif
 #---------------------------------------------------------
 # End of windows specific targets.
@@ -464,7 +421,7 @@ $(BINDIR)/heart@EXEEXT@: $(OBJDIR)/heart.o $(ENTRY_OBJ)
 	$(LD) $(LDFLAGS) $(ENTRY_LDFLAGS) -o $@ $(OBJDIR)/heart.o \
 		$(RTLIBS) $(ENTRY_OBJ) $(WINDSOCK)
 
-$(OBJDIR)/heart.o: heart.c
+$(OBJDIR)/heart.o: heart.c $(RC_GENERATED)
 	$(CC) $(CFLAGS) -o $@ -c heart.c
 
 endif
@@ -492,6 +449,84 @@ $(OBJDIR)/vxcall.o: $(VXETC)/vxcall.c
 
 
 
+#
+# Objects & executables
+#
+#$(OBJDIR)/%.o: %.c
+#	$(CC) $(CFLAGS) -o $@ -c $<
+#
+#$(OBJDIR)/%.o: ../unix/%.c
+#	$(CC) $(CFLAGS) -o $@ -c $<
+#
+#$(BINDIR)/%: $(OBJDIR)/%.o
+#	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $< $(LIBS)
+
+$(OBJDIR)/inet_gethost.o: inet_gethost.c $(RC_GENERATED)
+	$(CC) $(CFLAGS) -o $@ -c inet_gethost.c
+
+$(BINDIR)/inet_gethost@EXEEXT@: $(OBJDIR)/inet_gethost.o $(ENTRY_OBJ) $(ERTS_LIB)
+	$(PURIFY) $(LD) $(LDFLAGS) $(ENTRY_LDFLAGS) -o $@ $(OBJDIR)/inet_gethost.o $(ENTRY_OBJ) $(LIBS) $(ERTS_INTERNAL_LIBS)
+
+$(BINDIR)/run_erl: $(OBJDIR)/safe_string.o $(OBJDIR)/run_erl.o
+	$(LD) $(LDFLAGS) -o $@ $(OBJDIR)/safe_string.o $(OBJDIR)/run_erl.o $(LIBS)
+
+$(OBJDIR)/run_erl.o: ../unix/run_erl.c $(RC_GENERATED)
+	$(CC) $(CFLAGS) -o $@ -c ../unix/run_erl.c
+
+$(BINDIR)/to_erl: $(OBJDIR)/safe_string.o $(OBJDIR)/to_erl.o
+	$(LD) $(LDFLAGS) -o $@  $(OBJDIR)/safe_string.o $(OBJDIR)/to_erl.o
+
+$(OBJDIR)/to_erl.o: ../unix/to_erl.c $(RC_GENERATED)
+	$(CC) $(CFLAGS) -o $@ -c ../unix/to_erl.c
+
+$(BINDIR)/dyn_erl: $(OBJDIR)/safe_string.o $(OBJDIR)/dyn_erl.o
+	$(LD) $(LDFLAGS) -o $@  $(OBJDIR)/safe_string.o $(OBJDIR)/dyn_erl.o
+
+$(OBJDIR)/dyn_erl.o: ../unix/dyn_erl.c $(RC_GENERATED)
+	$(CC) $(CFLAGS) -o $@ -c ../unix/dyn_erl.c
+
+$(OBJDIR)/safe_string.o: ../unix/safe_string.c $(RC_GENERATED)
+	$(CC) $(CFLAGS) -o $@ -c ../unix/safe_string.c
+
+ifneq ($(TARGET),win32)
+$(BINDIR)/$(ERLEXEC): $(OBJDIR)/$(ERLEXEC).o $(ERTS_LIB)
+	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/$(ERLEXEC).o $(ERTS_INTERNAL_LIBS)
+
+$(OBJDIR)/$(ERLEXEC).o: $(ERLEXECDIR)/$(ERLEXEC).c $(RC_GENERATED)
+	$(CC) -I$(EMUDIR) $(CFLAGS) -o $@ -c $(ERLEXECDIR)/$(ERLEXEC).c
+endif
+$(BINDIR)/erlc@EXEEXT@: $(OBJDIR)/erlc.o $(ERTS_LIB)
+	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/erlc.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS)
+
+$(OBJDIR)/erlc.o: erlc.c $(RC_GENERATED)
+	$(CC) $(CFLAGS) -o $@ -c erlc.c
+
+$(BINDIR)/dialyzer@EXEEXT@: $(OBJDIR)/dialyzer.o $(ERTS_LIB)
+	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/dialyzer.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS)
+
+$(OBJDIR)/dialyzer.o: dialyzer.c $(RC_GENERATED)
+	$(CC) $(CFLAGS) -o $@ -c dialyzer.c
+
+$(BINDIR)/typer@EXEEXT@: $(OBJDIR)/typer.o $(ERTS_LIB)
+	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/typer.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS)
+
+$(OBJDIR)/typer.o: typer.c $(RC_GENERATED)
+	$(CC) $(CFLAGS) -o $@ -c typer.c
+
+$(BINDIR)/escript@EXEEXT@: $(OBJDIR)/escript.o $(ERTS_LIB)
+	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/escript.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS)
+
+$(OBJDIR)/escript.o: escript.c $(RC_GENERATED)
+	$(CC) $(CFLAGS) -o $@ -c escript.c
+
+$(BINDIR)/ct_run@EXEEXT@: $(OBJDIR)/ct_run.o $(ERTS_LIB)
+	$(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/ct_run.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS)
+
+$(OBJDIR)/ct_run.o: ct_run.c $(RC_GENERATED)
+	$(CC) $(CFLAGS) -o $@ -c ct_run.c
+
+
+
 Install:	../unix/Install.src ../../vsn.mk $(TARGET)/Makefile
 	sed -e 's;%I_VSN%;$(VSN);' \
 	    -e 's;%EMULATOR%;$(EMULATOR);' \
@@ -510,55 +545,52 @@ erl.src: ../unix/erl.src.src ../../vsn.mk $(TARGET)/Makefile
 # ---------------------------------------------------- 
 include $(ERL_TOP)/make/otp_release_targets.mk
 
+.PHONY: release_spec
 release_spec: etc
 ifneq ($(INSTALL_OBJS),)
-	$(INSTALL_DIR) $(RELEASE_PATH)/erts-$(VSN)/obj
-	$(INSTALL_DATA) $(INSTALL_OBJS) $(RELEASE_PATH)/erts-$(VSN)/obj
+	$(INSTALL_DIR) "$(RELEASE_PATH)/erts-$(VSN)/obj"
+	$(INSTALL_DATA) $(INSTALL_OBJS) "$(RELEASE_PATH)/erts-$(VSN)/obj"
 endif
-	$(INSTALL_DIR) $(RELEASE_PATH)/erts-$(VSN)/bin
+	$(INSTALL_DIR) "$(RELEASE_PATH)/erts-$(VSN)/bin"
 ifneq ($(TARGET), win32)
 ifneq ($(findstring vxworks,$(TARGET)), vxworks)
 ifneq ($(findstring ose,$(TARGET)), ose)
-	$(INSTALL_SCRIPT) erl.src $(RELEASE_PATH)/erts-$(VSN)/bin
+	$(INSTALL_SCRIPT) erl.src "$(RELEASE_PATH)/erts-$(VSN)/bin"
 endif
 endif
 endif
 ifneq ($(INSTALL_PROGS),)
-	$(INSTALL_PROGRAM) $(INSTALL_PROGS) $(RELEASE_PATH)/erts-$(VSN)/bin
+	$(INSTALL_PROGRAM) $(INSTALL_PROGS) "$(RELEASE_PATH)/erts-$(VSN)/bin"
 endif
 ifneq ($(INSTALL_TOP),)
-	$(INSTALL_SCRIPT) $(INSTALL_TOP) $(RELEASE_PATH)
+	$(INSTALL_SCRIPT) $(INSTALL_TOP) "$(RELEASE_PATH)"
 endif
 ifneq ($(INSTALL_TOP_BIN),)
-	$(INSTALL_PROGRAM) $(INSTALL_TOP_BIN) $(RELEASE_PATH)
+	$(INSTALL_PROGRAM) $(INSTALL_TOP_BIN) "$(RELEASE_PATH)"
 endif
 ifneq ($(INSTALL_MISC),)
-	$(INSTALL_DIR) $(RELEASE_PATH)/misc
-	$(INSTALL_SCRIPT) $(INSTALL_MISC) $(RELEASE_PATH)/misc
+	$(INSTALL_DIR) "$(RELEASE_PATH)/misc"
+	$(INSTALL_SCRIPT) $(INSTALL_MISC) "$(RELEASE_PATH)/misc"
 endif
 ifneq ($(INSTALL_ERL_OSE),)
-	$(INSTALL_DIR) $(RELEASE_PATH)/build_erl_ose
+	$(INSTALL_DIR) "$(RELEASE_PATH)/build_erl_ose"
 	cd $(OSEETC) && $(TAR) erl_ose_$(SYSTEM_VSN).tar $(INSTALL_ERL_OSE)
-	cd $(OSEETC) && $(INSTALL_SCRIPT) erl_ose_$(SYSTEM_VSN).tar $(RELEASE_PATH)/build_erl_ose
+	cd $(OSEETC) && $(INSTALL_SCRIPT) erl_ose_$(SYSTEM_VSN).tar "$(RELEASE_PATH)/build_erl_ose"
 endif
 ifneq ($(INSTALL_SRC),)
-	$(INSTALL_DIR) $(RELEASE_PATH)/erts-$(VSN)/src
-	$(INSTALL_DATA) $(INSTALL_SRC) $(RELEASE_PATH)/erts-$(VSN)/src
+	$(INSTALL_DIR) "$(RELEASE_PATH)/erts-$(VSN)/src"
+	$(INSTALL_DATA) $(INSTALL_SRC) "$(RELEASE_PATH)/erts-$(VSN)/src"
 endif
 ifneq ($(INSTALL_EMBEDDED_DATA),)
-	$(INSTALL_DATA) $(INSTALL_EMBEDDED_DATA) $(RELEASE_PATH)/erts-$(VSN)/bin
+	$(INSTALL_DATA) $(INSTALL_EMBEDDED_DATA) "$(RELEASE_PATH)/erts-$(VSN)/bin"
 endif
 ifneq ($(INSTALL_LIBS),)
-	$(INSTALL_DATA) $(INSTALL_LIBS) $(RELEASE_PATH)/erts-$(VSN)/bin
+	$(INSTALL_DATA) $(INSTALL_LIBS) "$(RELEASE_PATH)/erts-$(VSN)/bin"
 endif
 ifneq ($(INSTALL_INCLUDES),)
-	$(INSTALL_DIR) $(RELEASE_PATH)/erts-$(VSN)/include
-	$(INSTALL_DATA) $(INSTALL_INCLUDES) $(RELEASE_PATH)/erts-$(VSN)/include
+	$(INSTALL_DIR) "$(RELEASE_PATH)/erts-$(VSN)/include"
+	$(INSTALL_DATA) $(INSTALL_INCLUDES) "$(RELEASE_PATH)/erts-$(VSN)/include"
 endif
 
+.PHONY: release_docs_spec
 release_docs_spec:
-
-
-
-
-
diff --git a/erts/etc/common/ct_run.c b/erts/etc/common/ct_run.c
new file mode 100644
index 0000000000..7aaab716f7
--- /dev/null
+++ b/erts/etc/common/ct_run.c
@@ -0,0 +1,545 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * Purpose: Common Test front-end.
+ */
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include "sys.h"
+#ifdef __WIN32__
+#include <winbase.h>
+#endif
+
+#include <ctype.h>
+
+#define NO 0
+#define YES 1
+
+#define ASIZE(a) (sizeof(a)/sizeof(a[0]))
+
+static int debug = 0;		/* Bit flags for debug printouts. */
+
+static char** eargv_base;	/* Base of vector. */
+static char** eargv;		/* First argument for erl. */
+
+static int eargc;		/* Number of arguments in eargv. */
+
+#ifdef __WIN32__
+#  define QUOTE(s) possibly_quote(s)
+#  define IS_DIRSEP(c) ((c) == '/' || (c) == '\\')
+#  define ERL_NAME "erl.exe"
+#else
+#  define QUOTE(s) s
+#  define IS_DIRSEP(c) ((c) == '/')
+#  define ERL_NAME "erl"
+#endif
+
+#define UNSHIFT(s) eargc++, eargv--; eargv[0] = QUOTE(s)
+#define PUSH(s) eargv[eargc++] = QUOTE(s)
+#define PUSH2(s, t) PUSH(s); PUSH(t)
+#define PUSH3(s, t, u) PUSH2(s, t); PUSH(u)
+#define PUSH4(s, t, u, v) PUSH2(s, t); PUSH2(u, v)
+
+/*
+ * The possible modes to start Common Test
+ */
+
+#define NORMAL_MODE 0
+#define VTS_MODE 1
+#define CT_SHELL_MODE 2
+#define MASTER_MODE 3
+#define ERL_SHELL_MODE 4
+
+/*
+ * Distribution
+ */
+
+#define SHORT_NAME 0
+#define FULL_NAME 1
+
+/*
+ * Local functions.
+ */
+
+static void error(char* format, ...);
+static char* emalloc(size_t size);
+static char* strsave(char* string);
+static void push_words(char* src);
+static int run_erlang(char* name, char** argv);
+static char* get_default_emulator(char* progname);
+static void print_deprecation_warning(char *progname); 
+#ifdef __WIN32__
+static char* possibly_quote(char* arg);
+#endif
+
+/*
+ * Supply a strerror() function if libc doesn't.
+ */
+#ifndef HAVE_STRERROR
+
+extern int sys_nerr;
+
+#ifndef SYS_ERRLIST_DECLARED
+extern const char * const sys_errlist[];
+#endif /* !SYS_ERRLIST_DECLARED */
+
+char *strerror(int errnum)
+{
+  static char *emsg[1024];
+
+  if (errnum != 0) {
+    if (errnum > 0 && errnum < sys_nerr)
+      sprintf((char *) &emsg[0], "(%s)", sys_errlist[errnum]);
+    else
+      sprintf((char *) &emsg[0], "errnum = %d ", errnum);
+  }
+  else {
+    emsg[0] = '\0';
+  }
+  return (char *) &emsg[0];
+}
+#endif /* !HAVE_STRERROR */
+
+int
+main(int argc, char** argv)
+{
+    int eargv_size;
+    int eargc_base;		/* How many arguments in the base of eargv. */
+    char* emulator;
+    char nodename[100];
+    char browser[100];
+    int ct_mode;
+    int dist_mode;
+    int cnt;
+    int erl_args;
+    char** argv0 = argv;
+
+    print_deprecation_warning(argv[0]);
+
+    emulator = get_default_emulator(argv[0]);
+
+    /*
+     * Allocate the argv vector to be used for arguments to Erlang.
+     * Arrange for starting to pushing information in the middle of
+     * the array, to allow easy addition of commands in the beginning.
+     */
+
+    eargv_size = argc*4+100;
+    eargv_base = (char **) emalloc(eargv_size*sizeof(char*));
+    eargv = eargv_base;
+    eargc = 0;
+    push_words(emulator);
+    eargc_base = eargc;
+    eargv = eargv + eargv_size/2;
+    eargc = 0;
+
+    strcpy(nodename, "ct");
+    dist_mode = SHORT_NAME;
+    browser[0] = '\0';
+    ct_mode = NORMAL_MODE;
+    erl_args = argc;
+    cnt = 1;
+
+    /*
+     * Check various flags before building command line
+     */
+
+    while (cnt < argc) {
+	if (strcmp(argv[1], "-erl_args") == 0) {
+	    erl_args = cnt;
+	}
+	else if (strcmp(argv[1], "-sname") == 0) {
+	    strncpy(nodename, argv[2], sizeof(nodename));
+	    nodename[sizeof(nodename)-1] = '\0';
+	    cnt++, argv++;
+	}
+	else if (strcmp(argv[1], "-name") == 0) {
+	    strncpy(nodename, argv[2], sizeof(nodename));
+	    nodename[sizeof(nodename)-1] = '\0';
+	    dist_mode = FULL_NAME;
+	    cnt++, argv++;
+	}
+	else {
+	    if (cnt < erl_args) {
+		if (strcmp(argv[1], "-vts") == 0) {
+		    ct_mode = VTS_MODE;
+		}
+		else if (strcmp(argv[1], "-browser") == 0) {
+		    strncpy(browser, argv[2], sizeof(browser));
+		    browser[sizeof(browser)-1] = '\0';
+		    cnt++, argv++;
+		}
+		else if (strcmp(argv[1], "-shell") == 0) {
+		    ct_mode = CT_SHELL_MODE;
+		}
+		else if (strcmp(argv[1], "-ctmaster") == 0) {
+		    strcpy(nodename, "ct_master");
+		    ct_mode = MASTER_MODE;
+		}
+		else if (strcmp(argv[1], "-ctname") == 0) {
+		    strncpy(nodename, argv[2], sizeof(nodename));
+		    nodename[sizeof(nodename)-1] = '\0';
+		    ct_mode = ERL_SHELL_MODE;
+		    cnt++, argv++;
+		}
+	    }
+	}
+	cnt++, argv++;
+    }
+
+    argv = argv0;
+
+    /*
+     * Push initial arguments.
+     */
+
+    if (dist_mode == FULL_NAME) {
+	PUSH2("-name", nodename);
+    }
+    else {
+	PUSH2("-sname", nodename);
+    }
+
+    /*
+     * Push everything else
+     */
+
+    if (ct_mode == VTS_MODE) {
+	PUSH4("-s", "webtool", "script_start", "vts");
+	if (browser[0] != '\0') PUSH(browser);
+	PUSH3("-s", "ct_run", "script_start");
+    }
+    else if (ct_mode == CT_SHELL_MODE) {
+	PUSH3("-s", "ct_run", "script_start");
+    }
+    else if (ct_mode == NORMAL_MODE) {
+	PUSH3("-s", "ct_run", "script_start");
+	PUSH3("-s", "erlang", "halt");
+    }
+
+    cnt = 1;
+    while (cnt < argc) {
+	if (strcmp(argv[1], "-erl_args") == 0) {
+	    PUSH("-ct_erl_args");
+	}
+	else if ((strcmp(argv[1], "-sname") == 0) || (strcmp(argv[1], "-name") == 0)) {
+	    cnt++, argv++;
+	}
+	else if (cnt < erl_args) {
+	    if (strcmp(argv[1], "-config") == 0)
+		PUSH("-ct_config");
+	    else if (strcmp(argv[1], "-decrypt_key") == 0)
+		PUSH("-ct_decrypt_key");
+	    else if (strcmp(argv[1], "-decrypt_file") == 0)
+		PUSH("-ct_decrypt_file");
+	    else
+		PUSH(argv[1]);
+	}
+	else {
+	    PUSH(argv[1]);
+	}
+	cnt++, argv++;
+    }
+
+    /*
+     * Move up the commands for invoking the emulator and adjust eargv
+     * accordingly.
+     */
+
+    while (--eargc_base >= 0) {
+	UNSHIFT(eargv_base[eargc_base]);
+    }
+
+    /*
+     * Invoke Erlang with the collected options.
+     */
+
+    PUSH(NULL);
+
+    return run_erlang(eargv[0], eargv);
+}
+
+static void
+push_words(char* src)
+{
+    char sbuf[MAXPATHLEN];
+    char* dst;
+
+    dst = sbuf;
+    while ((*dst++ = *src++) != '\0') {
+	if (isspace((int)*src)) {
+	    *dst = '\0';
+	    PUSH(strsave(sbuf));
+	    dst = sbuf;
+	    do {
+		src++;
+	    } while (isspace((int)*src));
+	}
+    }
+    if (sbuf[0])
+	PUSH(strsave(sbuf));
+}
+#ifdef __WIN32__
+char *make_commandline(char **argv)
+{
+    static char *buff = NULL;
+    static int siz = 0;
+    int num = 0;
+    char **arg, *p;
+
+    if (*argv == NULL) {
+	return "";
+    }
+    for (arg = argv; *arg != NULL; ++arg) {
+	num += strlen(*arg)+1;
+    }
+    if (!siz) {
+	siz = num;
+	buff = malloc(siz*sizeof(char));
+    } else if (siz < num) {
+	siz = num;
+	buff = realloc(buff,siz*sizeof(char));
+    }
+    p = buff;
+    for (arg = argv; *arg != NULL; ++arg) {
+	strcpy(p,*arg);
+	p+=strlen(*arg);
+	*p++=' ';
+    }
+    *(--p) = '\0';
+
+    if (debug) {
+	printf("Processed commandline:%s\n",buff);
+    }
+    return buff;
+}
+
+int my_spawnvp(char **argv)
+{
+    STARTUPINFO siStartInfo;
+    PROCESS_INFORMATION piProcInfo;
+    DWORD ec;
+
+    memset(&siStartInfo,0,sizeof(STARTUPINFO));
+    siStartInfo.cb = sizeof(STARTUPINFO);
+    siStartInfo.dwFlags = STARTF_USESTDHANDLES;
+    siStartInfo.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
+    siStartInfo.hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE);
+    siStartInfo.hStdError = GetStdHandle(STD_ERROR_HANDLE);
+    siStartInfo.wShowWindow = SW_HIDE;
+    siStartInfo.dwFlags |= STARTF_USESHOWWINDOW;
+
+
+    if (!CreateProcess(NULL,
+		       make_commandline(argv),
+		       NULL,
+		       NULL,
+		       TRUE,
+		       0,
+		       NULL,
+		       NULL,
+		       &siStartInfo,
+		       &piProcInfo)) {
+	return -1;
+    }
+    CloseHandle(piProcInfo.hThread);
+
+    WaitForSingleObject(piProcInfo.hProcess,INFINITE);
+    if (!GetExitCodeProcess(piProcInfo.hProcess,&ec)) {
+	return 0;
+    }
+    return (int) ec;
+}
+#endif /* __WIN32__ */
+
+
+static int
+run_erlang(char* progname, char** argv)
+{
+#ifdef __WIN32__
+    int status;
+#endif
+
+    if (debug) {
+	int i = 0;
+	while (argv[i] != NULL)
+	    printf(" %s", argv[i++]);
+	printf("\n");
+    }
+
+#ifdef __WIN32__
+    /*
+     * Alas, we must wait here for the program to finish.
+     * Otherwise, the shell from which we were executed will think
+     * we are finished and print a prompt and read keyboard input.
+     */
+
+    status = my_spawnvp(argv)/*_spawnvp(_P_WAIT,progname,argv)*/;
+    if (status == -1) {
+	fprintf(stderr, "ct_run: Error executing '%s': %d", progname,
+		GetLastError());
+    }
+    return status;
+#else
+    execvp(progname, argv);
+    error("Error %d executing \'%s\'.", errno, progname);
+    return 2;
+#endif
+}
+
+static void
+error(char* format, ...)
+{
+    char sbuf[1024];
+    va_list ap;
+
+    va_start(ap, format);
+    erts_vsnprintf(sbuf, sizeof(sbuf), format, ap);
+    va_end(ap);
+    fprintf(stderr, "ct_run: %s\n", sbuf);
+    exit(1);
+}
+
+static char*
+emalloc(size_t size)
+{
+  char *p = malloc(size);
+  if (p == NULL)
+    error("Insufficient memory");
+  return p;
+}
+
+static char*
+strsave(char* string)
+{
+    char* p = emalloc(strlen(string)+1);
+    strcpy(p, string);
+    return p;
+}
+
+/* Instead of making sure basename exists, we do our own */
+static char *simple_basename(char *path)
+{
+    char *ptr;
+    for (ptr = path; *ptr != '\0'; ++ptr) {
+	if (*ptr == '/' || *ptr == '\\') {
+	    path = ptr + 1;
+	}
+    }
+    return path;
+}
+
+static void print_deprecation_warning(char* progpath)
+{
+  char *basename = simple_basename(progpath);
+  if(strcmp(basename,"run_test") == 0 ||
+       strcmp(basename, "run_test.exe") == 0) {
+    printf("---***---\nDeprecated: run_test is deprecated and will be removed in R16B,\n            please use ct_run instead\n---***---\n");
+  }
+}
+
+static char*
+get_default_emulator(char* progname)
+{
+    char sbuf[MAXPATHLEN];
+    char* s;
+
+    if (strlen(progname) >= sizeof(sbuf))
+        return ERL_NAME;
+
+    strcpy(sbuf, progname);
+    for (s = sbuf+strlen(sbuf); s >= sbuf; s--) {
+	if (IS_DIRSEP(*s)) {
+	    strcpy(s+1, ERL_NAME);
+#ifdef __WIN32__
+	    if (_access(sbuf, 0) != -1) {
+		return strsave(sbuf);
+	    }
+#else
+	    if (access(sbuf, 1) != -1) {
+		return strsave(sbuf);
+	    }
+#endif
+	    break;
+	}
+    }
+    return ERL_NAME;
+}
+
+#ifdef __WIN32__
+static char*
+possibly_quote(char* arg)
+{
+    int mustQuote = NO;
+    int n = 0;
+    char* s;
+    char* narg;
+
+    if (arg == NULL) {
+	return arg;
+    }
+
+    /*
+     * Scan the string to find out if it needs quoting and return
+     * the original argument if not.
+     */
+
+    for (s = arg; *s; s++, n++) {
+	switch(*s) {
+	case ' ':
+	    mustQuote = YES;
+	    continue;
+	case '"':
+	    mustQuote = YES;
+	    n++;
+	    continue;
+	case '\\':
+	    if(s[1] == '"')
+		n++;
+	    continue;
+	default:
+	    continue;
+	}
+    }
+    if (!mustQuote) {
+	return arg;
+    }
+
+    /*
+     * Insert the quotes and put a backslash in front of every quote
+     * inside the string.
+     */
+
+    s = narg = emalloc(n+2+1);
+    for (*s++ = '"'; *arg; arg++, s++) {
+	if (*arg == '"' || (*arg == '\\' && arg[1] == '"')) {
+	    *s++ = '\\';
+	}
+	*s = *arg;
+    }
+    if (s[-1] == '\\') {
+	*s++ ='\\';
+    }
+    *s++ = '"';
+    *s = '\0';
+    return narg;
+}
+#endif /* __WIN32__ */
diff --git a/erts/etc/common/dialyzer.c b/erts/etc/common/dialyzer.c
index 4b4c1124f1..70b11b1b36 100644
--- a/erts/etc/common/dialyzer.c
+++ b/erts/etc/common/dialyzer.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -147,6 +147,9 @@ main(int argc, char** argv)
     env = get_env("DIALYZER_EMULATOR");
     emulator = env ? env : get_default_emulator(argv[0]);
 
+    if (strlen(emulator) >= MAXPATHLEN)
+        error("Value of environment variable DIALYZER_EMULATOR is too large");
+
     /*
      * Allocate the argv vector to be used for arguments to Erlang.
      * Arrange for starting to pushing information in the middle of
@@ -186,6 +189,18 @@ main(int argc, char** argv)
 	argc--, argv++;
     }
 
+    if (argc > 2 && strcmp(argv[1], "+P") == 0) {
+	PUSH2("+P", argv[2]);
+	argc--, argv++;
+	argc--, argv++;
+    } else PUSH2("+P", "1000000");
+
+    if (argc > 2 && strcmp(argv[1], "+sbt") == 0) {
+	PUSH2("+sbt", argv[2]);
+	argc--, argv++;
+	argc--, argv++;
+    }
+
     PUSH("+B");
     PUSH2("-boot", "start_clean");
     PUSH3("-run", "dialyzer", "plain_cl");
@@ -228,7 +243,7 @@ main(int argc, char** argv)
 static void
 push_words(char* src)
 {
-    char sbuf[1024];
+    char sbuf[MAXPATHLEN];
     char* dst;
 
     dst = sbuf;
@@ -360,7 +375,7 @@ error(char* format, ...)
     va_list ap;
     
     va_start(ap, format);
-    vsprintf(sbuf, format, ap);
+    erts_vsnprintf(sbuf, sizeof(sbuf), format, ap);
     va_end(ap);
     fprintf(stderr, "dialyzer: %s\n", sbuf);
     exit(1);
@@ -389,6 +404,9 @@ get_default_emulator(char* progname)
     char sbuf[MAXPATHLEN];
     char* s;
 
+    if (strlen(progname) >= sizeof(sbuf))
+        return ERL_NAME;
+
     strcpy(sbuf, progname);
     for (s = sbuf+strlen(sbuf); s >= sbuf; s--) {
 	if (IS_DIRSEP(*s)) {
diff --git a/erts/etc/common/erlc.c b/erts/etc/common/erlc.c
index 09aca19e6c..0223cc5274 100644
--- a/erts/etc/common/erlc.c
+++ b/erts/etc/common/erlc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1997-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -28,6 +28,7 @@
 #include <winbase.h>
 /* FIXE ME config_win32.h? */
 #define HAVE_STRERROR 1
+#define snprintf _snprintf
 #endif
 
 #include <ctype.h>
@@ -148,10 +149,6 @@ int
 main(int argc, char** argv)
 {
     char cwd[MAXPATHLEN];	/* Current working directory. */
-    char** rpc_eargv;		/* Pointer to the beginning of arguments
-				 * if calling a running Erlang system
-				 * via erl_rpc().
-				 */
     int eargv_size;
     int eargc_base;		/* How many arguments in the base of eargv. */
     char* emulator;
@@ -160,6 +157,9 @@ main(int argc, char** argv)
     env = get_env("ERLC_EMULATOR");
     emulator = env ? env : get_default_emulator(argv[0]);
 
+    if (strlen(emulator) >= MAXPATHLEN)
+        error("Value of environment variable ERLC_EMULATOR is too large");
+
     /*
      * Allocate the argv vector to be used for arguments to Erlang.
      * Arrange for starting to pushing information in the middle of
@@ -170,7 +170,7 @@ main(int argc, char** argv)
      * base of the eargv vector, and move it up later.
      */
 
-    eargv_size = argc*4+100;
+    eargv_size = argc*6+100;
     eargv_base = (char **) emalloc(eargv_size*sizeof(char*));
     eargv = eargv_base;
     eargc = 0;
@@ -185,11 +185,11 @@ main(int argc, char** argv)
      * Push initial arguments.
      */
 
+    PUSH("+sbtu");
     PUSH("-noinput");
     PUSH2("-mode", "minimal");
     PUSH2("-boot", "start_clean");
     PUSH3("-s", "erl_compile", "compile_cmdline");
-    rpc_eargv = eargv+eargc;
 
     /*
      * Push standard arguments to Erlang.
@@ -252,16 +252,98 @@ main(int argc, char** argv)
 		    }
 		}
 		break;
-	    case 'h':
-		if (strcmp(argv[1], "-hybrid") == 0) {
-		    UNSHIFT(argv[1]);
-		} else {
-		    usage();
-		}
-		break;
 	    case 'I':
 		PUSH2("@i", process_opt(&argc, &argv, 0));
 		break;
+	    case 'M':
+		{
+		    char *buf, *key, *val;
+		    size_t buf_len;
+
+		    if (argv[1][2] == '\0') { /* -M */
+			/* Push the following options:
+			 *   o  'makedep'
+			 *   o  {makedep_output, standard_io}
+			 */
+			buf = strsave("makedep");
+			PUSH2("@option", buf);
+
+			key = "makedep_output";
+			val = "standard_io";
+			buf_len = 1 + strlen(key) + 1 + strlen(val) + 1 + 1;
+			buf = emalloc(buf_len);
+			snprintf(buf, buf_len, "{%s,%s}", key, val);
+			PUSH2("@option", buf);
+		    } else if (argv[1][3] == '\0') {
+			switch(argv[1][2]) {
+			case 'D': /* -MD */
+			    /* Push the following options:
+			     *   o  'makedep'
+			     */
+			    buf = strsave("makedep");
+			    PUSH2("@option", buf);
+			    break;
+			case 'F': /* -MF <file> */
+			    /* Push the following options:
+			     *   o  'makedep'
+			     *   o  {makedep_output, <file>}
+			     */
+			    buf = strsave("makedep");
+			    PUSH2("@option", buf);
+
+			    key = "makedep_output";
+			    val = process_opt(&argc, &argv, 1);
+			    buf_len = 1 + strlen(key) + 2 + strlen(val) + 2 + 1;
+			    buf = emalloc(buf_len);
+			    snprintf(buf, buf_len, "{%s,\"%s\"}", key, val);
+			    PUSH2("@option", buf);
+			    break;
+			case 'T': /* -MT <target> */
+			    /* Push the following options:
+			     *   o  {makedep_target, <target>}
+			     */
+			    key = "makedep_target";
+			    val = process_opt(&argc, &argv, 1);
+			    buf_len = 1 + strlen(key) + 2 + strlen(val) + 2 + 1;
+			    buf = emalloc(buf_len);
+			    snprintf(buf, buf_len, "{%s,\"%s\"}", key, val);
+			    PUSH2("@option", buf);
+			    break;
+			case 'Q': /* -MQ <target> */
+			    /* Push the following options:
+			     *   o  {makedep_target, <target>}
+			     *   o  makedep_quote_target
+			     */
+			    key = "makedep_target";
+			    val = process_opt(&argc, &argv, 1);
+			    buf_len = 1 + strlen(key) + 2 + strlen(val) + 2 + 1;
+			    buf = emalloc(buf_len);
+			    snprintf(buf, buf_len, "{%s,\"%s\"}", key, val);
+			    PUSH2("@option", buf);
+
+			    buf = strsave("makedep_quote_target");
+			    PUSH2("@option", buf);
+			    break;
+			case 'G': /* -MG */
+			    /* Push the following options:
+			     *   o  makedep_add_missing
+			     */
+			    buf = strsave("makedep_add_missing");
+			    PUSH2("@option", buf);
+			    break;
+			case 'P': /* -MP */
+			    /* Push the following options:
+			     *   o  makedep_phony
+			     */
+			    buf = strsave("makedep_add_missing");
+			    PUSH2("@option", buf);
+			    break;
+			default:
+			    goto error;
+			}
+		    }
+		}
+		break;
 	    case 'o':
 		PUSH2("@outdir", process_opt(&argc, &argv, 0));
 		break;
@@ -419,7 +501,7 @@ process_opt(int* pArgc, char*** pArgv, int offset)
 static void
 push_words(char* src)
 {
-    char sbuf[1024];
+    char sbuf[MAXPATHLEN];
     char* dst;
 
     dst = sbuf;
@@ -560,9 +642,17 @@ usage(void)
 	{"-d", "turn on debugging of erlc itself"},
 	{"-Dname", "define name"},
 	{"-Dname=value", "define name to have value"},
-	{"-hybrid", "compile using hybrid-heap emulator"},
 	{"-help", "shows this help text"},
 	{"-I path", "where to search for include files"},
+	{"-M", "generate a rule for make(1) describing the dependencies"},
+	{"-MF file", "write the dependencies to 'file'"},
+	{"-MT target", "change the target of the rule emitted by dependency "
+		"generation"},
+	{"-MQ target", "same as -MT but quote characters special to make(1)"},
+	{"-MG", "consider missing headers as generated files and add them to "
+		"the dependencies"},
+	{"-MP", "add a phony target for each dependency"},
+	{"-MD", "same as -M -MT file (with default 'file')"},
 	{"-o name", "name output directory or file"},
 	{"-pa path", "add path to the front of Erlang's code path"},
 	{"-pz path", "add path to the end of Erlang's code path"},
@@ -595,7 +685,7 @@ error(char* format, ...)
     va_list ap;
     
     va_start(ap, format);
-    vsprintf(sbuf, format, ap);
+    erts_vsnprintf(sbuf, sizeof(sbuf), format, ap);
     va_end(ap);
     fprintf(stderr, "erlc: %s\n", sbuf);
     exit(1);
@@ -624,6 +714,9 @@ get_default_emulator(char* progname)
     char sbuf[MAXPATHLEN];
     char* s;
 
+    if (strlen(progname) >= sizeof(sbuf))
+        return ERL_NAME;
+
     strcpy(sbuf, progname);
     for (s = sbuf+strlen(sbuf); s >= sbuf; s--) {
 	if (IS_DIRSEP(*s)) {
diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c
index f79f5cc978..790b0ed400 100644
--- a/erts/etc/common/erlexec.c
+++ b/erts/etc/common/erlexec.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2010. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -64,8 +64,10 @@
 static const char plusM_au_allocs[]= {
     'u',	/* all alloc_util allocators */
     'B',	/* binary_alloc		*/
+    'C',	/* sbmbc_alloc		*/
     'D',	/* std_alloc		*/
     'E',	/* ets_alloc		*/
+    'F',	/* fix_alloc		*/
     'H',	/* eheap_alloc		*/
     'L',	/* ll_alloc		*/
     'R',	/* driver_alloc		*/
@@ -93,6 +95,8 @@ static char *plusM_au_alloc_switches[] = {
     "rsbcst",
     "sbct",
     "smbcs",
+    "sbmbcs",
+    "sbmbct",
     NULL
 };
 
@@ -107,8 +111,6 @@ static char *plusM_other_switches[] = {
     "Mamcbf",
     "Mrmcbf",
     "Mmcs",
-    "Mcci",
-    "Fe",
     "Ye",
     "Ym",
     "Ytp",
@@ -119,7 +121,11 @@ static char *plusM_other_switches[] = {
 /* +s arguments with values */
 static char *pluss_val_switches[] = {
     "bt",
+    "bwt",
+    "cl",
     "ct",
+    "wt",
+    "ws",
     "ss",
     NULL
 };
@@ -131,6 +137,18 @@ static char *plush_val_switches[] = {
     NULL
 };
 
+/* +r arguments with values */
+static char *plusr_val_switches[] = {
+    "g",
+    NULL
+};
+
+/* +z arguments with values */
+static char *plusz_val_switches[] = {
+    "dbbl",
+    NULL
+};
+
 
 /*
  * Define sleep(seconds) in terms of Sleep() on Windows.
@@ -141,20 +159,13 @@ static char *plush_val_switches[] = {
 #endif
 
 #define SMP_SUFFIX	  ".smp"
-#define HYBRID_SUFFIX	  ".hybrid"
-
-#ifdef __WIN32__
 #define DEBUG_SUFFIX      ".debug"
-#define EMU_TYPE_SUFFIX_LENGTH  (strlen(HYBRID_SUFFIX)+(strlen(DEBUG_SUFFIX)))
-#else
-/* The length of the longest memory architecture suffix. */
-#define EMU_TYPE_SUFFIX_LENGTH  strlen(HYBRID_SUFFIX)
-#endif
+#define EMU_TYPE_SUFFIX_LENGTH  strlen(DEBUG_SUFFIX)
+
 /*
  * Define flags for different memory architectures.
  */
 #define EMU_TYPE_SMP		0x0001
-#define EMU_TYPE_HYBRID		0x0002
 
 #ifdef __WIN32__
 #define EMU_TYPE_DEBUG		0x0004
@@ -168,7 +179,7 @@ void error(char* format, ...);
  * Local functions.
  */
 
-#if !defined(ERTS_HAVE_SMP_EMU) || !defined(ERTS_HAVE_HYBRID_EMU)
+#if !defined(ERTS_HAVE_SMP_EMU)
 static void usage_notsup(const char *switchname);
 #endif
 static void usage_msg(const char *msg);
@@ -302,7 +313,7 @@ free_env_val(char *value)
 }
 
 /*
- * Add the arcitecture suffix to the program name if needed,
+ * Add the architecture suffix to the program name if needed,
  * except on Windows, where we insert it just before ".DLL".
  */
 static char*
@@ -351,9 +362,6 @@ add_extra_suffixes(char *prog, int type)
    if (type == EMU_TYPE_SMP) {
        p = write_str(p, SMP_SUFFIX);
    }
-   else if (type == EMU_TYPE_HYBRID) {
-       p = write_str(p, HYBRID_SUFFIX);
-   }
 #ifdef __WIN32__
    if (dll) {
        p = write_str(p, DLL_EXT);
@@ -381,6 +389,7 @@ int main(int argc, char **argv)
     int print_args_exit = 0;
     int print_qouted_cmd_exit = 0;
     erts_cpu_info_t *cpuinfo = NULL;
+    char* emu_name;
 
 #ifdef __WIN32__
     this_module_handle = module;
@@ -518,13 +527,6 @@ int main(int argc, char **argv)
 		emu_type_passed |= EMU_TYPE_DEBUG;
 		emu_type |= EMU_TYPE_DEBUG;
 #endif
-	    } else if (strcmp(argv[i], "-hybrid") == 0) {
-		emu_type_passed |= EMU_TYPE_HYBRID;
-#ifdef ERTS_HAVE_HYBRID_EMU
-		emu_type |= EMU_TYPE_HYBRID;
-#else
-		usage_notsup("-hybrid");
-#endif
 	    } else if (strcmp(argv[i], "-extra") == 0) {
 		break;
 	    }
@@ -535,25 +537,13 @@ int main(int argc, char **argv)
     erts_cpu_info_destroy(cpuinfo);
     cpuinfo = NULL;
 
-    if ((emu_type & EMU_TYPE_HYBRID) && (emu_type & EMU_TYPE_SMP)) {
-	/*
-	 * We have a conflict. Only using explicitly passed arguments
-	 * may solve it...
-	 */
-	emu_type &= emu_type_passed;
-	if ((emu_type & EMU_TYPE_HYBRID) && (emu_type & EMU_TYPE_SMP)) {
-	    usage_msg("Hybrid heap emulator with SMP support selected. The "
-		      "combination hybrid heap and SMP support is currently "
-		      "not supported.");
-	}
-    }
-
     if (malloc_lib) {
 	if (strcmp(malloc_lib, "libc") != 0)
 	    usage("+MYm");
     }
     emu = add_extra_suffixes(emu, emu_type);
-    sprintf(tmpStr, "%s" DIRSEP "%s" BINARY_EXT, bindir, emu);
+    emu_name = strsave(emu);
+    erts_snprintf(tmpStr, sizeof(tmpStr), "%s" DIRSEP "%s" BINARY_EXT, bindir, emu);
     emu = strsave(tmpStr);
 
     add_Eargs(emu);		/* Will be argv[0] -- necessary! */
@@ -564,12 +554,12 @@ int main(int argc, char **argv)
 
     s = get_env("PATH");
     if (!s) {
-	sprintf(tmpStr, "%s" PATHSEP "%s" DIRSEP "bin", bindir, rootdir);
+	erts_snprintf(tmpStr, sizeof(tmpStr), "%s" PATHSEP "%s" DIRSEP "bin", bindir, rootdir);
     } else if (strstr(s, bindir) == NULL) {
-	sprintf(tmpStr, "%s" PATHSEP "%s" DIRSEP "bin" PATHSEP "%s", bindir, 
+	erts_snprintf(tmpStr, sizeof(tmpStr), "%s" PATHSEP "%s" DIRSEP "bin" PATHSEP "%s", bindir,
 		rootdir, s);
     } else {
-	sprintf(tmpStr, "%s", s);
+	erts_snprintf(tmpStr, sizeof(tmpStr), "%s", s);
     }
     free_env_val(s);
     set_env("PATH", tmpStr);
@@ -669,6 +659,9 @@ int main(int argc, char **argv)
 			verbose = 1;
 		    } else if (strcmp(argv[i], "-emu_args_exit") == 0) {
 			print_args_exit = 1;
+		    } else if (strcmp(argv[i], "-emu_name_exit") == 0) {
+			printf("%s\n", emu_name);
+			exit(0);
 		    } else if (strcmp(argv[i], "-emu_qouted_cmd_exit") == 0) {
 			print_qouted_cmd_exit = 1;
 		    } else if (strcmp(argv[i], "-env") == 0) { /* -env VARNAME VARVALUE */
@@ -707,7 +700,7 @@ int main(int argc, char **argv)
 			error("-man not supported on Windows");
 #else
 			argv[i] = "man";
-			sprintf(tmpStr, "%s/man", rootdir);
+			erts_snprintf(tmpStr, sizeof(tmpStr), "%s/man", rootdir);
 			set_env("MANPATH", tmpStr);
 			execvp("man", argv+i);
 			error("Could not execute the 'man' command.");
@@ -872,6 +865,21 @@ int main(int argc, char **argv)
 			  i++;
 		      }
 		      break;
+		  case 'r':
+		      if (!is_one_of_strings(&argv[i][2],
+					     plusr_val_switches))
+			  goto the_default;
+		      else {
+			  if (i+1 >= argc
+			      || argv[i+1][0] == '-'
+			      || argv[i+1][0] == '+')
+			      usage(argv[i]);
+			  argv[i][0] = '-';
+			  add_Eargs(argv[i]);
+			  add_Eargs(argv[i+1]);
+			  i++;
+		      }
+		      break;
 		  case 's':
 		      if (!is_one_of_strings(&argv[i][2],
 					     pluss_val_switches))
@@ -887,6 +895,20 @@ int main(int argc, char **argv)
 			  i++;
 		      }
 		      break;
+		  case 'z':
+		      if (!is_one_of_strings(&argv[i][2], plusz_val_switches)) {
+			  goto the_default;
+		      } else {
+			  if (i+1 >= argc
+			      || argv[i+1][0] == '-'
+			      || argv[i+1][0] == '+')
+			      usage(argv[i]);
+			  argv[i][0] = '-';
+			  add_Eargs(argv[i]);
+			  add_Eargs(argv[i+1]);
+			  i++;
+		      }
+		      break;
 		  default:
 		  the_default:
 		    argv[i][0] = '-'; /* Change +option to -option. */
@@ -1065,15 +1087,13 @@ usage_aux(void)
 	  "]"
 #endif
 	  "] "
-#ifdef ERTS_HAVE_HYBRID_EMU
-	  "[-hybrid] "
-#endif
 	  "[-make] [-man [manopts] MANPAGE] [-x] [-emu_args] "
-	  "[-args_file FILENAME] "
-	  "[+A THREADS] [+a SIZE] [+B[c|d|i]] [+c] [+h HEAP_SIZE_OPTION] [+K BOOLEAN] "
+	  "[-args_file FILENAME] [+A THREADS] [+a SIZE] [+B[c|d|i]] [+c] "
+	  "[+h HEAP_SIZE_OPTION] [+K BOOLEAN] "
 	  "[+l] [+M<SUBSWITCH> <ARGUMENT>] [+P MAX_PROCS] [+R COMPAT_REL] "
-	  "[+r] [+s SCHEDULER_OPTION] [+S NO_SCHEDULERS:NO_SCHEDULERS_ONLINE] [+T LEVEL] [+V] [+v] [+W<i|w>] "
-	  "[args ...]\n");
+	  "[+r] [+rg READER_GROUPS_LIMIT] [+s SCHEDULER_OPTION] "
+	  "[+S NO_SCHEDULERS:NO_SCHEDULERS_ONLINE] [+T LEVEL] [+V] [+v] "
+	  "[+W<i|w>] [+z MISC_OPTION] [args ...]\n");
   exit(1);
 }
 
@@ -1084,7 +1104,7 @@ usage(const char *switchname)
     usage_aux();
 }
 
-#if !defined(ERTS_HAVE_SMP_EMU) || !defined(ERTS_HAVE_HYBRID_EMU)
+#if !defined(ERTS_HAVE_SMP_EMU)
 static void
 usage_notsup(const char *switchname)
 {
@@ -1122,10 +1142,10 @@ start_epmd(char *epmd)
     if (!epmd) {
 	epmd = epmd_cmd;
 #ifdef __WIN32__
-	sprintf(epmd_cmd, "%s" DIRSEP "epmd", bindir);
+	erts_snprintf(epmd_cmd, sizeof(epmd_cmd), "%s" DIRSEP "epmd", bindir);
 	arg1 = "-daemon";
 #else
-	sprintf(epmd_cmd, "%s" DIRSEP "epmd -daemon", bindir);
+	erts_snprintf(epmd_cmd, sizeof(epmd_cmd), "\"%s" DIRSEP "epmd\" -daemon", bindir);
 #endif
     } 
 #ifdef __WIN32__
@@ -1201,7 +1221,7 @@ void error(char* format, ...)
     va_list ap;
 
     va_start(ap, format);
-    vsprintf(sbuf, format, ap);
+    erts_vsnprintf(sbuf, sizeof(sbuf), format, ap);
     va_end(ap);
     fprintf(stderr, "erlexec: %s\n", sbuf);
     exit(1);
@@ -1281,14 +1301,14 @@ static void get_start_erl_data(char *file)
     if (env)
 	reldir = strsave(env);
     else {
-	sprintf(tmpbuffer, "%s/releases", rootdir);
+	erts_snprintf(tmpbuffer, sizeof(tmpbuffer), "%s/releases", rootdir);
 	reldir = strsave(tmpbuffer);
     }
     free_env_val(env);
     if (file == NULL)
-       sprintf(start_erl_data, "%s/start_erl.data", reldir);
+       erts_snprintf(start_erl_data, sizeof(start_erl_data), "%s/start_erl.data", reldir);
     else
-       sprintf(start_erl_data, "%s", file);
+       erts_snprintf(start_erl_data, sizeof(start_erl_data), "%s", file);
     fp = _open(start_erl_data, _O_RDONLY );
     if( fp == -1 )
 	error( "open failed on %s",start_erl_data );
@@ -1318,16 +1338,16 @@ static void get_start_erl_data(char *file)
     }
 	
     bindir = emalloc(512);
-    sprintf(bindir,"%s/erts-%s/bin",rootdir,tmpbuffer);
+    erts_snprintf(bindir,512,"%s/erts-%s/bin",rootdir,tmpbuffer);
     /* BINDIR=$ROOTDIR/erts-$ERTS_VSN/bin */
     tprogname = progname;
     progname = emalloc(strlen(tprogname) + 20);
-    sprintf(progname,"%s -start_erl",tprogname);
+    erts_snprintf(progname,strlen(tprogname) + 20,"%s -start_erl",tprogname);
 
     boot_script = emalloc(512);
     config_script = emalloc(512);
-    sprintf(boot_script, "%s/%s/start", reldir, otpstring);
-    sprintf(config_script, "%s/%s/sys", reldir, otpstring);
+    erts_snprintf(boot_script, 512, "%s/%s/start", reldir, otpstring);
+    erts_snprintf(config_script, 512, "%s/%s/sys", reldir, otpstring);
        
 }
 
@@ -1335,7 +1355,7 @@ static void get_start_erl_data(char *file)
 static char *replace_filename(char *path, char *new_base) 
 {
     int plen = strlen(path);
-    char *res = malloc((plen+strlen(new_base)+1)*sizeof(char));
+    char *res = emalloc((plen+strlen(new_base)+1)*sizeof(char));
     char *p;
 
     strcpy(res,path);
@@ -1350,7 +1370,7 @@ static char *path_massage(char *long_path)
 {
      char *p;
 
-     p = malloc(MAX_PATH+1);
+     p = emalloc(MAX_PATH+1);
      strcpy(p, long_path);
      GetShortPathName(p, p, MAX_PATH);
      return p;
@@ -1486,7 +1506,8 @@ get_parameters(int argc, char** argv)
 	/* Determine bindir from absolute path to executable */
 	char *p;
 	char buffer[PATH_MAX];
-	strcpy(buffer, argv[0]);
+	strncpy(buffer, argv[0], sizeof(buffer));
+	buffer[sizeof(buffer)-1] = '\0';
 	
 	for (p = buffer+strlen(buffer)-1 ; p >= buffer && *p != '/'; --p)
 	    ;
@@ -1499,7 +1520,8 @@ get_parameters(int argc, char** argv)
 	/* Determine rootdir from absolute path to bindir */
 	char *p;
 	char buffer[PATH_MAX];
-	strcpy(buffer, bindir);
+	strncpy(buffer, bindir, sizeof(buffer));
+	buffer[sizeof(buffer)-1] = '\0';
 	
 	for (p = buffer+strlen(buffer)-1; p >= buffer && *p != '/'; --p)
 	    ;
@@ -1925,6 +1947,11 @@ initial_argv_massage(int *argc, char ***argv)
      */
 
     vix = 0;
+
+    av = build_args_from_env("ERL_" OTP_SYSTEM_VERSION "_FLAGS");
+    if (av)
+	avv[vix++].argv = av;
+
     av = build_args_from_env("ERL_AFLAGS");
     if (av)
 	avv[vix++].argv = av;
@@ -1939,10 +1966,6 @@ initial_argv_massage(int *argc, char ***argv)
     if (av)
 	avv[vix++].argv = av;
 
-    av = build_args_from_env("ERL_" OTP_SYSTEM_VERSION "_FLAGS");
-    if (av)
-	avv[vix++].argv = av;
-
     av = build_args_from_env("ERL_ZFLAGS");
     if (av)
 	avv[vix++].argv = av;
diff --git a/erts/etc/common/escript.c b/erts/etc/common/escript.c
index 1bc5eb7651..9e80ec6656 100644
--- a/erts/etc/common/escript.c
+++ b/erts/etc/common/escript.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2007-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -151,6 +151,9 @@ find_prog(char *origpath)
     char relpath[PMAX];
     char abspath[PMAX];
 
+    if (strlen(origpath) >= sizeof(relpath))
+        error("Path too long");
+
     strcpy(relpath, origpath);
 
     if (strstr(relpath, DIRSEPSTR) == NULL) {
@@ -180,19 +183,21 @@ find_prog(char *origpath)
                 end = strstr(beg, PATHSEPSTR);
                 if (end != NULL) {
                     sz = end - beg;
-                    strncpy(dir, beg, sz);
-                    dir[sz] = '\0';
                 } else {
                     sz = strlen(beg);
-                    strcpy(dir, beg);
                     look_for_sep = FALSE;
                 }
+                if (sz >= sizeof(dir)) {
+                    beg = end + 1;
+                    continue;
+                }
+                strncpy(dir, beg, sz);
+                dir[sz] = '\0';
                 beg = end + 1;
 
 #ifdef __WIN32__
-		strcpy(wildcard, dir);
-		strcat(wildcard, DIRSEPSTR);
-		strcat(wildcard, relpath); /* basename */
+		erts_snprintf(wildcard, sizeof(wildcard), "%s" DIRSEPSTR "%s",
+            dir, relpath /* basename */);
 		dir_handle = FindFirstFile(wildcard, &find_data);
 		if (dir_handle == INVALID_HANDLE_VALUE) {
 		    /* Try next directory in path */
@@ -217,9 +222,8 @@ find_prog(char *origpath)
 
                         if (strcmp(origpath, dirp->d_name) == 0) {
                             /* Wow we found the executable. */
-                            strcpy(relpath, dir);
-                            strcat(relpath, DIRSEPSTR);
-                            strcat(relpath, dirp->d_name);
+                            erts_snprintf(relpath, sizeof(relpath), "%s" DIRSEPSTR "%s",
+                                dir, dirp->d_name);
                             closedir(dp);
 			    look_for_sep = FALSE;
                             break;
@@ -291,7 +295,7 @@ append_shebang_args(char* scriptname)
 		    
 		    /* Find end of arg */
 		    end = beg;
-		    while (end && end[0] != ' ') {
+		    while (end && end < (linebuf+LINEBUFSZ-1) && end[0] != ' ') {
 			if (end[0] == '\n') {
 			    newline = TRUE;
 			    end[0]= '\0';
@@ -335,13 +339,16 @@ main(int argc, char** argv)
 	emulator = get_default_emulator(argv[0]);
     }
 
+    if (strlen(emulator) >= PMAX)
+        error("Value of environment variable ESCRIPT_EMULATOR is too large");
+
     /*
      * Allocate the argv vector to be used for arguments to Erlang.
      * Arrange for starting to pushing information in the middle of
      * the array, to allow easy addition of commands in the beginning.
      */
 
-    eargv_size = argc*4+1000;
+    eargv_size = argc*4+1000+LINEBUFSZ/2;
     eargv_base = (char **) emalloc(eargv_size*sizeof(char*));
     eargv = eargv_base;
     eargc = 0;
@@ -370,7 +377,8 @@ main(int argc, char** argv)
     last_opt = argv;
 
 #ifdef __WIN32__
-    if (_stricmp(basename, "escript.exe") == 0) {
+    if ( (_stricmp(basename, "escript.exe") == 0)
+       ||(_stricmp(basename, "escript") == 0)) {
 #else
     if (strcmp(basename, "escript") == 0) {
 #endif
@@ -387,7 +395,8 @@ main(int argc, char** argv)
 	if (argc <= 1) {
 	    error("Missing filename\n");
 	}
-	strcpy(scriptname, argv[1]);
+	strncpy(scriptname, argv[1], sizeof(scriptname));
+	scriptname[sizeof(scriptname)-1] = '\0';
 	argc--;
 	argv++;
     } else {
@@ -395,16 +404,17 @@ main(int argc, char** argv)
 	int len;
 #endif
 	absname = find_prog(argv[0]);
-	strcpy(scriptname, absname);
-	efree(absname);
 #ifdef __WIN32__
-	len = strlen(scriptname);
-	if (len >= 4 && _stricmp(scriptname+len-4, ".exe") == 0) {
-	    scriptname[len-4] = '\0';
+	len = strlen(absname);
+	if (len >= 4 && _stricmp(absname+len-4, ".exe") == 0) {
+	    absname[len-4] = '\0';
 	}
 #endif
 
-	strcat(scriptname, ".escript");
+	erts_snprintf(scriptname, sizeof(scriptname), "%s.escript",
+        absname);
+	efree(absname);
+
     }
 
     /*
@@ -455,7 +465,7 @@ main(int argc, char** argv)
 static void
 push_words(char* src)
 {
-    char sbuf[1024];
+    char sbuf[PMAX];
     char* dst;
 
     dst = sbuf;
@@ -584,7 +594,7 @@ error(char* format, ...)
     va_list ap;
     
     va_start(ap, format);
-    vsprintf(sbuf, format, ap);
+    erts_vsnprintf(sbuf, sizeof(sbuf), format, ap);
     va_end(ap);
     fprintf(stderr, "escript: %s\n", sbuf);
     exit(1);
@@ -619,6 +629,9 @@ get_default_emulator(char* progname)
     char sbuf[MAXPATHLEN];
     char* s;
 
+    if (strlen(progname) >= sizeof(sbuf))
+        return ERL_NAME;
+
     strcpy(sbuf, progname);
     for (s = sbuf+strlen(sbuf); s >= sbuf; s--) {
 	if (IS_DIRSEP(*s)) {
diff --git a/erts/etc/common/heart.c b/erts/etc/common/heart.c
index c1f6551d21..70c2b3bb23 100644
--- a/erts/etc/common/heart.c
+++ b/erts/etc/common/heart.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -372,7 +372,8 @@ main(int argc, char **argv)
     _setmode(erlin_fd,_O_BINARY);
     _setmode(erlout_fd,_O_BINARY);
 #endif
-    strcpy(program_name, argv[0]);
+    strncpy(program_name, argv[0], sizeof(program_name));
+    program_name[sizeof(program_name)-1] = '\0';
     notify_ack(erlout_fd);
     cmd[0] = '\0';
     do_terminate(message_loop(erlin_fd,erlout_fd));
@@ -682,14 +683,16 @@ do_terminate(reason)
 	  print_error("Would reboot. Terminating.");
 	else {
 	  kill_old_erlang();
-	  system(command);
+	  /* suppress gcc warning with 'if' */
+	  if(system(command));
 	  print_error("Executed \"%s\". Terminating.",command);
 	}
 	free_env_val(command);
       }
       else {
 	kill_old_erlang();
-	system((char*)&cmd[0]);
+	/* suppress gcc warning with 'if' */
+	if(system((char*)&cmd[0]));
 	print_error("Executed \"%s\". Terminating.",cmd);
       }
     }
@@ -724,12 +727,16 @@ static int
 heart_cmd_reply(int fd, char *s)
 {
   struct msg m;
-  int len = strlen(s) + 1;	/* Include \0 */
+  int len = strlen(s);
 
-  /* FIXME if s >= MSG_BODY_SIZE error */
+  /* if s >= MSG_BODY_SIZE, return a write
+   * failure immediately.
+   */
+  if (len >= sizeof(m.fill))
+      return -1;
 
   m.op = HEART_CMD;
-  m.len = htons(len + 2);	/* Include Op */
+  m.len = htons(len + 1);	/* Include Op */
   strcpy((char*)m.fill, s);
 
   return write_message(fd, &m);
diff --git a/erts/etc/common/inet_gethost.c b/erts/etc/common/inet_gethost.c
index ff16ee02c4..e923233ce9 100644
--- a/erts/etc/common/inet_gethost.c
+++ b/erts/etc/common/inet_gethost.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 1998-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /*
@@ -52,20 +52,21 @@
 #  include "config.h"
 #endif
 
+#include "erl_printf.h"
+
 #ifdef WIN32
 
 #define WIN32_LEAN_AND_MEAN
 #include <winsock2.h>
 #include <windows.h>
+#include <ws2tcpip.h>
 #include <process.h>
 #include <stdio.h>
 #include <stdlib.h>
 
 /* These are not used even if they would exist which they should not */
-#undef HAVE_GETADDRINFO
 #undef HAVE_GETIPNODEBYNAME
 #undef HAVE_GETHOSTBYNAME2
-#undef HAVE_GETNAMEINFO
 #undef HAVE_GETIPNODEBYADDR
 
 #else /* Unix */
@@ -1140,14 +1141,12 @@ static Worker *pick_worker(void)
 static Worker *pick_worker_greedy(AddrByte *domainbuff)
 {
     int i;
-    int ql = 0;
     int found = -1;
     for (i=0; i < num_busy_workers; ++i) {
 	if (domaineq(busy_workers[i].domain, domainbuff)) {
 	    if ((found < 0) || (busy_workers[i].que_size < 
 				busy_workers[found].que_size)) {
 		found = i;
-		ql = busy_workers[i].que_size;
 	    }
 	}
     }
@@ -1296,7 +1295,7 @@ static int read_request(AddrByte **buff, size_t *buff_size)
     }
 
     if (siz > *buff_size) {
-	if (buff_size == 0) {
+	if (*buff_size == 0) {
 	    *buff = ALLOC((*buff_size = siz));
 	} else {
 	    *buff = REALLOC(*buff, (*buff_size = siz));
@@ -1759,7 +1758,7 @@ static int worker_loop(void)
 		struct addrinfo hints;
 		
 		memset(&hints, 0, sizeof(hints));
-		hints.ai_flags = (AI_CANONNAME|AI_V4MAPPED|AI_ADDRCONFIG);
+		hints.ai_flags = AI_CANONNAME;
 		hints.ai_socktype = SOCK_STREAM;
 		hints.ai_family = AF_INET6;
 		DEBUGF(5, ("Starting getaddrinfo(%s, ...)", data));
@@ -1944,12 +1943,14 @@ static int worker_loop(void)
 	}
 	m = NULL;
 #else
-	write(1, reply, data_size); /* No signals expected */
+	/* expect no signals */
+	if (write(1, reply, data_size) < 0)
+	    goto fail;
 #endif
     } /* for (;;) */
 
-#ifdef WIN32
  fail:
+#ifdef WIN32
     if (m != NULL) {
 	FREE(m);
     }
@@ -1958,8 +1959,8 @@ static int worker_loop(void)
     if (reply) {
 	FREE(reply);
     }
-    return 1;
 #endif
+    return 1;
 }
 
 static int map_netdb_error(int netdb_code)
@@ -2552,7 +2553,7 @@ static void debugf(char *format, ...)
     sprintf(buff,"%s[%d] (DEBUG):",program_name,(int) getpid());
 #endif
     ptr = buff + strlen(buff);
-    vsprintf(ptr,format,ap);
+    erts_vsnprintf(ptr,sizeof(buff)-strlen(buff)-2,format,ap);
     strcat(ptr,"\r\n");
 #ifdef WIN32
     if (debug_console_allocated != INVALID_HANDLE_VALUE) {
@@ -2560,7 +2561,8 @@ static void debugf(char *format, ...)
 	WriteFile(debug_console_allocated,buff,strlen(buff),&res,NULL);
     }
 #else
-    write(2,buff,strlen(buff));
+    /* suppress warning with 'if' */
+    if(write(2,buff,strlen(buff)));
 #endif
     va_end(ap);
 }
@@ -2574,7 +2576,7 @@ static void warning(char *format, ...)
     va_start(ap,format);
     sprintf(buff,"%s[%d]: WARNING:",program_name, (int) getpid());
     ptr = buff + strlen(buff);
-    vsprintf(ptr,format,ap);
+    erts_vsnprintf(ptr,sizeof(buff)-strlen(buff)-2,format,ap);
     strcat(ptr,"\r\n");
 #ifdef WIN32
     {
@@ -2582,7 +2584,8 @@ static void warning(char *format, ...)
 	WriteFile(GetStdHandle(STD_ERROR_HANDLE),buff,strlen(buff),&res,NULL);
     }
 #else
-    write(2,buff,strlen(buff));
+    /* suppress warning with 'if' */
+    if(write(2,buff,strlen(buff)));
 #endif
     va_end(ap);
 }
@@ -2596,7 +2599,7 @@ static void fatal(char *format, ...)
     va_start(ap,format);
     sprintf(buff,"%s[%d]: FATAL ERROR:",program_name, (int) getpid());
     ptr = buff + strlen(buff);
-    vsprintf(ptr,format,ap);
+    erts_vsnprintf(ptr,sizeof(buff)-strlen(buff)-2,format,ap);
     strcat(ptr,"\r\n");
 #ifdef WIN32
     {
@@ -2604,7 +2607,8 @@ static void fatal(char *format, ...)
 	WriteFile(GetStdHandle(STD_ERROR_HANDLE),buff,strlen(buff),&res,NULL);
     }
 #else
-    write(2,buff,strlen(buff));
+    /* suppress warning with 'if' */
+    if(write(2,buff,strlen(buff)));
 #endif
     va_end(ap);
 #ifndef WIN32
diff --git a/erts/etc/common/typer.c b/erts/etc/common/typer.c
index c2567cb8b4..c95959d52d 100644
--- a/erts/etc/common/typer.c
+++ b/erts/etc/common/typer.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -175,7 +175,7 @@ main(int argc, char** argv)
 static void
 push_words(char* src)
 {
-    char sbuf[1024];
+    char sbuf[MAXPATHLEN];
     char* dst;
 
     dst = sbuf;
@@ -307,7 +307,7 @@ error(char* format, ...)
     va_list ap;
     
     va_start(ap, format);
-    vsprintf(sbuf, format, ap);
+    erts_vsnprintf(sbuf, sizeof(sbuf), format, ap);
     va_end(ap);
     fprintf(stderr, "typer: %s\n", sbuf);
     exit(1);
@@ -336,6 +336,9 @@ get_default_emulator(char* progname)
     char sbuf[MAXPATHLEN];
     char* s;
 
+    if (strlen(progname) >= sizeof(sbuf))
+        return ERL_NAME;
+
     strcpy(sbuf, progname);
     for (s = sbuf+strlen(sbuf); s >= sbuf; s--) {
 	if (IS_DIRSEP(*s)) {
diff --git a/erts/etc/unix/Install.src b/erts/etc/unix/Install.src
index 83f9690782..58f7b38ed0 100644
--- a/erts/etc/unix/Install.src
+++ b/erts/etc/unix/Install.src
@@ -66,12 +66,12 @@ then
     exit 1
 fi
 
-if [ ! -d $ERL_ROOT/bin ]
+if [ ! -d "$ERL_ROOT/bin" ]
 then
-    mkdir $ERL_ROOT/bin
+    mkdir "$ERL_ROOT/bin"
 fi
 
-cd $ERL_ROOT/erts-%I_VSN%/bin
+cd "$ERL_ROOT/erts-%I_VSN%/bin"
 
 sed -e "s;%FINAL_ROOTDIR%;$TARGET_ERL_ROOT;" erl.src > erl
 chmod 755 erl
@@ -79,17 +79,21 @@ chmod 755 erl
 #
 # Create start file for embedded system use,
 #
-(cd $ERL_ROOT/erts-%I_VSN%/bin;
+(cd "$ERL_ROOT/erts-%I_VSN%/bin";
   sed -e "s;%FINAL_ROOTDIR%;$TARGET_ERL_ROOT;" start.src > start;
   chmod 755 start)
 
-cd $ERL_ROOT/bin
+cd "$ERL_ROOT/bin"
 
-cp -p $ERL_ROOT/erts-%I_VSN%/bin/erl .
-cp -p $ERL_ROOT/erts-%I_VSN%/bin/erlc .
-cp -p $ERL_ROOT/erts-%I_VSN%/bin/dialyzer .
-cp -p $ERL_ROOT/erts-%I_VSN%/bin/typer .
-cp -p $ERL_ROOT/erts-%I_VSN%/bin/escript .
+cp -p "$ERL_ROOT/erts-%I_VSN%/bin/erl" .
+cp -p "$ERL_ROOT/erts-%I_VSN%/bin/erlc" .
+cp -p "$ERL_ROOT/erts-%I_VSN%/bin/dialyzer" .
+cp -p "$ERL_ROOT/erts-%I_VSN%/bin/typer" .
+cp -p "$ERL_ROOT/erts-%I_VSN%/bin/ct_run" .
+cp -p "$ERL_ROOT/erts-%I_VSN%/bin/escript" .
+
+# Remove in R16B
+ln -s ct_run run_test
 
 #
 # Set a soft link to epmd
@@ -103,15 +107,15 @@ fi
 
 ln -s ../erts-%I_VSN%/bin/epmd epmd
 
-cp -p $ERL_ROOT/erts-%I_VSN%/bin/run_erl .
-cp -p $ERL_ROOT/erts-%I_VSN%/bin/to_erl .
-cp -p $ERL_ROOT/erts-%I_VSN%/bin/start .
-sed -e "s;%EMU%;%EMULATOR%%EMULATOR_NUMBER%;" $ERL_ROOT/erts-%I_VSN%/bin/start_erl.src > start_erl
+cp -p "$ERL_ROOT/erts-%I_VSN%/bin/run_erl" .
+cp -p "$ERL_ROOT/erts-%I_VSN%/bin/to_erl" .
+cp -p "$ERL_ROOT/erts-%I_VSN%/bin/start" .
+sed -e "s;%EMU%;%EMULATOR%%EMULATOR_NUMBER%;" "$ERL_ROOT/erts-%I_VSN%/bin/start_erl.src" > start_erl
 chmod 755 start_erl
 echo ""
 
-echo %I_VSN% %I_SYSTEM_VSN% > $ERL_ROOT/releases/start_erl.data
-sed -e "s;%ERL_ROOT%;$TARGET_ERL_ROOT;" $ERL_ROOT/releases/RELEASES.src > $ERL_ROOT/releases/RELEASES
+echo %I_VSN% %I_SYSTEM_VSN% > "$ERL_ROOT/releases/start_erl.data"
+sed -e "s;%ERL_ROOT%;$TARGET_ERL_ROOT;" "$ERL_ROOT/releases/RELEASES.src" > "$ERL_ROOT/releases/RELEASES"
 
 if [ "$start_option" = "query" ]
 then
@@ -143,10 +147,10 @@ cp -p ../releases/%I_SYSTEM_VSN%/$Name.script start.script
 # Fixing the man pages
 #
 
-if [ -d $ERL_ROOT/man ]
+if [ -d "$ERL_ROOT/man" ]
 then
-    cd $ERL_ROOT
-    ./misc/format_man_pages $ERL_ROOT
+    cd "$ERL_ROOT"
+    ./misc/format_man_pages "$ERL_ROOT"
 fi
 
 exit 0
diff --git a/erts/etc/unix/cerl.src b/erts/etc/unix/cerl.src
index f81ef6b0fe..ffd48d5811 100644
--- a/erts/etc/unix/cerl.src
+++ b/erts/etc/unix/cerl.src
@@ -1,20 +1,20 @@
 #!/bin/sh
 #
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2003-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2003-2011. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 #
 #
@@ -62,12 +62,21 @@ cxargs_add() {
     done
 }
 
+eeargs=
+eeargs_add() {
+    while [ $# -gt 0 ]; do
+	cargs="$cargs $1"
+	eeargs="$eeargs $1"
+	shift
+    done
+}
+
 core=
 
 GDB=
 GDBBP=
+GDBARGS=
 TYPE=
-EMU_TYPE=
 debug=
 run_valgrind=no
 
@@ -126,34 +135,37 @@ while [ $# -gt 0 ]; do
 	    ;;
 	"-smp")
 	    shift
-	    cargs="$cargs -smp"
-	    EMU_TYPE=.smp
+	    if [ $# -le 0 ]; then
+		eeargs_add -smp
+	    else
+		case $1 in
+		    disable)
+			shift
+			eeargs_add -smpdisable
+			;;
+		    enable)
+			shift
+			eeargs_add -smp
+			;;
+		    *)
+			eeargs_add -smp
+		esac
+	    fi
+	    ;;
+	"-smpdisable")
+	    shift
+	    eeargs_add -smpdisable
 	    ;;
 	"-lcnt")
 	    shift
 	    cargs="$cargs -lcnt"
 	    TYPE=.lcnt
 	    ;;
-	"-frag")
-	    shift
-	    cargs="$cargs -frag"
-	    EMU_TYPE=.frag
-	    ;;
-	"-smp_frag")
-	    shift
-	    cargs="$cargs -smp_frag"
-	    EMU_TYPE=.smp_frag
-	    ;;
 	"-gprof")
 	    shift
 	    cargs="$cargs -gprof"
 	    TYPE=.gprof
 	    ;;
-	"-hybrid")
-	    shift
-	    cargs="$cargs -hybrid"
-	    EMU_TYPE=.hybrid
-	    ;;
 	"-debug")
 	    shift
 	    cargs="$cargs -debug"
@@ -179,11 +191,6 @@ while [ $# -gt 0 ]; do
 #	    shift
 #	    GDB=xxgdb
 #	    ;;
-	"-shared")
-	    shift
-	    cargs="$cargs -shared"
-	    TYPE=.shared
-	    ;;
 	"-purify")
 	    shift
 	    cargs="$cargs -purify"
@@ -220,24 +227,39 @@ done
 PATH=$BINDIR:$ROOTDIR/bin:$PATH
 EXEC=$BINDIR/erlexec
 
-PROGNAME="$PROGNAME $cargs"
-EMU=$EMU$TYPE$EMU_TYPE
+PROGNAME="$PROGNAME$cargs"
+EMU="$EMU$TYPE"
+EMU_NAME=`$EXEC -emu_name_exit $eeargs`
+
 if [ $run_valgrind != yes ]; then
     xargs="$xargs -pz $PRELOADED --"
 fi
 if [ "x$GDB" = "x" ]; then
     if [ $run_valgrind = yes ]; then
+	valversion=`valgrind --version`
+	valmajor=`echo $valversion | sed 's,[a-z]*\-\([0-9]*\).*,\1,'`
+        valminor=`echo $valversion | sed 's,[a-z]*\-[0-9]*.\([0-9]*\).*,\1,'`
 	emu_xargs=`echo $xargs | sed "s|+|-|g"`
-	if [ "x$VALGRIND_LOG_DIR" = "x" ]; then
-	    valgrind_log=
-	else
-	    valgrind_log="--log-file=$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU.log"
-	fi
 	if [ "x$VALGRIND_LOG_XML" = "x" ]; then
 	    valgrind_xml=
+	    log_file_prefix="--log-file="
 	else
 	    export VALGRIND_LOG_XML
 	    valgrind_xml="--xml=yes"
+	    if [ $valmajor -gt 2 -a $valminor -gt 4 ]; then
+		log_file_prefix="--xml-file="
+	    else
+		log_file_prefix="--log-file="
+	    fi
+	fi
+	if [ "x$VALGRIND_LOG_DIR" = "x" ]; then
+	    valgrind_log=
+	else
+	    if [ $valmajor -gt 2 -a $valminor -gt 4 ]; then
+		valgrind_log="$log_file_prefix$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU_NAME.log.$$"
+	    else
+		valgrind_log="$log_file_prefix$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU_NAME.log"
+	    fi
 	fi
 	if [ "x$VALGRIND_MISC_FLAGS" = "x" ]; then
 	    valgrind_misc_flags=
@@ -249,9 +271,9 @@ if [ "x$GDB" = "x" ]; then
 	early_beam_args=`echo $beam_args | sed "s|^\(.*-progname\).*$|\1|g"`
 	late_beam_args=`echo $beam_args | sed "s|^$pre_beam_args.*\(-- -home.*\)$|\1|g"`
 	
-	exec valgrind $valgrind_xml $valgrind_log $valgrind_misc_flags $BINDIR/$EMU $emu_xargs $early_beam_args "$PROGNAME" $late_beam_args -pz $PRELOADED
+	exec valgrind $valgrind_xml $valgrind_log $valgrind_misc_flags $BINDIR/$EMU_NAME $emu_xargs $early_beam_args "$PROGNAME" $late_beam_args -pz $PRELOADED
     else
-	exec $EXEC $xargs ${1+"$@"}
+	exec $EXEC $eeargs $xargs ${1+"$@"}
     fi
 else
     if [ "x$EMACS" = "x" ]; then
@@ -277,9 +299,14 @@ else
 	    ;;
     esac
 
+    # Set annotation level for gdb in emacs 22 and higher.
+    emacs_major=`$EMACS --version | head -1 | sed 's,^[^0-9]*\([0-9]*\).*,\1,g'`
+    if [ '!' -z "$emacs_major" -a $emacs_major -gt 21 ]; then
+	GDBARGS="--annotate=1 "
+    fi
     gdbcmd="$gdbcmd $GDBBP \
             (insert-string \"source $ROOTDIR/erts/etc/unix/etp-commands\") \
             (comint-send-input)"
     # Fire up gdb in emacs...
-    exec $EMACS --eval "(progn (gdb \"gdb $EMU\") $gdbcmd)"
+    exec $EMACS --eval "(progn (gdb \"gdb $GDBARGS$EMU_NAME\") $gdbcmd)"
 fi
diff --git a/erts/etc/unix/erl.src.src b/erts/etc/unix/erl.src.src
index 50603f12f4..fa187c5509 100644
--- a/erts/etc/unix/erl.src.src
+++ b/erts/etc/unix/erl.src.src
@@ -17,7 +17,7 @@
 # 
 # %CopyrightEnd%
 #
-ROOTDIR=%FINAL_ROOTDIR%
+ROOTDIR="%FINAL_ROOTDIR%"
 BINDIR=$ROOTDIR/erts-%VSN%/bin
 EMU=%EMULATOR%%EMULATOR_NUMBER%
 PROGNAME=`echo $0 | sed 's/.*\///'`
@@ -25,4 +25,4 @@ export EMU
 export ROOTDIR
 export BINDIR
 export PROGNAME
-exec $BINDIR/erlexec ${1+"$@"}
+exec "$BINDIR/erlexec" ${1+"$@"}
diff --git a/erts/etc/unix/etp-commands b/erts/etc/unix/etp-commands
index 6a01e0b7e0..1c886620bb 100644
--- a/erts/etc/unix/etp-commands
+++ b/erts/etc/unix/etp-commands
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 2005-2009. All Rights Reserved.
+# Copyright Ericsson AB 2005-2012. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -1883,6 +1883,28 @@ document etp-ets-tables
 %---------------------------------------------------------------------------
 end
 
+define etp-ets-obj
+# Args: DbTerm*
+#
+  set $etp_ets_obj_i = 1
+  while $etp_ets_obj_i <= (($arg0)->tpl[0] >> 6)
+    if $etp_ets_obj_i == 1      
+      printf "{"
+    else
+      printf ", "
+    end
+    set $etp_ets_elem = ($arg0)->tpl[$etp_ets_obj_i]
+    if ($etp_ets_elem & 3) == 0
+      printf "<compressed>"
+    else
+      etp-1 $etp_ets_elem 0
+    end
+    set $etp_ets_obj_i++
+  end
+  printf "}"
+end
+
+
 define etp-ets-tabledump
 # Args: int tableindex
 #
@@ -1896,10 +1918,10 @@ define etp-ets-tabledump
   if $etp_ets_tabledump_t->common.status & 0x130
     # Hash table
     set $etp_ets_tabledump_h = $etp_ets_tabledump_t->hash
-    printf "%% nitems=%d\n", $etp_ets_tabledump_t->common.nitems
-    while $etp_ets_tabledump_i < $etp_ets_tabledump_h->nactive
-      set $etp_ets_tabledump_l = $etp_ets_tabledump_h->seg \
-            [$etp_ets_tabledump_i>>8][$etp_ets_tabledump_i&0xFF]
+    printf "%% nitems=%d\n", (long) $etp_ets_tabledump_t->common.nitems
+    while $etp_ets_tabledump_i < (long) $etp_ets_tabledump_h->nactive
+      set $etp_ets_tabledump_seg = ((struct segment**)$etp_ets_tabledump_h->segtab)[$etp_ets_tabledump_i>>8]
+      set $etp_ets_tabledump_l = $etp_ets_tabledump_seg->buckets[$etp_ets_tabledump_i&0xFF]
       if $etp_ets_tabledump_l
         printf "%% Slot %d:\n", $etp_ets_tabledump_i
         while $etp_ets_tabledump_l
@@ -1909,7 +1931,7 @@ define etp-ets-tabledump
             printf "["
           end
           set $etp_ets_tabledump_n++
-          etp-1 ((Eterm)($etp_ets_tabledump_l->dbterm.tpl)|0x2) 0
+	  etp-ets-obj &($etp_ets_tabledump_l->dbterm)
           if $etp_ets_tabledump_l->hvalue == ((unsigned long)-1)
             printf "% *\n"
           else
diff --git a/erts/etc/unix/format_man_pages b/erts/etc/unix/format_man_pages
index 2c4f6eee4f..93dcdcd8fa 100644
--- a/erts/etc/unix/format_man_pages
+++ b/erts/etc/unix/format_man_pages
@@ -3,7 +3,7 @@
 #
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 1996-2009. All Rights Reserved.
+# Copyright Ericsson AB 1996-2010. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -59,34 +59,21 @@ esac
 # Create the 'cat' directories (probably not needed)
 #
 
-cd $ERL_ROOT
+cd $ERL_ROOT/man
 
-if [ ! -d man/cat1 ]
-then
-    mkdir man/cat1
-fi
+for d in 0 1 2 3 4 5 6 7 8 9
+do
+  if [ ! -d cat$d ]
+      then
+      mkdir cat$d
+  fi
 
-if [ ! -d man/cat3 ]
-then
-    mkdir man/cat3
-fi
-
-if [ ! -d man/cat4 ]
-then
-    mkdir man/cat4
-fi
-
-if [ ! -d man/cat6 ]
-then
-    mkdir man/cat6
-fi
+done
 
 #
 # Cleanup old formatting
 #
 
-cd $ERL_ROOT/man
-
 rm -f whatis windex
 
 # Remove old cat files
diff --git a/erts/etc/unix/run_erl.c b/erts/etc/unix/run_erl.c
index 4bb148df98..6b350e8bd5 100644
--- a/erts/etc/unix/run_erl.c
+++ b/erts/etc/unix/run_erl.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -75,6 +75,9 @@
 #ifdef HAVE_SYS_IOCTL_H
 #  include <sys/ioctl.h>
 #endif
+#if defined(__sun) && defined(__SVR4)
+#  include <stropts.h>
+#endif
 
 #include "run_erl.h"
 #include "safe_string.h"    /* sn_printf, strn_cpy, strn_cat, etc */
@@ -123,7 +126,7 @@
 /* prototypes */
 static void usage(char *);
 static int create_fifo(char *name, int perm);
-static int open_pty_master(char **name);
+static int open_pty_master(char **name, int *sfd);
 static int open_pty_slave(char *name);
 static void pass_on(pid_t);
 static void exec_shell(char **);
@@ -147,6 +150,10 @@ static int write_all(int fd, const char* buf, int len);
 static int extract_ctrl_seq(char* buf, int len);
 static void set_window_size(unsigned col, unsigned row);
 
+static ssize_t sf_write(int fd, const void *buffer, size_t len);
+static ssize_t sf_read(int fd, void *buffer, size_t len);
+static int sf_open(const char *path, int flags, mode_t mode);
+static int sf_close(int fd);
 
 #ifdef DEBUG
 static void show_terminal_settings(struct termios *t);
@@ -213,11 +220,13 @@ static char* outbuf_in;
 int main(int argc, char **argv)
 {
   int childpid;
-  int sfd;
+  int sfd = -1;
   int fd;
   char *p, *ptyslave=NULL;
   int i = 1;
   int off_argv;
+  int calculated_pipename = 0;
+  int highest_pipe_num = 0;
 
   program_name = argv[0];
 
@@ -295,10 +304,10 @@ int main(int argc, char **argv)
   if(*pipename && pipename[strlen(pipename)-1] == '/') {
     /* The user wishes us to find a unique pipe name in the specified */
     /* directory */
-    int highest_pipe_num = 0;
     DIR *dirp;
     struct dirent *direntp;
 
+    calculated_pipename = 1;
     dirp = opendir(pipename);
     if(!dirp) {
       ERRNO_ERR1(LOG_ERR,"Can't access pipe directory '%s'.", pipename);
@@ -319,35 +328,44 @@ int main(int argc, char **argv)
 	      PIPE_STUBNAME, highest_pipe_num+1);
   } /* if */
 
-  /* write FIFO - is read FIFO for `to_erl' program */
-  strn_cpy(fifo1, sizeof(fifo1), pipename);
-  strn_cat(fifo1, sizeof(fifo1), ".r");
-  if (create_fifo(fifo1, PERM) < 0) {
-    ERRNO_ERR1(LOG_ERR,"Cannot create FIFO %s for writing.", fifo1);
-    exit(1);
-  }
-
-  /* read FIFO - is write FIFO for `to_erl' program */
-  strn_cpy(fifo2, sizeof(fifo2), pipename);
-  strn_cat(fifo2, sizeof(fifo2), ".w");
-
-  /* Check that nobody is running run_erl already */
-  if ((fd = open (fifo2, O_WRONLY|DONT_BLOCK_PLEASE, 0)) >= 0) {
-    /* Open as client succeeded -- run_erl is already running! */
-    fprintf(stderr, "Erlang already running on pipe %s.\n", pipename);
-    close(fd);
-    exit(1);
-  }
-  if (create_fifo(fifo2, PERM) < 0) { 
-    ERRNO_ERR1(LOG_ERR,"Cannot create FIFO %s for reading.", fifo2);
-    exit(1);
+  for(;;) {
+      /* write FIFO - is read FIFO for `to_erl' program */
+      strn_cpy(fifo1, sizeof(fifo1), pipename);
+      strn_cat(fifo1, sizeof(fifo1), ".r");
+      if (create_fifo(fifo1, PERM) < 0) {
+	  ERRNO_ERR1(LOG_ERR,"Cannot create FIFO %s for writing.", fifo1);
+	  exit(1);
+      }
+      
+      /* read FIFO - is write FIFO for `to_erl' program */
+      strn_cpy(fifo2, sizeof(fifo2), pipename);
+      strn_cat(fifo2, sizeof(fifo2), ".w");
+      
+      /* Check that nobody is running run_erl already */
+      if ((fd = sf_open(fifo2, O_WRONLY|DONT_BLOCK_PLEASE, 0)) >= 0) {
+	  /* Open as client succeeded -- run_erl is already running! */
+	  sf_close(fd);
+	  if (calculated_pipename) {
+	      ++highest_pipe_num;
+	      strn_catf(pipename, sizeof(pipename), "%s.%d",
+			PIPE_STUBNAME, highest_pipe_num+1);
+	      continue;
+	  } 
+	  fprintf(stderr, "Erlang already running on pipe %s.\n", pipename);
+	  exit(1);
+      }
+      if (create_fifo(fifo2, PERM) < 0) { 
+	  ERRNO_ERR1(LOG_ERR,"Cannot create FIFO %s for reading.", fifo2);
+	  exit(1);
+      }
+      break;
   }
 
   /*
    * Open master pseudo-terminal
    */
 
-  if ((mfd = open_pty_master(&ptyslave)) < 0) {
+  if ((mfd = open_pty_master(&ptyslave, &sfd)) < 0) {
     ERRNO_ERR0(LOG_ERR,"Could not open pty master");
     exit(1);
   }
@@ -362,7 +380,7 @@ int main(int argc, char **argv)
   }
   if (childpid == 0) {
     /* Child */
-    close(mfd);
+    sf_close(mfd);
     /* disassociate from control terminal */
 #ifdef USE_SETPGRP_NOARGS       /* SysV */
     setpgrp();
@@ -372,15 +390,30 @@ int main(int argc, char **argv)
     setsid();
 #endif
     /* Open the slave pty */
-    if ((sfd = open_pty_slave(ptyslave)) < 0) {
-      ERRNO_ERR1(LOG_ERR,"Could not open pty slave '%s'", ptyslave);
-      exit(1);
+    if (sfd < 0) {
+	/* not allocated by open_pty_master */
+	if ((sfd = open_pty_slave(ptyslave)) < 0) {
+	    ERRNO_ERR1(LOG_ERR,"Could not open pty slave '%s'", ptyslave);
+	    exit(1);
+	}
+	/* But sfd may be one of the stdio fd's now, and we should be unmodern and not use dup2... */
+	/* easiest to dup it up... */
+	while (sfd < 3) {
+	    sfd = dup(sfd);
+	}
     }
-    /* But sfd may be one of the stdio fd's now, and we should be unmodern and not use dup2... */
-    /* easiest to dup it up... */
-    while (sfd < 3) {
-	sfd = dup(sfd);
+#if defined(HAVE_OPENPTY) && defined(TIOCSCTTY)
+    else {
+	/* sfd is from open_pty_master 
+	 * openpty -> fork -> login_tty (forkpty)
+	 * 
+	 * It would be preferable to implement a portable 
+	 * forkpty instead of open_pty_master / open_pty_slave
+	 */
+	/* login_tty(sfd);  <- FAIL */
+	ioctl(sfd, TIOCSCTTY, (char *)NULL);
     }
+#endif
 
 #ifndef NO_SYSLOG
     /* Before fiddling with file descriptors we make sure syslog is turned off
@@ -393,14 +426,14 @@ int main(int argc, char **argv)
 #endif
 
     /* Close stdio */
-    close(0);
-    close(1);
-    close(2);
+    sf_close(0);
+    sf_close(1);
+    sf_close(2);
 
     if (dup(sfd) != 0 || dup(sfd) != 1 || dup(sfd) != 2) {
       status("Cannot dup\n");
     }
-    close(sfd);
+    sf_close(sfd);
     exec_shell(argv+off_argv); /* exec_shell expects argv[2] to be */
                         /* the command name, so we have to */
                         /* adjust. */
@@ -452,7 +485,7 @@ static void pass_on(pid_t childpid)
      * We can't open the writing side because nobody is reading and 
      * we'd either hang or get an error.
      */
-    if ((rfd = open(fifo2, O_RDONLY|DONT_BLOCK_PLEASE, 0)) < 0) {
+    if ((rfd = sf_open(fifo2, O_RDONLY|DONT_BLOCK_PLEASE, 0)) < 0) {
 	ERRNO_ERR1(LOG_ERR,"Could not open FIFO '%s' for reading.", fifo2);
 	exit(1);
     }
@@ -545,7 +578,7 @@ static void pass_on(pid_t childpid)
 	    char* buf = outbuf_first();
 
 	    len = outbuf_size();
-	    written = write(wfd, buf, len);
+	    written = sf_write(wfd, buf, len);
 	    if (written < 0 && errno == EAGAIN) {
 		/*
 		 * Nothing was written - this is really strange because
@@ -556,7 +589,7 @@ static void pass_on(pid_t childpid)
 		 * A write error. Assume that to_erl has terminated.
 		 */
 		clear_outbuf();
-		close(wfd);
+		sf_close(wfd);
 		wfd = 0;
 	    } else {
 		/* Delete the written part (or all) from the buffer. */
@@ -571,10 +604,10 @@ static void pass_on(pid_t childpid)
 #ifdef DEBUG
 	    status("Pty master read; ");
 #endif
-	    if ((len = read(mfd, buf, BUFSIZ)) <= 0) {
-		close(rfd);
-		if(wfd) close(wfd);
-		close(mfd);
+	    if ((len = sf_read(mfd, buf, BUFSIZ)) <= 0) {
+		sf_close(rfd);
+		if(wfd) sf_close(wfd);
+		sf_close(mfd);
 		unlink(fifo1);
 		unlink(fifo2);
 		if (len < 0) {
@@ -605,10 +638,10 @@ static void pass_on(pid_t childpid)
 #ifdef DEBUG
 	    status("FIFO read; ");
 #endif
-	    if ((len = read(rfd, buf, BUFSIZ)) < 0) {
-		close(rfd);
-		if(wfd) close(wfd);
-		close(mfd);
+	    if ((len = sf_read(rfd, buf, BUFSIZ)) < 0) {
+		sf_close(rfd);
+		if(wfd) sf_close(wfd);
+		sf_close(mfd);
 		unlink(fifo1);
 		unlink(fifo2);
 		ERRNO_ERR0(LOG_ERR,"Error in reading from FIFO.");
@@ -617,8 +650,8 @@ static void pass_on(pid_t childpid)
 
 	    if(!len) {
 		/* to_erl closed its end of the pipe */
-		close(rfd);
-		rfd = open(fifo2, O_RDONLY|DONT_BLOCK_PLEASE, 0);
+		sf_close(rfd);
+		rfd = sf_open(fifo2, O_RDONLY|DONT_BLOCK_PLEASE, 0);
 		if (rfd < 0) {
 		    ERRNO_ERR1(LOG_ERR,"Could not open FIFO '%s' for reading.", fifo2);
 		    exit(1);
@@ -631,11 +664,11 @@ static void pass_on(pid_t childpid)
 		     * from to_erl, to_erl should already be reading this pipe - open
 		     * should succeed. But in case of error, we just ignore it.
 		     */
-		    if ((wfd = open(fifo1, O_WRONLY|DONT_BLOCK_PLEASE, 0)) < 0) {
+		    if ((wfd = sf_open(fifo1, O_WRONLY|DONT_BLOCK_PLEASE, 0)) < 0) {
 			status("Client expected on FIFO %s, but can't open (len=%d)\n",
 			       fifo1, len);
-			close(rfd);
-			rfd = open(fifo2, O_RDONLY|DONT_BLOCK_PLEASE, 0);
+			sf_close(rfd);
+			rfd = sf_open(fifo2, O_RDONLY|DONT_BLOCK_PLEASE, 0);
 			if (rfd < 0) {
 			    ERRNO_ERR1(LOG_ERR,"Could not open FIFO '%s' for reading.", fifo2);
 			    exit(1);
@@ -669,9 +702,9 @@ static void pass_on(pid_t childpid)
 		} 
 		else if (len>0 && write_all(mfd, buf, len) != len) {
 		    ERRNO_ERR0(LOG_ERR,"Error in writing to terminal.");
-		    close(rfd);
-		    if(wfd) close(wfd);
-		    close(mfd);
+		    sf_close(rfd);
+		    if(wfd) sf_close(wfd);
+		    sf_close(mfd);
 		    exit(1);
 		}
 	    }
@@ -783,7 +816,7 @@ static int open_log(int log_num, int flags)
 
   /* Create or continue on the current log file */
   sn_printf(buf, sizeof(buf), "%s/%s%d", log_dir, LOG_STUBNAME, log_num);
-  if((lfd = open(buf, flags, LOG_PERM))<0){
+  if((lfd = sf_open(buf, flags, LOG_PERM))<0){
       ERRNO_ERR1(LOG_ERR,"Can't open log file '%s'.", buf);
     exit(1);
   }
@@ -827,7 +860,7 @@ static void write_to_log(int* lfd, int* log_num, char* buf, int len)
   
   size = lseek(*lfd,0,SEEK_END);
   if(size+len > log_maxsize) {
-    close(*lfd);
+    sf_close(*lfd);
     *log_num = next_log(*log_num);
     *lfd = open_log(*log_num, O_RDWR|O_CREAT|O_TRUNC|O_SYNC); 
   }
@@ -858,26 +891,32 @@ static int create_fifo(char *name, int perm)
  * Find a master device, open and return fd and slave device name.
  */
 
-static int open_pty_master(char **ptyslave)
+static int open_pty_master(char **ptyslave, int *sfdp)
 {
   int mfd;
 
 /* Use the posix_openpt if working, as this guarantees creation of the 
    slave device properly. */
-#ifdef HAVE_WORKING_POSIX_OPENPT
+#if defined(HAVE_WORKING_POSIX_OPENPT) || (defined(__sun) && defined(__SVR4))
+#  ifdef HAVE_WORKING_POSIX_OPENPT
   if ((mfd = posix_openpt(O_RDWR)) >= 0) {
+#  elif defined(__sun) && defined(__SVR4)
+  mfd = sf_open("/dev/ptmx", O_RDWR, 0);
+
+  if (mfd >= 0) {
+#  endif
       if ((*ptyslave = ptsname(mfd)) != NULL &&
 	  grantpt(mfd) == 0 && 
 	  unlockpt(mfd) == 0) {
 
 	  return mfd;
       }
-      close(mfd);
+      sf_close(mfd);
   }
   /* fallback to openpty if it exist */
 #endif
 
-#ifdef HAVE_OPENPTY
+#if defined(HAVE_OPENPTY)
 #  ifdef PATH_MAX
 #    define SLAVE_SIZE PATH_MAX
 #  else
@@ -885,11 +924,8 @@ static int open_pty_master(char **ptyslave)
 #  endif
   {
       static char slave[SLAVE_SIZE];
-      int sfd;
 #  undef SLAVE_SIZE
-
-      if (openpty(&mfd, &sfd, slave, NULL, NULL) == 0) {
-	  close(sfd);
+      if (openpty(&mfd, sfdp, slave, NULL, NULL) == 0) {
 	  *ptyslave = slave;
 	  return mfd;
       }
@@ -921,7 +957,8 @@ static int open_pty_master(char **ptyslave)
 
     for (minor = minorchars; *minor; minor++) {
       ptyname[10] = *minor;
-      if ((mfd = open(ptyname, O_RDWR, 0)) >= 0) {
+
+      if ((mfd = sf_open(ptyname, O_RDWR, 0)) >= 0) {
 	ptyname[9] = 's';
 	*ptyslave = ptyname;
 	return mfd;
@@ -939,7 +976,7 @@ static int open_pty_master(char **ptyslave)
       ptyname[13] = *major;
       for (minor = minorchars; *minor; minor++) {
 	ptyname[14] = *minor;
-	if ((mfd = open(ptyname, O_RDWR, 0)) >= 0) {
+	if ((mfd = sf_open(ptyname, O_RDWR, 0)) >= 0) {
 	  ttyname[12] = *major;
 	  ttyname[13] = *minor;
 	  *ptyslave = ttyname;
@@ -958,7 +995,7 @@ static int open_pty_master(char **ptyslave)
       ptyname[8] = *major;
       for (minor = minorchars; *minor; minor++) {
 	ptyname[9] = *minor;
-	if ((mfd = open(ptyname, O_RDWR, 0)) >= 0) {
+	if ((mfd = sf_open(ptyname, O_RDWR, 0)) >= 0) {
 	  ptyname[5] = 't';
 	  *ptyslave = ptyname;
 	  return mfd;
@@ -973,16 +1010,53 @@ static int open_pty_master(char **ptyslave)
 static int open_pty_slave(char *name)
 {
   int sfd;
-#ifdef DEBUG
   struct termios tty_rmode;
-#endif
 
-  if ((sfd = open(name, O_RDWR, 0)) < 0) {
+  if ((sfd = sf_open(name, O_RDWR, 0)) < 0) {
     return -1;
   }
 
+#if defined(__sun) && defined(__SVR4)
+  /* Load the necessary STREAMS modules for Solaris */
+  if ((ioctl(sfd, I_FIND, "ldterm")) < 0) {
+    ERROR0(LOG_ERR, "Failed to find ldterm STREAMS module");
+    return -1;
+  }
+  if (ioctl(sfd, I_PUSH, "ptem") < 0) {
+    ERROR0(LOG_ERR, "Failed to push ptem STREAMS module");
+    return -1;
+  }
+  if (ioctl(sfd, I_PUSH, "ldterm") < 0) {
+    ERROR0(LOG_ERR, "Failed to push ldterm STREAMS module");
+    return -1;
+  }
+  if (ioctl(sfd, I_PUSH, "ttcompat") < 0) {
+    ERROR0(LOG_ERR, "Failed to push ttcompat STREAMS module");
+    return -1;
+  }
+#endif
+
+  if (getenv("RUN_ERL_DISABLE_FLOWCNTRL")) {
+    if (tcgetattr(sfd, &tty_rmode) < 0) {
+      fprintf(stderr, "Cannot get terminal's current mode\n");
+      exit(-1);
+    }
+
+    tty_rmode.c_iflag &= ~IXOFF;
+    if (tcsetattr(sfd, TCSANOW, &tty_rmode) < 0) {
+      fprintf(stderr, "Cannot disable terminal's flow control on input\n");
+      exit(-1);
+    }
+
+    tty_rmode.c_iflag &= ~IXON;
+    if (tcsetattr(sfd, TCSANOW, &tty_rmode) < 0) {
+      fprintf(stderr, "Cannot disable terminal's flow control on output\n");
+      exit(-1);
+    }
+  }
+
 #ifdef DEBUG
-  if (tcgetattr(sfd, &tty_rmode) , 0) {
+  if (tcgetattr(sfd, &tty_rmode) < 0) {
     fprintf(stderr, "Cannot get terminals current mode\n");
     exit(-1);
   }
@@ -1065,7 +1139,7 @@ static void daemon_init(void)
        would be backward incompatible */
 
     for (i = 0; i < maxfd; ++i ) {
-	close(i);
+	sf_close(i);
     }
 
     OPEN_SYSLOG();
@@ -1191,7 +1265,7 @@ static int write_all(int fd, const char* buf, int len)
     int left = len;
     int written;
     for (;;) {
-	written = write(fd,buf,left);
+	written = sf_write(fd,buf,left);
 	if (written == left) {
 	    return len;
 	}
@@ -1203,6 +1277,36 @@ static int write_all(int fd, const char* buf, int len)
     }
 }
 
+static ssize_t sf_read(int fd, void *buffer, size_t len) {
+    ssize_t n = 0;
+
+    do { n = read(fd, buffer, len); } while (n < 0 && errno == EINTR);
+
+    return n;
+}
+
+static ssize_t sf_write(int fd, const void *buffer, size_t len) {
+    ssize_t n = 0;
+
+    do { n = write(fd, buffer, len); } while (n < 0 && errno == EINTR);
+
+    return n;
+}
+
+static int sf_open(const char *path, int type, mode_t mode) {
+    int fd = 0;
+
+    do { fd = open(path, type, mode); } while(fd < 0 && errno == EINTR);
+
+    return fd;
+}
+static int sf_close(int fd) {
+    int res = 0;
+
+    do { res = close(fd); } while(fd < 0 && errno == EINTR);
+
+    return res;
+}
 /* Extract any control sequences that are ment only for run_erl
  * and should not be forwarded to the pty.
  */
diff --git a/erts/etc/unix/to_erl.c b/erts/etc/unix/to_erl.c
index 886b301997..754b349338 100644
--- a/erts/etc/unix/to_erl.c
+++ b/erts/etc/unix/to_erl.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1996-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -125,7 +125,7 @@ static void usage(char *pname)
 int main(int argc, char **argv)
 {
     char  FIFO1[FILENAME_MAX], FIFO2[FILENAME_MAX];
-    int i, len, wfd, rfd, result = 0;
+    int i, len, wfd, rfd;
     fd_set readfds;
     char buf[BUFSIZ];
     char pipename[FILENAME_MAX];
@@ -352,7 +352,10 @@ int main(int argc, char **argv)
      * to trigger the version handshaking between to_erl and run_erl
      * at the start of every new to_erl-session.
      */
-    write(wfd, "\022", 1);
+
+    if (write(wfd, "\022", 1) < 0) {
+	fprintf(stderr, "Error in writing ^R to FIFO.\n");
+    }
 
     /*
      * read and write
@@ -367,7 +370,6 @@ int main(int argc, char **argv)
 	    }
 	    else {
 		fprintf(stderr, "Error in select.\n");
-		result = -1;
 		break;
 	    }
 	}
@@ -398,7 +400,6 @@ int main(int argc, char **argv)
 		close(wfd);
 		if (len < 0) {
 		    fprintf(stderr, "Error in reading from stdin.\n");
-		    result = -1;
 		} else {
 		    fprintf(stderr, "[EOF]\n\r");
 		}
@@ -414,13 +415,12 @@ int main(int argc, char **argv)
 
 	if (len) {
 #ifdef DEBUG
-	    write(1, buf, len);
+	    if(write(1, buf, len));
 #endif
 	    if (write_all(wfd, buf, len) != len) {
 		fprintf(stderr, "Error in writing to FIFO.\n");
 		close(rfd);
 		close(wfd);
-		result = -1;
 		break;
 	    }
 	    STATUS("\" OK\r\n");
@@ -447,7 +447,6 @@ int main(int argc, char **argv)
 		close(wfd);
 		if (len < 0) {
 		    fprintf(stderr, "Error in reading from FIFO.\n");
-		    result = -1;
 		} else
 		    fprintf(stderr, "[End]\n\r");
 		break;
@@ -456,7 +455,6 @@ int main(int argc, char **argv)
 		    if ((len=version_handshake(buf,len,wfd)) < 0) {
 			close(rfd);
 			close(wfd);
-			result = -1;
 			break;
 		    }
 		    if (protocol_ver >= 1) {
@@ -475,7 +473,6 @@ int main(int argc, char **argv)
 		    fprintf(stderr, "Error in writing to terminal.\n");
 		    close(rfd);
 		    close(wfd);
-		    result = -1;
 		    break;
 		}
 		STATUS("\" OK\r\n");
diff --git a/erts/etc/win32/Install.c b/erts/etc/win32/Install.c
index 4a559cd8a2..d680b67dd6 100644
--- a/erts/etc/win32/Install.c
+++ b/erts/etc/win32/Install.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2003-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2003-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /*
@@ -45,8 +45,8 @@ int main(int argc, char **argv)
     InitSection *ini_section;
     HANDLE module = GetModuleHandle(NULL);
     char *binaries[] = { "erl.exe", "werl.exe", "erlc.exe",
-			     "dialyzer.exe", "typer.exe",
-			     "escript.exe", NULL };
+			 "dialyzer.exe", "typer.exe",
+			 "escript.exe", "ct_run.exe", NULL };
     char *scripts[] = { "start_clean.boot", "start_sasl.boot", NULL };
     char fromname[MAX_PATH];
     char toname[MAX_PATH];
@@ -172,6 +172,20 @@ int main(int argc, char **argv)
 	}
     }
 
+    // Remove in R16B
+    sprintf(fromname,"%s\\%s",bin_dir,"ct_run.exe");
+    sprintf(toname,"%s\\%s",bin_dir,"run_test.exe");
+    if (GetFileAttributes(fromname) == 0xFFFFFFFF) {
+      fprintf(stderr,"Could not find file %s\n",
+	      fromname);
+      exit(1);
+    }
+    if (!CopyFile(fromname,toname,FALSE)) {
+      fprintf(stderr,"Could not copy file %s to %s\n",
+	      fromname,toname);
+      fprintf(stderr,"Continuing installation anyway...\n");
+    }
+    
     for (i = 0; scripts[i] != NULL; ++i) {
 	sprintf(fromname,"%s\\%s",release_dir,scripts[i]);
 	sprintf(toname,"%s\\%s",bin_dir,scripts[i]);
@@ -199,6 +213,9 @@ int main(int argc, char **argv)
 	fprintf(stderr,"Cannot continue installation, bailing out.\n");
 	exit(1);
     }
+
+    /* OBS!!! If the format of the init file is changed, do not forget
+       to update release_handler:write_ini_file(...) */
     ini_file = create_init_file();
     ini_section = create_init_section("erlang");
     add_init_section(ini_file,ini_section);
diff --git a/erts/etc/win32/Makefile b/erts/etc/win32/Makefile
index 400e5c5bba..4218be1eff 100644
--- a/erts/etc/win32/Makefile
+++ b/erts/etc/win32/Makefile
@@ -59,14 +59,14 @@ opt debug all clean depend:
 include $(ERL_TOP)/make/otp_release_targets.mk
 
 release_spec:
-	$(INSTALL_DIR) $(RELSYSDIR)/bin
-	$(INSTALL_DIR) $(RELSYSDIR)/src
-	$(INSTALL_DIR) $(ROOTSYSDIR)/usr/include
-	$(INSTALL_DIR) $(ROOTSYSDIR)/usr/lib
-	$(INSTALL_DIR) $(ROOTSYSDIR)/usr/lib/icons
-	$(INSTALL_PROGRAM) $(INSTALL_PROGS) $(RELSYSDIR)/bin
-	$(INSTALL_DATA) $(INSTALL_SRC) $(RELSYSDIR)/src
-	$(INSTALL_DATA) $(INSTALL_ICONS) $(ROOTSYSDIR)/usr/lib/icons
+	$(INSTALL_DIR) "$(RELSYSDIR)/bin"
+	$(INSTALL_DIR) "$(RELSYSDIR)/src"
+	$(INSTALL_DIR) "$(ROOTSYSDIR)/usr/include"
+	$(INSTALL_DIR) "$(ROOTSYSDIR)/usr/lib"
+	$(INSTALL_DIR) "$(ROOTSYSDIR)/usr/lib/icons"
+	$(INSTALL_PROGRAM) $(INSTALL_PROGS) "$(RELSYSDIR)/bin"
+	$(INSTALL_DATA) $(INSTALL_SRC) "$(RELSYSDIR)/src"
+	$(INSTALL_DATA) $(INSTALL_ICONS) "$(ROOTSYSDIR)/usr/lib/icons"
 
 release_docs release_docs_spec docs:
 
diff --git a/erts/etc/win32/cygwin_tools/vc/emu_cc.sh b/erts/etc/win32/cygwin_tools/vc/emu_cc.sh
index c74c35111b..6c179aed00 100755
--- a/erts/etc/win32/cygwin_tools/vc/emu_cc.sh
+++ b/erts/etc/win32/cygwin_tools/vc/emu_cc.sh
@@ -2,7 +2,7 @@
 # 
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 2002-2009. All Rights Reserved.
+# Copyright Ericsson AB 2002-2011. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -17,6 +17,11 @@
 # 
 # %CopyrightEnd%
 # 
+if [ X"$CONFIG_SUBTYPE" = X"win64" ]; then
+    GCC="x86_64-w64-mingw32-gcc.exe"
+else 
+    GCC="gcc"
+fi
 TOOLDIR=$ERL_TOP/erts/etc/win32/cygwin_tools/vc
 COFFIX=$TOOLDIR/coffix
 WTOOLDIR=`(cygpath -d $TOOLDIR 2>/dev/null || cygpath -w $TOOLDIR)`
@@ -71,7 +76,7 @@ if [ $SKIP_COFFIX = false ]; then
     if [ "X$EMU_CC_SH_DEBUG_LOG" != "X" ]; then
 	echo "gcc -o $TEMPFILE -D__WIN32__ -DWIN32 -DWINDOWS -fomit-frame-pointer $CMD" >> $EMU_CC_SH_DEBUG_LOG 2>&1
     fi
-    eval gcc -o $TEMPFILE -D__WIN32__ -DWIN32 -DWINDOWS -fomit-frame-pointer $CMD
+    eval $GCC -o $TEMPFILE -D__WIN32__ -DWIN32 -DWINDOWS -fomit-frame-pointer $CMD
     RES=$?
     if [ $RES = 0 ]; then
 	$COFFIX.exe -e `(cygpath -d $TEMPFILE 2>/dev/null || cygpath -w $TEMPFILE)`
@@ -85,6 +90,6 @@ if [ $SKIP_COFFIX = false ]; then
     rm -f $TEMPFILE
     exit $RES
 else
-    eval gcc -D__WIN32__ -DWIN32 -DWINDOWS -fomit-frame-pointer -fno-tree-copyrename $CMD 2>/dev/null
+    eval $GCC -D__WIN32__ -DWIN32 -DWINDOWS -fomit-frame-pointer -fno-tree-copyrename $CMD 2>/dev/null
     exit $?
 fi
diff --git a/erts/etc/win32/cygwin_tools/vc/ld.sh b/erts/etc/win32/cygwin_tools/vc/ld.sh
index ac39bf871c..406c63ffee 100755
--- a/erts/etc/win32/cygwin_tools/vc/ld.sh
+++ b/erts/etc/win32/cygwin_tools/vc/ld.sh
@@ -3,7 +3,7 @@
 # 
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 2002-2009. All Rights Reserved.
+# Copyright Ericsson AB 2002-2010. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -53,7 +53,7 @@ while test -n "$1" ; do
 	    STDLIB_FORCED=true;
 	    STDLIB=LIBCMTD.LIB;; 
 	-lsocket)
-	    DEFAULT_LIBRARIES="$DEFAULT_LIBRARIES WS2_32.LIB";;
+	    DEFAULT_LIBRARIES="$DEFAULT_LIBRARIES WS2_32.LIB IPHLPAPI.LIB";;
 	-l*)
 	    y=`echo $x | sed 's,^-l\(.*\),\1,g'`;
 	    MPATH=`cygpath -m $y`;
@@ -158,7 +158,7 @@ else
 fi    
 	    
 p=$$
-CMD="$linktype -nologo -incremental:no $CMD $STDLIB $DEFAULT_LIBRARIES"
+CMD="$linktype -nologo -incremental:no -largeaddressaware $CMD $STDLIB $DEFAULT_LIBRARIES"
 if [ "X$LD_SH_DEBUG_LOG" != "X" ]; then
     echo ld.sh "$SAVE" >>$LD_SH_DEBUG_LOG
     echo link.exe $CMD >>$LD_SH_DEBUG_LOG
@@ -167,6 +167,10 @@ eval link.exe "$CMD"  >/tmp/link.exe.${p}.1 2>/tmp/link.exe.${p}.2
 RES=$?
 CMANIFEST=`cygpath $MANIFEST`
 if [ "$RES" = "0" -a -f "$CMANIFEST" ]; then
+    # Add stuff to manifest to turn off "virtualization"
+    sed -n -i '1h;1!H;${;g;s,<trustInfo.*</trustInfo>.,,g;p;}' $CMANIFEST
+    sed -i "s/<\/assembly>/ <ms_asmv2:trustInfo xmlns:ms_asmv2=\"urn:schemas-microsoft-com:asm.v2\">\n  <ms_asmv2:security>\n   <ms_asmv2:requestedPrivileges>\n    <ms_asmv2:requestedExecutionLevel level=\"AsInvoker\" uiAccess=\"false\"\/>\n   <\/ms_asmv2:requestedPrivileges>\n  <\/ms_asmv2:security>\n <\/ms_asmv2:trustInfo>\n<\/assembly>/" $CMANIFEST
+
     eval mt.exe -nologo -manifest "$MANIFEST" -outputresource:"$OUTPUTRES" >>/tmp/link.exe.${p}.1 2>>/tmp/link.exe.${p}.2
     RES=$?
     if [ "$RES" != "0" ]; then
diff --git a/erts/etc/win32/cygwin_tools/vc/rc.sh b/erts/etc/win32/cygwin_tools/vc/rc.sh
index 6a6921c49e..054c672e64 100755
--- a/erts/etc/win32/cygwin_tools/vc/rc.sh
+++ b/erts/etc/win32/cygwin_tools/vc/rc.sh
@@ -2,20 +2,20 @@
 # set -x
 # 
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2002-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2002-2010. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
+#
 # %CopyrightEnd%
 # 
 # Save the command line for debug outputs
@@ -80,7 +80,7 @@ if [ "X$RC_SH_DEBUG_LOG" != "X" ]; then
 fi
 eval $RCC "$CMD"  >/tmp/rc.exe.${p}.1 2>/tmp/rc.exe.${p}.2
 RES=$?
-tail +2 /tmp/rc.exe.${p}.2 >&2
+tail -n +2 /tmp/rc.exe.${p}.2 >&2
 cat /tmp/rc.exe.${p}.1
 rm -f /tmp/rc.exe.${p}.2 /tmp/rc.exe.${p}.1
 exit $RES
diff --git a/erts/etc/win32/erlsrv/erlsrv_interactive.c b/erts/etc/win32/erlsrv/erlsrv_interactive.c
index 13e029b364..736eabac79 100644
--- a/erts/etc/win32/erlsrv/erlsrv_interactive.c
+++ b/erts/etc/win32/erlsrv/erlsrv_interactive.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -135,7 +135,12 @@ void print_last_error(void){
     fprintf(stderr,"Error: %s",mes);
     LocalFree(mes);
 }
-    
+
+static int get_last_error(void)
+{
+  return (last_error) ? last_error : GetLastError();
+}
+
 static BOOL install_service(void){
   SC_HANDLE scm;
   SC_HANDLE service;
@@ -508,7 +513,7 @@ int do_usage(char *arg0){
 	 "\t[{-sn[ame] | -n[ame]} [<nodename>]]\n"
 	 "\t[-d[ebugtype] [{new|reuse|console}]]\n"
 	 "\t[-ar[gs] [<limited erl arguments>]]\n\n"
-	 "%s {start | stop | disable | enable} <servicename>\n\n"
+	 "%s {start | start_disabled | stop | disable | enable} <servicename>\n\n"
 	 "%s remove <servicename>\n\n"
 	 "%s rename <servicename> <servicename>\n\n"
 	 "%s list [<servicename>]\n\n"
@@ -561,6 +566,45 @@ int do_manage(int argc,char **argv){
       return 0;
     }
   }
+  if(!_stricmp(action,"start_disabled")){
+    if(!enable_service()){
+      fprintf(stderr,"%s: Failed to enable service %s.\n",
+	      argv[0],service_name);
+      print_last_error();
+      return 1;
+    } 
+    if(!start_service() && get_last_error() != ERROR_SERVICE_ALREADY_RUNNING){
+      fprintf(stderr,"%s: Failed to start service %s.\n",
+	      argv[0],service_name);
+      print_last_error();
+      goto failure_starting;
+    }
+    
+    if(!wait_service_trans(SERVICE_STOPPED, SERVICE_START_PENDING,
+			   SERVICE_RUNNING, 60)){
+      fprintf(stderr,"%s: Failed to start service %s.\n",
+	      argv[0],service_name);
+      print_last_error();
+      goto failure_starting;
+    }
+
+    if(!disable_service()){
+      fprintf(stderr,"%s: Failed to disable service %s.\n",
+	      argv[0],service_name);
+      print_last_error();
+      return 1;
+    } 
+    printf("%s: Service %s started.\n",
+	   argv[0],service_name);
+    return 0;
+  failure_starting:
+    if(!disable_service()){
+      fprintf(stderr,"%s: Failed to disable service %s.\n",
+	      argv[0],service_name);
+      print_last_error();
+    } 
+    return 1;
+  }
   if(!_stricmp(action,"stop")){
     if(!stop_service()){
       fprintf(stderr,"%s: Failed to stop service %s.\n",
@@ -841,6 +885,7 @@ int do_add_or_set(int argc, char **argv){
 	   argv[0], service_name);
   return 0;
 }
+
 int do_rename(int argc, char **argv){
   RegEntry *current = empty_reg_tab();
   RegEntry *dummy = empty_reg_tab();
@@ -1129,35 +1174,131 @@ void read_arguments(int *pargc, char ***pargv){
     *pargc = argc;
     *pargv = argv;
 }
+
+/* Create a free-for-all ACL to set on the semaphore */
+PACL get_acl(PSECURITY_DESCRIPTOR secdescp)  
+{
+  DWORD acl_length = 0;  
+  PSID auth_users_sidp = NULL;  
+  PACL aclp = NULL;  
+  SID_IDENTIFIER_AUTHORITY ntauth = SECURITY_NT_AUTHORITY;  
+  
+  if(!InitializeSecurityDescriptor(secdescp, SECURITY_DESCRIPTOR_REVISION)) {
+    return NULL;
+  }
+  
+  if(!AllocateAndInitializeSid(&ntauth, 
+			       1, 
+			       SECURITY_AUTHENTICATED_USER_RID, 
+			       0, 0, 0, 0, 0, 0, 0, 
+			       &auth_users_sidp)) {
+    return NULL;
+  }
+  
+  acl_length =   sizeof(ACL) +  
+    sizeof(ACCESS_ALLOWED_ACE) - sizeof(DWORD) +  
+    GetLengthSid(auth_users_sidp);  
+  
+  if((aclp = (PACL) HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, acl_length)) == NULL) {
+    FreeSid(auth_users_sidp);  
+    return NULL;
+  }
+  
+  if(!InitializeAcl(aclp, acl_length, ACL_REVISION)) {
+    FreeSid(auth_users_sidp);  
+    HeapFree(GetProcessHeap(), 0, aclp);
+    return NULL;
+  }
+  
+  if(!AddAccessAllowedAce(aclp, ACL_REVISION, SEMAPHORE_ALL_ACCESS, auth_users_sidp)) {
+    FreeSid(auth_users_sidp);  
+    HeapFree(GetProcessHeap(), 0, aclp);
+    return NULL;
+  }
+  
+  if(!SetSecurityDescriptorDacl(secdescp, TRUE, aclp, FALSE)) {
+    FreeSid(auth_users_sidp);  
+    HeapFree(GetProcessHeap(), 0, aclp);
+    return NULL;
+  }
+  return aclp;  
+} 
+
+static HANDLE lock_semaphore = NULL;
+
+int take_lock(void) {
+  SECURITY_ATTRIBUTES attr;  
+  PACL aclp;  
+  SECURITY_DESCRIPTOR secdesc;  
+  
+  if ((aclp = get_acl(&secdesc)) == NULL) {
+    return -1;
+  }
+  
+  memset(&attr,0,sizeof(attr));
+  attr.nLength = sizeof(attr);  
+  attr.lpSecurityDescriptor = &secdesc;  
+  attr.bInheritHandle = FALSE;  
+  
+  if ((lock_semaphore = CreateSemaphore(&attr, 1, 1, ERLSRV_INTERACTIVE_GLOBAL_SEMAPHORE)) == NULL) {
+    return -1;
+  }
+  
+  if (WaitForSingleObject(lock_semaphore,INFINITE) != WAIT_OBJECT_0) {
+    return -1;
+  }
+  
+  HeapFree(GetProcessHeap(), 0, aclp);
+  return 0;
+}
+
+void release_lock(void) {
+  ReleaseSemaphore(lock_semaphore,1,NULL);
+}
     
 
+
 int interactive_main(int argc, char **argv){
   char *action = argv[1];
-
+  int res;
+  
+  if (take_lock() != 0) {
+    fprintf(stderr,"%s: unable to acquire global lock (%s).\n",argv[0],
+	    ERLSRV_INTERACTIVE_GLOBAL_SEMAPHORE);
+    return 1;
+  }
+  
   if(!_stricmp(action,"readargs")){
-      read_arguments(&argc,&argv);
-      action = argv[1];
+    read_arguments(&argc,&argv);
+    action = argv[1];
   }
   if(!_stricmp(action,"set") || !_stricmp(action,"add"))
-    return do_add_or_set(argc,argv);
-  if(!_stricmp(action,"rename"))
-    return do_rename(argc,argv);
-  if(!_stricmp(action,"remove"))
-    return do_remove(argc,argv);
-  if(!_stricmp(action,"list"))
-    return do_list(argc,argv);
-  if(!_stricmp(action,"start") ||
-     !_stricmp(action,"stop") ||
-     !_stricmp(action,"enable") ||
-     !_stricmp(action,"disable"))
-    return do_manage(argc,argv);
-  if(_stricmp(action,"?") &&
-     _stricmp(action,"/?") &&
-     _stricmp(action,"-?") &&
-     *action != 'h' &&
-     *action != 'H')
+    res = do_add_or_set(argc,argv);
+  else if(!_stricmp(action,"rename"))
+    res = do_rename(argc,argv);
+  else if(!_stricmp(action,"remove"))
+    res = do_remove(argc,argv);
+  else if(!_stricmp(action,"list"))
+    res = do_list(argc,argv);
+  else if(!_stricmp(action,"start") ||
+	  !_stricmp(action,"start_disabled") ||
+	  !_stricmp(action,"stop") ||
+	  !_stricmp(action,"enable") ||
+	  !_stricmp(action,"disable"))
+    res =  do_manage(argc,argv);
+  else if(_stricmp(action,"?") &&
+	  _stricmp(action,"/?") &&
+	  _stricmp(action,"-?") &&
+	  *action != 'h' &&
+	  *action != 'H') {
     fprintf(stderr,"%s: action %s not implemented.\n",argv[0],action);
-  do_usage(argv[0]);
-  return 1;
+    do_usage(argv[0]);
+    res = 1;
+  } else {
+    do_usage(argv[0]);
+    res = 0;
+  }
+  release_lock();
+  return res;
 }
 
diff --git a/erts/etc/win32/erlsrv/erlsrv_interactive.h b/erts/etc/win32/erlsrv/erlsrv_interactive.h
index deacf81899..23e69e508d 100644
--- a/erts/etc/win32/erlsrv/erlsrv_interactive.h
+++ b/erts/etc/win32/erlsrv/erlsrv_interactive.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -19,6 +19,8 @@
 #ifndef _ERLSRV_INTERACTIVE_H
 #define _ERLSRV_INTERACTIVE_H
 
+#define ERLSRV_INTERACTIVE_GLOBAL_SEMAPHORE "{468d6954-e355-415f-968f-d257cb0feef4}"
+
 int interactive_main(int argc, char **argv);
 
 #endif /* _ERLSRV_INTERACTIVE_H */
diff --git a/erts/etc/win32/erlsrv/erlsrv_service.c b/erts/etc/win32/erlsrv/erlsrv_service.c
index a58ee862c5..242e2905a9 100644
--- a/erts/etc/win32/erlsrv/erlsrv_service.c
+++ b/erts/etc/win32/erlsrv/erlsrv_service.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -523,7 +523,7 @@ static BOOL start_a_service(ServerInfo *srvi){
 		    srvi->keys[WorkDir].data.bytes : NULL,
 		    &start,
 		    &(srvi->info))){
-    sprintf(errbuff,"Could not start erlang service"
+    sprintf(errbuff,"Could not start erlang service "
 	    "with commandline \"%s\".",
 	    service_name,
 	    execbuff
@@ -924,7 +924,7 @@ static VOID WINAPI service_main_loop(DWORD argc, char **argv){
     } else {
       DWORD ecode = NO_ERROR;
       if(success_wait == NO_SUCCESS_WAIT){
-	log_warning("Erlang machine volountarily stopped. "
+	log_warning("Erlang machine voluntarily stopped. "
 		    "The service is not restarted as OnFail "
 		    "is set to ignore.");
       } else {
diff --git a/erts/etc/win32/msys_tools/erl b/erts/etc/win32/msys_tools/erl
new file mode 100644
index 0000000000..525253fd84
--- /dev/null
+++ b/erts/etc/win32/msys_tools/erl
@@ -0,0 +1,42 @@
+#! /bin/sh
+# 
+# %CopyrightBegin%
+# 
+# Copyright Ericsson AB 2002-2011. All Rights Reserved.
+# 
+# The contents of this file are subject to the Erlang Public License,
+# Version 1.1, (the "License"); you may not use this file except in
+# compliance with the License. You should have received a copy of the
+# Erlang Public License along with this software. If not, it can be
+# retrieved online at http://www.erlang.org/.
+# 
+# Software distributed under the License is distributed on an "AS IS"
+# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+# the License for the specific language governing rights and limitations
+# under the License.
+# 
+# %CopyrightEnd%
+# 
+# Note! This shellscript expects to be run in a cygwin environment,
+# it converts erlc command lines to native windows erlc commands, which
+# basically means running the command cygpath on whatever is a path...
+
+CMD=""
+for x in "$@"; do 
+    case "$x" in
+	-I/*|-o/*)
+	    y=`echo $x | sed 's,^-[Io]\(/.*\),\1,g'`;
+	    z=`echo $x | sed 's,^-\([Io]\)\(/.*\),\1,g'`;
+	    MPATH=`msys2win_path.sh -m $y`;
+	    CMD="$CMD -$z\"$MPATH\"";; 
+	/*)
+	    MPATH=`msys2win_path.sh -m $x`;
+	    CMD="$CMD \"$MPATH\"";; 
+	*)
+	    y=`echo $x | sed 's,",\\\",g'`;
+	    CMD="$CMD \"$y\"";;
+    esac
+done
+ERL_TOP=`msys2win_path.sh -m $ERL_TOP`
+export ERL_TOP
+eval erl.exe $CMD
diff --git a/erts/etc/win32/msys_tools/erlc b/erts/etc/win32/msys_tools/erlc
new file mode 100644
index 0000000000..3f53ef7f4f
--- /dev/null
+++ b/erts/etc/win32/msys_tools/erlc
@@ -0,0 +1,59 @@
+#! /bin/sh
+# 
+# %CopyrightBegin%
+# 
+# Copyright Ericsson AB 2002-2011. All Rights Reserved.
+# 
+# The contents of this file are subject to the Erlang Public License,
+# Version 1.1, (the "License"); you may not use this file except in
+# compliance with the License. You should have received a copy of the
+# Erlang Public License along with this software. If not, it can be
+# retrieved online at http://www.erlang.org/.
+# 
+# Software distributed under the License is distributed on an "AS IS"
+# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+# the License for the specific language governing rights and limitations
+# under the License.
+# 
+# %CopyrightEnd%
+# 
+
+
+CMD=""
+ECHO_ONLY=false
+for x in "$@"; do 
+    case "$x" in
+	--echo_only)
+	    ECHO_ONLY=true;;
+	-I/*|-o/*)
+	    y=`echo $x | sed 's,^-[Io]\(/.*\),\1,g'`;
+	    z=`echo $x | sed 's,^-\([Io]\)\(/.*\),\1,g'`;
+	    MPATH=`msys2win_path.sh -m $y`;
+	    CMD="$CMD -$z$MPATH";;
+	-pa/*)
+	    y=`echo $x | sed 's,^-pa\(/.*\),\1,g'`;
+	    MPATH=`msys2win_path.sh -m $y`;
+	    CMD="$CMD -pa $MPATH";;
+	/*)
+	    MPATH=`msys2win_path.sh -m $x`;
+	    CMD="$CMD \"$MPATH\"";; 
+# Needed for  +'{preproc_flags,whatever}'
+	+{preproc_flags,*})
+	     y=`echo $x | sed 's,^+{preproc_flags\,"\(.*\)"},\1,g'`;
+	     z=`eval $0 --echo_only $y`;
+	     case "$z" in # Dont "doubledoublequote"
+		 \"*\") 
+		 CMD="$CMD +'{preproc_flags,$z}'";;
+		 *)
+		 CMD="$CMD +'{preproc_flags,\"$z\"}'";;
+	     esac;;
+	*)
+	    y=`echo $x | sed 's,",\\\",g'`;
+	    CMD="$CMD \"$y\"";;
+    esac
+done
+if [ $ECHO_ONLY = true ]; then
+    echo $CMD
+else
+    eval erlc.exe $CMD
+fi
diff --git a/erts/etc/win32/msys_tools/javac.sh b/erts/etc/win32/msys_tools/javac.sh
new file mode 100644
index 0000000000..2d884bc2c8
--- /dev/null
+++ b/erts/etc/win32/msys_tools/javac.sh
@@ -0,0 +1,65 @@
+#! /bin/sh
+# 
+# %CopyrightBegin%
+# 
+# Copyright Ericsson AB 2002-2011. All Rights Reserved.
+# 
+# The contents of this file are subject to the Erlang Public License,
+# Version 1.1, (the "License"); you may not use this file except in
+# compliance with the License. You should have received a copy of the
+# Erlang Public License along with this software. If not, it can be
+# retrieved online at http://www.erlang.org/.
+# 
+# Software distributed under the License is distributed on an "AS IS"
+# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+# the License for the specific language governing rights and limitations
+# under the License.
+# 
+# %CopyrightEnd%
+# 
+# Note! This shellscript expects to be run in a cygwin environment,
+# it converts erlc command lines to native windows erlc commands, which
+# basically means running the command cygpath on whatever is a path...
+
+CMD=""
+save_IFS=$IFS
+IFS=":"
+NEWCLASSPATH=""
+for x in $CLASSPATH; do
+  TMP=`msys2win_path.sh -m $x`
+  if [ -z "$NEWCLASSPATH" ]; then
+      NEWCLASSPATH="$TMP"
+  else
+      NEWCLASSPATH="$NEWCLASSPATH;$TMP"
+  fi
+done
+IFS=$save_IFS
+CLASSPATH="$NEWCLASSPATH"
+export CLASSPATH
+#echo "CLASSPATH=$CLASSPATH"
+SAVE="$@"
+while test -n "$1" ; do
+    x="$1"
+    case "$x" in
+	-I/*|-o/*|-d/*)
+	    y=`echo $x | sed 's,^-[Iod]\(/.*\),\1,g'`;
+	    z=`echo $x | sed 's,^-\([Iod]\)\(/.*\),\1,g'`;
+	    #echo "Foooo:$z"
+	    MPATH=`msys2win_path.sh -m $y`;
+	    CMD="$CMD -$z\"$MPATH\"";; 
+	-d|-I|-o)
+	    shift;
+	    MPATH=`msys2win_path.sh -m $1`;
+	    CMD="$CMD $x $MPATH";; 
+	/*)
+	    #echo "absolute:"$x;
+	    MPATH=`msys2win_path.sh -m $x`;
+	    CMD="$CMD \"$MPATH\"";; 
+	*)
+	    y=`echo $x | sed 's,",\\\",g'`;
+	    CMD="$CMD \"$y\"";;
+    esac
+    shift
+done
+#echo javac.exe "$CMD"
+eval javac.exe "$CMD"
diff --git a/erts/etc/win32/msys_tools/make_bootstrap_ini.sh b/erts/etc/win32/msys_tools/make_bootstrap_ini.sh
new file mode 100644
index 0000000000..b61965f546
--- /dev/null
+++ b/erts/etc/win32/msys_tools/make_bootstrap_ini.sh
@@ -0,0 +1,44 @@
+#! /bin/bash
+# 
+# %CopyrightBegin%
+# 
+# Copyright Ericsson AB 2003-2011. All Rights Reserved.
+# 
+# The contents of this file are subject to the Erlang Public License,
+# Version 1.1, (the "License"); you may not use this file except in
+# compliance with the License. You should have received a copy of the
+# Erlang Public License along with this software. If not, it can be
+# retrieved online at http://www.erlang.org/.
+# 
+# Software distributed under the License is distributed on an "AS IS"
+# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+# the License for the specific language governing rights and limitations
+# under the License.
+# 
+# %CopyrightEnd%
+# 
+# Create a local init-file for erlang in the build environment.
+if [ -z "$1" ]; then
+	echo "error: $0: No rootdir given"
+ 	exit 1
+else
+    RDIR=$1
+fi
+if [ -z "$2" ]; then
+	echo "error: $0: No bindir given"
+ 	exit 1
+else
+    BDIR=$2
+fi
+
+DRDIR=`msys2win_path.sh $RDIR | sed 's,\\\,\\\\\\\\,g'`
+DBDIR=`msys2win_path.sh $BDIR | sed 's,\\\,\\\\\\\\,g'`
+
+
+cat > $RDIR/bin/erl.ini <<EOF
+[erlang]
+Bindir=$DBDIR
+Progname=erl
+Rootdir=$DRDIR
+EOF
+
diff --git a/erts/etc/win32/msys_tools/make_local_ini.sh b/erts/etc/win32/msys_tools/make_local_ini.sh
new file mode 100644
index 0000000000..6c5d84c4f5
--- /dev/null
+++ b/erts/etc/win32/msys_tools/make_local_ini.sh
@@ -0,0 +1,41 @@
+#! /bin/bash
+# 
+# %CopyrightBegin%
+# 
+# Copyright Ericsson AB 2003-2011. All Rights Reserved.
+# 
+# The contents of this file are subject to the Erlang Public License,
+# Version 1.1, (the "License"); you may not use this file except in
+# compliance with the License. You should have received a copy of the
+# Erlang Public License along with this software. If not, it can be
+# retrieved online at http://www.erlang.org/.
+# 
+# Software distributed under the License is distributed on an "AS IS"
+# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+# the License for the specific language governing rights and limitations
+# under the License.
+# 
+# %CopyrightEnd%
+# 
+# Create a local init-file for erlang in the build environment.
+if [ -z "$1" ]; then
+    if [ -z $ERL_TOP ]; then
+	echo "error: $0: No rootdir available"
+ 	exit 1
+    else
+	RDIR=$ERL_TOP
+    fi
+else
+    RDIR=$1
+fi
+
+DDIR=`msys2win_path.sh $RDIR | sed 's,\\\,\\\\\\\\,g'`
+
+
+cat > $RDIR/bin/erl.ini <<EOF
+[erlang]
+Bindir=$DDIR\\\\bin\\\\win32
+Progname=erl
+Rootdir=$DDIR
+EOF
+
diff --git a/erts/etc/win32/msys_tools/msys2win_path.sh b/erts/etc/win32/msys_tools/msys2win_path.sh
new file mode 100644
index 0000000000..679a9d6a8f
--- /dev/null
+++ b/erts/etc/win32/msys_tools/msys2win_path.sh
@@ -0,0 +1,44 @@
+#! /bin/bash

+MIXED=false

+ABSOLUTE=false

+done=false

+while [ $done = false ]; do

+    case "$1" in

+	-m) 

+	    MIXED=true;

+	    shift;;

+	-a)

+	    ABSOLUTE=true;

+	    shift;;

+	*)

+	    done=true;;

+    esac

+done

+if [ -z "$1" ]; then

+    echo "Usage: $0 <path>" >&2

+    exit 1;

+fi

+MSYS_PATH=`win2msys_path.sh "$1"` # Clean up spaces

+if [ $ABSOLUTE = true ]; then

+    MSYS_DIR=`dirname "$MSYS_PATH"`

+    MSYS_FILE=`basename "$MSYS_PATH"`

+    if [ X"$MSYS_FILE" = X".." ]; then

+	MSYS_DIR="$MSYS_DIR/$MSYS_FILE"

+	WIN_ADD=""

+    else

+	WIN_ADD="/$MSYS_FILE"

+    fi

+    SAVEDIR=`pwd`

+    cd $MSYS_DIR

+    CURRENT=`pwd`

+    cd $SAVEDIR

+    WINPATH=`cmd //C echo $CURRENT`

+    WINPATH="$WINPATH$WIN_ADD"

+else

+    WINPATH=`cmd //c echo $MSYS_PATH`

+fi

+if [ $MIXED = true ]; then

+    echo $WINPATH

+else

+    echo $WINPATH | sed 's,/,\\,g'

+fi
\ No newline at end of file
diff --git a/erts/etc/win32/msys_tools/reg_query.sh b/erts/etc/win32/msys_tools/reg_query.sh
new file mode 100644
index 0000000000..ae6d5c3218
--- /dev/null
+++ b/erts/etc/win32/msys_tools/reg_query.sh
@@ -0,0 +1,24 @@
+#! /bin/sh

+BAT_FILE=/tmp/w$$.bat

+if [ -z "$1" -o -z "$2" ]; then

+    echo "Usage:" "$0" '<key> <valuename>'

+    exit 1

+fi

+BACKED=`echo "$1" | sed 's,/,\\\\,g'`

+# We need to get the 64bit part of the registry, hence we need to execute 

+# a 64bit reg.exe, but c:\windows\system32 is redirected to 32bit versions

+# if we ate in the 32bit virtual environment, why we need to use the 

+# sysnative trick to get to the 64bit executable of reg.exe (ouch!)

+if [ -d $WINDIR/sysnative ]; then

+    REG_CMD="$WINDIR\\sysnative\\reg.exe"

+else

+    REG_CMD="reg"

+fi

+cat > $BAT_FILE <<EOF

+@echo off

+$REG_CMD query "$BACKED" /v "$2"

+EOF

+#echo $BAT_FILE

+#cat $BAT_FILE

+RESULT=`cmd //C $BAT_FILE`

+echo $RESULT | sed "s,.*$2 REG_[^ ]* ,,"

diff --git a/erts/etc/win32/msys_tools/vc/ar.sh b/erts/etc/win32/msys_tools/vc/ar.sh
new file mode 100644
index 0000000000..f4c61e1d92
--- /dev/null
+++ b/erts/etc/win32/msys_tools/vc/ar.sh
@@ -0,0 +1,47 @@
+#! /bin/sh
+# 
+# %CopyrightBegin%
+# 
+# Copyright Ericsson AB 2002-2011. All Rights Reserved.
+# 
+# The contents of this file are subject to the Erlang Public License,
+# Version 1.1, (the "License"); you may not use this file except in
+# compliance with the License. You should have received a copy of the
+# Erlang Public License along with this software. If not, it can be
+# retrieved online at http://www.erlang.org/.
+# 
+# Software distributed under the License is distributed on an "AS IS"
+# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+# the License for the specific language governing rights and limitations
+# under the License.
+# 
+# %CopyrightEnd%
+# 
+CMD=""
+while test -n "$1" ; do
+    x="$1"
+    case "$x" in
+	-out:)
+	    shift
+	    case "$1" in
+		/*)
+		    MPATH=`msys2win_path.sh -m $1`;;
+		 *)
+		    MPATH=$1;;
+	    esac
+	    CMD="$CMD -out:\"$MPATH\"";; 
+	-out:/*)
+	    y=`echo $x | sed 's,^-out:\(/.*\),\1,g'`;
+	    MPATH=`msys2win_path.sh -m $y`;
+	    CMD="$CMD -out:\"$MPATH\"";; 
+	/*)
+	    MPATH=`msys2win_path.sh -m $x`;
+	    CMD="$CMD \"$MPATH\"";; 
+	*)
+	    y=`echo $x | sed 's,",\\\",g'`;
+	    CMD="$CMD \"$y\"";;
+    esac
+    shift
+done
+
+eval lib.exe $CMD
diff --git a/erts/etc/win32/msys_tools/vc/cc.sh b/erts/etc/win32/msys_tools/vc/cc.sh
new file mode 100644
index 0000000000..38b3d2ee81
--- /dev/null
+++ b/erts/etc/win32/msys_tools/vc/cc.sh
@@ -0,0 +1,320 @@
+#! /bin/sh
+# 
+# %CopyrightBegin%
+# 
+# Copyright Ericsson AB 2002-2011. All Rights Reserved.
+# 
+# The contents of this file are subject to the Erlang Public License,
+# Version 1.1, (the "License"); you may not use this file except in
+# compliance with the License. You should have received a copy of the
+# Erlang Public License along with this software. If not, it can be
+# retrieved online at http://www.erlang.org/.
+# 
+# Software distributed under the License is distributed on an "AS IS"
+# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+# the License for the specific language governing rights and limitations
+# under the License.
+# 
+# %CopyrightEnd%
+# 
+# Icky cl wrapper that does it's best to behave like a Unixish cc.
+# Made to work for Erlang builds and to make configure happy, not really 
+# general I suspect.
+# set -x
+# Save the command line for debug outputs
+
+SAVE="$@"
+
+# Constants
+COMMON_CFLAGS="-nologo -D__WIN32__ -DWIN32 -DWINDOWS -D_WIN32 -DNT -D_CRT_SECURE_NO_DEPRECATE"
+
+# Variables
+# The stdout and stderr for the compiler
+MSG_FILE=/tmp/cl.exe.$$.1
+ERR_FILE=/tmp/cl.exe.$$.2
+
+# "Booleans" determined during "command line parsing"
+# If the stdlib option is explicitly passed to this program
+MD_FORCED=false
+# If we're preprocession (only) i.e. -E
+PREPROCESSING=false
+# If we're generating dependencies (implies preprocesing)
+DEPENDENCIES=false
+# If this is supposed to be a debug build
+DEBUG_BUILD=false
+# If this is supposed to be an optimized build (there can only be one...)
+OPTIMIZED_BUILD=false
+# If we're linking or only compiling
+LINKING=true
+
+# This data is accumulated during command line "parsing"
+# The stdlibrary option, default multithreaded dynamic
+MD=-MD
+# Flags for debug compilation
+DEBUG_FLAGS=""
+# Flags for optimization
+OPTIMIZE_FLAGS=""
+# The specified output filename (if any), may be either object or exe.
+OUTFILE=""
+# Unspecified command line options for the compiler
+CMD=""
+# All the c source files, in unix style
+SOURCES=""
+# All the options to pass to the linker, kept in Unix style
+LINKCMD=""
+
+
+# Loop through the parameters and set the above variables accordingly
+# Also convert some cygwin filenames to "mixed style" dito (understood by the
+# compiler very well), except for anything passed to the linker, that script
+# handles those and the sources, which are also kept unixish for now
+
+while test -n "$1" ; do
+    x="$1"
+    case "$x" in
+	-Wall)
+	    ;;
+	-c) 
+	    LINKING=false;;
+	    #CMD="$CMD -c";;
+	-MM)
+	    PREPROCESSING=true;
+	    LINKING=false;
+	    DEPENDENCIES=true;;
+	-E)
+	    PREPROCESSING=true;
+	    LINKING=false;; # Obviously...
+	    #CMD="$CMD -E";;
+	-Owx)
+	    # Optimization hardcoded of wxErlang, needs to disable debugging too
+	    OPTIMIZE_FLAGS="-Ob2ity -Gs -Zi";
+	    DEBUG_FLAGS="";
+	    DEBUG_BUILD=false;
+	    if [ $MD_FORCED = false ]; then
+		MD=-MD;
+	    fi
+	    OPTIMIZED_BUILD=true;;
+	-O*)
+	    # Optimization hardcoded, needs to disable debugging too
+	    OPTIMIZE_FLAGS="-Ox -Zi";
+	    DEBUG_FLAGS="";
+	    DEBUG_BUILD=false;
+	    if [ $MD_FORCED = false ]; then
+		MD=-MD;
+	    fi
+	    OPTIMIZED_BUILD=true;;
+	-g|-ggdb)
+	    if [ $OPTIMIZED_BUILD = false ];then
+		# Hardcoded windows debug flags
+		DEBUG_FLAGS="-Z7";
+		if [ $MD_FORCED = false ]; then
+		    MD=-MDd;
+		fi
+		LINKCMD="$LINKCMD -g";
+		DEBUG_BUILD=true;
+	    fi;;
+	# Allow forcing of stdlib
+	-mt|-MT)
+	    MD="-MT";
+	    MD_FORCED=true;;
+	-md|-MD)
+	    MD="-MD";
+	    MD_FORCED=true;;
+	-ml|-ML)
+	    MD="-ML";
+	    MD_FORCED=true;;
+	-mdd|-MDD|-MDd)
+	    MD="-MDd";
+	    MD_FORCED=true;;
+	-mtd|-MTD|-MTd)
+	    MD="-MTd";
+	    MD_FORCED=true;;
+	-mld|-MLD|-MLd)
+	    MD="-MLd";
+	    MD_FORCED=true;;
+	-o)
+	    shift;
+	    OUTFILE="$1";;
+	-o*)
+	    y=`echo $x | sed 's,^-[Io]\(.*\),\1,g'`;
+	    OUTFILE="$y";;
+	-I/*)
+	    y=`echo $x | sed 's,^-[Io]\(/.*\),\1,g'`;
+	    z=`echo $x | sed 's,^-\([Io]\)\(/.*\),\1,g'`;
+	    MPATH=`echo $y`;
+	    CMD="$CMD -$z\"$MPATH\"";; 
+	-I*)
+	    y=`echo $x | sed 's,",\\\",g'`;
+	    CMD="$CMD $y";;
+	-D*)
+	    y=`echo $x | sed 's,",\\\",g'`;
+	    CMD="$CMD $y";;
+	-EH*)
+	    y=`echo $x | sed 's,",\\\",g'`;
+	    CMD="$CMD $y";;
+	-TP|-Tp)
+	    y=`echo $x | sed 's,",\\\",g'`;
+	    CMD="$CMD $y";;
+	-l*)
+	    y=`echo $x | sed 's,^-l\(.*\),\1,g'`;
+	    LINKCMD="$LINKCMD $x";;
+	/*.c)
+	    SOURCES="$SOURCES $x";;
+	*.c)
+	    SOURCES="$SOURCES $x";;
+	/*.cc)
+	    SOURCES="$SOURCES $x";;
+	*.cc)
+	    SOURCES="$SOURCES $x";;
+	/*.cpp)
+	    SOURCES="$SOURCES $x";;
+	*.cpp)
+	    SOURCES="$SOURCES $x";;
+	/*.o)
+	    LINKCMD="$LINKCMD $x";;
+	*.o)
+	    LINKCMD="$LINKCMD $x";;
+	*)
+	    # Try to quote uninterpreted options
+	    y=`echo $x | sed 's,",\\\",g'`;
+	    LINKCMD="$LINKCMD $y";;
+    esac
+    shift
+done
+
+#Return code from compiler, linker.sh and finally this script...
+RES=0
+
+# Accumulated object names
+ACCUM_OBJECTS=""
+
+# A temporary object file location
+TMPOBJDIR=/tmp/tmpobj$$
+mkdir $TMPOBJDIR
+
+# Compile
+for x in $SOURCES; do
+    start_time=`date '+%s'` 
+    # Compile each source
+    if [ $LINKING = false ]; then
+	# We should have an output defined, which is a directory 
+	# or an object file
+	case $OUTFILE in
+	    /*.o)
+		# Simple output, SOURCES should be one single
+		n=`echo $SOURCES | wc -w`;
+		if [ $n -gt 1 ]; then
+		    echo "cc.sh:Error, multiple sources, one object output.";
+		    exit 1;
+		else
+		    output_filename=`echo $OUTFILE`;
+		fi;;
+	    *.o)
+		# Relative path needs no translation
+		n=`echo $SOURCES | wc -w`
+		if [ $n -gt 1 ]; then
+		    echo "cc.sh:Error, multiple sources, one object output."
+		    exit 1
+		else
+		    output_filename=$OUTFILE
+		fi;;
+	    /*)
+		# Absolute directory
+		o=`echo $x | sed 's,.*/,,' | sed 's,\.c$,.o,'`
+		output_filename=`echo $OUTFILE`
+		output_filename="$output_filename/${o}";;
+	    *)
+		# Relative_directory or empty string (.//x.o is valid)
+		o=`echo $x | sed 's,.*/,,' | sed 's,\.cp*$,.o,'`
+		output_filename="./${OUTFILE}/${o}";;
+	esac
+    else
+	# We are linking, which means we build objects in a temporary 
+	# directory and link from there. We should retain the basename
+	# of each source to make examining the exe easier...
+	o=`echo $x | sed 's,.*/,,' | sed 's,\.c$,.o,'`
+	output_filename=$TMPOBJDIR/$o
+	ACCUM_OBJECTS="$ACCUM_OBJECTS $output_filename"
+    fi
+    # Now we know enough, lets try a compilation...
+    MPATH=`echo $x`
+    if [ $PREPROCESSING = true ]; then
+	output_flag="-E"
+    else
+	output_flag="-c -Fo`cmd //C echo ${output_filename}`"
+    fi
+    params="$COMMON_CFLAGS $MD $DEBUG_FLAGS $OPTIMIZE_FLAGS \
+	    $CMD ${output_flag} $MPATH"
+    if [ "X$CC_SH_DEBUG_LOG" != "X" ]; then
+	echo cc.sh "$SAVE" >>$CC_SH_DEBUG_LOG
+	echo cl.exe $params >>$CC_SH_DEBUG_LOG
+    fi
+    eval cl.exe $params >$MSG_FILE 2>$ERR_FILE
+    RES=$?
+    if test $PREPROCESSING = false; then
+	cat $ERR_FILE >&2
+	tail -n +2 $MSG_FILE
+    else
+	tail -n +2 $ERR_FILE >&2
+	if test $DEPENDENCIES = true; then
+	    if test `grep -v $x $MSG_FILE | grep -c '#line'` != "0"; then
+		o=`echo $x | sed 's,.*/,,' | sed 's,\.cp*$,.o,'`
+		echo -n $o':'
+#		cat $MSG_FILE | grep '#line' | grep -v $x | awk -F\" '{printf("%s\n",$2)}' | sort -u | grep -v " " | xargs -n 1 win2msys_path.sh | awk '{printf("\\\n %s ",$0)}' 
+		cat $MSG_FILE | grep '#line' | grep -v $x | awk -F\" '{printf("%s\n",$2)}' | sort -u | grep -v " " | sed 's,^\([A-Za-z]\):[\\/]*,/\1/,;s,\\\\*,/,g'| awk '{printf("\\\n %s ",$0)}' 
+		echo
+		echo 
+		after_sed=`date '+%s'`
+		echo Made dependencises for $x':' `expr $after_sed '-' $start_time` 's' >&2
+	    fi 
+	else
+	    cat $MSG_FILE
+	fi
+    fi
+    rm -f $ERR_FILE $MSG_FILE
+    if [ $RES != 0 ]; then
+	rm -rf $TMPOBJDIR
+	exit $RES
+    fi
+done
+
+# If we got here, we succeeded in compiling (if there were anything to compile)
+# The output filename should name an executable if we're linking
+if [ $LINKING = true ]; then
+    case $OUTFILE in
+	"")
+	    # Use the first source name to name the executable
+	    first_source=""
+	    for x in $SOURCES; do first_source=$x; break; done;
+	    if [ -n "$first_source" ]; then
+		e=`echo $x | sed 's,.*/,,' | sed 's,\.c$,.exe,'`;
+		out_spec="-o $e";
+	    else
+		out_spec="";
+	    fi;;
+	*)
+	    out_spec="-o $OUTFILE";;
+    esac
+    # Descide which standard library to link against
+    case $MD in
+	-ML)
+	    stdlib="-lLIBC";;
+	-MLd)
+	    stdlib="-lLIBCD";;
+	-MD)
+	    stdlib="-lMSVCRT";;
+	-MDd)
+	    stdlib="-lMSVCRTD";;
+	-MT)
+	    stdlib="-lLIBCMT";;
+	-MTd)
+	    stdlib="-lLIBMTD";;
+    esac
+    # And finally call the next script to do the linking...
+    params="$out_spec $LINKCMD $stdlib" 
+    eval ld.sh $ACCUM_OBJECTS $params
+    RES=$?
+fi
+rm -rf $TMPOBJDIR
+
+exit $RES
diff --git a/erts/etc/win32/msys_tools/vc/coffix.c b/erts/etc/win32/msys_tools/vc/coffix.c
new file mode 100644
index 0000000000..1773b222fe
--- /dev/null
+++ b/erts/etc/win32/msys_tools/vc/coffix.c
@@ -0,0 +1,161 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 1999-2011. All Rights Reserved.
+ * 
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ * 
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ * 
+ * %CopyrightEnd%
+ */
+/*
+** This mini tool fixes an incompatibility between 
+** Microsoft's tools, who dont like the virtual size being put in
+** the physical address field, but rely on the raw size field for
+** sizing the ".bss" section.
+** This fixes some of the problems with linking gcc compiled objects
+** together with MSVC dito.
+**
+** Courtesy DJ Delorie for describing the COFF file format on 
+** http://www.delorie.com/djgpp/doc/coff/
+** The coff structures are fetched from Microsofts headers though.
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdarg.h>
+
+#include <windows.h>
+#include <winnt.h> /* Structure definitions for PE (COFF) */
+
+static int dump_edit(char *filename, int edit);
+static int v_printf(char *format, ...);
+
+
+char *progname;
+int verbouse = 0;
+
+int main(int argc, char **argv) 
+{
+    int findex = 1;
+    int edit = 0;
+    int ret;
+
+    progname = argv[0];
+    if (argc == 1) {
+	fprintf(stderr,"Format : %s [-e] [-v] <filename>\n", progname);
+	return 1;
+    }
+    for (findex = 1; 
+	 findex < argc && (*argv[findex] == '-' || *argv[findex] == '/');
+	 ++findex)
+	switch (argv[findex][1]) {
+	case 'e':
+	case 'E':
+	    edit = 1;
+	    break;
+	case 'v':
+	case 'V':
+	    verbouse = 1;
+	default:
+	    fprintf(stderr, "%s: unknown option %s\n", progname, argv[findex]);
+	    break;
+	}
+    if (findex == argc) {
+	fprintf(stderr,"%s: No filenames given.\n", progname);
+	return 1;
+    }
+    for(; findex < argc; ++findex)
+	if ((ret = dump_edit(argv[findex],edit)) != 0)
+	    return ret;
+    return 0;
+}
+
+int dump_edit(char *filename, int edit) 
+{
+    FILE *f = fopen(filename, (edit) ? "r+b" : "rb");
+    IMAGE_FILE_HEADER filhdr;
+    IMAGE_SECTION_HEADER scnhdr;
+    int i;
+
+    if (f == NULL) {
+	fprintf(stderr, "%s: cannot open %s.\n", progname, filename);
+	return 1;
+    }
+    
+    if (fread(&filhdr, sizeof(filhdr), 1, f) == 0) {
+	fprintf(stderr,"%s: Could not read COFF header from %s,"
+		" is this a PE (COFF) file?\n", progname, filename);
+	fclose(f);
+	return 1;
+    }
+    v_printf("File: %s\n", filename);
+    v_printf("Magic number: 0x%08x\n", filhdr.Machine);
+    v_printf("Number of sections: %d\n",filhdr.NumberOfSections);
+    
+    if (fseek(f, (long) filhdr.SizeOfOptionalHeader, SEEK_CUR) != 0) {
+	fprintf(stderr,"%s: Could not read COFF optional header from %s,"
+		" is this a PE (COFF) file?\n", progname, filename);
+	fclose(f);
+	return 1;
+    }
+    
+    for (i = 0; i < filhdr.NumberOfSections; ++i) {
+	if (fread(&scnhdr, sizeof(scnhdr), 1, f) == 0) {
+	    fprintf(stderr,"%s: Could not read section header from %s,"
+		    " is this a PE (COFF) file?\n", progname, filename);
+	    fclose(f);
+	    return 1;
+	}
+	v_printf("Section %s:\n", scnhdr.Name);
+	v_printf("Physical address: 0x%08x\n", scnhdr.Misc.PhysicalAddress);
+	v_printf("Size: 0x%08x\n", scnhdr.SizeOfRawData);
+	if (scnhdr.Misc.PhysicalAddress != 0 &&
+	    scnhdr.SizeOfRawData == 0) {
+	    printf("Section header %s in file %s will confuse MSC linker, "
+		   "virtual size is 0x%08x and raw size is 0\n",
+		   scnhdr.Name, filename, scnhdr.Misc.PhysicalAddress, 
+		   scnhdr.SizeOfRawData);
+	    if (edit) {
+		scnhdr.SizeOfRawData = scnhdr.Misc.PhysicalAddress;
+		scnhdr.Misc.PhysicalAddress = 0;
+		if (fseek(f, (long) -((long)sizeof(scnhdr)), SEEK_CUR) != 0 ||
+		    fwrite(&scnhdr, sizeof(scnhdr), 1, f) == 0) {
+		    fprintf(stderr,"%s: could not edit file %s.\n",
+			    progname, filename);
+		    fclose(f);
+		    return 1;
+		}
+		printf("Edited object, virtual size is now 0, and "
+		       "raw size is 0x%08x.\n", scnhdr.SizeOfRawData);
+	    } else {
+		printf("Specify option '-e' to correct the problem.\n");
+	    }
+	}
+    }
+    fclose(f);
+    return 0;
+}
+	    
+
+static int v_printf(char *format, ...)
+{
+    va_list ap;
+    int ret = 0;
+    if (verbouse) {
+	va_start(ap, format);
+	ret = vfprintf(stdout, format, ap);
+	va_end(ap);
+    }
+    return ret;
+}
+
diff --git a/erts/etc/win32/msys_tools/vc/emu_cc.sh b/erts/etc/win32/msys_tools/vc/emu_cc.sh
new file mode 100644
index 0000000000..68ce4e359f
--- /dev/null
+++ b/erts/etc/win32/msys_tools/vc/emu_cc.sh
@@ -0,0 +1,95 @@
+#! /bin/sh
+# 
+# %CopyrightBegin%
+# 
+# Copyright Ericsson AB 2002-2011. All Rights Reserved.
+# 
+# The contents of this file are subject to the Erlang Public License,
+# Version 1.1, (the "License"); you may not use this file except in
+# compliance with the License. You should have received a copy of the
+# Erlang Public License along with this software. If not, it can be
+# retrieved online at http://www.erlang.org/.
+# 
+# Software distributed under the License is distributed on an "AS IS"
+# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+# the License for the specific language governing rights and limitations
+# under the License.
+# 
+# %CopyrightEnd%
+# 
+if [ X"$CONFIG_SUBTYPE" = X"win64" ]; then
+    GCC="x86_64-w64-mingw32-gcc.exe"
+else 
+    GCC="gcc"
+fi
+TOOLDIR=$ERL_TOP/erts/etc/win32/msys_tools/vc
+COFFIX=$TOOLDIR/coffix
+WTOOLDIR0=`win2msys_path.sh "$TOOLDIR"`
+WTOOLDIR=`cmd //C echo $WTOOLDIR0`
+# Do primitive 'make'
+newer_exe=`find $TOOLDIR -newer $COFFIX.c -name coffix.exe -print`
+if [ -z $newer_exe ]; then
+    echo recompiling $COFFIX.exe
+    cl.exe -Fe${WTOOLDIR}/coffix.exe ${WTOOLDIR}/coffix.c
+    rm -f $COFFIX.obj coffix.obj $COFFIX.pdb coffix.pdb
+fi
+
+# Try to find out the output filename and remove it from command line
+CMD=""
+OUTFILE=""
+INFILE=""
+SKIP_COFFIX=false
+while test -n "$1" ; do
+    x="$1"
+    case "$x" in
+	-o/*)
+	    OUTFILE=`echo $x | sed 's,^-[Io]\(/.*\),\1,g'`;;
+	-o)
+	    shift
+	    OUTFILE=$1;;
+	-MM)
+	    SKIP_COFFIX=true
+	    CMD="$CMD \"$x\"";;
+	*.c)
+	    INFILE="$INFILE $x";
+	    CMD="$CMD \"$x\"";;
+	*)
+	    CMD="$CMD \"$x\"";;
+    esac
+    shift
+done
+if [ -z "$INFILE" ]; then
+    echo 'emu_cc.sh: please give an input filename for the compiler' >&2
+    exit 1
+fi    
+if [ -z "$OUTFILE" ]; then
+    OUTFILE=`echo $INFILE | sed 's,\.c$,.o,'`
+fi
+
+if [ $SKIP_COFFIX = false ]; then
+    n=`echo $INFILE | wc -w`;
+    if [ $n -gt 1 ]; then
+	echo "emu_cc.sh:Error, multiple sources, one object output.";
+	exit 1;
+    fi
+    TEMPFILE=/tmp/tmp_emu_cc$$.o
+    if [ "X$EMU_CC_SH_DEBUG_LOG" != "X" ]; then
+	echo "$GCC -o $TEMPFILE -D__WIN32__ -DWIN32 -DWINDOWS -fomit-frame-pointer $CMD" >> $EMU_CC_SH_DEBUG_LOG 2>&1
+    fi
+    eval $GCC -o $TEMPFILE -D__WIN32__ -DWIN32 -DWINDOWS -fomit-frame-pointer $CMD
+    RES=$?
+    if [ $RES = 0 ]; then
+	$COFFIX.exe -e `win2msys_path.sh $TEMPFILE`
+	RES=$?
+	if [ $RES = 0 ]; then
+	    cp $TEMPFILE $OUTFILE
+	else
+	    echo "emu_cc.sh: fatal: coffix failed!" >&2
+	fi
+    fi
+    rm -f $TEMPFILE
+    exit $RES
+else
+    eval $GCC -D__WIN32__ -DWIN32 -DWINDOWS -fomit-frame-pointer -fno-tree-copyrename $CMD 2>/dev/null
+    exit $?
+fi
diff --git a/erts/etc/win32/msys_tools/vc/ld.sh b/erts/etc/win32/msys_tools/vc/ld.sh
new file mode 100644
index 0000000000..0fcbf6f7d9
--- /dev/null
+++ b/erts/etc/win32/msys_tools/vc/ld.sh
@@ -0,0 +1,198 @@
+#! /bin/sh
+# set -x
+# 
+# %CopyrightBegin%
+# 
+# Copyright Ericsson AB 2002-2011. All Rights Reserved.
+# 
+# The contents of this file are subject to the Erlang Public License,
+# Version 1.1, (the "License"); you may not use this file except in
+# compliance with the License. You should have received a copy of the
+# Erlang Public License along with this software. If not, it can be
+# retrieved online at http://www.erlang.org/.
+# 
+# Software distributed under the License is distributed on an "AS IS"
+# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+# the License for the specific language governing rights and limitations
+# under the License.
+# 
+# %CopyrightEnd%
+# 
+# Save the command line for debug outputs
+
+SAVE="$@"
+kernel_libs="kernel32.lib advapi32.lib"
+gdi_libs="gdi32.lib user32.lib comctl32.lib comdlg32.lib shell32.lib"
+DEFAULT_LIBRARIES="$kernel_libs $gdi_libs"
+
+CMD=""
+STDLIB=MSVCRT.LIB
+DEBUG_BUILD=false
+STDLIB_FORCED=false
+BUILD_DLL=false
+OUTPUT_FILENAME=""
+
+while test -n "$1" ; do
+    x="$1"
+    case "$x" in
+	-dll| -DLL)
+	    BUILD_DLL=true;; 
+	-L/*|-L.*)
+	    y=`echo $x | sed 's,^-L\(.*\),\1,g'`;
+	    MPATH=`msys2win_path.sh -m $y`;
+	    CMD="$CMD -libpath:\"$MPATH\"";; 
+	-lMSVCRT|-lmsvcrt)
+	    STDLIB_FORCED=true;
+	    STDLIB=MSVCRT.LIB;; 
+	-lMSVCRTD|-lmsvcrtd)
+	    STDLIB_FORCED=true;
+	    STDLIB=MSVCRTD.LIB;; 
+	-lLIBCMT|-llibcmt)
+	    STDLIB_FORCED=true;
+	    STDLIB=LIBCMT.LIB;; 
+	-lLIBCMTD|-llibcmtd)
+	    STDLIB_FORCED=true;
+	    STDLIB=LIBCMTD.LIB;; 
+	-lsocket)
+	    DEFAULT_LIBRARIES="$DEFAULT_LIBRARIES WS2_32.LIB IPHLPAPI.LIB";;
+	-l*)
+	    y=`echo $x | sed 's,^-l\(.*\),\1,g'`;
+	    MPATH=`msys2win_path.sh -m $y`;
+	    CMD="$CMD \"${MPATH}.lib\"";; 
+	-g)
+	    DEBUG_BUILD=true;;
+	-pdb:none|-incremental:no)
+	    ;;
+	-implib:*)
+	    y=`echo $x | sed 's,^-implib:\(.*\),\1,g'`;
+	    MPATH=`msys2win_path.sh -m $y`;
+	    CMD="$CMD -implib:\"${MPATH}\"";; 
+	-def:*)
+	    y=`echo $x | sed 's,^-def:\(.*\),\1,g'`;
+	    MPATH=`msys2win_path.sh -m $y`;
+	    CMD="$CMD -def:\"${MPATH}\"";; 
+	-o)
+	    shift
+	    MPATH=`msys2win_path.sh -m $1`;
+	    OUTPUT_FILENAME="$MPATH";;
+	-o/*)
+	    y=`echo $x | sed 's,^-[Io]\(/.*\),\1,g'`;
+	    MPATH=`msys2win_path.sh -m $y`;
+	    OUTPUT_FILENAME="$MPATH";;
+	/*)
+	    MPATH=`msys2win_path.sh -m $x`;
+	    CMD="$CMD \"$MPATH\"";; 
+	*)
+	    y=`echo $x | sed 's,",\\\",g'`;
+	    CMD="$CMD \"$y\"";;
+    esac
+    shift
+done
+if [ $DEBUG_BUILD = true ]; then
+    linktype="-debug -pdb:none"
+    if [ $STDLIB_FORCED = false ]; then
+	STDLIB=MSVCRTD.LIB
+    fi
+fi
+# Generate a PDB 
+linkadd_pdb=""
+case "$OUTPUT_FILENAME" in
+    *.exe|*.EXE)
+	    fn=`echo "$OUTPUT_FILENAME" | sed 's,[eE][xX][eE]$,,g'`;
+	    linkadd_pdb="-pdb:\"${fn}pdb\"";;
+    *.dll|*.DLL)
+	    fn=`echo "$OUTPUT_FILENAME" | sed 's,[dD][lL][lL]$,,g'`;
+	    linkadd_pdb="-pdb:\"${fn}pdb\"";;
+    "")
+	    linkadd_pdb="-pdb:\"a.pdb\"";;
+    *)
+	    linkadd_pdb="-pdb:\"${OUTPUT_FILENAME}.pdb\"";;
+esac
+	
+    linktype="-debug $linkadd_pdb"
+
+CHMOD_FILE=""
+
+if [ $BUILD_DLL = true ];then
+    case "$OUTPUT_FILENAME" in
+	*.exe|*.EXE)
+	    echo "Warning, output set to .exe when building DLL" >&2
+	    CHMOD_FILE="$OUTPUT_FILENAME";
+	    CMD="-dll -out:\"$OUTPUT_FILENAME\" $CMD";
+	    OUTPUTRES="${OUTPUT_FILENAME}\;2";
+	    MANIFEST="${OUTPUT_FILENAME}.manifest";;
+	*.dll|*.DLL)
+	    CMD="-dll -out:\"$OUTPUT_FILENAME\" $CMD";
+	    OUTPUTRES="${OUTPUT_FILENAME}\;2";
+	    MANIFEST="${OUTPUT_FILENAME}.manifest";;
+	"")
+	    CMD="-dll -out:\"a.dll\" $CMD";
+	    OUTPUTRES="a.dll\;2";
+	    MANIFEST="a.dll.manifest";;
+	*)
+	    CMD="-dll -out:\"${OUTPUT_FILENAME}.dll\" $CMD";
+	    OUTPUTRES="${OUTPUT_FILENAME}.dll\;2";
+	    MANIFEST="${OUTPUT_FILENAME}.dll.manifest";;
+    esac
+else
+    case "$OUTPUT_FILENAME" in
+	*.exe|*.EXE)
+	    CHMOD_FILE="$OUTPUT_FILENAME";
+	    CMD="-out:\"$OUTPUT_FILENAME\" $CMD";
+	    OUTPUTRES="${OUTPUT_FILENAME}\;1"
+	    MANIFEST="${OUTPUT_FILENAME}.manifest";;
+	*.dll|*.DLL)
+	    echo "Warning, output set to .dll when building EXE" >&2
+	    CMD="-out:\"$OUTPUT_FILENAME\" $CMD";
+	    OUTPUTRES="${OUTPUT_FILENAME}\;1";
+	    MANIFEST="${OUTPUT_FILENAME}.manifest";;
+	"")
+	    CHMOD_FILE="a.exe";
+	    CMD="-out:\"a.exe\" $CMD";
+	    OUTPUTRES="a.exe\;1";
+	    MANIFEST="a.exe.manifest";;
+	*)
+	    CMD="-out:\"${OUTPUT_FILENAME}.exe\" $CMD";
+	    OUTPUTRES="${OUTPUT_FILENAME}.exe\;1";
+	    MANIFEST="${OUTPUT_FILENAME}.exe.manifest";;
+    esac
+fi    
+	    
+p=$$
+CMD="$linktype -nologo -incremental:no $CMD $STDLIB $DEFAULT_LIBRARIES"
+if [ "X$LD_SH_DEBUG_LOG" != "X" ]; then
+    echo ld.sh "$SAVE" >>$LD_SH_DEBUG_LOG
+    echo link.exe $CMD >>$LD_SH_DEBUG_LOG
+fi
+eval link.exe "$CMD"  >/tmp/link.exe.${p}.1 2>/tmp/link.exe.${p}.2
+RES=$?
+
+CMANIFEST=`win2msys_path.sh $MANIFEST`
+if [ "$RES" = "0" -a -f "$CMANIFEST" ]; then
+    # Add stuff to manifest to turn off "virtualization"
+    sed -n -i '1h;1!H;${;g;s,<trustInfo.*</trustInfo>.,,g;p;}' $CMANIFEST 2>/dev/null
+    sed -i "s/<\/assembly>/ <ms_asmv2:trustInfo xmlns:ms_asmv2=\"urn:schemas-microsoft-com:asm.v2\">\n  <ms_asmv2:security>\n   <ms_asmv2:requestedPrivileges>\n    <ms_asmv2:requestedExecutionLevel level=\"AsInvoker\" uiAccess=\"false\"\/>\n   <\/ms_asmv2:requestedPrivileges>\n  <\/ms_asmv2:security>\n <\/ms_asmv2:trustInfo>\n<\/assembly>/" $CMANIFEST 2>/dev/null
+
+    eval mt.exe -nologo -manifest "$MANIFEST" -outputresource:"$OUTPUTRES" >>/tmp/link.exe.${p}.1 2>>/tmp/link.exe.${p}.2
+    RES=$?
+    if [ "$RES" != "0" ]; then
+	REMOVE=`echo "$OUTPUTRES" | sed 's,\\\;[12]$,,g'`
+	CREMOVE=`cygpath $REMOVE`
+	rm -f "$CREMOVE"
+    fi
+    rm -f "$CMANIFEST"
+fi
+
+# This works around some strange behaviour 
+# in cygwin 1.7 Beta on Windows 7 with samba drive.
+# Configure will think the compiler failed if test -x fails, 
+# which it might do as we might not be the owner of the
+# file.
+if [ '!' -z "$CHMOD_FILE" -a -s "$CHMOD_FILE" -a '!' -x "$CHMOD_FILE" ]; then
+    chmod +x $CHMOD_FILE
+fi
+    
+tail -n +2 /tmp/link.exe.${p}.2 >&2
+cat /tmp/link.exe.${p}.1
+rm -f /tmp/link.exe.${p}.2 /tmp/link.exe.${p}.1
+exit $RES
diff --git a/erts/etc/win32/msys_tools/vc/mc.sh b/erts/etc/win32/msys_tools/vc/mc.sh
new file mode 100644
index 0000000000..27d985f73e
--- /dev/null
+++ b/erts/etc/win32/msys_tools/vc/mc.sh
@@ -0,0 +1,87 @@
+#! /bin/sh
+# set -x
+# 
+# %CopyrightBegin%
+# 
+# Copyright Ericsson AB 2002-2011. All Rights Reserved.
+# 
+# The contents of this file are subject to the Erlang Public License,
+# Version 1.1, (the "License"); you may not use this file except in
+# compliance with the License. You should have received a copy of the
+# Erlang Public License along with this software. If not, it can be
+# retrieved online at http://www.erlang.org/.
+# 
+# Software distributed under the License is distributed on an "AS IS"
+# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+# the License for the specific language governing rights and limitations
+# under the License.
+# 
+# %CopyrightEnd%
+# 
+# Save the command line for debug outputs
+SAVE="$@"
+CMD=""
+OUTPUT_DIRNAME=""
+
+# Find the correct mc.exe. This could be done by the configure script,
+# But as we seldom use the message compiler, it might as well be done here...
+MCC=""
+save_ifs=$IFS
+IFS=:
+for p in $PATH; do 
+    if [ -f $p/mc.exe ]; then 
+	if [ -n "`$p/mc.exe -? 2>&1 >/dev/null </dev/null \
+                 | grep -i \"message compiler\"`" ]; then 
+	    MCC=`echo "$p/mc.exe" | sed 's/ /\\\\ /g'`
+	fi
+    fi
+done
+IFS=$save_ifs
+
+if [ -z "$MCC" ]; then
+    echo 'mc.exe not found!' >&2
+    exit 1
+fi
+
+while test -n "$1" ; do
+    x="$1"
+    case "$x" in
+	-o)
+	    shift
+	    OUTPUT_DIRNAME="$1";;
+	-o/*)
+	    y=`echo $x | sed 's,^-[Io]\(/.*\),\1,g'`;
+	    OUTPUT_DIRNAME="$y";;
+	-I)
+	    shift
+	    MPATH=`msys2win_path.sh -m $1`;
+	    CMD="$CMD -I\"$MPATH\"";;
+	-I/*)
+	    y=`echo $x | sed 's,^-[Io]\(/.*\),\1,g'`;
+	    MPATH=`msys2win_path.sh -m $y`;
+	    CMD="$CMD -I\"$MPATH\"";;
+	*)
+	    MPATH=`msys2win_path.sh -m -a $x`;
+	    CMD="$CMD \"$MPATH\"";; 
+    esac
+    shift
+done
+p=$$
+if [ "X$MC_SH_DEBUG_LOG" != "X" ]; then
+    echo mc.sh "$SAVE" >>$MC_SH_DEBUG_LOG
+    echo mc.exe $CMD >>$MC_SH_DEBUG_LOG
+fi
+if [ -n "$OUTPUT_DIRNAME" ]; then
+    cd $OUTPUT_DIRNAME
+    RES=$?
+    if [ "$RES" != "0" ]; then
+	echo "mc.sh: Error: could not cd to $OUTPUT_DIRNAME">&2
+	exit $RES
+    fi
+fi
+eval $MCC "$CMD"  >/tmp/mc.exe.${p}.1 2>/tmp/mc.exe.${p}.2
+RES=$?
+tail +2 /tmp/mc.exe.${p}.2 >&2
+cat /tmp/mc.exe.${p}.1
+rm -f /tmp/mc.exe.${p}.2 /tmp/mc.exe.${p}.1
+exit $RES
diff --git a/erts/etc/win32/msys_tools/vc/rc.sh b/erts/etc/win32/msys_tools/vc/rc.sh
new file mode 100644
index 0000000000..dfa9e324db
--- /dev/null
+++ b/erts/etc/win32/msys_tools/vc/rc.sh
@@ -0,0 +1,86 @@
+#! /bin/sh
+# set -x
+# 
+# %CopyrightBegin%
+# 
+# Copyright Ericsson AB 2002-2011. All Rights Reserved.
+# 
+# The contents of this file are subject to the Erlang Public License,
+# Version 1.1, (the "License"); you may not use this file except in
+# compliance with the License. You should have received a copy of the
+# Erlang Public License along with this software. If not, it can be
+# retrieved online at http://www.erlang.org/.
+# 
+# Software distributed under the License is distributed on an "AS IS"
+# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+# the License for the specific language governing rights and limitations
+# under the License.
+# 
+# %CopyrightEnd%
+# 
+# Save the command line for debug outputs
+SAVE="$@"
+CMD=""
+OUTPUT_FILENAME=""
+
+# Find the correct rc.exe. This could be done by the configure script,
+# But as we seldom use the resource compiler, it might as well be done here...
+RCC=""
+save_ifs=$IFS
+IFS=:
+for p in $PATH; do 
+    if [ -f $p/rc.exe ]; then 
+	if [ -n "`$p/rc.exe -? 2>&1 | grep -i "resource compiler"`" ]; then
+	    RCC=`echo "$p/rc.exe" | sed 's/ /\\\\ /g'`
+	fi
+    fi
+done
+IFS=$save_ifs
+
+if [ -z "$RCC" ]; then
+    echo 'rc.exe not found!' >&2
+    exit 1
+fi
+
+while test -n "$1" ; do
+    x="$1"
+    case "$x" in
+	-o)
+	    shift
+	    MPATH=`msys2win_path.sh -m $1`;
+	    OUTPUT_FILENAME="$MPATH";;
+	-o/*)
+	    y=`echo $x | sed 's,^-[Io]\(/.*\),\1,g'`;
+	    MPATH=`msys2win_path.sh -m $y`;
+	    OUTPUT_FILENAME="$MPATH";;
+	-I)
+	    shift
+	    MPATH=`msys2win_path.sh -m $1`;
+	    CMD="$CMD -I\"$MPATH\"";;
+	-I/*)
+	    y=`echo $x | sed 's,^-[Io]\(/.*\),\1,g'`;
+	    MPATH=`msys2win_path.sh -m $y`;
+	    CMD="$CMD -I\"$MPATH\"";;
+	/*)
+	    MPATH=`msys2win_path.sh -m $x`;
+	    CMD="$CMD \"$MPATH\"";; 
+	*)
+	    y=`echo $x | sed 's,",\\\",g'`;
+	    CMD="$CMD \"$y\"";;
+    esac
+    shift
+done
+p=$$
+if [ -n "$OUTPUT_FILENAME" ]; then
+    CMD="-Fo$OUTPUT_FILENAME $CMD"
+fi
+if [ "X$RC_SH_DEBUG_LOG" != "X" ]; then
+    echo rc.sh "$SAVE" >>$RC_SH_DEBUG_LOG
+    echo rc.exe $CMD >>$RC_SH_DEBUG_LOG
+fi
+eval $RCC "$CMD"  >/tmp/rc.exe.${p}.1 2>/tmp/rc.exe.${p}.2
+RES=$?
+tail +2 /tmp/rc.exe.${p}.2 >&2
+cat /tmp/rc.exe.${p}.1
+rm -f /tmp/rc.exe.${p}.2 /tmp/rc.exe.${p}.1
+exit $RES
diff --git a/erts/etc/win32/msys_tools/win2msys_path.sh b/erts/etc/win32/msys_tools/win2msys_path.sh
new file mode 100644
index 0000000000..6558a15ecd
--- /dev/null
+++ b/erts/etc/win32/msys_tools/win2msys_path.sh
@@ -0,0 +1,38 @@
+#! /bin/bash

+if [ -z "$1" ]; then

+    echo "Usage: $0 <path>" >&2

+    exit 1;

+fi

+

+MSYS_PATH=`echo "$1" | sed 's,^\([a-zA-Z]\):\\\\,/\L\1/,;s,\\\\,/,g'`

+if [ -z "$MSYS_PATH" ]; then

+    echo "$0: Could not translate $1 to msys format" >&2

+    exit 2;

+fi

+

+DELBLANK=`echo "$MSYS_PATH" | sed 's, ,,g'`

+

+if [ "X$DELBLANK" != "X$MSYS_PATH" ]; then

+    if [ -d "$MSYS_PATH" ]; then

+	C1=`(cd "$MSYS_PATH" && cmd //C "for %i in (".") do @echo %~fsi")`

+	MSYS_PATH=`echo "$C1" | sed 's,^\([a-zA-Z]\):\\\\,/\L\1/,;s,\\\\,/,g'`

+    else

+	MSYS_DIR=`dirname "$MSYS_PATH"`

+	MSYS_FILE=`basename "$MSYS_PATH"`

+	if [ -d "$MSYS_DIR" ]; then

+	    	C1=`(cd "$MSYS_DIR" && cmd //C "for %i in (".") do @echo %~fsi")`	

+		BAT_FILE=/tmp/w$$.bat

+		# I simply cannot get the quoting right for this, 

+		# need an intermediate bat file

+		cat > $BAT_FILE <<EOF

+@echo off

+for %%i in ("$MSYS_FILE") do @echo %%~snxi

+EOF

+		C2=`(cd "$MSYS_DIR" && cmd //C $BAT_FILE)`

+		rm -f $BAT_FILE

+		MSYS_PATH=`echo "$C1/$C2" | sed 's,^\([a-zA-Z]\):\\\\,/\L\1/,;s,\\\\,/,g;s," ", ,g'`

+	fi

+    fi

+fi		

+echo $MSYS_PATH

+exit 0
\ No newline at end of file
diff --git a/erts/etc/win32/nsis/Makefile b/erts/etc/win32/nsis/Makefile
index ebb3ad9a96..4ed4ef2700 100644
--- a/erts/etc/win32/nsis/Makefile
+++ b/erts/etc/win32/nsis/Makefile
@@ -1,7 +1,7 @@
 # 
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 2003-2009. All Rights Reserved.
+# Copyright Ericsson AB 2003-2011. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -40,11 +40,31 @@ clean:
 include $(ERL_TOP)/make/otp_release_targets.mk
 
 TARGET_DIR = $(RELEASE_PATH)
-WTESTROOT=$(shell (cygpath -d $(RELEASE_PATH) 2>/dev/null || cygpath -w $(RELEASE_PATH)))
-WTARGET_DIR=$(shell (cygpath -d $(TARGET_DIR) 2>/dev/null || cygpath -d $(TARGET_DIR)))
+
+ifeq ($(MSYSTEM),MINGW32)
+
+  MAKENSISFLAGS = //V2
+  WTESTROOT=$(shell (msys2win_path.sh "$(RELEASE_PATH)"))
+  WTARGET_DIR=$(shell (msys2win_path.sh "$(TARGET_DIR)"))
+
+else
+
+  MAKENSISFLAGS = /V2
+  WTESTROOT=$(shell (cygpath -d "$(RELEASE_PATH)" 2>/dev/null || cygpath -w "$(RELEASE_PATH)"))
+  WTARGET_DIR=$(shell (cygpath -d "$(TARGET_DIR)" 2>/dev/null || cygpath -d "$(TARGET_DIR)"))
+
+endif
+
+ifeq ($(CONFIG_SUBTYPE),win64)
+  WINTYPE=win64
+else
+  WINTYPE=win32
+endif
 
 REDIST_FILE=$(shell (sh ./find_redist.sh || echo ""))
+REDIST_TARGET=$(shell (sh ./find_redist.sh -n || echo ""))
 REDIST_DLL_VERSION=$(shell (sh ./dll_version_helper.sh || echo ""))
+REDIST_DLL_NAME=$(shell (sh ./dll_version_helper.sh -n || echo ""))
 
 release_spec:
 	@NSIS_VER=`makensis /hdrinfo | head -1 | awk '{print $$2}'`; \
@@ -54,7 +74,7 @@ release_spec:
 	  echo '!define ERTS_VERSION "$(VSN)"' >> $(VERSION_HEADER);\
 	  echo '!define TESTROOT "$(WTESTROOT)"' >> $(VERSION_HEADER);\
 	  echo '!define OUTFILEDIR "$(WTARGET_DIR)"' >> $(VERSION_HEADER);\
-	  if [ -f $(RELEASE_PATH)/docs/doc/index.html ];\
+	  if [ -f "$(RELEASE_PATH)/docs/doc/index.html" ];\
 	  then\
 		echo '!define HAVE_DOCS 1' >> $(VERSION_HEADER); \
 	  fi;\
@@ -64,17 +84,20 @@ release_spec:
 	  echo '!define ERTS_VERSION "$(VSN)"' >> $(VERSION_HEADER);\
 	  echo '!define TESTROOT "$(WTESTROOT)"' >> $(VERSION_HEADER);\
 	  echo '!define OUTFILEDIR "$(WTARGET_DIR)"' >> $(VERSION_HEADER);\
+	  echo '!define WINTYPE "$(WINTYPE)"' >> $(VERSION_HEADER);\
 	  if [ -f $(CUSTOM_MODERN) ];\
 	  then \
 		echo '!define HAVE_CUSTOM_MODERN 1' >> $(VERSION_HEADER); \
 	  fi;\
 	  if [ '!' -z "$(REDIST_FILE)" -a '!' -z "$(REDIST_DLL_VERSION)" ];\
 	  then \
-		cp $(REDIST_FILE) $(RELEASE_PATH)/vcredist_x86.exe;\
+		cp $(REDIST_FILE) "$(RELEASE_PATH)/$(REDIST_TARGET);"\
 		echo '!define HAVE_REDIST_FILE 1'  >> $(VERSION_HEADER); \
 		echo '!define REDIST_DLL_VERSION "$(REDIST_DLL_VERSION)"' >> $(VERSION_HEADER);\
+		echo '!define REDIST_DLL_NAME "$(REDIST_DLL_NAME)"' >> $(VERSION_HEADER);\
+		echo '!define REDIST_EXECUTABLE "$(REDIST_TARGET)"' >> $(VERSION_HEADER);\
 	  fi;\
-	  if [ -f $(RELEASE_PATH)/docs/doc/index.html ];\
+	  if [ -f "$(RELEASE_PATH)/docs/doc/index.html" ];\
 	  then \
 		echo '!define HAVE_DOCS 1' >> $(VERSION_HEADER); \
 	  fi;\
diff --git a/erts/etc/win32/nsis/dll_version_helper.sh b/erts/etc/win32/nsis/dll_version_helper.sh
index e0047dea8b..96e4532b7a 100755
--- a/erts/etc/win32/nsis/dll_version_helper.sh
+++ b/erts/etc/win32/nsis/dll_version_helper.sh
@@ -2,7 +2,7 @@
 # 
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 2007-2009. All Rights Reserved.
+# Copyright Ericsson AB 2007-2011. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -26,6 +26,7 @@
 # exit 0
 
 cat > hello.c <<EOF
+#include <windows.h>
 #include <stdio.h>
 
 int main(void)
@@ -35,15 +36,77 @@ int main(void)
 }
 
 EOF
-cl /MD hello.c > /dev/null 2>&1
+cl -MD hello.c  > /dev/null 2>&1
 if [ '!' -f hello.exe.manifest ]; then
-    echo "This compiler does not generate manifest files - OK if using mingw" >&2
-    exit 0
+    # Gah - VC 2010 changes the way it handles DLL's and manifests... Again...
+    # need another way of getting the version
+    DLLNAME=`dumpbin.exe -imports hello.exe | egrep MSVCR.*dll`
+    DLLNAME=`echo $DLLNAME`
+    cat > helper.c <<EOF
+#include <windows.h>
+#include <stdio.h>
+
+#define REQ_MODULE "$DLLNAME"
+
+int main(void)
+{
+  DWORD dummy;
+  DWORD versize;
+  int i,n;
+  unsigned char *versinfo;
+  char buff[100];
+
+  char *vs_verinfo;
+  unsigned int vs_ver_size;
+  
+  WORD *translate;
+  unsigned int tr_size;
+  
+  if (!(versize = GetFileVersionInfoSize(REQ_MODULE,&dummy))) {
+    fprintf(stderr,"No version info size in %s!\n",REQ_MODULE);
+    exit(1);
+  }
+  versinfo=malloc(versize);
+  if (!GetFileVersionInfo(REQ_MODULE,dummy,versize,versinfo)) {
+    fprintf(stderr,"No version info in %s!\n",REQ_MODULE);
+    exit(2);
+  }
+  if (!VerQueryValue(versinfo,"\\\\VarFileInfo\\\\Translation",&translate,&tr_size)) {
+    fprintf(stderr,"No translation info in %s!\n",REQ_MODULE);
+    exit(3);
+  }
+  n = tr_size/(2*sizeof(*translate));
+  for(i=0; i < n; ++i) {
+    sprintf(buff,"\\\\StringFileInfo\\\\%04x%04x\\\\FileVersion",
+	    translate[i*2],translate[i*2+1]);
+    if (VerQueryValue(versinfo,buff,&vs_verinfo,&vs_ver_size)) {
+      printf("%s\n",(char *) vs_verinfo);
+      return 0;
+    }
+  } 
+  fprintf(stderr,"Failed to find file version of %s\n",REQ_MODULE);
+  return 0;
+}
+EOF
+    cl -MD helper.c version.lib > /dev/null 2>&1
+    if [ '!' -f helper.exe ]; then
+	echo "Failed to build helper program." >&2
+	exit 1
+    fi
+    NAME=$DLLNAME
+    VERSION=`./helper`
+else
+    VERSION=`grep '<assemblyIdentity' hello.exe.manifest | sed 's,.*version=.\([0-9\.]*\).*,\1,g' | grep -v '<'`
+    NAME=`grep '<assemblyIdentity' hello.exe.manifest | sed 's,.*name=.[A-Za-z\.]*\([0-9]*\).*,msvcr\1.dll,g' | grep -v '<'`
+fi
+#rm -f hello.c hello.obj hello.exe hello.exe.manifest helper.c helper.obj helper.exe helper.exe.manifest
+if [ "$1" = "-n" ]; then
+    ASKEDFOR=$NAME
+else
+    ASKEDFOR=$VERSION
 fi
-VERSION=`grep '<assemblyIdentity' hello.exe.manifest | sed 's,.*version=.\([0-9\.]*\).*,\1,g' | grep -v '<'`
-rm -f hello.c hello.obj hello.exe hello.exe.manifest
-if [ -z "$VERSION" ]; then
+if [ -z "$ASKEDFOR" ]; then
     exit 1
 fi
-echo $VERSION
+echo $ASKEDFOR
 exit 0
diff --git a/erts/etc/win32/nsis/erlang20.nsi b/erts/etc/win32/nsis/erlang20.nsi
index 43e5d91604..c5ada9e3b3 100644
--- a/erts/etc/win32/nsis/erlang20.nsi
+++ b/erts/etc/win32/nsis/erlang20.nsi
@@ -4,6 +4,25 @@
 ; Original example written by Joost Verburg

 ; Modified for Erlang by Patrik

 

+;

+; %CopyrightBegin%

+;

+; Copyright Ericsson AB 2012. All Rights Reserved.

+;

+; The contents of this file are subject to the Erlang Public License,

+; Version 1.1, (the "License"); you may not use this file except in

+; compliance with the License. You should have received a copy of the

+; Erlang Public License along with this software. If not, it can be

+; retrieved online at http://www.erlang.org/.

+;

+; Software distributed under the License is distributed on an "AS IS"

+; basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See

+; the License for the specific language governing rights and limitations

+; under the License.

+;

+; %CopyrightEnd%

+;

+

 ; Verbosity does not come naturally with MUI, have to set it back now and then.

 	!verbose 1

 	!define MUI_MANUALVERBOSE 1

@@ -31,17 +50,24 @@ Var STARTMENU_FOLDER
 !define MY_STARTMENUPAGE_REGISTRY_VALUENAME "Start Menu Folder"

 

 ;General

-	OutFile "${OUTFILEDIR}\otp_win32_${OTP_VERSION}.exe"

+	OutFile "${OUTFILEDIR}\otp_${WINTYPE}_${OTP_VERSION}.exe"

 

 ;Folder selection page

+!if ${WINTYPE} == "win64"

+  	InstallDir "$PROGRAMFILES64\erl${ERTS_VERSION}"

+!else

   	InstallDir "$PROGRAMFILES\erl${ERTS_VERSION}"

-  

+!endif  

 ;Remember install folder

   	InstallDirRegKey HKLM "SOFTWARE\Ericsson\Erlang\${ERTS_VERSION}" ""

   

 ; Set the default start menu folder

 

+!if ${WINTYPE} == "win64"

+	!define MUI_STARTMENUPAGE_DEFAULTFOLDER "${OTP_PRODUCT} ${OTP_VERSION} (x64)"

+!else

 	!define MUI_STARTMENUPAGE_DEFAULTFOLDER "${OTP_PRODUCT} ${OTP_VERSION}"

+!endif  

 

 ;--------------------------------

 ;Modern UI Configuration

@@ -98,12 +124,15 @@ Var STARTMENU_FOLDER
 Section "Microsoft redistributable libraries." SecMSRedist

 

   	SetOutPath "$INSTDIR"

-	File "${TESTROOT}\vcredist_x86.exe"

+	File "${TESTROOT}\${REDIST_EXECUTABLE}"

   

 ; Set back verbosity...

   	!verbose 1

-; Run the setup program  

-  	ExecWait '"$INSTDIR\vcredist_x86.exe"'

+; Run the setup program

+	IfSilent +3

+	    ExecWait '"$INSTDIR\${REDIST_EXECUTABLE}"'

+	Goto +2

+	    ExecWait '"$INSTDIR\${REDIST_EXECUTABLE}" /q'

 

   	!verbose 1

 SectionEnd ; MSRedist

@@ -310,24 +339,32 @@ Function DllVersionGoodEnough
 FunctionEnd

 

 Function .onInit

-   SectionGetFlags 0 $MYTEMP 

-;   MessageBox MB_YESNO "Found $SYSDIR\msvcr80.dll" IDYES FoundLbl

-   IfFileExists $SYSDIR\msvcr80.dll MaybeFoundInSystemLbl

-   SearchSxsLbl:	

-        FindFirst $0 $1 $WINDIR\WinSxS\x86*

+   Var /GLOBAL archprefix

+   Var /GLOBAL sysnativedir

+   SectionGetFlags 0 $MYTEMP

+   StrCmpS ${WINTYPE} "win64" +1 +4

+	StrCpy $archprefix "amd64"

+	StrCpy $sysnativedir "$WINDIR\sysnative"

+   Goto +3

+	StrCpy $archprefix "x86"

+	StrCpy $sysnativedir $SYSDIR

+   ;MessageBox MB_YESNO "Found $sysnativedir\${REDIST_DLL_NAME}" IDYES FoundLbl

+   IfFileExists $sysnativedir\${REDIST_DLL_NAME} MaybeFoundInSystemLbl

+   SearchSxSLbl:	

+        FindFirst $0 $1 $WINDIR\WinSxS\$archprefix*

         LoopLbl:

 	    StrCmp $1 "" NotFoundLbl

-	    IfFileExists $WINDIR\WinSxS\$1\msvcr80.dll MaybeFoundInSxsLbl

+	    IfFileExists $WINDIR\WinSxS\$1\${REDIST_DLL_NAME} MaybeFoundInSxSLbl

 	    FindNext $0 $1

 	    Goto LoopLbl

-        MaybeFoundInSxsLbl:

-	    GetDllVersion $WINDIR\WinSxS\$1\msvcr80.dll $R0 $R1

+        MaybeFoundInSxSLbl:

+	    GetDllVersion $WINDIR\WinSxS\$1\${REDIST_DLL_NAME} $R0 $R1

 	    Call DllVersionGoodEnough

 	    FindNext $0 $1

 	    IntCmp 2 $R0 LoopLbl

 	    Goto FoundLbl  

    MaybeFoundInSystemLbl:

-	GetDllVersion $SYSDIR\msvcr80.dll $R0 $R1

+	GetDllVersion $sysnativedir\${REDIST_DLL_NAME} $R0 $R1

 	Call DllVersionGoodEnough

 	IntCmp 2 $R0 SearchSxSLbl  

    FoundLbl:

diff --git a/erts/etc/win32/nsis/erlang_uninst.ico b/erts/etc/win32/nsis/erlang_uninst.ico
index edbd8a6f2c..edbd8a6f2c 100755..100644
--- a/erts/etc/win32/nsis/erlang_uninst.ico
+++ b/erts/etc/win32/nsis/erlang_uninst.ico
diff --git a/erts/etc/win32/nsis/find_redist.sh b/erts/etc/win32/nsis/find_redist.sh
index c5572839c5..7c449c9e4e 100755
--- a/erts/etc/win32/nsis/find_redist.sh
+++ b/erts/etc/win32/nsis/find_redist.sh
@@ -2,7 +2,7 @@
 # 
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 2007-2009. All Rights Reserved.
+# Copyright Ericsson AB 2007-2011. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -65,7 +65,6 @@ remove_path_element()
     else
 	echo "${ACC}/$1"
     fi
-
     #echo "ACC=$ACC" >&2
     #echo "1=$1" >&2
 }
@@ -89,34 +88,124 @@ add_path_element()
     echo "$PA"
 }
 
+
 CLPATH=`lookup_prog_in_path cl`
 
 if [ -z "$CLPATH" ]; then 
-    echo "Can not locate cl.exe and vcredist_x86.exe - OK if using mingw" >&2
+    echo "Can not locate cl.exe and vcredist_x86/x64.exe - OK if using mingw" >&2
     exit 1
 fi
 
-#echo $CLPATH
+# Look to see if it's 64bit
+XX=`remove_path_element cl "$CLPATH"`
+YY=`remove_path_element amd64 "$XX"`
+if [ "$YY" != "$XX" ]; then
+    AMD64DIR=true
+    VCREDIST=vcredist_x64
+    COMPONENTS="cl amd64 bin vc"
+else
+    AMD64DIR=false
+    VCREDIST=vcredist_x86
+    COMPONENTS="cl bin vc"
+fi
+
+if [ X"$1" = X"-n" ]; then
+    echo $VCREDIST.exe
+    exit 0
+fi
+
+# echo $CLPATH
 BPATH=$CLPATH
-for x in cl bin vc; do
-    #echo $x
+for x in $COMPONENTS; do
+    # echo $x
     NBPATH=`remove_path_element $x "$BPATH"`
     if [ "$NBPATH" = "$BPATH" ]; then
-	echo "Failed to locate vcredist_x86.exe because cl.exe was in an unexpected location" >&2
+	echo "Failed to locate $VCREDIST.exe because cl.exe was in an unexpected location" >&2
 	exit 2
     fi
     BPATH="$NBPATH"
 done
-#echo $BPATH
-for x in sdk v2.0 bootstrapper packages vcredist_x86 vcredist_x86.exe; do
-    #echo "x=$x"
-    #echo "BPATH=$BPATH"
-    NBPATH=`add_path_element $x "$BPATH"`
-    if [ "$NBPATH" = "$BPATH" ]; then
-	echo "Failed to locate vcredist_x86.exe because directory structure was unexpected" >&2
-	exit 3
+BPATH_LIST=$BPATH
+
+# rc.exe is in the Microsoft SDK directory of VS2008
+RCPATH=`lookup_prog_in_path rc`
+fail=false
+if [ '!' -z "$RCPATH" ]; then 
+    BPATH=$RCPATH
+    allow_fail=false
+    if [ $AMD64DIR = true ]; then
+	COMPONENTS="rc x64 bin @ANY v6.0A v7.0A v7.1"
+    else
+	COMPONENTS="rc bin @ANY v6.0A v7.0A v7.1"
+    fi
+    for x in $COMPONENTS; do
+	if [ $x = @ANY ]; then
+	    allow_fail=true
+	else
+	    NBPATH=`remove_path_element $x "$BPATH"`
+	    if [ $allow_fail = false -a "$NBPATH" = "$BPATH" ]; then
+		fail=true
+		break;
+	    fi
+	    BPATH="$NBPATH"
+	fi
+    done
+    if [ $fail = false ]; then
+	BPATH_LIST="$BPATH_LIST $BPATH"
+    fi
+fi
+# echo "BPATH_LIST=$BPATH_LIST"
+
+# Frantic search through two roots with different 
+# version directories. We want to be very specific about the
+# directory structures as we wouldnt want to find the wrong 
+# redistributables...
+
+#echo $BPATH_LIST
+for BP in $BPATH_LIST; do
+    for verdir in "sdk v2.0" "sdk v3.5" "v6.0A" "v7.0"  "v7.0A" "v7.1"; do
+	BPATH=$BP
+	fail=false
+	allow_fail=false
+	for x in $verdir @ANY bootstrapper packages $VCREDIST Redist VC @ALL $VCREDIST.exe; do
+	    #echo "x=$x"
+	    #echo "BPATH=$BPATH"
+	    #echo "allow_fail=$allow_fail"
+	    if [ $x = @ANY ]; then
+		allow_fail=true
+	    elif [ $x = @ALL ]; then
+		allow_fail=false
+	    else
+		NBPATH=`add_path_element $x "$BPATH"`
+		if [ $allow_fail = false -a "$NBPATH" = "$BPATH" ]; then
+		    fail=true
+		    break;
+		fi
+		BPATH="$NBPATH"
+	    fi
+	done
+	if [ $fail = false ]; then
+	    break;
+	fi
+    done
+    if [ $fail = false ]; then
+	echo $BPATH
+	exit 0
     fi
-    BPATH="$NBPATH"
 done
-echo $BPATH
-exit 0
\ No newline at end of file
+
+# shortcut for locating $VCREDIST.exe is to put it into $ERL_TOP
+if [ -f $ERL_TOP/$VCREDIST.exe ]; then
+    echo $ERL_TOP/$VCREDIST.exe
+    exit 0
+fi
+
+# or $ERL_TOP/.. to share across multiple builds
+if [ -f $ERL_TOP/../$VCREDIST.exe ]; then
+    echo $ERL_TOP/../$VCREDIST.exe
+    exit 0
+fi
+
+echo "Failed to locate $VCREDIST.exe because directory structure was unexpected" >&2
+exit 3
+
diff --git a/erts/etc/win32/start_erl.c b/erts/etc/win32/start_erl.c
index dcf8c8b281..28c8e55bd3 100644
--- a/erts/etc/win32/start_erl.c
+++ b/erts/etc/win32/start_erl.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1998-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -44,6 +44,8 @@ char *progname;
 #endif
 
 #define RELEASE_SUBDIR "\\releases"
+#define ERTS_SUBDIR_PREFIX "\\erts-"
+#define BIN_SUBDIR "\\bin"
 #define REGISTRY_BASE "Software\\Ericsson\\Erlang\\"
 #define DEFAULT_DATAFILE "start_erl.data"
 
@@ -101,7 +103,8 @@ void exit_help(char *err)
     printf("Usage:\n%s\n"
 	   "      [<erlang options>] ++\n"
 	   "      [-data <datafile>]\n"
-	   "      [-reldir <releasedir>]\n"
+	   "      {-rootdir <erlang root directory> | \n"
+	   "       -reldir <releasedir>}\n"
 	   "      [-bootflags <bootflagsfile>]\n"
 	   "      [-noconfig]\n", progname);
 
@@ -177,8 +180,9 @@ void split_commandline(void)
  */
 char * unquote_optionarg(char *str, char **strp)
 {
-    char	*newstr = (char *)malloc(strlen(str)+1);  /* This one is realloc:ed later */
-    int		i=0, inquote=0;
+    char	*newstr = (char *)malloc(strlen(str)+1);  /* This one is 
+							     realloc:ed later */
+    int		i = 0, inquote = 0;
 
     assert(newstr);
     assert(str);
@@ -223,8 +227,8 @@ char * unquote_optionarg(char *str, char **strp)
 
 
 /*
- *	Parses MyCommandLine and tries to fill in all the required option variables
- *	(one way or another).
+ * Parses MyCommandLine and tries to fill in all the required option 
+ * variables (in one way or another).
  */
 void parse_commandline(void)
 {
@@ -237,6 +241,11 @@ void parse_commandline(void)
 	    *cmdline++;
 	    if( strnicmp(cmdline, "data", 4) == 0) {
 		DataFileName = unquote_optionarg(cmdline+4, &cmdline);
+	    } else if( strnicmp(cmdline, "rootdir", 7) == 0) {
+		RootDir = unquote_optionarg(cmdline+7, &cmdline);
+#ifdef _DEBUG
+		fprintf(stderr, "RootDir: '%s'\n", RootDir);
+#endif
 	    } else if( strnicmp(cmdline, "reldir", 6) == 0) {
 		RelDir = unquote_optionarg(cmdline+6, &cmdline);
 #ifdef _DEBUG
@@ -266,8 +275,8 @@ void parse_commandline(void)
  * Read the data file specified and get the version and release number
  * from it.
  *
- * This function also construct the correct RegistryKey from the version information
- * retrieved.
+ * This function also construct the correct RegistryKey from the version 
+ * information retrieved.
  */
 void read_datafile(void)
 {
@@ -325,88 +334,6 @@ void read_datafile(void)
 
 
 /*
- * Read the registry keys we need
- */
-void read_registry_keys(void)
-{
-    HKEY	hReg;
-    ULONG	lLen;
-
-    /* Create the RegistryKey name */
-    RegistryKey = (char *) malloc(strlen(REGISTRY_BASE) + 
-				  strlen(Version) + 1);
-    assert(RegistryKey);
-    sprintf(RegistryKey, REGISTRY_BASE "%s", Version);
-
-    /* We always need to find BinDir */
-    if( (RegOpenKeyEx(HKEY_LOCAL_MACHINE, 
-		      RegistryKey, 
-		      0, 
-		      KEY_READ, 
-		      &hReg)) != ERROR_SUCCESS ) {
-	exit_help("Could not open registry key.");
-    }
-
-    /* First query size of data */
-    if( (RegQueryValueEx(hReg, 
-			 "Bindir", 
-			 NULL, 
-			 NULL, 
-			 NULL, 
-			 &lLen)) != ERROR_SUCCESS) {
-	exit_help("Failed to query BinDir of release.\n");
-    }
-
-    /* Allocate enough space */
-    BinDir = (char *)malloc(lLen+1);
-    assert(BinDir);
-    /* Retrieve the value */
-    if( (RegQueryValueEx(hReg, 
-			 "Bindir", 
-			 NULL, 
-			 NULL, 
-			 (unsigned char *) BinDir, 
-			 &lLen)) != ERROR_SUCCESS) {
-	exit_help("Failed to query BinDir of release (2).\n");
-    }
-
-#ifdef _DEBUG
-    fprintf(stderr, "Bindir: '%s'\n", BinDir);
-#endif
-
-    /* We also need the rootdir, in case we need to build RelDir later */
-	
-    /* First query size of data */
-    if( (RegQueryValueEx(hReg, 
-			 "Rootdir", 
-			 NULL, 
-			 NULL, 
-			 NULL, 
-			 &lLen)) != ERROR_SUCCESS) {
-	exit_help("Failed to query RootDir of release.\n");
-    }
-
-    /* Allocate enough space */
-    RootDir = (char *) malloc(lLen+1);
-    assert(RootDir);
-    /* Retrieve the value */
-    if( (RegQueryValueEx(hReg, 
-			 "Rootdir", 
-			 NULL, 
-			 NULL, 
-			 (unsigned char *) RootDir, 
-			 &lLen)) != ERROR_SUCCESS) {
-	exit_help("Failed to query RootDir of release (2).\n");
-    }
-
-#ifdef _DEBUG
-    fprintf(stderr, "Rootdir: '%s'\n", RootDir);
-#endif
-
-    RegCloseKey(hReg);
-}
-
-/*
  * Read the bootflags. This file contains extra command line options to erl.exe
  */
 void read_bootflags(void)
@@ -424,7 +351,8 @@ void read_bootflags(void)
 		exit_help("Need -reldir when -bootflags "
 			  "filename has relative path.");
 	    } else {
-		newname = (char *)malloc(strlen(BootFlagsFile)+strlen(RelDir)+strlen(Release)+3);
+		newname = (char *)malloc(strlen(BootFlagsFile)+
+					 strlen(RelDir)+strlen(Release)+3);
 		assert(newname);
 		sprintf(newname, "%s\\%s\\%s", RelDir, Release, BootFlagsFile);
 		free(BootFlagsFile);
@@ -436,8 +364,6 @@ void read_bootflags(void)
 	fprintf(stderr, "BootFlagsFile: '%s'\n", BootFlagsFile);
 #endif
 	
-	
-	
 	if( (fp=fopen(BootFlagsFile, "rb")) == NULL) {
 	    exit_help("Could not open BootFlags file.");
 	}
@@ -605,32 +531,49 @@ void complete_options(void)
 		sz = nsz;
 	}
 	if (RelDir == NULL) {
-	    if(DataFileName){
-		/* Needs to be absolute for this to work, but we
-		   can try... */
-		read_datafile();
-		read_registry_keys();
-	    } else {
-		/* Impossible to find all data... */
-		exit_help("Need either Release directory or an absolute "
-			  "datafile name.");
-	    }
-	    /* Ok, construct our own RelDir from RootDir */
-	    RelDir = (char *) malloc(strlen(RootDir)+strlen(RELEASE_SUBDIR)+1);
-	    assert(RelDir);
-	    sprintf(RelDir, "%s" RELEASE_SUBDIR, RootDir);
+	  if (!RootDir) {
+	    /* Impossible to find all data... */
+	    exit_help("Need either Root directory nor Release directory.");
+	  }
+	  /* Ok, construct our own RelDir from RootDir */
+	  RelDir = (char *) malloc(strlen(RootDir)+strlen(RELEASE_SUBDIR)+1);
+	  assert(RelDir);
+	  sprintf(RelDir, "%s" RELEASE_SUBDIR, RootDir);
+	  read_datafile();
 	} else {
 	    read_datafile();
-	    read_registry_keys();
 	}
     } else {
 	read_datafile();
-	read_registry_keys();
     }
+    if( !RootDir ) {
+	/* Try to construct RootDir from RelDir */
+	char *p;
+	RootDir = malloc(strlen(RelDir)+1);
+	strcpy(RootDir,RelDir);
+	p = RootDir+strlen(RootDir)-1;
+	if (p >= RootDir && (*p == '/' || *p == '\\'))
+	    --p;
+	while (p >= RootDir && *p != '/' &&  *p != '\\')
+	    --p;
+	if (p <= RootDir) { /* Empty RootDir is also an error */
+	    exit_help("Cannot determine Root directory from "
+		      "Release directory.");
+	}
+	*p = '\0';
+    }
+	    
+    
+    BinDir = (char *) malloc(strlen(RootDir)+strlen(ERTS_SUBDIR_PREFIX)+
+			     strlen(Version)+strlen(BIN_SUBDIR)+1);
+    assert(BinDir);
+    sprintf(BinDir, "%s" ERTS_SUBDIR_PREFIX "%s" BIN_SUBDIR, RootDir, Version);
+
     read_bootflags();
     
 #ifdef _DEBUG
     fprintf(stderr, "RelDir: '%s'\n", RelDir);
+    fprintf(stderr, "BinDir: '%s'\n", BinDir);
 #endif
 }
 
diff --git a/erts/etc/win32/win_erlexec.c b/erts/etc/win32/win_erlexec.c
index 0eed8e28b9..11cc6a30f7 100644
--- a/erts/etc/win32/win_erlexec.c
+++ b/erts/etc/win32/win_erlexec.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 1997-2009. All Rights Reserved.
+ * Copyright Ericsson AB 1997-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -30,6 +30,9 @@
 #include <winuser.h>
 #include <wincon.h>
 #include <process.h>
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
 #include "sys.h"
 #include "erl_driver.h"
 
diff --git a/erts/example/matrix_nif.c b/erts/example/matrix_nif.c
index c5e01dade5..404329e36c 100644
--- a/erts/example/matrix_nif.c
+++ b/erts/example/matrix_nif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -31,7 +31,19 @@ typedef struct
     unsigned nrows;
     unsigned ncols;
     double* data;
-}Matrix;
+} Matrix;
+
+/*
+ * Use a union for pointer type conversion to avoid compiler warnings
+ * about strict-aliasing violations with gcc-4.1. gcc >= 4.2 does not
+ * emit the warning.
+ * TODO: Reconsider use of union once gcc-4.1 is obsolete?
+ */
+typedef union
+{
+    void* vp;
+    Matrix* p;
+} mx_t;
 
 #define POS(MX, ROW, COL) ((MX)->data[(ROW)* (MX)->ncols + (COL)])
 
@@ -44,8 +56,9 @@ static ErlNifResourceType* resource_type = NULL;
 
 static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
 {
-    ErlNifResourceType* rt = enif_open_resource_type(env, "matrix_nif_example",
-						     matrix_dtor, 
+    ErlNifResourceType* rt = enif_open_resource_type(env, NULL,
+						     "matrix_nif_example",
+						     matrix_dtor,
 						     ERL_NIF_RT_CREATE, NULL);
     if (rt == NULL) {
 	return -1;
@@ -90,12 +103,12 @@ static ERL_NIF_TERM create(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     }
 
     ret = enif_make_resource(env, mx);
-    enif_release_resource(env, mx);
+    enif_release_resource(mx);
     return ret;
 
 badarg:
     if (mx != NULL) {
-	enif_release_resource(env,mx);
+	enif_release_resource(mx);
     }
     return enif_make_badarg(env);
 }
@@ -104,14 +117,14 @@ badarg:
 static ERL_NIF_TERM pos(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     /* pos(Matrix, Row, Column) -> float() */
-    Matrix* mx;
+    mx_t mx;
     unsigned i, j;
-    if (!enif_get_resource(env, argv[0], resource_type, (void**)&mx) ||
-	!enif_get_uint(env, argv[1], &i) || (--i >= mx->nrows) ||
-	!enif_get_uint(env, argv[2], &j) || (--j >= mx->ncols)) {
+    if (!enif_get_resource(env, argv[0], resource_type, &mx.vp) ||
+	!enif_get_uint(env, argv[1], &i) || (--i >= mx.p->nrows) ||
+	!enif_get_uint(env, argv[2], &j) || (--j >= mx.p->ncols)) {
 	return enif_make_badarg(env);
     }
-    return enif_make_double(env, POS(mx, i,j));
+    return enif_make_double(env, POS(mx.p, i,j));
 }
 
 static ERL_NIF_TERM add(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
@@ -119,37 +132,38 @@ static ERL_NIF_TERM add(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     /* add(Matrix_A, Matrix_B) -> Matrix_Sum */
     unsigned i, j;
     ERL_NIF_TERM ret;
-    Matrix* mxA = NULL;
-    Matrix* mxB = NULL;
-    Matrix* mxS = NULL;
+    mx_t mxA, mxB, mxS;
+    mxA.p = NULL;
+    mxB.p = NULL;
+    mxS.p = NULL;
 
-    if (!enif_get_resource(env, argv[0], resource_type, (void**)&mxA) ||
-	!enif_get_resource(env, argv[1], resource_type, (void**)&mxB) ||
-	mxA->nrows != mxB->nrows ||
-	mxB->ncols != mxB->ncols) {
+    if (!enif_get_resource(env, argv[0], resource_type, &mxA.vp) ||
+	!enif_get_resource(env, argv[1], resource_type, &mxB.vp) ||
+	mxA.p->nrows != mxB.p->nrows ||
+	mxB.p->ncols != mxB.p->ncols) {
 
     	return enif_make_badarg(env);
     }
-    mxS = alloc_matrix(env, mxA->nrows, mxA->ncols);
-    for (i = 0; i < mxA->nrows; i++) {
-	for (j = 0; j < mxA->ncols; j++) {
-	    POS(mxS, i, j) = POS(mxA, i, j) + POS(mxB, i, j);
+    mxS.p = alloc_matrix(env, mxA.p->nrows, mxA.p->ncols);
+    for (i = 0; i < mxA.p->nrows; i++) {
+	for (j = 0; j < mxA.p->ncols; j++) {
+	    POS(mxS.p, i, j) = POS(mxA.p, i, j) + POS(mxB.p, i, j);
 	}
     }
-    ret = enif_make_resource(env, mxS);
-    enif_release_resource(env, mxS);
+    ret = enif_make_resource(env, mxS.p);
+    enif_release_resource(mxS.p);
     return ret;
 }
 
 static ERL_NIF_TERM size_of(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     /* size(Matrix) -> {Nrows, Ncols} */
-    Matrix* mx;
-    if (!enif_get_resource(env, argv[0], resource_type, (void**)&mx)) {
+    mx_t mx;
+    if (!enif_get_resource(env, argv[0], resource_type, &mx.vp)) {
 	return enif_make_badarg(env);
     }
-    return enif_make_tuple2(env, enif_make_uint(env, mx->nrows),
-			    enif_make_uint(env, mx->ncols));
+    return enif_make_tuple2(env, enif_make_uint(env, mx.p->nrows),
+			    enif_make_uint(env, mx.p->ncols));
 }
 
 static ERL_NIF_TERM to_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
@@ -157,16 +171,17 @@ static ERL_NIF_TERM to_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     /* to_term(Matrix) -> [[first row], [second row], ...,[last row]] */
     unsigned i, j;
     ERL_NIF_TERM res;
-    Matrix* mx = NULL;
+    mx_t mx;
+    mx.p = NULL;
 
-    if (!enif_get_resource(env, argv[0], resource_type, (void**)&mx)) {	
+    if (!enif_get_resource(env, argv[0], resource_type, &mx.vp)) {
     	return enif_make_badarg(env);
     }
     res = enif_make_list(env, 0);
-    for (i = mx->nrows; i-- > 0; ) {
+    for (i = mx.p->nrows; i-- > 0; ) {
 	ERL_NIF_TERM row = enif_make_list(env, 0);
-	for (j = mx->ncols; j-- > 0; ) {
-	    row = enif_make_list_cell(env, enif_make_double(env, POS(mx,i,j)),
+	for (j = mx.p->ncols; j-- > 0; ) {
+	    row = enif_make_list_cell(env, enif_make_double(env, POS(mx.p,i,j)),
 				      row);
 	}
 	res = enif_make_list_cell(env, row, res);
@@ -183,17 +198,17 @@ static int get_number(ErlNifEnv* env, ERL_NIF_TERM term, double* dp)
 
 static Matrix* alloc_matrix(ErlNifEnv* env, unsigned nrows, unsigned ncols)
 {
-    Matrix* mx = enif_alloc_resource(env, resource_type, sizeof(Matrix));
+    Matrix* mx = enif_alloc_resource(resource_type, sizeof(Matrix));
     mx->nrows = nrows;
     mx->ncols = ncols;
-    mx->data = enif_alloc(env, nrows*ncols*sizeof(double));
+    mx->data = enif_alloc(nrows*ncols*sizeof(double));
     return mx;
 }
 
 static void matrix_dtor(ErlNifEnv* env, void* obj)
 {
     Matrix* mx = (Matrix*) obj;
-    enif_free(env, mx->data);
+    enif_free(mx->data);
     mx->data = NULL;
 }
 
diff --git a/erts/example/next_perm.cc b/erts/example/next_perm.cc
index ee81cb0404..d7c8f1ad97 100644
--- a/erts/example/next_perm.cc
+++ b/erts/example/next_perm.cc
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -120,7 +120,7 @@ static void ready_async(ErlDrvData drv_data, ErlDrvThreadData async_data)
     ErlDrvPort port = reinterpret_cast<ErlDrvPort>(drv_data);
     our_async_data* d = reinterpret_cast<our_async_data*>(async_data);
     int n = d->data.size(), result_n = n*2 + 5;
-    ErlDrvTermData* result = new ErlDrvTermData[result_n], * rp = result;
+    ErlDrvTermData *result = new ErlDrvTermData[result_n], *rp = result;
     *rp++ = ERL_DRV_PORT;
     *rp++ = driver_mk_port(port);
     for (vector<int>::iterator i = d->data.begin();
diff --git a/erts/include/erl_int_sizes_config.h.in b/erts/include/erl_int_sizes_config.h.in
index ef49995732..14d7c6a702 100644
--- a/erts/include/erl_int_sizes_config.h.in
+++ b/erts/include/erl_int_sizes_config.h.in
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -31,3 +31,9 @@
 
 /* The number of bytes in a long long.  */
 #undef SIZEOF_LONG_LONG
+
+/* The size of a pointer. */
+#undef SIZEOF_VOID_P
+
+/* Define if building a halfword-heap 64bit emulator (needed for NIF's) */
+#undef HALFWORD_HEAP_EMULATOR
diff --git a/erts/include/internal/erl_misc_utils.h b/erts/include/internal/erl_misc_utils.h
index 82e9ba3798..507e1726f4 100644
--- a/erts/include/internal/erl_misc_utils.h
+++ b/erts/include/internal/erl_misc_utils.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2010. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -34,7 +34,7 @@ typedef struct {
 
 erts_cpu_info_t *erts_cpu_info_create(void);
 void erts_cpu_info_destroy(erts_cpu_info_t *cpuinfo);
-void erts_cpu_info_update(erts_cpu_info_t *cpuinfo);
+int erts_cpu_info_update(erts_cpu_info_t *cpuinfo);
 int erts_get_cpu_configured(erts_cpu_info_t *cpuinfo);
 int erts_get_cpu_online(erts_cpu_info_t *cpuinfo);
 int erts_get_cpu_available(erts_cpu_info_t *cpuinfo);
@@ -50,4 +50,9 @@ int erts_unbind_from_cpu_str(char *str);
 
 int erts_milli_sleep(long);
 
+#ifdef __WIN32__
+int erts_map_win_error_to_errno(DWORD win_error);
+int erts_get_last_win_errno(void);
+#endif
+
 #endif /* #ifndef ERL_MISC_UTILS_H_ */
diff --git a/erts/include/internal/erl_printf_format.h b/erts/include/internal/erl_printf_format.h
index 45818079ea..064c4a5c09 100644
--- a/erts/include/internal/erl_printf_format.h
+++ b/erts/include/internal/erl_printf_format.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -40,7 +40,7 @@ extern int erts_printf_ulong(fmtfn_t, void*, char, int, int, unsigned long);
 extern int erts_printf_slong(fmtfn_t, void*, char, int, int, signed long);
 extern int erts_printf_double(fmtfn_t, void *, char, int, int, double);
 
-extern int (*erts_printf_eterm_func)(fmtfn_t, void*, unsigned long, long);
+extern int (*erts_printf_eterm_func)(fmtfn_t, void*, unsigned long, long, unsigned long*);
 
 #endif
 
diff --git a/erts/include/internal/ethr_atomics.h b/erts/include/internal/ethr_atomics.h
new file mode 100644
index 0000000000..612894b8c1
--- /dev/null
+++ b/erts/include/internal/ethr_atomics.h
@@ -0,0 +1,9167 @@
+/*
+ * --------------- DO NOT EDIT THIS FILE! ---------------
+ * This file was automatically generated by the
+ * $ERL_TOP/erts/lib_src/utils/make_atomics_api script.
+ * If you need to make changes, edit the script and
+ * regenerate this file.
+ * --------------- DO NOT EDIT THIS FILE! ---------------
+ */
+
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: The ethread atomics API
+ * Author: Rickard Green
+ */
+
+/*
+ * This file maps native atomic implementations to ethread
+ * API atomics. If no native atomic implementation
+ * is available, a less efficient fallback is used instead.
+ * The API consists of 32-bit size, word size (pointer size),
+ * and double word size atomics.
+ *
+ * The following atomic operations are implemented for
+ * 32-bit size, and word size atomics:
+ * - cmpxchg
+ * - xchg
+ * - set
+ * - init
+ * - add_read
+ * - read
+ * - inc_read
+ * - dec_read
+ * - add
+ * - inc
+ * - dec
+ * - read_band
+ * - read_bor
+ *
+ * The following atomic operations are implemented for
+ * double word size atomics:
+ * - cmpxchg
+ * - set
+ * - read
+ * - init
+ *
+ * Appart from a function implementing the atomic operation
+ * with unspecified memory barrier semantics, there are
+ * functions implementing each operation with the following
+ * implied memory barrier semantics:
+ * - mb   - Full memory barrier. Orders both loads, and
+ *          stores before, and after the atomic operation.
+ *          No load or store is allowed to be reordered
+ *          over the atomic operation.
+ * - relb - Release barrier. Orders both loads, and
+ *          stores appearing *before* the atomic
+ *          operation. These are not allowed to be
+ *          reordered over the atomic operation.
+ * - acqb - Acquire barrier. Orders both loads, and stores
+ *          appearing *after* the atomic operation. These
+ *          are not allowed to be reordered over the
+ *          atomic operation.
+ * - wb   - Write barrier. Orders *only* stores. These are
+ *          not allowed to be reordered over the barrier.
+ *          Store in atomic operation is ordered *after*
+ *          the barrier.
+ * - rb   - Read barrier. Orders *only* loads. These are
+ *          not allowed to be reordered over the barrier.
+ *          Load in atomic operation is ordered *before*
+ *          the barrier. 
+ * - ddrb - Data dependency read barrier. Orders *only*
+ *          loads according to data dependency across the
+ *          barrier. Load in atomic operation is ordered
+ *          before the barrier.
+ *
+ * We implement all of these operation/barrier
+ * combinations, regardless of whether they are useful
+ * or not (some of them are useless).
+ *
+ * Double word size atomic functions are on the followning
+ * form:
+ *   ethr_dw_atomic_<OP>[_<BARRIER>]
+ *
+ * Word size atomic functions are on the followning
+ * form:
+ *   ethr_atomic_<OP>[_<BARRIER>]
+ *
+ * 32-bit size atomic functions are on the followning
+ * form:
+ *   ethr_atomic32_<OP>[_<BARRIER>]
+ *
+ * Apart from the operation/barrier functions
+ * described above also 'addr' functions are implemented
+ * which return the actual memory address used of the
+ * atomic variable. The 'addr' functions have no barrier
+ * versions.
+ *
+ * The native atomic implementation does not need to
+ * implement all operation/barrier combinations.
+ * Functions that have no native implementation will be
+ * constructed from existing native functionality. These
+ * functions will perform the wanted operation and will
+ * produce sufficient memory barriers, but may
+ * in some cases be less efficient than pure native
+ * versions.
+ *
+ * When we create ethread API operation/barrier functions by
+ * adding barriers before and after native operations it is
+ * assumed that:
+ * - A native read operation begins, and ends with a load.
+ * - A native set operation begins, and ends with a store.
+ * - An init operation begins with either a load, or a store,
+ *   and ends with either a load, or a store.
+ * - All other operations begins with a load, and ends with
+ *   either a load, or a store.
+ *
+ * This is the minimum functionality that a native
+ * implementation needs to provide:
+ *
+ * - Functions that need to be implemented:
+ *
+ *   - ethr_native_[dw_|su_dw_]atomic[BITS]_addr
+ *   - ethr_native_[dw_|su_dw_]atomic[BITS]_cmpxchg[_<BARRIER>]
+ *     (at least one cmpxchg of optional barrier)
+ *
+ * - Macros that needs to be defined:
+ *
+ *   A macro informing about the presence of the native
+ *   implementation:
+ *
+ *   - ETHR_HAVE_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]
+ *
+ *   A macro naming (a string constant) the implementation:
+ *
+ *   - ETHR_NATIVE_[DW_]ATOMIC[BITS]_IMPL
+ *
+ *   Each implemented native atomic function has to
+ *   be accompanied by a defined macro on the following
+ *   form informing about its presence:
+ *
+ *   - ETHR_HAVE_ETHR_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]_<OP>[_<BARRIER>]
+ *
+ *   A (sparc-v9 style) membar macro:
+ *
+ *   - ETHR_MEMBAR(B)
+ *
+ *     Which takes a combination of the following macros
+ *     or:ed (using |) together:
+ *
+ *     - ETHR_LoadLoad
+ *     - ETHR_LoadStore
+ *     - ETHR_StoreLoad
+ *     - ETHR_StoreStore
+ *
+ */
+
+#ifndef ETHR_ATOMICS_H__
+#define ETHR_ATOMICS_H__
+
+#undef ETHR_AMC_FALLBACK__
+#undef ETHR_AMC_NO_ATMCS__
+#undef ETHR_AMC_ATMC_T__
+#undef ETHR_AMC_ATMC_FUNC__
+
+/* -- 32-bit atomics -- */
+
+#undef ETHR_NAINT32_T__
+#undef ETHR_NATMC32_FUNC__
+#undef ETHR_NATMC32_ADDR_FUNC__
+#undef ETHR_NATMC32_BITS__
+#if defined(ETHR_HAVE_NATIVE_ATOMIC32)
+#    define ETHR_NEED_NATMC32_ADDR
+#  define ETHR_NATMC32_ADDR_FUNC__ ethr_native_atomic32_addr
+typedef ethr_native_atomic32_t ethr_atomic32_t;
+#  define ETHR_NAINT32_T__ ethr_sint32_t
+#  define ETHR_NATMC32_FUNC__(X) ethr_native_atomic32_ ## X
+#  define ETHR_NATMC32_BITS__ 32
+#elif defined(ETHR_HAVE_NATIVE_ATOMIC64)
+#  define ETHR_NEED_NATMC64_ADDR
+#ifdef ETHR_BIGENDIAN
+#  define ETHR_NATMC32_ADDR_FUNC__(VAR) \
+  (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1)
+#else
+#  define ETHR_NATMC32_ADDR_FUNC__(VAR) \
+  ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR)))
+#endif
+typedef ethr_native_atomic64_t ethr_atomic32_t;
+#  define ETHR_NAINT32_T__ ethr_sint64_t
+#  define ETHR_NATMC32_FUNC__(X) ethr_native_atomic64_ ## X
+#  define ETHR_NATMC32_BITS__ 64
+#else
+/*
+ * No native atomics usable for 32-bits atomics :(
+ * Use fallback...
+ */
+typedef ethr_sint32_t ethr_atomic32_t;
+#endif
+
+#undef ETHR_ATMC32_INLINE__
+#ifdef ETHR_NATMC32_BITS__
+#  ifdef ETHR_TRY_INLINE_FUNCS
+#    define ETHR_ATMC32_INLINE__
+#  endif
+#  define ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS
+#endif
+
+#if !defined(ETHR_ATMC32_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
+#  define ETHR_NEED_ATMC32_PROTOTYPES__
+#endif
+
+#ifndef ETHR_INLINE_ATMC32_FUNC_NAME_
+#  define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X
+#endif
+
+#undef ETHR_ATMC32_FUNC__
+#define ETHR_ATMC32_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
+
+
+/* -- Word size atomics -- */
+
+#undef ETHR_NEED_NATMC32_ADDR
+#undef ETHR_NEED_NATMC64_ADDR
+
+#undef ETHR_NAINT_T__
+#undef ETHR_NATMC_FUNC__
+#undef ETHR_NATMC_ADDR_FUNC__
+#undef ETHR_NATMC_BITS__
+#if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_NATIVE_ATOMIC64)
+#  ifndef ETHR_NEED_NATMC64_ADDR
+#    define ETHR_NEED_NATMC64_ADDR
+#  endif
+#  define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic64_addr
+typedef ethr_native_atomic64_t ethr_atomic_t;
+#  define ETHR_NAINT_T__ ethr_sint64_t
+#  define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#  define ETHR_NATMC_BITS__ 64
+#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC32)
+#  ifndef ETHR_NEED_NATMC64_ADDR
+#    define ETHR_NEED_NATMC32_ADDR
+#  endif
+#  define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic32_addr
+typedef ethr_native_atomic32_t ethr_atomic_t;
+#  define ETHR_NAINT_T__ ethr_sint32_t
+#  define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+#  define ETHR_NATMC_BITS__ 32
+#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64)
+#  ifndef ETHR_NEED_NATMC64_ADDR
+#    define ETHR_NEED_NATMC64_ADDR
+#  endif
+#ifdef ETHR_BIGENDIAN
+#  define ETHR_NATMC_ADDR_FUNC__(VAR) \
+  (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1)
+#else
+#  define ETHR_NATMC_ADDR_FUNC__(VAR) \
+  ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR)))
+#endif
+typedef ethr_native_atomic64_t ethr_atomic_t;
+#  define ETHR_NATMC_T__ ethr_native_atomic64_t
+#  define ETHR_NAINT_T__ ethr_sint64_t
+#  define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#  define ETHR_NATMC_BITS__ 64
+#else
+/*
+ * No native atomics usable for pointer size atomics :(
+ * Use fallback...
+ */
+
+#  if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+#    define ETHR_AMC_FALLBACK__
+#    define ETHR_AMC_NO_ATMCS__ 2
+#    define ETHR_AMC_SINT_T__ ethr_sint32_t
+#    define ETHR_AMC_ATMC_T__ ethr_atomic32_t
+#    define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
+typedef struct {
+    ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__];
+} ethr_amc_t;
+typedef struct {
+    ethr_amc_t amc;
+    ethr_sint_t sint;
+} ethr_atomic_t;
+#  else /* locked fallback */
+typedef ethr_sint_t ethr_atomic_t;
+#  endif
+#endif
+
+#undef ETHR_ATMC_INLINE__
+#ifdef ETHR_NATMC_BITS__
+#  ifdef ETHR_TRY_INLINE_FUNCS
+#    define ETHR_ATMC_INLINE__
+#  endif
+#  define ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS
+#endif
+
+#if !defined(ETHR_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
+#  define ETHR_NEED_ATMC_PROTOTYPES__
+#endif
+
+#ifndef ETHR_INLINE_ATMC_FUNC_NAME_
+#  define ETHR_INLINE_ATMC_FUNC_NAME_(X) X
+#endif
+
+#undef ETHR_ATMC_FUNC__
+#define ETHR_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X)
+
+/* -- Double word atomics -- */
+
+#undef ETHR_SU_DW_NAINT_T__
+#undef ETHR_SU_DW_NATMC_FUNC__
+#undef ETHR_SU_DW_NATMC_ADDR_FUNC__
+#undef ETHR_DW_NATMC_FUNC__
+#undef ETHR_DW_NATMC_ADDR_FUNC__
+#undef ETHR_DW_NATMC_BITS__
+#if defined(ETHR_HAVE_NATIVE_DW_ATOMIC) || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)
+#  define ETHR_NEED_DW_NATMC_ADDR
+#  define ETHR_DW_NATMC_ADDR_FUNC__ ethr_native_dw_atomic_addr
+#  define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_dw_atomic_t
+#  define ETHR_DW_NATMC_FUNC__(X) ethr_native_dw_atomic_ ## X
+#  define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_su_dw_atomic_ ## X
+#  if ETHR_SIZEOF_PTR == 8
+#    define ETHR_DW_NATMC_BITS__ 128
+#  elif ETHR_SIZEOF_PTR == 4
+#    define ETHR_DW_NATMC_BITS__ 64
+#  else
+#    error "Word size not supported"
+#  endif
+#  ifdef ETHR_NATIVE_SU_DW_SINT_T
+#    define ETHR_SU_DW_NAINT_T__ ETHR_NATIVE_SU_DW_SINT_T
+#  endif
+#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64)
+#  define ETHR_HAVE_NATIVE_SU_DW_ATOMIC
+#  ifndef ETHR_NEED_NATMC64_ADDR
+#    define ETHR_NEED_NATMC64_ADDR
+#  endif
+#  define ETHR_DW_NATMC_ADDR_FUNC__(VAR) \
+  ((ethr_dw_sint_t *) ethr_native_atomic64_addr((VAR)))
+#  define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_atomic64_t
+#  define ETHR_SU_DW_NAINT_T__ ethr_sint64_t
+#  define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#  define ETHR_DW_NATMC_BITS__ 64
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+#define ETHR_DW_ATOMIC_FUNC__(X) ethr_dw_atomic_ ## X ## _fallback__
+#else
+#define ETHR_DW_ATOMIC_FUNC__(X) ethr_dw_atomic_ ## X
+#endif
+
+#if !defined(ETHR_DW_NATMC_BITS__) || defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+#  define ETHR_NEED_DW_FALLBACK__
+#endif
+
+#if defined(ETHR_NEED_DW_FALLBACK__)
+/*
+ * No native atomics usable for double word atomics :(
+ * Use fallback...
+ */
+
+#  ifndef ETHR_AMC_FALLBACK__
+#    if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+#      define ETHR_AMC_FALLBACK__
+#      define ETHR_AMC_NO_ATMCS__ 1
+#      define ETHR_AMC_SINT_T__ ethr_sint_t
+#      define ETHR_AMC_ATMC_T__ ethr_atomic_t
+#      define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X)
+#    elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+#      define ETHR_AMC_FALLBACK__
+#      define ETHR_AMC_NO_ATMCS__ 2
+#      define ETHR_AMC_SINT_T__ ethr_sint32_t
+#      define ETHR_AMC_ATMC_T__ ethr_atomic32_t
+#      define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
+#    endif
+#    ifdef ETHR_AMC_FALLBACK__
+typedef struct {
+    ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__];
+} ethr_amc_t;
+#    endif
+#  endif
+
+typedef struct {
+#ifdef ETHR_AMC_FALLBACK__
+    ethr_amc_t amc;
+#endif
+    ethr_sint_t sint[2];
+} ethr_dw_atomic_fallback_t;
+
+#endif
+
+typedef union {
+#ifdef ETHR_NATIVE_DW_ATOMIC_T__
+    ETHR_NATIVE_DW_ATOMIC_T__ native;
+#endif
+#ifdef ETHR_NEED_DW_FALLBACK__
+    ethr_dw_atomic_fallback_t fallback;
+#endif
+    ethr_sint_t sint[2];
+} ethr_dw_atomic_t;
+
+typedef union {
+#ifdef ETHR_SU_DW_NAINT_T__
+    ETHR_SU_DW_NAINT_T__ dw_sint;
+#endif
+    ethr_sint_t sint[2];
+} ethr_dw_sint_t;
+
+#ifdef ETHR_BIGENDIAN
+#  define ETHR_DW_SINT_LOW_WORD 1
+#  define ETHR_DW_SINT_HIGH_WORD 0
+#else
+#  define ETHR_DW_SINT_LOW_WORD 0
+#  define ETHR_DW_SINT_HIGH_WORD 1
+#endif
+
+#undef ETHR_DW_ATMC_INLINE__
+#ifdef ETHR_DW_NATMC_BITS__
+#  ifdef ETHR_TRY_INLINE_FUNCS
+#    define ETHR_ATMC32_INLINE__
+#  endif
+#  define ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS
+#endif
+
+#if !defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
+#  define ETHR_NEED_DW_ATMC_PROTOTYPES__
+#endif
+
+#ifndef ETHR_INLINE_DW_ATMC_FUNC_NAME_
+#  define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X
+#endif
+
+#undef ETHR_DW_ATMC_FUNC__
+#define ETHR_DW_ATMC_FUNC__(X) ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_dw_atomic_ ## X)
+
+#if defined(ETHR_NEED_DW_ATMC_PROTOTYPES__)
+int ethr_have_native_dw_atomic(void);
+#endif
+#if defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
+static ETHR_INLINE int
+ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_have_native_dw_atomic)(void)
+{
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    return ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__;
+#elif defined(ETHR_DW_NATMC_BITS__)
+    return 1;
+#else
+    return 0;
+#endif
+}
+#endif
+
+/* -- Misc -- */
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+/*
+ * Unusual values are used by read() fallbacks implemented via cmpxchg().
+ * We want to use an unusual value in hope that it is more efficient
+ * not to match the value in memory.
+ *
+ * - Negative integer values are probably more unusual.
+ * - Very large absolute integer values are probably more unusual.
+ * - Odd pointers are probably more unusual (only char pointers can be odd).
+ */
+#  define ETHR_UNUSUAL_SINT32_VAL__ ((ethr_sint32_t) 0x81818181)
+#  if ETHR_SIZEOF_PTR == 4
+#    define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) ETHR_UNUSUAL_SINT32_VAL__)
+#  elif ETHR_SIZEOF_PTR == 8
+#    define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) 0x8181818181818181L)
+#  else
+#    error "Word size not supported"
+#  endif
+#  if defined(ETHR_NEED_DW_NATMC_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR)
+#    error "No ethr_native_dw_atomic_addr() available"
+#  endif
+#  if defined(ETHR_NEED_NATMC32_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR)
+#    error "No ethr_native_atomic32_addr() available"
+#  endif
+#  if defined(ETHR_NEED_NATMC64_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR)
+#    error "No ethr_native_atomic64_addr() available"
+#  endif
+#endif
+
+#if defined(__GNUC__)
+#  ifndef ETHR_COMPILER_BARRIER
+#    define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory")
+#  endif
+#elif defined(ETHR_WIN32_THREADS)
+#  ifndef ETHR_COMPILER_BARRIER
+#    include <intrin.h>
+#    pragma intrinsic(_ReadWriteBarrier)
+#    define ETHR_COMPILER_BARRIER _ReadWriteBarrier()
+#  endif
+#endif
+
+void ethr_compiler_barrier_fallback(void);
+#ifndef ETHR_COMPILER_BARRIER
+#  define ETHR_COMPILER_BARRIER ethr_compiler_barrier_fallback()
+#endif
+
+int ethr_init_atomics(void);
+
+/* info */
+char **ethr_native_atomic32_ops(void);
+char **ethr_native_atomic64_ops(void);
+char **ethr_native_dw_atomic_ops(void);
+char **ethr_native_su_dw_atomic_ops(void);
+
+#if !defined(ETHR_DW_NATMC_BITS__) && !defined(ETHR_NATMC_BITS__) && !defined(ETHR_NATMC32_BITS__)
+/*
+ * ETHR_*MEMORY_BARRIER orders between locked and atomic accesses only,
+ * i.e. when no native atomic implementation exist and only our lock
+ * based atomic fallback is used, a noop is sufficient.
+ */
+#  undef ETHR_MEMORY_BARRIER
+#  undef ETHR_WRITE_MEMORY_BARRIER
+#  undef ETHR_READ_MEMORY_BARRIER
+#  undef ETHR_READ_DEPEND_MEMORY_BARRIER
+#  undef ETHR_MEMBAR
+#  define ETHR_MEMBAR(B) do { } while (0)
+#endif
+
+#ifndef ETHR_MEMBAR
+#  error "No ETHR_MEMBAR defined"
+#endif
+
+#define ETHR_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore)
+#define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMBAR(ETHR_StoreStore)
+#define ETHR_READ_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad)
+#ifdef ETHR_READ_DEPEND_MEMORY_BARRIER
+#  undef ETHR_ORDERED_READ_DEPEND
+#else
+#  define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER
+#  define ETHR_ORDERED_READ_DEPEND
+#endif
+
+
+/* ---------- Double word size atomic implementation ---------- */
+
+
+#ifdef ETHR_NEED_DW_ATMC_PROTOTYPES__
+ethr_sint_t *ethr_dw_atomic_addr(ethr_dw_atomic_t *var);
+int ethr_dw_atomic_cmpxchg(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ethr_dw_atomic_cmpxchg_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ethr_dw_atomic_cmpxchg_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ethr_dw_atomic_cmpxchg_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ethr_dw_atomic_cmpxchg_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ethr_dw_atomic_cmpxchg_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ethr_dw_atomic_cmpxchg_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+void ethr_dw_atomic_set(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_set_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_set_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_set_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_set_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_set_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_set_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_read(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_read_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_read_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_read_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_read_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_read_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_read_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_init(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_init_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_init_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_init_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_init_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_init_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_init_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+ethr_sint_t *ETHR_DW_ATOMIC_FUNC__(addr)(ethr_dw_atomic_t *var);
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+void ETHR_DW_ATOMIC_FUNC__(set)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(set_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(set_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(set_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(set_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(read)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(read_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(read_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(read_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(read_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(init)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(init_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(init_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(init_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(init_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+#endif
+#endif /* ETHR_NEED_DW_ATMC_PROTOTYPES__ */
+
+#if (defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) \
+     && (defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)))
+
+#if !defined(ETHR_DW_NATMC_BITS__)
+#  error "Missing native atomic implementation"
+#elif defined(ETHR_HAVE_NATIVE_DW_ATOMIC)  || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)
+#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG
+#  undef ETHR_HAVE_DW_NATMC_CMPXCHG
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG
+#    define ETHR_HAVE_DW_NATMC_CMPXCHG 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG
+#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB
+#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RB
+#    define ETHR_HAVE_DW_NATMC_CMPXCHG_RB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RB
+#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB
+#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_WB
+#    define ETHR_HAVE_DW_NATMC_CMPXCHG_WB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_WB
+#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB
+#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_ACQB
+#    define ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_ACQB
+#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB
+#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RELB
+#    define ETHR_HAVE_DW_NATMC_CMPXCHG_RELB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RELB
+#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB
+#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB
+#    define ETHR_HAVE_DW_NATMC_CMPXCHG_MB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB
+#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_SET
+#  undef ETHR_HAVE_DW_NATMC_SET
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET
+#    define ETHR_HAVE_DW_NATMC_SET 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET
+#    define ETHR_HAVE_SU_DW_NATMC_SET 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_SET_RB
+#  undef ETHR_HAVE_DW_NATMC_SET_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RB
+#    define ETHR_HAVE_DW_NATMC_SET_RB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RB
+#    define ETHR_HAVE_SU_DW_NATMC_SET_RB 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_SET_WB
+#  undef ETHR_HAVE_DW_NATMC_SET_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_WB
+#    define ETHR_HAVE_DW_NATMC_SET_WB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_WB
+#    define ETHR_HAVE_SU_DW_NATMC_SET_WB 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_SET_ACQB
+#  undef ETHR_HAVE_DW_NATMC_SET_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_ACQB
+#    define ETHR_HAVE_DW_NATMC_SET_ACQB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_ACQB
+#    define ETHR_HAVE_SU_DW_NATMC_SET_ACQB 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_SET_RELB
+#  undef ETHR_HAVE_DW_NATMC_SET_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RELB
+#    define ETHR_HAVE_DW_NATMC_SET_RELB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RELB
+#    define ETHR_HAVE_SU_DW_NATMC_SET_RELB 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_SET_MB
+#  undef ETHR_HAVE_DW_NATMC_SET_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_MB
+#    define ETHR_HAVE_DW_NATMC_SET_MB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_MB
+#    define ETHR_HAVE_SU_DW_NATMC_SET_MB 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_READ
+#  undef ETHR_HAVE_DW_NATMC_READ
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ
+#    define ETHR_HAVE_DW_NATMC_READ 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ
+#    define ETHR_HAVE_SU_DW_NATMC_READ 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_READ_RB
+#  undef ETHR_HAVE_DW_NATMC_READ_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RB
+#    define ETHR_HAVE_DW_NATMC_READ_RB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RB
+#    define ETHR_HAVE_SU_DW_NATMC_READ_RB 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_READ_WB
+#  undef ETHR_HAVE_DW_NATMC_READ_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_WB
+#    define ETHR_HAVE_DW_NATMC_READ_WB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_WB
+#    define ETHR_HAVE_SU_DW_NATMC_READ_WB 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_READ_ACQB
+#  undef ETHR_HAVE_DW_NATMC_READ_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_ACQB
+#    define ETHR_HAVE_DW_NATMC_READ_ACQB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_ACQB
+#    define ETHR_HAVE_SU_DW_NATMC_READ_ACQB 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_READ_RELB
+#  undef ETHR_HAVE_DW_NATMC_READ_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RELB
+#    define ETHR_HAVE_DW_NATMC_READ_RELB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RELB
+#    define ETHR_HAVE_SU_DW_NATMC_READ_RELB 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_READ_MB
+#  undef ETHR_HAVE_DW_NATMC_READ_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_MB
+#    define ETHR_HAVE_DW_NATMC_READ_MB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_MB
+#    define ETHR_HAVE_SU_DW_NATMC_READ_MB 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_INIT
+#  undef ETHR_HAVE_DW_NATMC_INIT
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT
+#    define ETHR_HAVE_DW_NATMC_INIT 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT
+#    define ETHR_HAVE_SU_DW_NATMC_INIT 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_INIT_RB
+#  undef ETHR_HAVE_DW_NATMC_INIT_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RB
+#    define ETHR_HAVE_DW_NATMC_INIT_RB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RB
+#    define ETHR_HAVE_SU_DW_NATMC_INIT_RB 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_INIT_WB
+#  undef ETHR_HAVE_DW_NATMC_INIT_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_WB
+#    define ETHR_HAVE_DW_NATMC_INIT_WB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_WB
+#    define ETHR_HAVE_SU_DW_NATMC_INIT_WB 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_INIT_ACQB
+#  undef ETHR_HAVE_DW_NATMC_INIT_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_ACQB
+#    define ETHR_HAVE_DW_NATMC_INIT_ACQB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_ACQB
+#    define ETHR_HAVE_SU_DW_NATMC_INIT_ACQB 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_INIT_RELB
+#  undef ETHR_HAVE_DW_NATMC_INIT_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RELB
+#    define ETHR_HAVE_DW_NATMC_INIT_RELB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RELB
+#    define ETHR_HAVE_SU_DW_NATMC_INIT_RELB 1
+#  endif
+#  undef ETHR_HAVE_SU_DW_NATMC_INIT_MB
+#  undef ETHR_HAVE_DW_NATMC_INIT_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_MB
+#    define ETHR_HAVE_DW_NATMC_INIT_MB 1
+#  endif
+#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_MB
+#    define ETHR_HAVE_SU_DW_NATMC_INIT_MB 1
+#  endif
+#elif ETHR_DW_NATMC_BITS__ == 64
+#  undef ETHR_HAVE_DW_NATMC_CMPXCHG
+#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG
+#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_RB
+#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB
+#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_WB
+#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB
+#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB
+#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB
+#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_RELB
+#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB
+#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_MB
+#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB
+#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_SET
+#  undef ETHR_HAVE_SU_DW_NATMC_SET
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET
+#    define ETHR_HAVE_SU_DW_NATMC_SET 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_SET_RB
+#  undef ETHR_HAVE_SU_DW_NATMC_SET_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB
+#    define ETHR_HAVE_SU_DW_NATMC_SET_RB 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_SET_WB
+#  undef ETHR_HAVE_SU_DW_NATMC_SET_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB
+#    define ETHR_HAVE_SU_DW_NATMC_SET_WB 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_SET_ACQB
+#  undef ETHR_HAVE_SU_DW_NATMC_SET_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB
+#    define ETHR_HAVE_SU_DW_NATMC_SET_ACQB 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_SET_RELB
+#  undef ETHR_HAVE_SU_DW_NATMC_SET_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB
+#    define ETHR_HAVE_SU_DW_NATMC_SET_RELB 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_SET_MB
+#  undef ETHR_HAVE_SU_DW_NATMC_SET_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB
+#    define ETHR_HAVE_SU_DW_NATMC_SET_MB 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_READ
+#  undef ETHR_HAVE_SU_DW_NATMC_READ
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ
+#    define ETHR_HAVE_SU_DW_NATMC_READ 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_READ_RB
+#  undef ETHR_HAVE_SU_DW_NATMC_READ_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB
+#    define ETHR_HAVE_SU_DW_NATMC_READ_RB 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_READ_WB
+#  undef ETHR_HAVE_SU_DW_NATMC_READ_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB
+#    define ETHR_HAVE_SU_DW_NATMC_READ_WB 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_READ_ACQB
+#  undef ETHR_HAVE_SU_DW_NATMC_READ_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB
+#    define ETHR_HAVE_SU_DW_NATMC_READ_ACQB 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_READ_RELB
+#  undef ETHR_HAVE_SU_DW_NATMC_READ_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB
+#    define ETHR_HAVE_SU_DW_NATMC_READ_RELB 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_READ_MB
+#  undef ETHR_HAVE_SU_DW_NATMC_READ_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB
+#    define ETHR_HAVE_SU_DW_NATMC_READ_MB 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_INIT
+#  undef ETHR_HAVE_SU_DW_NATMC_INIT
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT
+#    define ETHR_HAVE_SU_DW_NATMC_INIT 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_INIT_RB
+#  undef ETHR_HAVE_SU_DW_NATMC_INIT_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB
+#    define ETHR_HAVE_SU_DW_NATMC_INIT_RB 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_INIT_WB
+#  undef ETHR_HAVE_SU_DW_NATMC_INIT_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB
+#    define ETHR_HAVE_SU_DW_NATMC_INIT_WB 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_INIT_ACQB
+#  undef ETHR_HAVE_SU_DW_NATMC_INIT_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB
+#    define ETHR_HAVE_SU_DW_NATMC_INIT_ACQB 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_INIT_RELB
+#  undef ETHR_HAVE_SU_DW_NATMC_INIT_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB
+#    define ETHR_HAVE_SU_DW_NATMC_INIT_RELB 1
+#  endif
+#  undef ETHR_HAVE_DW_NATMC_INIT_MB
+#  undef ETHR_HAVE_SU_DW_NATMC_INIT_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB
+#    define ETHR_HAVE_SU_DW_NATMC_INIT_MB 1
+#  endif
+#else
+#  error "Invalid native atomic size"
+#endif
+
+
+#if defined(ETHR_HAVE_NATIVE_DW_ATOMIC)
+
+#if (!defined(ETHR_HAVE_DW_NATMC_CMPXCHG) \
+     && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) \
+     && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) \
+     && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) \
+     && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) \
+     && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB))
+#  error "No native cmpxchg() op available"
+#endif
+
+
+/*
+ * Read op used together with cmpxchg() fallback when no native op present.
+ */
+#if defined(ETHR_HAVE_DW_NATMC_READ)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+  ETHR_DW_NATMC_FUNC__(read)(VAR, VAL)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+  VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(VAR)
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+  ETHR_DW_NATMC_FUNC__(read_rb)(VAR, VAL)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+  VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(VAR)
+#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+  ETHR_DW_NATMC_FUNC__(read_wb)(VAR, VAL)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+  VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(VAR)
+#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+  ETHR_DW_NATMC_FUNC__(read_acqb)(VAR, VAL)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+  VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(VAR)
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+  ETHR_DW_NATMC_FUNC__(read_relb)(VAR, VAL)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+  VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(VAR)
+#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+  ETHR_DW_NATMC_FUNC__(read_mb)(VAR, VAL)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+  VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(VAR)
+#else
+/*
+ * We have no native read() op; guess zero and then use the
+ * the atomics actual value returned from cmpxchg().
+ */
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
+do { \
+  VAL.sint[0] = (ethr_sint_t) 0; \
+  VAL.sint[1] = (ethr_sint_t) 0; \
+} while (0)
+#endif
+
+/*
+ * Native cmpxchg() fallback used when no native op present.
+ */
+#define ETHR_DW_NATMC_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \
+do { \
+    int res__; \
+    ethr_dw_sint_t AVAL, exp_act__; \
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, exp_act__); \
+    do { \
+	AVAL.sint[0] = exp_act__.sint[0]; \
+	AVAL.sint[1] = exp_act__.sint[1]; \
+	{ OPS; } \
+	res__ = CMPXCHG(VAR, AVAL.sint, exp_act__.sint); \
+    } while (__builtin_expect(res__ == 0, 0)); \
+} while (0)
+
+
+#elif defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)
+
+#if (!defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) \
+     && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) \
+     && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) \
+     && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) \
+     && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) \
+     && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB))
+#  error "No native cmpxchg() op available"
+#endif
+
+
+/*
+ * Read op used together with cmpxchg() fallback when no native op present.
+ */
+#if defined(ETHR_HAVE_SU_DW_NATMC_READ)
+#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+  ETHR_SU_DW_NATMC_FUNC__(read)(VAR)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
+#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+  ETHR_SU_DW_NATMC_FUNC__(read_rb)(VAR)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
+#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+  ETHR_SU_DW_NATMC_FUNC__(read_wb)(VAR)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
+#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+  ETHR_SU_DW_NATMC_FUNC__(read_acqb)(VAR)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
+#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+  ETHR_SU_DW_NATMC_FUNC__(read_relb)(VAR)
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
+#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+  ETHR_SU_DW_NATMC_FUNC__(read_mb)(VAR)
+#else
+/*
+ * We have no native read() op; guess zero and then use the
+ * the atomics actual value returned from cmpxchg().
+ */
+#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+  ((ETHR_SU_DW_NAINT_T__) 0)
+#endif
+
+/*
+ * Native cmpxchg() fallback used when no native op present.
+ */
+#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \
+do { \
+    ETHR_SU_DW_NAINT_T__ AVAL; \
+    ETHR_SU_DW_NAINT_T__ new__, act__, exp__; \
+    act__ = ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR); \
+    do { \
+	exp__ = act__; \
+	AVAL = (ETHR_SU_DW_NAINT_T__) act__; \
+	{ OPS; } \
+	new__ = (ETHR_SU_DW_NAINT_T__) AVAL; \
+	act__ = CMPXCHG(VAR, new__, exp__); \
+    } while (__builtin_expect(act__ != exp__, 0)); \
+} while (0)
+
+
+#else
+#  error "?!?"
+#endif
+
+
+
+/* --- addr() --- */
+
+static ETHR_INLINE ethr_sint_t *ETHR_DW_ATMC_FUNC__(addr)(ethr_dw_atomic_t *var)
+{
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+    return (ethr_sint_t *) ETHR_DW_NATMC_ADDR_FUNC__((&var->native));
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { return ETHR_DW_ATOMIC_FUNC__(addr)(var); }
+#endif
+
+}
+
+
+/* --- cmpxchg() --- */
+
+
+static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+    int res;
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
+#else
+#error "Missing implementation of ethr_dw_atomic_cmpxchg()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg)(var, val, old_val); }
+#endif
+
+    return res;
+}
+
+static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+    int res;
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_dw_atomic_cmpxchg_rb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(var, val, old_val); }
+#endif
+
+    return res;
+}
+
+static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+    int res;
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+    ETHR_SU_DW_NAINT_T__ act;
+    ETHR_MEMBAR(ETHR_StoreStore);
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+    ETHR_SU_DW_NAINT_T__ act;
+    ETHR_MEMBAR(ETHR_StoreStore);
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_SU_DW_NAINT_T__ act;
+    ETHR_MEMBAR(ETHR_StoreStore);
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+    ETHR_SU_DW_NAINT_T__ act;
+    ETHR_MEMBAR(ETHR_StoreStore);
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
+#else
+#error "Missing implementation of ethr_dw_atomic_cmpxchg_wb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)(var, val, old_val); }
+#endif
+
+    return res;
+}
+
+static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+    int res;
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_dw_atomic_cmpxchg_acqb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)(var, val, old_val); }
+#endif
+
+    return res;
+}
+
+static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+    int res;
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+    ETHR_SU_DW_NAINT_T__ act;
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+    ETHR_SU_DW_NAINT_T__ act;
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+    ETHR_SU_DW_NAINT_T__ act;
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_SU_DW_NAINT_T__ act;
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
+#else
+#error "Missing implementation of ethr_dw_atomic_cmpxchg_relb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)(var, val, old_val); }
+#endif
+
+    return res;
+}
+
+static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+    int res;
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+    ETHR_SU_DW_NAINT_T__ act;
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_SU_DW_NAINT_T__ act;
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+    ETHR_SU_DW_NAINT_T__ act;
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+    ETHR_SU_DW_NAINT_T__ act;
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+    ETHR_SU_DW_NAINT_T__ act;
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    res = (act == old_val->dw_sint);
+    old_val->dw_sint = act;
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_dw_atomic_cmpxchg_mb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)(var, val, old_val); }
+#endif
+
+    return res;
+}
+
+static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_DW_ATMC_FUNC__(cmpxchg)(var, val, old_val);
+#else
+    return ETHR_DW_ATMC_FUNC__(cmpxchg_rb)(var, val, old_val);
+#endif
+}
+
+
+/* --- set() --- */
+
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_SET)
+    ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
+    ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
+    ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
+    ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
+    ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
+    ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET)
+    ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
+    ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
+    ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
+    ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
+    ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
+    ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#else
+#error "Missing implementation of ethr_dw_atomic_set()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { ETHR_DW_ATOMIC_FUNC__(set)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
+    ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
+    ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
+    ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
+    ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
+    ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
+    ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
+    ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET)
+    ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
+    ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
+    ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
+    ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
+    ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_dw_atomic_set_rb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { ETHR_DW_ATOMIC_FUNC__(set_rb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
+    ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
+    ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
+    ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
+    ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#else
+#error "Missing implementation of ethr_dw_atomic_set_wb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { ETHR_DW_ATOMIC_FUNC__(set_wb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
+    ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
+    ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
+    ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
+    ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
+    ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
+    ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
+    ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
+    ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_SET)
+    ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
+    ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
+    ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
+    ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_dw_atomic_set_acqb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { ETHR_DW_ATOMIC_FUNC__(set_acqb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
+    ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
+    ETHR_MEMBAR(ETHR_LoadStore);
+    ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
+    ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
+    ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
+    ETHR_MEMBAR(ETHR_LoadStore);
+    ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
+    ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#else
+#error "Missing implementation of ethr_dw_atomic_set_relb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { ETHR_DW_ATOMIC_FUNC__(set_relb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
+    ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
+    ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
+    ETHR_MEMBAR(ETHR_LoadStore);
+    ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
+    ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
+    ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
+    ETHR_MEMBAR(ETHR_LoadStore);
+    ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_SET)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_dw_atomic_set_mb()!"
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { ETHR_DW_ATOMIC_FUNC__(set_mb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_DW_ATMC_FUNC__(set)(var, val);
+#else
+    ETHR_DW_ATMC_FUNC__(set_rb)(var, val);
+#endif
+}
+
+
+/* --- read() --- */
+
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_READ)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
+#elif defined(ETHR_HAVE_DW_NATMC_READ)
+    ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
+    ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
+    ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
+    ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
+    ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
+    ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
+#else
+    ethr_dw_sint_t tmp;
+    tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+    tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+    val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+    val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+    (void) ETHR_DW_ATMC_FUNC__(cmpxchg)(var, &tmp, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { ETHR_DW_ATOMIC_FUNC__(read)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
+    ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ)
+    ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
+    ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
+    ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
+    ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
+    ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    ethr_dw_sint_t tmp;
+    tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+    tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+    val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+    val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+    (void) ETHR_DW_ATMC_FUNC__(cmpxchg_rb)(var, &tmp, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { ETHR_DW_ATOMIC_FUNC__(read_rb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
+    ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
+    ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
+#else
+    ethr_dw_sint_t tmp;
+    tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+    tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+    val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+    val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+    (void) ETHR_DW_ATMC_FUNC__(cmpxchg_wb)(var, &tmp, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { ETHR_DW_ATOMIC_FUNC__(read_wb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
+    ETHR_MEMBAR(ETHR_LoadStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
+    ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
+    ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadStore);
+#elif defined(ETHR_HAVE_DW_NATMC_READ)
+    ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
+    ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
+    ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
+    ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#else
+    ethr_dw_sint_t tmp;
+    tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+    tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+    val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+    val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+    (void) ETHR_DW_ATMC_FUNC__(cmpxchg_acqb)(var, &tmp, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { ETHR_DW_ATOMIC_FUNC__(read_acqb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
+    ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
+    ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
+#else
+    ethr_dw_sint_t tmp;
+    tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+    tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+    val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+    val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+    (void) ETHR_DW_ATMC_FUNC__(cmpxchg_relb)(var, &tmp, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { ETHR_DW_ATOMIC_FUNC__(read_relb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
+    ETHR_MEMBAR(ETHR_LoadStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
+    ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
+    ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadStore);
+#elif defined(ETHR_HAVE_DW_NATMC_READ)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#else
+    ethr_dw_sint_t tmp;
+    tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+    tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+    val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
+    val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
+    (void) ETHR_DW_ATMC_FUNC__(cmpxchg_mb)(var, &tmp, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { ETHR_DW_ATOMIC_FUNC__(read_mb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_DW_ATMC_FUNC__(read)(var, val);
+#else
+    ETHR_DW_ATMC_FUNC__(read_rb)(var, val);
+#endif
+}
+
+
+/* --- init() --- */
+
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_INIT)
+    ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
+    ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
+    ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
+    ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
+    ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
+    ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT)
+    ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
+    ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
+    ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
+    ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
+    ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
+    ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
+#else
+    ETHR_DW_ATMC_FUNC__(set)(var, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { ETHR_DW_ATOMIC_FUNC__(init)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
+    ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
+    ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
+    ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
+    ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
+    ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
+    ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
+    ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT)
+    ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
+    ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
+    ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
+    ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
+    ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    ETHR_DW_ATMC_FUNC__(set_rb)(var, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { ETHR_DW_ATOMIC_FUNC__(init_rb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
+    ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
+    ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
+    ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
+    ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
+#else
+    ETHR_DW_ATMC_FUNC__(set_wb)(var, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { ETHR_DW_ATOMIC_FUNC__(init_wb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
+    ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
+    ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
+    ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
+    ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
+    ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
+    ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
+    ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
+    ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT)
+    ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
+    ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
+    ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
+    ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_DW_ATMC_FUNC__(set_acqb)(var, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { ETHR_DW_ATOMIC_FUNC__(init_acqb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
+    ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
+    ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
+    ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
+    ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
+    ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
+    ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
+#else
+    ETHR_DW_ATMC_FUNC__(set_relb)(var, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { ETHR_DW_ATOMIC_FUNC__(init_relb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
+#endif
+
+#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
+    ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
+    ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
+    ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
+    ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
+    ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
+    ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_DW_NATMC_INIT)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_DW_ATMC_FUNC__(set_mb)(var, val);
+#endif
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    } else { ETHR_DW_ATOMIC_FUNC__(init_mb)(var, val); }
+#endif
+
+}
+
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_DW_ATMC_FUNC__(init)(var, val);
+#else
+    ETHR_DW_ATMC_FUNC__(init_rb)(var, val);
+#endif
+}
+
+#endif /* ETHR_DW_ATMC_INLINE__ */
+
+
+/* ---------- Word size atomic implementation ---------- */
+
+
+#ifdef ETHR_NEED_ATMC_PROTOTYPES__
+ethr_sint_t *ethr_atomic_addr(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
+ethr_sint_t ethr_atomic_cmpxchg_ddrb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
+ethr_sint_t ethr_atomic_cmpxchg_rb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
+ethr_sint_t ethr_atomic_cmpxchg_wb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
+ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
+ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
+ethr_sint_t ethr_atomic_cmpxchg_mb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
+ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_xchg_ddrb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_xchg_rb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_xchg_wb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_xchg_acqb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_xchg_relb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_xchg_mb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_set_ddrb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_set_rb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_set_wb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_set_acqb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_set_mb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_init_ddrb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_init_rb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_init_wb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_init_acqb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_init_relb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_init_mb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_add_read_ddrb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_add_read_rb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_add_read_wb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_add_read_acqb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_add_read_relb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_add_read_mb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_read_ddrb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_read_rb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_read_wb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_read_relb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_read_mb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_inc_read_ddrb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_inc_read_rb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_inc_read_wb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_inc_read_relb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_inc_read_mb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_dec_read_ddrb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_dec_read_rb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_dec_read_wb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_dec_read_acqb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_dec_read_mb(ethr_atomic_t *var);
+void ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_add_ddrb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_add_rb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_add_wb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_add_acqb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_add_relb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_add_mb(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_inc(ethr_atomic_t *var);
+void ethr_atomic_inc_ddrb(ethr_atomic_t *var);
+void ethr_atomic_inc_rb(ethr_atomic_t *var);
+void ethr_atomic_inc_wb(ethr_atomic_t *var);
+void ethr_atomic_inc_acqb(ethr_atomic_t *var);
+void ethr_atomic_inc_relb(ethr_atomic_t *var);
+void ethr_atomic_inc_mb(ethr_atomic_t *var);
+void ethr_atomic_dec(ethr_atomic_t *var);
+void ethr_atomic_dec_ddrb(ethr_atomic_t *var);
+void ethr_atomic_dec_rb(ethr_atomic_t *var);
+void ethr_atomic_dec_wb(ethr_atomic_t *var);
+void ethr_atomic_dec_acqb(ethr_atomic_t *var);
+void ethr_atomic_dec_relb(ethr_atomic_t *var);
+void ethr_atomic_dec_mb(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_band_ddrb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_band_rb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_band_wb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_band_acqb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_band_relb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_band_mb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_bor_ddrb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_bor_rb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_bor_wb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_bor_acqb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_bor_relb(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_bor_mb(ethr_atomic_t *var, ethr_sint_t val);
+#endif /* ETHR_NEED_ATMC_PROTOTYPES__ */
+
+#if (defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \
+     && (defined(ETHR_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)))
+
+#if !defined(ETHR_NATMC_BITS__)
+#  error "Missing native atomic implementation"
+#elif ETHR_NATMC_BITS__ == 64
+#  undef ETHR_HAVE_NATMC_CMPXCHG
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG
+#    define ETHR_HAVE_NATMC_CMPXCHG 1
+#  endif
+#  undef ETHR_HAVE_NATMC_CMPXCHG_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB
+#    define ETHR_HAVE_NATMC_CMPXCHG_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_CMPXCHG_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB
+#    define ETHR_HAVE_NATMC_CMPXCHG_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_CMPXCHG_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB
+#    define ETHR_HAVE_NATMC_CMPXCHG_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_CMPXCHG_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB
+#    define ETHR_HAVE_NATMC_CMPXCHG_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_CMPXCHG_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB
+#    define ETHR_HAVE_NATMC_CMPXCHG_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_XCHG
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG
+#    define ETHR_HAVE_NATMC_XCHG 1
+#  endif
+#  undef ETHR_HAVE_NATMC_XCHG_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RB
+#    define ETHR_HAVE_NATMC_XCHG_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_XCHG_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_WB
+#    define ETHR_HAVE_NATMC_XCHG_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_XCHG_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_ACQB
+#    define ETHR_HAVE_NATMC_XCHG_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_XCHG_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RELB
+#    define ETHR_HAVE_NATMC_XCHG_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_XCHG_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB
+#    define ETHR_HAVE_NATMC_XCHG_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_SET
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET
+#    define ETHR_HAVE_NATMC_SET 1
+#  endif
+#  undef ETHR_HAVE_NATMC_SET_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB
+#    define ETHR_HAVE_NATMC_SET_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_SET_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB
+#    define ETHR_HAVE_NATMC_SET_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_SET_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB
+#    define ETHR_HAVE_NATMC_SET_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_SET_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB
+#    define ETHR_HAVE_NATMC_SET_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_SET_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB
+#    define ETHR_HAVE_NATMC_SET_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INIT
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT
+#    define ETHR_HAVE_NATMC_INIT 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INIT_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB
+#    define ETHR_HAVE_NATMC_INIT_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INIT_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB
+#    define ETHR_HAVE_NATMC_INIT_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INIT_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB
+#    define ETHR_HAVE_NATMC_INIT_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INIT_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB
+#    define ETHR_HAVE_NATMC_INIT_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INIT_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB
+#    define ETHR_HAVE_NATMC_INIT_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_RETURN
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN
+#    define ETHR_HAVE_NATMC_ADD_RETURN 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_RETURN_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RB
+#    define ETHR_HAVE_NATMC_ADD_RETURN_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_RETURN_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_WB
+#    define ETHR_HAVE_NATMC_ADD_RETURN_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_RETURN_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB
+#    define ETHR_HAVE_NATMC_ADD_RETURN_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_RETURN_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB
+#    define ETHR_HAVE_NATMC_ADD_RETURN_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_RETURN_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB
+#    define ETHR_HAVE_NATMC_ADD_RETURN_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_READ
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ
+#    define ETHR_HAVE_NATMC_READ 1
+#  endif
+#  undef ETHR_HAVE_NATMC_READ_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB
+#    define ETHR_HAVE_NATMC_READ_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_READ_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB
+#    define ETHR_HAVE_NATMC_READ_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_READ_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB
+#    define ETHR_HAVE_NATMC_READ_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_READ_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB
+#    define ETHR_HAVE_NATMC_READ_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_READ_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB
+#    define ETHR_HAVE_NATMC_READ_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_RETURN
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN
+#    define ETHR_HAVE_NATMC_INC_RETURN 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_RETURN_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RB
+#    define ETHR_HAVE_NATMC_INC_RETURN_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_RETURN_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_WB
+#    define ETHR_HAVE_NATMC_INC_RETURN_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_RETURN_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB
+#    define ETHR_HAVE_NATMC_INC_RETURN_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_RETURN_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB
+#    define ETHR_HAVE_NATMC_INC_RETURN_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_RETURN_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB
+#    define ETHR_HAVE_NATMC_INC_RETURN_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_RETURN
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN
+#    define ETHR_HAVE_NATMC_DEC_RETURN 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_RETURN_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RB
+#    define ETHR_HAVE_NATMC_DEC_RETURN_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_RETURN_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_WB
+#    define ETHR_HAVE_NATMC_DEC_RETURN_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_RETURN_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB
+#    define ETHR_HAVE_NATMC_DEC_RETURN_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_RETURN_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB
+#    define ETHR_HAVE_NATMC_DEC_RETURN_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_RETURN_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB
+#    define ETHR_HAVE_NATMC_DEC_RETURN_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD
+#    define ETHR_HAVE_NATMC_ADD 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RB
+#    define ETHR_HAVE_NATMC_ADD_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_WB
+#    define ETHR_HAVE_NATMC_ADD_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_ACQB
+#    define ETHR_HAVE_NATMC_ADD_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RELB
+#    define ETHR_HAVE_NATMC_ADD_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB
+#    define ETHR_HAVE_NATMC_ADD_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC
+#    define ETHR_HAVE_NATMC_INC 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RB
+#    define ETHR_HAVE_NATMC_INC_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_WB
+#    define ETHR_HAVE_NATMC_INC_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_ACQB
+#    define ETHR_HAVE_NATMC_INC_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RELB
+#    define ETHR_HAVE_NATMC_INC_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB
+#    define ETHR_HAVE_NATMC_INC_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC
+#    define ETHR_HAVE_NATMC_DEC 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RB
+#    define ETHR_HAVE_NATMC_DEC_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_WB
+#    define ETHR_HAVE_NATMC_DEC_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_ACQB
+#    define ETHR_HAVE_NATMC_DEC_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RELB
+#    define ETHR_HAVE_NATMC_DEC_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB
+#    define ETHR_HAVE_NATMC_DEC_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_AND_RETOLD
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD
+#    define ETHR_HAVE_NATMC_AND_RETOLD 1
+#  endif
+#  undef ETHR_HAVE_NATMC_AND_RETOLD_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RB
+#    define ETHR_HAVE_NATMC_AND_RETOLD_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_AND_RETOLD_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_WB
+#    define ETHR_HAVE_NATMC_AND_RETOLD_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_AND_RETOLD_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_ACQB
+#    define ETHR_HAVE_NATMC_AND_RETOLD_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_AND_RETOLD_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RELB
+#    define ETHR_HAVE_NATMC_AND_RETOLD_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_AND_RETOLD_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB
+#    define ETHR_HAVE_NATMC_AND_RETOLD_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_OR_RETOLD
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD
+#    define ETHR_HAVE_NATMC_OR_RETOLD 1
+#  endif
+#  undef ETHR_HAVE_NATMC_OR_RETOLD_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RB
+#    define ETHR_HAVE_NATMC_OR_RETOLD_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_OR_RETOLD_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_WB
+#    define ETHR_HAVE_NATMC_OR_RETOLD_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_OR_RETOLD_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_ACQB
+#    define ETHR_HAVE_NATMC_OR_RETOLD_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_OR_RETOLD_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RELB
+#    define ETHR_HAVE_NATMC_OR_RETOLD_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_OR_RETOLD_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB
+#    define ETHR_HAVE_NATMC_OR_RETOLD_MB 1
+#  endif
+#elif ETHR_NATMC_BITS__ == 32
+#  undef ETHR_HAVE_NATMC_CMPXCHG
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG
+#    define ETHR_HAVE_NATMC_CMPXCHG 1
+#  endif
+#  undef ETHR_HAVE_NATMC_CMPXCHG_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB
+#    define ETHR_HAVE_NATMC_CMPXCHG_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_CMPXCHG_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB
+#    define ETHR_HAVE_NATMC_CMPXCHG_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_CMPXCHG_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB
+#    define ETHR_HAVE_NATMC_CMPXCHG_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_CMPXCHG_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB
+#    define ETHR_HAVE_NATMC_CMPXCHG_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_CMPXCHG_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB
+#    define ETHR_HAVE_NATMC_CMPXCHG_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_XCHG
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG
+#    define ETHR_HAVE_NATMC_XCHG 1
+#  endif
+#  undef ETHR_HAVE_NATMC_XCHG_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RB
+#    define ETHR_HAVE_NATMC_XCHG_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_XCHG_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_WB
+#    define ETHR_HAVE_NATMC_XCHG_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_XCHG_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB
+#    define ETHR_HAVE_NATMC_XCHG_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_XCHG_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RELB
+#    define ETHR_HAVE_NATMC_XCHG_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_XCHG_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB
+#    define ETHR_HAVE_NATMC_XCHG_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_SET
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET
+#    define ETHR_HAVE_NATMC_SET 1
+#  endif
+#  undef ETHR_HAVE_NATMC_SET_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RB
+#    define ETHR_HAVE_NATMC_SET_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_SET_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB
+#    define ETHR_HAVE_NATMC_SET_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_SET_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_ACQB
+#    define ETHR_HAVE_NATMC_SET_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_SET_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB
+#    define ETHR_HAVE_NATMC_SET_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_SET_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB
+#    define ETHR_HAVE_NATMC_SET_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INIT
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT
+#    define ETHR_HAVE_NATMC_INIT 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INIT_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RB
+#    define ETHR_HAVE_NATMC_INIT_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INIT_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_WB
+#    define ETHR_HAVE_NATMC_INIT_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INIT_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_ACQB
+#    define ETHR_HAVE_NATMC_INIT_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INIT_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RELB
+#    define ETHR_HAVE_NATMC_INIT_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INIT_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_MB
+#    define ETHR_HAVE_NATMC_INIT_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_RETURN
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN
+#    define ETHR_HAVE_NATMC_ADD_RETURN 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_RETURN_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RB
+#    define ETHR_HAVE_NATMC_ADD_RETURN_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_RETURN_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_WB
+#    define ETHR_HAVE_NATMC_ADD_RETURN_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_RETURN_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB
+#    define ETHR_HAVE_NATMC_ADD_RETURN_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_RETURN_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB
+#    define ETHR_HAVE_NATMC_ADD_RETURN_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_RETURN_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB
+#    define ETHR_HAVE_NATMC_ADD_RETURN_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_READ
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ
+#    define ETHR_HAVE_NATMC_READ 1
+#  endif
+#  undef ETHR_HAVE_NATMC_READ_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB
+#    define ETHR_HAVE_NATMC_READ_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_READ_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_WB
+#    define ETHR_HAVE_NATMC_READ_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_READ_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB
+#    define ETHR_HAVE_NATMC_READ_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_READ_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RELB
+#    define ETHR_HAVE_NATMC_READ_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_READ_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB
+#    define ETHR_HAVE_NATMC_READ_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_RETURN
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN
+#    define ETHR_HAVE_NATMC_INC_RETURN 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_RETURN_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RB
+#    define ETHR_HAVE_NATMC_INC_RETURN_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_RETURN_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_WB
+#    define ETHR_HAVE_NATMC_INC_RETURN_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_RETURN_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB
+#    define ETHR_HAVE_NATMC_INC_RETURN_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_RETURN_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB
+#    define ETHR_HAVE_NATMC_INC_RETURN_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_RETURN_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB
+#    define ETHR_HAVE_NATMC_INC_RETURN_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_RETURN
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN
+#    define ETHR_HAVE_NATMC_DEC_RETURN 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_RETURN_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RB
+#    define ETHR_HAVE_NATMC_DEC_RETURN_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_RETURN_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_WB
+#    define ETHR_HAVE_NATMC_DEC_RETURN_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_RETURN_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB
+#    define ETHR_HAVE_NATMC_DEC_RETURN_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_RETURN_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB
+#    define ETHR_HAVE_NATMC_DEC_RETURN_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_RETURN_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB
+#    define ETHR_HAVE_NATMC_DEC_RETURN_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD
+#    define ETHR_HAVE_NATMC_ADD 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RB
+#    define ETHR_HAVE_NATMC_ADD_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_WB
+#    define ETHR_HAVE_NATMC_ADD_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_ACQB
+#    define ETHR_HAVE_NATMC_ADD_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RELB
+#    define ETHR_HAVE_NATMC_ADD_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_ADD_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB
+#    define ETHR_HAVE_NATMC_ADD_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC
+#    define ETHR_HAVE_NATMC_INC 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RB
+#    define ETHR_HAVE_NATMC_INC_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_WB
+#    define ETHR_HAVE_NATMC_INC_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_ACQB
+#    define ETHR_HAVE_NATMC_INC_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RELB
+#    define ETHR_HAVE_NATMC_INC_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_INC_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB
+#    define ETHR_HAVE_NATMC_INC_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC
+#    define ETHR_HAVE_NATMC_DEC 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RB
+#    define ETHR_HAVE_NATMC_DEC_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_WB
+#    define ETHR_HAVE_NATMC_DEC_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_ACQB
+#    define ETHR_HAVE_NATMC_DEC_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RELB
+#    define ETHR_HAVE_NATMC_DEC_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_DEC_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB
+#    define ETHR_HAVE_NATMC_DEC_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_AND_RETOLD
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD
+#    define ETHR_HAVE_NATMC_AND_RETOLD 1
+#  endif
+#  undef ETHR_HAVE_NATMC_AND_RETOLD_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RB
+#    define ETHR_HAVE_NATMC_AND_RETOLD_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_AND_RETOLD_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_WB
+#    define ETHR_HAVE_NATMC_AND_RETOLD_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_AND_RETOLD_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB
+#    define ETHR_HAVE_NATMC_AND_RETOLD_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_AND_RETOLD_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB
+#    define ETHR_HAVE_NATMC_AND_RETOLD_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_AND_RETOLD_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB
+#    define ETHR_HAVE_NATMC_AND_RETOLD_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_OR_RETOLD
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD
+#    define ETHR_HAVE_NATMC_OR_RETOLD 1
+#  endif
+#  undef ETHR_HAVE_NATMC_OR_RETOLD_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RB
+#    define ETHR_HAVE_NATMC_OR_RETOLD_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_OR_RETOLD_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_WB
+#    define ETHR_HAVE_NATMC_OR_RETOLD_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_OR_RETOLD_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB
+#    define ETHR_HAVE_NATMC_OR_RETOLD_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_OR_RETOLD_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB
+#    define ETHR_HAVE_NATMC_OR_RETOLD_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC_OR_RETOLD_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB
+#    define ETHR_HAVE_NATMC_OR_RETOLD_MB 1
+#  endif
+#else
+#  error "Invalid native atomic size"
+#endif
+
+#if (!defined(ETHR_HAVE_NATMC_CMPXCHG) \
+     && !defined(ETHR_HAVE_NATMC_CMPXCHG_RB) \
+     && !defined(ETHR_HAVE_NATMC_CMPXCHG_WB) \
+     && !defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) \
+     && !defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) \
+     && !defined(ETHR_HAVE_NATMC_CMPXCHG_MB))
+#  error "No native cmpxchg() op available"
+#endif
+
+
+/*
+ * Read op used together with cmpxchg() fallback when no native op present.
+ */
+#if defined(ETHR_HAVE_NATMC_READ)
+#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+  ETHR_NATMC_FUNC__(read)(VAR)
+#elif defined(ETHR_HAVE_NATMC_READ_RB)
+#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+  ETHR_NATMC_FUNC__(read_rb)(VAR)
+#elif defined(ETHR_HAVE_NATMC_READ_WB)
+#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+  ETHR_NATMC_FUNC__(read_wb)(VAR)
+#elif defined(ETHR_HAVE_NATMC_READ_ACQB)
+#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+  ETHR_NATMC_FUNC__(read_acqb)(VAR)
+#elif defined(ETHR_HAVE_NATMC_READ_RELB)
+#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+  ETHR_NATMC_FUNC__(read_relb)(VAR)
+#elif defined(ETHR_HAVE_NATMC_READ_MB)
+#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+  ETHR_NATMC_FUNC__(read_mb)(VAR)
+#else
+/*
+ * We have no native read() op; guess zero and then use the
+ * the atomics actual value returned from cmpxchg().
+ */
+#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
+  ((ETHR_NAINT_T__) 0)
+#endif
+
+/*
+ * Native cmpxchg() fallback used when no native op present.
+ */
+#define ETHR_NATMC_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \
+do { \
+    ethr_sint_t AVAL; \
+    ETHR_NAINT_T__ new__, act__, exp__; \
+    act__ = ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR); \
+    do { \
+	exp__ = act__; \
+	AVAL = (ethr_sint_t) act__; \
+	{ OPS; } \
+	new__ = (ETHR_NAINT_T__) AVAL; \
+	act__ = CMPXCHG(VAR, new__, exp__); \
+    } while (__builtin_expect(act__ != exp__, 0)); \
+} while (0)
+
+
+
+/* --- addr() --- */
+
+static ETHR_INLINE ethr_sint_t *ETHR_ATMC_FUNC__(addr)(ethr_atomic_t *var)
+{
+    return (ethr_sint_t *) ETHR_NATMC_ADDR_FUNC__(var);
+
+}
+
+
+/* --- cmpxchg() --- */
+
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_CMPXCHG)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#else
+#error "Missing implementation of ethr_atomic_cmpxchg()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_rb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_atomic_cmpxchg_rb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_wb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#else
+#error "Missing implementation of ethr_atomic_cmpxchg_wb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_acqb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic_cmpxchg_acqb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_relb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#else
+#error "Missing implementation of ethr_atomic_cmpxchg_relb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_mb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic_cmpxchg_mb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_ddrb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC_FUNC__(cmpxchg)(var, val, old_val);
+#else
+    return ETHR_ATMC_FUNC__(cmpxchg_rb)(var, val, old_val);
+#endif
+}
+
+
+/* --- xchg() --- */
+
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_XCHG)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#else
+#error "Missing implementation of ethr_atomic_xchg()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_rb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_XCHG_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_XCHG_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_atomic_xchg_rb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_wb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_XCHG_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+#else
+#error "Missing implementation of ethr_atomic_xchg_wb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_acqb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_XCHG_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_XCHG)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_XCHG_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic_xchg_acqb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_relb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_XCHG_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+#else
+#error "Missing implementation of ethr_atomic_xchg_relb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_mb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_XCHG_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_XCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_XCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_XCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic_xchg_mb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC_FUNC__(xchg)(var, val);
+#else
+    return ETHR_ATMC_FUNC__(xchg_rb)(var, val);
+#endif
+}
+
+
+/* --- set() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(set)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_SET)
+    ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_RB)
+    ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_WB)
+    ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_ACQB)
+    ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_RELB)
+    ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_MB)
+    ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
+#else
+    (void) ETHR_ATMC_FUNC__(xchg)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(set_rb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_SET_RB)
+    ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET)
+    ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_SET_MB)
+    ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_WB)
+    ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_SET_ACQB)
+    ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_SET_RELB)
+    ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    (void) ETHR_ATMC_FUNC__(xchg_rb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(set_wb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_SET_WB)
+    ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_MB)
+    ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
+#else
+    (void) ETHR_ATMC_FUNC__(xchg_wb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(set_acqb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_SET_ACQB)
+    ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_RB)
+    ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_SET)
+    ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_SET_MB)
+    ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_WB)
+    ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_SET_RELB)
+    ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    (void) ETHR_ATMC_FUNC__(xchg_acqb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(set_relb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_SET_RELB)
+    ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_WB)
+    ETHR_MEMBAR(ETHR_LoadStore);
+    ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_MB)
+    ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_RB)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_ACQB)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
+#else
+    (void) ETHR_ATMC_FUNC__(xchg_relb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(set_mb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_SET_MB)
+    ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_RELB)
+    ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_SET_ACQB)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_SET_WB)
+    ETHR_MEMBAR(ETHR_LoadStore);
+    ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_SET_RB)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_SET)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    (void) ETHR_ATMC_FUNC__(xchg_mb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(set_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC_FUNC__(set)(var, val);
+#else
+    ETHR_ATMC_FUNC__(set_rb)(var, val);
+#endif
+}
+
+
+/* --- init() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(init)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_INIT)
+    ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_RB)
+    ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_WB)
+    ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
+    ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_RELB)
+    ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_MB)
+    ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
+#else
+    ETHR_ATMC_FUNC__(set)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(init_rb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_INIT_RB)
+    ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT)
+    ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_INIT_MB)
+    ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_WB)
+    ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
+    ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_INIT_RELB)
+    ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    ETHR_ATMC_FUNC__(set_rb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(init_wb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_INIT_WB)
+    ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_MB)
+    ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
+#else
+    ETHR_ATMC_FUNC__(set_wb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(init_acqb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_INIT_ACQB)
+    ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_RB)
+    ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INIT)
+    ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INIT_MB)
+    ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_WB)
+    ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INIT_RELB)
+    ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_ATMC_FUNC__(set_acqb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(init_relb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_INIT_RELB)
+    ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_MB)
+    ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
+#else
+    ETHR_ATMC_FUNC__(set_relb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(init_mb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_INIT_MB)
+    ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_RELB)
+    ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_INIT_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INIT_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INIT)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_ATMC_FUNC__(set_mb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(init_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC_FUNC__(init)(var, val);
+#else
+    ETHR_ATMC_FUNC__(init_rb)(var, val);
+#endif
+}
+
+
+/* --- add_read() --- */
+
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_ADD_RETURN)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#else
+#error "Missing implementation of ethr_atomic_add_read()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_rb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_atomic_add_read_rb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_wb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+#else
+#error "Missing implementation of ethr_atomic_add_read_wb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_acqb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic_add_read_acqb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_relb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+#else
+#error "Missing implementation of ethr_atomic_add_read_relb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_mb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic_add_read_mb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC_FUNC__(add_read)(var, val);
+#else
+    return ETHR_ATMC_FUNC__(add_read_rb)(var, val);
+#endif
+}
+
+
+/* --- read() --- */
+
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read)(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_READ)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
+#else
+    res = ETHR_ATMC_FUNC__(cmpxchg)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_rb)(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_READ_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_READ_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_READ_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_READ_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    res = ETHR_ATMC_FUNC__(cmpxchg_rb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_wb)(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_READ_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
+#else
+    res = ETHR_ATMC_FUNC__(cmpxchg_wb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_acqb)(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_READ_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC_READ)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC_READ_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC_READ_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#else
+    res = ETHR_ATMC_FUNC__(cmpxchg_acqb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_relb)(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_READ_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
+#else
+    res = ETHR_ATMC_FUNC__(cmpxchg_relb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_mb)(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_READ_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC_READ_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_READ_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC_READ_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC_READ)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#else
+    res = ETHR_ATMC_FUNC__(cmpxchg_mb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_ddrb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC_FUNC__(read)(var);
+#else
+    return ETHR_ATMC_FUNC__(read_rb)(var);
+#endif
+}
+
+
+/* --- inc_read() --- */
+
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read)(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_INC_RETURN)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
+#else
+    res = ETHR_ATMC_FUNC__(add_read)(var, (ethr_sint_t) 1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_rb)(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    res = ETHR_ATMC_FUNC__(add_read_rb)(var, (ethr_sint_t) 1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_wb)(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
+#else
+    res = ETHR_ATMC_FUNC__(add_read_wb)(var, (ethr_sint_t) 1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_acqb)(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    res = ETHR_ATMC_FUNC__(add_read_acqb)(var, (ethr_sint_t) 1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_relb)(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
+#else
+    res = ETHR_ATMC_FUNC__(add_read_relb)(var, (ethr_sint_t) 1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_mb)(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_RETURN)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    res = ETHR_ATMC_FUNC__(add_read_mb)(var, (ethr_sint_t) 1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_ddrb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC_FUNC__(inc_read)(var);
+#else
+    return ETHR_ATMC_FUNC__(inc_read_rb)(var);
+#endif
+}
+
+
+/* --- dec_read() --- */
+
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read)(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_DEC_RETURN)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
+#else
+    res = ETHR_ATMC_FUNC__(add_read)(var, (ethr_sint_t) -1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_rb)(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    res = ETHR_ATMC_FUNC__(add_read_rb)(var, (ethr_sint_t) -1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_wb)(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
+#else
+    res = ETHR_ATMC_FUNC__(add_read_wb)(var, (ethr_sint_t) -1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_acqb)(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    res = ETHR_ATMC_FUNC__(add_read_acqb)(var, (ethr_sint_t) -1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_relb)(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
+#else
+    res = ETHR_ATMC_FUNC__(add_read_relb)(var, (ethr_sint_t) -1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_mb)(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    res = ETHR_ATMC_FUNC__(add_read_mb)(var, (ethr_sint_t) -1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_ddrb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC_FUNC__(dec_read)(var);
+#else
+    return ETHR_ATMC_FUNC__(dec_read_rb)(var);
+#endif
+}
+
+
+/* --- add() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(add)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_ADD)
+    ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RB)
+    ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_WB)
+    ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
+    ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RELB)
+    ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_MB)
+    ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
+#else
+    (void) ETHR_ATMC_FUNC__(add_read)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(add_rb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_ADD_RB)
+    ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD)
+    ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_ADD_MB)
+    ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_WB)
+    ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
+    ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_ADD_RELB)
+    ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    (void) ETHR_ATMC_FUNC__(add_read_rb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(add_wb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_ADD_WB)
+    ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_MB)
+    ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
+#else
+    (void) ETHR_ATMC_FUNC__(add_read_wb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(add_acqb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_ADD_ACQB)
+    ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RB)
+    ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD)
+    ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_MB)
+    ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_WB)
+    ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_RELB)
+    ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    (void) ETHR_ATMC_FUNC__(add_read_acqb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(add_relb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_ADD_RELB)
+    ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_MB)
+    ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
+#else
+    (void) ETHR_ATMC_FUNC__(add_read_relb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(add_mb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_NATMC_ADD_MB)
+    ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_RELB)
+    ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_ADD_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_ADD)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    (void) ETHR_ATMC_FUNC__(add_read_mb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(add_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC_FUNC__(add)(var, val);
+#else
+    ETHR_ATMC_FUNC__(add_rb)(var, val);
+#endif
+}
+
+
+/* --- inc() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(inc)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_INC)
+    ETHR_NATMC_FUNC__(inc)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RB)
+    ETHR_NATMC_FUNC__(inc_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_WB)
+    ETHR_NATMC_FUNC__(inc_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_ACQB)
+    ETHR_NATMC_FUNC__(inc_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RELB)
+    ETHR_NATMC_FUNC__(inc_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_MB)
+    ETHR_NATMC_FUNC__(inc_mb)(var);
+#else
+    (void) ETHR_ATMC_FUNC__(inc_read)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_rb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_INC_RB)
+    ETHR_NATMC_FUNC__(inc_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC)
+    ETHR_NATMC_FUNC__(inc)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_INC_MB)
+    ETHR_NATMC_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_WB)
+    ETHR_NATMC_FUNC__(inc_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_INC_ACQB)
+    ETHR_NATMC_FUNC__(inc_acqb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_INC_RELB)
+    ETHR_NATMC_FUNC__(inc_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    (void) ETHR_ATMC_FUNC__(inc_read_rb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_wb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_INC_WB)
+    ETHR_NATMC_FUNC__(inc_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(inc)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_MB)
+    ETHR_NATMC_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(inc_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(inc_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(inc_relb)(var);
+#else
+    (void) ETHR_ATMC_FUNC__(inc_read_wb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_acqb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_INC_ACQB)
+    ETHR_NATMC_FUNC__(inc_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RB)
+    ETHR_NATMC_FUNC__(inc_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC)
+    ETHR_NATMC_FUNC__(inc)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_MB)
+    ETHR_NATMC_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_WB)
+    ETHR_NATMC_FUNC__(inc_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_RELB)
+    ETHR_NATMC_FUNC__(inc_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    (void) ETHR_ATMC_FUNC__(inc_read_acqb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_relb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_INC_RELB)
+    ETHR_NATMC_FUNC__(inc_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(inc_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(inc)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_MB)
+    ETHR_NATMC_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(inc_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(inc_acqb)(var);
+#else
+    (void) ETHR_ATMC_FUNC__(inc_read_relb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_mb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_INC_MB)
+    ETHR_NATMC_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_RELB)
+    ETHR_NATMC_FUNC__(inc_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(inc_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_INC_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(inc_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(inc_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_INC)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(inc)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    (void) ETHR_ATMC_FUNC__(inc_read_mb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_ddrb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC_FUNC__(inc)(var);
+#else
+    ETHR_ATMC_FUNC__(inc_rb)(var);
+#endif
+}
+
+
+/* --- dec() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(dec)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_DEC)
+    ETHR_NATMC_FUNC__(dec)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RB)
+    ETHR_NATMC_FUNC__(dec_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_WB)
+    ETHR_NATMC_FUNC__(dec_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
+    ETHR_NATMC_FUNC__(dec_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RELB)
+    ETHR_NATMC_FUNC__(dec_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_MB)
+    ETHR_NATMC_FUNC__(dec_mb)(var);
+#else
+    (void) ETHR_ATMC_FUNC__(dec_read)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_rb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_DEC_RB)
+    ETHR_NATMC_FUNC__(dec_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC)
+    ETHR_NATMC_FUNC__(dec)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_DEC_MB)
+    ETHR_NATMC_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_WB)
+    ETHR_NATMC_FUNC__(dec_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
+    ETHR_NATMC_FUNC__(dec_acqb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_DEC_RELB)
+    ETHR_NATMC_FUNC__(dec_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    (void) ETHR_ATMC_FUNC__(dec_read_rb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_wb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_DEC_WB)
+    ETHR_NATMC_FUNC__(dec_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(dec)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_MB)
+    ETHR_NATMC_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(dec_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(dec_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_FUNC__(dec_relb)(var);
+#else
+    (void) ETHR_ATMC_FUNC__(dec_read_wb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_acqb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_DEC_ACQB)
+    ETHR_NATMC_FUNC__(dec_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RB)
+    ETHR_NATMC_FUNC__(dec_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC)
+    ETHR_NATMC_FUNC__(dec)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_MB)
+    ETHR_NATMC_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_WB)
+    ETHR_NATMC_FUNC__(dec_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_RELB)
+    ETHR_NATMC_FUNC__(dec_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    (void) ETHR_ATMC_FUNC__(dec_read_acqb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_relb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_DEC_RELB)
+    ETHR_NATMC_FUNC__(dec_relb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(dec_wb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(dec)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_MB)
+    ETHR_NATMC_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(dec_rb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(dec_acqb)(var);
+#else
+    (void) ETHR_ATMC_FUNC__(dec_read_relb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_mb)(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_NATMC_DEC_MB)
+    ETHR_NATMC_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_RELB)
+    ETHR_NATMC_FUNC__(dec_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(dec_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC_DEC_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(dec_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(dec_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_DEC)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_FUNC__(dec)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    (void) ETHR_ATMC_FUNC__(dec_read_mb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_ddrb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC_FUNC__(dec)(var);
+#else
+    ETHR_ATMC_FUNC__(dec_rb)(var);
+#endif
+}
+
+
+/* --- read_band() --- */
+
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_AND_RETOLD)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#else
+#error "Missing implementation of ethr_atomic_read_band()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_rb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_atomic_read_band_rb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_wb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+#else
+#error "Missing implementation of ethr_atomic_read_band_wb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_acqb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic_read_band_acqb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_relb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+#else
+#error "Missing implementation of ethr_atomic_read_band_relb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_mb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic_read_band_mb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC_FUNC__(read_band)(var, val);
+#else
+    return ETHR_ATMC_FUNC__(read_band_rb)(var, val);
+#endif
+}
+
+
+/* --- read_bor() --- */
+
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_OR_RETOLD)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#else
+#error "Missing implementation of ethr_atomic_read_bor()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_rb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_atomic_read_bor_rb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_wb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+#else
+#error "Missing implementation of ethr_atomic_read_bor_wb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_acqb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic_read_bor_acqb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_relb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#else
+#error "Missing implementation of ethr_atomic_read_bor_relb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_mb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+#if defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic_read_bor_mb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC_FUNC__(read_bor)(var, val);
+#else
+    return ETHR_ATMC_FUNC__(read_bor_rb)(var, val);
+#endif
+}
+
+#endif /* ETHR_ATMC_INLINE__ */
+
+
+/* ---------- 32-bit atomic implementation ---------- */
+
+
+#ifdef ETHR_NEED_ATMC32_PROTOTYPES__
+ethr_sint32_t *ethr_atomic32_addr(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
+ethr_sint32_t ethr_atomic32_cmpxchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
+ethr_sint32_t ethr_atomic32_cmpxchg_rb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
+ethr_sint32_t ethr_atomic32_cmpxchg_wb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
+ethr_sint32_t ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
+ethr_sint32_t ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
+ethr_sint32_t ethr_atomic32_cmpxchg_mb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
+ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_xchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_xchg_rb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_xchg_wb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_xchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_xchg_relb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_xchg_mb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_set_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_set_rb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_set_wb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_set_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_set_mb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_init_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_init_rb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_init_wb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_init_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_init_relb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_init_mb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_add_read_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_add_read_rb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_add_read_wb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_add_read_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_add_read_relb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_add_read_mb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_read_ddrb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_read_rb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_read_wb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_read_acqb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_read_relb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_read_mb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_inc_read_ddrb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_inc_read_rb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_inc_read_wb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_inc_read_relb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_inc_read_mb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_dec_read_ddrb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_dec_read_rb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_dec_read_wb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_dec_read_acqb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_dec_read_relb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_dec_read_mb(ethr_atomic32_t *var);
+void ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_add_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_add_rb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_add_wb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_add_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_add_relb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_add_mb(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_inc(ethr_atomic32_t *var);
+void ethr_atomic32_inc_ddrb(ethr_atomic32_t *var);
+void ethr_atomic32_inc_rb(ethr_atomic32_t *var);
+void ethr_atomic32_inc_wb(ethr_atomic32_t *var);
+void ethr_atomic32_inc_acqb(ethr_atomic32_t *var);
+void ethr_atomic32_inc_relb(ethr_atomic32_t *var);
+void ethr_atomic32_inc_mb(ethr_atomic32_t *var);
+void ethr_atomic32_dec(ethr_atomic32_t *var);
+void ethr_atomic32_dec_ddrb(ethr_atomic32_t *var);
+void ethr_atomic32_dec_rb(ethr_atomic32_t *var);
+void ethr_atomic32_dec_wb(ethr_atomic32_t *var);
+void ethr_atomic32_dec_acqb(ethr_atomic32_t *var);
+void ethr_atomic32_dec_relb(ethr_atomic32_t *var);
+void ethr_atomic32_dec_mb(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_band_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_band_rb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_band_wb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_band_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_band_relb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_band_mb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_bor_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_bor_rb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_bor_wb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_bor_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_bor_relb(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_bor_mb(ethr_atomic32_t *var, ethr_sint32_t val);
+#endif /* ETHR_NEED_ATMC32_PROTOTYPES__ */
+
+#if (defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) \
+     && (defined(ETHR_ATMC32_INLINE__) || defined(ETHR_ATOMIC_IMPL__)))
+
+#if !defined(ETHR_NATMC32_BITS__)
+#  error "Missing native atomic implementation"
+#elif ETHR_NATMC32_BITS__ == 64
+#  undef ETHR_HAVE_NATMC32_CMPXCHG
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG
+#    define ETHR_HAVE_NATMC32_CMPXCHG 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_CMPXCHG_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB
+#    define ETHR_HAVE_NATMC32_CMPXCHG_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_CMPXCHG_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB
+#    define ETHR_HAVE_NATMC32_CMPXCHG_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_CMPXCHG_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB
+#    define ETHR_HAVE_NATMC32_CMPXCHG_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_CMPXCHG_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB
+#    define ETHR_HAVE_NATMC32_CMPXCHG_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_CMPXCHG_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB
+#    define ETHR_HAVE_NATMC32_CMPXCHG_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_XCHG
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG
+#    define ETHR_HAVE_NATMC32_XCHG 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_XCHG_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RB
+#    define ETHR_HAVE_NATMC32_XCHG_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_XCHG_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_WB
+#    define ETHR_HAVE_NATMC32_XCHG_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_XCHG_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_ACQB
+#    define ETHR_HAVE_NATMC32_XCHG_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_XCHG_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RELB
+#    define ETHR_HAVE_NATMC32_XCHG_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_XCHG_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB
+#    define ETHR_HAVE_NATMC32_XCHG_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_SET
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET
+#    define ETHR_HAVE_NATMC32_SET 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_SET_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB
+#    define ETHR_HAVE_NATMC32_SET_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_SET_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB
+#    define ETHR_HAVE_NATMC32_SET_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_SET_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB
+#    define ETHR_HAVE_NATMC32_SET_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_SET_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB
+#    define ETHR_HAVE_NATMC32_SET_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_SET_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB
+#    define ETHR_HAVE_NATMC32_SET_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INIT
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT
+#    define ETHR_HAVE_NATMC32_INIT 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INIT_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB
+#    define ETHR_HAVE_NATMC32_INIT_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INIT_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB
+#    define ETHR_HAVE_NATMC32_INIT_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INIT_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB
+#    define ETHR_HAVE_NATMC32_INIT_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INIT_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB
+#    define ETHR_HAVE_NATMC32_INIT_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INIT_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB
+#    define ETHR_HAVE_NATMC32_INIT_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_RETURN
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN
+#    define ETHR_HAVE_NATMC32_ADD_RETURN 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_RETURN_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RB
+#    define ETHR_HAVE_NATMC32_ADD_RETURN_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_RETURN_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_WB
+#    define ETHR_HAVE_NATMC32_ADD_RETURN_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_RETURN_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB
+#    define ETHR_HAVE_NATMC32_ADD_RETURN_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_RETURN_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB
+#    define ETHR_HAVE_NATMC32_ADD_RETURN_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_RETURN_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB
+#    define ETHR_HAVE_NATMC32_ADD_RETURN_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_READ
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ
+#    define ETHR_HAVE_NATMC32_READ 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_READ_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB
+#    define ETHR_HAVE_NATMC32_READ_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_READ_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB
+#    define ETHR_HAVE_NATMC32_READ_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_READ_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB
+#    define ETHR_HAVE_NATMC32_READ_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_READ_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB
+#    define ETHR_HAVE_NATMC32_READ_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_READ_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB
+#    define ETHR_HAVE_NATMC32_READ_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_RETURN
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN
+#    define ETHR_HAVE_NATMC32_INC_RETURN 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_RETURN_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RB
+#    define ETHR_HAVE_NATMC32_INC_RETURN_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_RETURN_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_WB
+#    define ETHR_HAVE_NATMC32_INC_RETURN_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_RETURN_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB
+#    define ETHR_HAVE_NATMC32_INC_RETURN_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_RETURN_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB
+#    define ETHR_HAVE_NATMC32_INC_RETURN_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_RETURN_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB
+#    define ETHR_HAVE_NATMC32_INC_RETURN_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_RETURN
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN
+#    define ETHR_HAVE_NATMC32_DEC_RETURN 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_RETURN_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RB
+#    define ETHR_HAVE_NATMC32_DEC_RETURN_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_RETURN_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_WB
+#    define ETHR_HAVE_NATMC32_DEC_RETURN_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_RETURN_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB
+#    define ETHR_HAVE_NATMC32_DEC_RETURN_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_RETURN_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB
+#    define ETHR_HAVE_NATMC32_DEC_RETURN_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_RETURN_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB
+#    define ETHR_HAVE_NATMC32_DEC_RETURN_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD
+#    define ETHR_HAVE_NATMC32_ADD 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RB
+#    define ETHR_HAVE_NATMC32_ADD_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_WB
+#    define ETHR_HAVE_NATMC32_ADD_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_ACQB
+#    define ETHR_HAVE_NATMC32_ADD_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RELB
+#    define ETHR_HAVE_NATMC32_ADD_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB
+#    define ETHR_HAVE_NATMC32_ADD_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC
+#    define ETHR_HAVE_NATMC32_INC 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RB
+#    define ETHR_HAVE_NATMC32_INC_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_WB
+#    define ETHR_HAVE_NATMC32_INC_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_ACQB
+#    define ETHR_HAVE_NATMC32_INC_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RELB
+#    define ETHR_HAVE_NATMC32_INC_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB
+#    define ETHR_HAVE_NATMC32_INC_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC
+#    define ETHR_HAVE_NATMC32_DEC 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RB
+#    define ETHR_HAVE_NATMC32_DEC_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_WB
+#    define ETHR_HAVE_NATMC32_DEC_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_ACQB
+#    define ETHR_HAVE_NATMC32_DEC_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RELB
+#    define ETHR_HAVE_NATMC32_DEC_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB
+#    define ETHR_HAVE_NATMC32_DEC_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_AND_RETOLD
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD
+#    define ETHR_HAVE_NATMC32_AND_RETOLD 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_AND_RETOLD_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RB
+#    define ETHR_HAVE_NATMC32_AND_RETOLD_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_AND_RETOLD_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_WB
+#    define ETHR_HAVE_NATMC32_AND_RETOLD_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_AND_RETOLD_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_ACQB
+#    define ETHR_HAVE_NATMC32_AND_RETOLD_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_AND_RETOLD_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RELB
+#    define ETHR_HAVE_NATMC32_AND_RETOLD_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_AND_RETOLD_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB
+#    define ETHR_HAVE_NATMC32_AND_RETOLD_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_OR_RETOLD
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD
+#    define ETHR_HAVE_NATMC32_OR_RETOLD 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_OR_RETOLD_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RB
+#    define ETHR_HAVE_NATMC32_OR_RETOLD_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_OR_RETOLD_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_WB
+#    define ETHR_HAVE_NATMC32_OR_RETOLD_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_OR_RETOLD_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_ACQB
+#    define ETHR_HAVE_NATMC32_OR_RETOLD_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_OR_RETOLD_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RELB
+#    define ETHR_HAVE_NATMC32_OR_RETOLD_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_OR_RETOLD_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB
+#    define ETHR_HAVE_NATMC32_OR_RETOLD_MB 1
+#  endif
+#elif ETHR_NATMC32_BITS__ == 32
+#  undef ETHR_HAVE_NATMC32_CMPXCHG
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG
+#    define ETHR_HAVE_NATMC32_CMPXCHG 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_CMPXCHG_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB
+#    define ETHR_HAVE_NATMC32_CMPXCHG_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_CMPXCHG_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB
+#    define ETHR_HAVE_NATMC32_CMPXCHG_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_CMPXCHG_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB
+#    define ETHR_HAVE_NATMC32_CMPXCHG_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_CMPXCHG_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB
+#    define ETHR_HAVE_NATMC32_CMPXCHG_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_CMPXCHG_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB
+#    define ETHR_HAVE_NATMC32_CMPXCHG_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_XCHG
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG
+#    define ETHR_HAVE_NATMC32_XCHG 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_XCHG_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RB
+#    define ETHR_HAVE_NATMC32_XCHG_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_XCHG_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_WB
+#    define ETHR_HAVE_NATMC32_XCHG_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_XCHG_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB
+#    define ETHR_HAVE_NATMC32_XCHG_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_XCHG_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RELB
+#    define ETHR_HAVE_NATMC32_XCHG_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_XCHG_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB
+#    define ETHR_HAVE_NATMC32_XCHG_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_SET
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET
+#    define ETHR_HAVE_NATMC32_SET 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_SET_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RB
+#    define ETHR_HAVE_NATMC32_SET_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_SET_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB
+#    define ETHR_HAVE_NATMC32_SET_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_SET_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_ACQB
+#    define ETHR_HAVE_NATMC32_SET_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_SET_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB
+#    define ETHR_HAVE_NATMC32_SET_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_SET_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB
+#    define ETHR_HAVE_NATMC32_SET_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INIT
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT
+#    define ETHR_HAVE_NATMC32_INIT 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INIT_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RB
+#    define ETHR_HAVE_NATMC32_INIT_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INIT_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_WB
+#    define ETHR_HAVE_NATMC32_INIT_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INIT_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_ACQB
+#    define ETHR_HAVE_NATMC32_INIT_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INIT_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RELB
+#    define ETHR_HAVE_NATMC32_INIT_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INIT_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_MB
+#    define ETHR_HAVE_NATMC32_INIT_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_RETURN
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN
+#    define ETHR_HAVE_NATMC32_ADD_RETURN 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_RETURN_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RB
+#    define ETHR_HAVE_NATMC32_ADD_RETURN_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_RETURN_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_WB
+#    define ETHR_HAVE_NATMC32_ADD_RETURN_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_RETURN_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB
+#    define ETHR_HAVE_NATMC32_ADD_RETURN_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_RETURN_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB
+#    define ETHR_HAVE_NATMC32_ADD_RETURN_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_RETURN_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB
+#    define ETHR_HAVE_NATMC32_ADD_RETURN_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_READ
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ
+#    define ETHR_HAVE_NATMC32_READ 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_READ_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB
+#    define ETHR_HAVE_NATMC32_READ_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_READ_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_WB
+#    define ETHR_HAVE_NATMC32_READ_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_READ_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB
+#    define ETHR_HAVE_NATMC32_READ_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_READ_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RELB
+#    define ETHR_HAVE_NATMC32_READ_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_READ_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB
+#    define ETHR_HAVE_NATMC32_READ_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_RETURN
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN
+#    define ETHR_HAVE_NATMC32_INC_RETURN 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_RETURN_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RB
+#    define ETHR_HAVE_NATMC32_INC_RETURN_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_RETURN_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_WB
+#    define ETHR_HAVE_NATMC32_INC_RETURN_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_RETURN_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB
+#    define ETHR_HAVE_NATMC32_INC_RETURN_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_RETURN_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB
+#    define ETHR_HAVE_NATMC32_INC_RETURN_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_RETURN_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB
+#    define ETHR_HAVE_NATMC32_INC_RETURN_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_RETURN
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN
+#    define ETHR_HAVE_NATMC32_DEC_RETURN 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_RETURN_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RB
+#    define ETHR_HAVE_NATMC32_DEC_RETURN_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_RETURN_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_WB
+#    define ETHR_HAVE_NATMC32_DEC_RETURN_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_RETURN_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB
+#    define ETHR_HAVE_NATMC32_DEC_RETURN_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_RETURN_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB
+#    define ETHR_HAVE_NATMC32_DEC_RETURN_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_RETURN_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB
+#    define ETHR_HAVE_NATMC32_DEC_RETURN_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD
+#    define ETHR_HAVE_NATMC32_ADD 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RB
+#    define ETHR_HAVE_NATMC32_ADD_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_WB
+#    define ETHR_HAVE_NATMC32_ADD_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_ACQB
+#    define ETHR_HAVE_NATMC32_ADD_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RELB
+#    define ETHR_HAVE_NATMC32_ADD_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_ADD_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB
+#    define ETHR_HAVE_NATMC32_ADD_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC
+#    define ETHR_HAVE_NATMC32_INC 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RB
+#    define ETHR_HAVE_NATMC32_INC_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_WB
+#    define ETHR_HAVE_NATMC32_INC_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_ACQB
+#    define ETHR_HAVE_NATMC32_INC_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RELB
+#    define ETHR_HAVE_NATMC32_INC_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_INC_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB
+#    define ETHR_HAVE_NATMC32_INC_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC
+#    define ETHR_HAVE_NATMC32_DEC 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RB
+#    define ETHR_HAVE_NATMC32_DEC_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_WB
+#    define ETHR_HAVE_NATMC32_DEC_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_ACQB
+#    define ETHR_HAVE_NATMC32_DEC_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RELB
+#    define ETHR_HAVE_NATMC32_DEC_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_DEC_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB
+#    define ETHR_HAVE_NATMC32_DEC_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_AND_RETOLD
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD
+#    define ETHR_HAVE_NATMC32_AND_RETOLD 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_AND_RETOLD_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RB
+#    define ETHR_HAVE_NATMC32_AND_RETOLD_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_AND_RETOLD_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_WB
+#    define ETHR_HAVE_NATMC32_AND_RETOLD_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_AND_RETOLD_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB
+#    define ETHR_HAVE_NATMC32_AND_RETOLD_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_AND_RETOLD_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB
+#    define ETHR_HAVE_NATMC32_AND_RETOLD_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_AND_RETOLD_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB
+#    define ETHR_HAVE_NATMC32_AND_RETOLD_MB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_OR_RETOLD
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD
+#    define ETHR_HAVE_NATMC32_OR_RETOLD 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_OR_RETOLD_RB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RB
+#    define ETHR_HAVE_NATMC32_OR_RETOLD_RB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_OR_RETOLD_WB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_WB
+#    define ETHR_HAVE_NATMC32_OR_RETOLD_WB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_OR_RETOLD_ACQB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB
+#    define ETHR_HAVE_NATMC32_OR_RETOLD_ACQB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_OR_RETOLD_RELB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB
+#    define ETHR_HAVE_NATMC32_OR_RETOLD_RELB 1
+#  endif
+#  undef ETHR_HAVE_NATMC32_OR_RETOLD_MB
+#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB
+#    define ETHR_HAVE_NATMC32_OR_RETOLD_MB 1
+#  endif
+#else
+#  error "Invalid native atomic size"
+#endif
+
+#if (!defined(ETHR_HAVE_NATMC32_CMPXCHG) \
+     && !defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) \
+     && !defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) \
+     && !defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) \
+     && !defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) \
+     && !defined(ETHR_HAVE_NATMC32_CMPXCHG_MB))
+#  error "No native cmpxchg() op available"
+#endif
+
+
+/*
+ * Read op used together with cmpxchg() fallback when no native op present.
+ */
+#if defined(ETHR_HAVE_NATMC32_READ)
+#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
+  ETHR_NATMC32_FUNC__(read)(VAR)
+#elif defined(ETHR_HAVE_NATMC32_READ_RB)
+#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
+  ETHR_NATMC32_FUNC__(read_rb)(VAR)
+#elif defined(ETHR_HAVE_NATMC32_READ_WB)
+#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
+  ETHR_NATMC32_FUNC__(read_wb)(VAR)
+#elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
+#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
+  ETHR_NATMC32_FUNC__(read_acqb)(VAR)
+#elif defined(ETHR_HAVE_NATMC32_READ_RELB)
+#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
+  ETHR_NATMC32_FUNC__(read_relb)(VAR)
+#elif defined(ETHR_HAVE_NATMC32_READ_MB)
+#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
+  ETHR_NATMC32_FUNC__(read_mb)(VAR)
+#else
+/*
+ * We have no native read() op; guess zero and then use the
+ * the atomics actual value returned from cmpxchg().
+ */
+#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
+  ((ETHR_NAINT32_T__) 0)
+#endif
+
+/*
+ * Native cmpxchg() fallback used when no native op present.
+ */
+#define ETHR_NATMC32_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \
+do { \
+    ethr_sint32_t AVAL; \
+    ETHR_NAINT32_T__ new__, act__, exp__; \
+    act__ = ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR); \
+    do { \
+	exp__ = act__; \
+	AVAL = (ethr_sint32_t) act__; \
+	{ OPS; } \
+	new__ = (ETHR_NAINT32_T__) AVAL; \
+	act__ = CMPXCHG(VAR, new__, exp__); \
+    } while (__builtin_expect(act__ != exp__, 0)); \
+} while (0)
+
+
+
+/* --- addr() --- */
+
+static ETHR_INLINE ethr_sint32_t *ETHR_ATMC32_FUNC__(addr)(ethr_atomic32_t *var)
+{
+    return (ethr_sint32_t *) ETHR_NATMC32_ADDR_FUNC__(var);
+
+}
+
+
+/* --- cmpxchg() --- */
+
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#else
+#error "Missing implementation of ethr_atomic32_cmpxchg()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_rb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_atomic32_cmpxchg_rb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_wb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#else
+#error "Missing implementation of ethr_atomic32_cmpxchg_wb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_acqb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic32_cmpxchg_acqb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_relb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#else
+#error "Missing implementation of ethr_atomic32_cmpxchg_relb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_mb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic32_cmpxchg_mb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC32_FUNC__(cmpxchg)(var, val, old_val);
+#else
+    return ETHR_ATMC32_FUNC__(cmpxchg_rb)(var, val, old_val);
+#endif
+}
+
+
+/* --- xchg() --- */
+
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_XCHG)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#else
+#error "Missing implementation of ethr_atomic32_xchg()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_XCHG_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_atomic32_xchg_rb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_XCHG_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+#else
+#error "Missing implementation of ethr_atomic32_xchg_wb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_XCHG)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic32_xchg_acqb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_XCHG_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+#else
+#error "Missing implementation of ethr_atomic32_xchg_relb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_XCHG_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_XCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic32_xchg_mb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC32_FUNC__(xchg)(var, val);
+#else
+    return ETHR_ATMC32_FUNC__(xchg_rb)(var, val);
+#endif
+}
+
+
+/* --- set() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(set)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_SET)
+    ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_RB)
+    ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_WB)
+    ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
+    ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_RELB)
+    ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_MB)
+    ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
+#else
+    (void) ETHR_ATMC32_FUNC__(xchg)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_SET_RB)
+    ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET)
+    ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_SET_MB)
+    ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_WB)
+    ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
+    ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_SET_RELB)
+    ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    (void) ETHR_ATMC32_FUNC__(xchg_rb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_SET_WB)
+    ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_MB)
+    ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
+#else
+    (void) ETHR_ATMC32_FUNC__(xchg_wb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_SET_ACQB)
+    ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_RB)
+    ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_SET)
+    ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_SET_MB)
+    ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_WB)
+    ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_SET_RELB)
+    ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    (void) ETHR_ATMC32_FUNC__(xchg_acqb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_SET_RELB)
+    ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_WB)
+    ETHR_MEMBAR(ETHR_LoadStore);
+    ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_MB)
+    ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_RB)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
+#else
+    (void) ETHR_ATMC32_FUNC__(xchg_relb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_SET_MB)
+    ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_RELB)
+    ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_SET_WB)
+    ETHR_MEMBAR(ETHR_LoadStore);
+    ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_SET_RB)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_SET)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    (void) ETHR_ATMC32_FUNC__(xchg_mb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC32_FUNC__(set)(var, val);
+#else
+    ETHR_ATMC32_FUNC__(set_rb)(var, val);
+#endif
+}
+
+
+/* --- init() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(init)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_INIT)
+    ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RB)
+    ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_WB)
+    ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
+    ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
+    ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_MB)
+    ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
+#else
+    ETHR_ATMC32_FUNC__(set)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_INIT_RB)
+    ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT)
+    ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_INIT_MB)
+    ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_WB)
+    ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
+    ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
+    ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    ETHR_ATMC32_FUNC__(set_rb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_INIT_WB)
+    ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_MB)
+    ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
+#else
+    ETHR_ATMC32_FUNC__(set_wb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_INIT_ACQB)
+    ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RB)
+    ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INIT)
+    ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INIT_MB)
+    ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_WB)
+    ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
+    ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_ATMC32_FUNC__(set_acqb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_INIT_RELB)
+    ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_MB)
+    ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
+#else
+    ETHR_ATMC32_FUNC__(set_relb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_INIT_MB)
+    ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
+    ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_INIT_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INIT_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INIT)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_ATMC32_FUNC__(set_mb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC32_FUNC__(init)(var, val);
+#else
+    ETHR_ATMC32_FUNC__(init_rb)(var, val);
+#endif
+}
+
+
+/* --- add_read() --- */
+
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_ADD_RETURN)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#else
+#error "Missing implementation of ethr_atomic32_add_read()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_atomic32_add_read_rb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+#else
+#error "Missing implementation of ethr_atomic32_add_read_wb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic32_add_read_acqb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+#else
+#error "Missing implementation of ethr_atomic32_add_read_relb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic32_add_read_mb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC32_FUNC__(add_read)(var, val);
+#else
+    return ETHR_ATMC32_FUNC__(add_read_rb)(var, val);
+#endif
+}
+
+
+/* --- read() --- */
+
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read)(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_READ)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
+#else
+    res = ETHR_ATMC32_FUNC__(cmpxchg)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_rb)(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_READ_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_READ_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_READ_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    res = ETHR_ATMC32_FUNC__(cmpxchg_rb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_wb)(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_READ_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
+#else
+    res = ETHR_ATMC32_FUNC__(cmpxchg_wb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_acqb)(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_READ_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC32_READ)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC32_READ_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC32_READ_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#else
+    res = ETHR_ATMC32_FUNC__(cmpxchg_acqb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_relb)(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_READ_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
+#else
+    res = ETHR_ATMC32_FUNC__(cmpxchg_relb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_mb)(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_READ_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_READ_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC32_READ_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore);
+#elif defined(ETHR_HAVE_NATMC32_READ)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#else
+    res = ETHR_ATMC32_FUNC__(cmpxchg_mb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_ddrb)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC32_FUNC__(read)(var);
+#else
+    return ETHR_ATMC32_FUNC__(read_rb)(var);
+#endif
+}
+
+
+/* --- inc_read() --- */
+
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read)(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_INC_RETURN)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
+#else
+    res = ETHR_ATMC32_FUNC__(add_read)(var, (ethr_sint32_t) 1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_rb)(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    res = ETHR_ATMC32_FUNC__(add_read_rb)(var, (ethr_sint32_t) 1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_wb)(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
+#else
+    res = ETHR_ATMC32_FUNC__(add_read_wb)(var, (ethr_sint32_t) 1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_acqb)(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    res = ETHR_ATMC32_FUNC__(add_read_acqb)(var, (ethr_sint32_t) 1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_relb)(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
+#else
+    res = ETHR_ATMC32_FUNC__(add_read_relb)(var, (ethr_sint32_t) 1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_mb)(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    res = ETHR_ATMC32_FUNC__(add_read_mb)(var, (ethr_sint32_t) 1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_ddrb)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC32_FUNC__(inc_read)(var);
+#else
+    return ETHR_ATMC32_FUNC__(inc_read_rb)(var);
+#endif
+}
+
+
+/* --- dec_read() --- */
+
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read)(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_DEC_RETURN)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
+#else
+    res = ETHR_ATMC32_FUNC__(add_read)(var, (ethr_sint32_t) -1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_rb)(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    res = ETHR_ATMC32_FUNC__(add_read_rb)(var, (ethr_sint32_t) -1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_wb)(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
+#else
+    res = ETHR_ATMC32_FUNC__(add_read_wb)(var, (ethr_sint32_t) -1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_acqb)(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    res = ETHR_ATMC32_FUNC__(add_read_acqb)(var, (ethr_sint32_t) -1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_relb)(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
+#else
+    res = ETHR_ATMC32_FUNC__(add_read_relb)(var, (ethr_sint32_t) -1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_mb)(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    res = ETHR_ATMC32_FUNC__(add_read_mb)(var, (ethr_sint32_t) -1);
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_ddrb)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC32_FUNC__(dec_read)(var);
+#else
+    return ETHR_ATMC32_FUNC__(dec_read_rb)(var);
+#endif
+}
+
+
+/* --- add() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(add)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_ADD)
+    ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RB)
+    ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_WB)
+    ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
+    ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
+    ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_MB)
+    ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
+#else
+    (void) ETHR_ATMC32_FUNC__(add_read)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_ADD_RB)
+    ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD)
+    ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_ADD_MB)
+    ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_WB)
+    ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
+    ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
+    ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    (void) ETHR_ATMC32_FUNC__(add_read_rb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_ADD_WB)
+    ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_MB)
+    ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
+#else
+    (void) ETHR_ATMC32_FUNC__(add_read_wb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_ADD_ACQB)
+    ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RB)
+    ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD)
+    ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_MB)
+    ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_WB)
+    ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
+    ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    (void) ETHR_ATMC32_FUNC__(add_read_acqb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_ADD_RELB)
+    ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_MB)
+    ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
+#else
+    (void) ETHR_ATMC32_FUNC__(add_read_relb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_NATMC32_ADD_MB)
+    ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
+    ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_ADD_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_ADD)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    (void) ETHR_ATMC32_FUNC__(add_read_mb)(var, val);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC32_FUNC__(add)(var, val);
+#else
+    ETHR_ATMC32_FUNC__(add_rb)(var, val);
+#endif
+}
+
+
+/* --- inc() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_INC)
+    ETHR_NATMC32_FUNC__(inc)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RB)
+    ETHR_NATMC32_FUNC__(inc_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_WB)
+    ETHR_NATMC32_FUNC__(inc_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
+    ETHR_NATMC32_FUNC__(inc_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RELB)
+    ETHR_NATMC32_FUNC__(inc_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_MB)
+    ETHR_NATMC32_FUNC__(inc_mb)(var);
+#else
+    (void) ETHR_ATMC32_FUNC__(inc_read)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_rb)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_INC_RB)
+    ETHR_NATMC32_FUNC__(inc_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC)
+    ETHR_NATMC32_FUNC__(inc)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_INC_MB)
+    ETHR_NATMC32_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_WB)
+    ETHR_NATMC32_FUNC__(inc_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
+    ETHR_NATMC32_FUNC__(inc_acqb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_INC_RELB)
+    ETHR_NATMC32_FUNC__(inc_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    (void) ETHR_ATMC32_FUNC__(inc_read_rb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_wb)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_INC_WB)
+    ETHR_NATMC32_FUNC__(inc_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(inc)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_MB)
+    ETHR_NATMC32_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(inc_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(inc_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(inc_relb)(var);
+#else
+    (void) ETHR_ATMC32_FUNC__(inc_read_wb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_acqb)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_INC_ACQB)
+    ETHR_NATMC32_FUNC__(inc_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RB)
+    ETHR_NATMC32_FUNC__(inc_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC)
+    ETHR_NATMC32_FUNC__(inc)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_MB)
+    ETHR_NATMC32_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_WB)
+    ETHR_NATMC32_FUNC__(inc_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_RELB)
+    ETHR_NATMC32_FUNC__(inc_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    (void) ETHR_ATMC32_FUNC__(inc_read_acqb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_relb)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_INC_RELB)
+    ETHR_NATMC32_FUNC__(inc_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(inc_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(inc)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_MB)
+    ETHR_NATMC32_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(inc_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(inc_acqb)(var);
+#else
+    (void) ETHR_ATMC32_FUNC__(inc_read_relb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_mb)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_INC_MB)
+    ETHR_NATMC32_FUNC__(inc_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_RELB)
+    ETHR_NATMC32_FUNC__(inc_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(inc_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_INC_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(inc_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(inc_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_INC)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(inc)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    (void) ETHR_ATMC32_FUNC__(inc_read_mb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_ddrb)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC32_FUNC__(inc)(var);
+#else
+    ETHR_ATMC32_FUNC__(inc_rb)(var);
+#endif
+}
+
+
+/* --- dec() --- */
+
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_DEC)
+    ETHR_NATMC32_FUNC__(dec)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RB)
+    ETHR_NATMC32_FUNC__(dec_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_WB)
+    ETHR_NATMC32_FUNC__(dec_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
+    ETHR_NATMC32_FUNC__(dec_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
+    ETHR_NATMC32_FUNC__(dec_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_MB)
+    ETHR_NATMC32_FUNC__(dec_mb)(var);
+#else
+    (void) ETHR_ATMC32_FUNC__(dec_read)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_rb)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_DEC_RB)
+    ETHR_NATMC32_FUNC__(dec_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC)
+    ETHR_NATMC32_FUNC__(dec)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_DEC_MB)
+    ETHR_NATMC32_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_WB)
+    ETHR_NATMC32_FUNC__(dec_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
+    ETHR_NATMC32_FUNC__(dec_acqb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
+    ETHR_NATMC32_FUNC__(dec_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    (void) ETHR_ATMC32_FUNC__(dec_read_rb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_wb)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_DEC_WB)
+    ETHR_NATMC32_FUNC__(dec_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(dec)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_MB)
+    ETHR_NATMC32_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(dec_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(dec_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_FUNC__(dec_relb)(var);
+#else
+    (void) ETHR_ATMC32_FUNC__(dec_read_wb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_acqb)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_DEC_ACQB)
+    ETHR_NATMC32_FUNC__(dec_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RB)
+    ETHR_NATMC32_FUNC__(dec_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC)
+    ETHR_NATMC32_FUNC__(dec)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_MB)
+    ETHR_NATMC32_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_WB)
+    ETHR_NATMC32_FUNC__(dec_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
+    ETHR_NATMC32_FUNC__(dec_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    (void) ETHR_ATMC32_FUNC__(dec_read_acqb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_relb)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_DEC_RELB)
+    ETHR_NATMC32_FUNC__(dec_relb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(dec_wb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(dec)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_MB)
+    ETHR_NATMC32_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(dec_rb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(dec_acqb)(var);
+#else
+    (void) ETHR_ATMC32_FUNC__(dec_read_relb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_mb)(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_NATMC32_DEC_MB)
+    ETHR_NATMC32_FUNC__(dec_mb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
+    ETHR_NATMC32_FUNC__(dec_relb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(dec_acqb)(var);
+#elif defined(ETHR_HAVE_NATMC32_DEC_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(dec_wb)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(dec_rb)(var);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_DEC)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_FUNC__(dec)(var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    (void) ETHR_ATMC32_FUNC__(dec_read_mb)(var);
+#endif
+}
+
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_ddrb)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC32_FUNC__(dec)(var);
+#else
+    ETHR_ATMC32_FUNC__(dec_rb)(var);
+#endif
+}
+
+
+/* --- read_band() --- */
+
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_AND_RETOLD)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#else
+#error "Missing implementation of ethr_atomic32_read_band()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_atomic32_read_band_rb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+#else
+#error "Missing implementation of ethr_atomic32_read_band_wb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic32_read_band_acqb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+#else
+#error "Missing implementation of ethr_atomic32_read_band_relb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic32_read_band_mb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC32_FUNC__(read_band)(var, val);
+#else
+    return ETHR_ATMC32_FUNC__(read_band_rb)(var, val);
+#endif
+}
+
+
+/* --- read_bor() --- */
+
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_OR_RETOLD)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#else
+#error "Missing implementation of ethr_atomic32_read_bor()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+#error "Missing implementation of ethr_atomic32_read_bor_rb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+#else
+#error "Missing implementation of ethr_atomic32_read_bor_wb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic32_read_bor_acqb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#else
+#error "Missing implementation of ethr_atomic32_read_bor_relb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+#error "Missing implementation of ethr_atomic32_read_bor_mb()!"
+#endif
+    return res;
+}
+
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC32_FUNC__(read_bor)(var, val);
+#else
+    return ETHR_ATMC32_FUNC__(read_bor_rb)(var, val);
+#endif
+}
+
+#endif /* ETHR_ATMC32_INLINE__ */
+
+#endif /* ETHR_ATOMICS_H__ */
diff --git a/erts/include/internal/ethr_internal.h b/erts/include/internal/ethr_internal.h
new file mode 100644
index 0000000000..c9b1db5b46
--- /dev/null
+++ b/erts/include/internal/ethr_internal.h
@@ -0,0 +1,71 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Internal ethread exports
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_INTERNAL_H__
+#define ETHR_INTERNAL_H__
+
+#include "erl_misc_utils.h"
+
+extern ethr_memory_allocators ethr_mem__;
+extern erts_cpu_info_t *ethr_cpu_info__;
+extern size_t ethr_pagesize__;
+extern size_t ethr_min_stack_size__; /* kilo words */
+extern size_t ethr_max_stack_size__; /* kilo words */
+extern int ethr_not_completely_inited__;
+extern int ethr_not_inited__;
+
+extern void *(*ethr_thr_prepare_func__)(void);
+extern void (*ethr_thr_parent_func__)(void *);
+extern void (*ethr_thr_child_func__)(void *);
+
+#define ETHR_PAGE_ALIGN(SZ) \
+  (((((size_t) (SZ)) - 1)/ethr_pagesize__ + 1)*ethr_pagesize__)
+#define ETHR_B2KW(B) ((((size_t) (B)) - 1)/(sizeof(void *)*1024) + 1)
+#define ETHR_KW2B(KW) (((size_t) (KW))*sizeof(void *)*1024)
+
+#undef ETHR_STACK_GUARD_SIZE 
+#ifdef ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE
+#  define ETHR_STACK_GUARD_SIZE (ethr_pagesize__)
+#endif
+
+/* implemented in lib_src/<thr-lib>/ethread.c */
+int ethr_set_tse__(ethr_ts_event *tsep);
+ethr_ts_event *ethr_get_tse__(void);
+ETHR_PROTO_NORETURN__ ethr_abort__(void);
+#ifdef ETHR_WIN32_THREADS
+int ethr_win_get_errno__(void);
+#endif
+
+/* implemented in lib_src/common/ethread_aux.c */
+int ethr_init_common__(ethr_init_data *id);
+int ethr_late_init_common__(ethr_late_init_data *lid);
+void ethr_run_exit_handlers__(void);
+void ethr_ts_event_destructor__(void *vtsep);
+
+#if defined(ETHR_X86_RUNTIME_CONF__)
+int ethr_x86_have_cpuid__(void);
+void ethr_x86_cpuid__(int *eax, int *ebx, int *ecx, int *edx);
+#endif
+
+#endif
diff --git a/erts/include/internal/ethr_mutex.h b/erts/include/internal/ethr_mutex.h
new file mode 100644
index 0000000000..86a1e9fbdf
--- /dev/null
+++ b/erts/include/internal/ethr_mutex.h
@@ -0,0 +1,785 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Mutex, rwmutex and condition variable implementation
+ * Author: Rickard Green
+ */
+
+/*
+ * IMPORTANT note about ethr_cond_signal() and ethr_cond_broadcast()
+ *
+ * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast'
+ * even though the associated mutex/mutexes isn't/aren't locked by the
+ * caller. We do not allow that by default in order to avoid a performance
+ * penalty on some platforms.
+ *
+ * Mutexes and condition variables can, however, be initialized as POSIX
+ * compliant. When initialized as such ethr_cond_signal(), and
+ * ethr_cond_broadcast() are allowed to be called even though the associated
+ * mutexes aren't locked. This will, however, incur a performance penalty on
+ * some platforms.
+ *
+ * POSIX compliant mutexes and condition variables *need* to be used together.
+ */
+
+#ifndef ETHR_MUTEX_H__
+#define ETHR_MUTEX_H__
+
+#define ETHR_RWMUTEX_INITIALIZED 	0x99999999
+#define ETHR_MUTEX_INITIALIZED		0x77777777
+#define ETHR_COND_INITIALIZED		0x55555555
+
+#if 0
+#  define ETHR_MTX_HARD_DEBUG
+#endif
+
+#if 0
+#  define ETHR_MTX_CHK_EXCL
+#if 1
+#    define ETHR_MTX_CHK_NON_EXCL
+#endif
+#endif
+
+/* #define ETHR_DBG_WIN_MTX_WITH_PTHREADS */
+#ifdef ETHR_DBG_WIN_MTX_WITH_PTHREADS
+typedef pthread_mutex_t CRITICAL_SECTION;
+int TryEnterCriticalSection(CRITICAL_SECTION *);
+void EnterCriticalSection(CRITICAL_SECTION *);
+void LeaveCriticalSection(CRITICAL_SECTION *);
+#endif
+
+#ifdef ETHR_MTX_HARD_DEBUG
+#  ifdef __GNUC__
+#    warning ETHR_MTX_HARD_DEBUG
+#  endif
+/*#  define ETHR_MTX_HARD_DEBUG_LFS*/
+/*#  define ETHR_MTX_HARD_DEBUG_FENCE*/
+/*#  define ETHR_MTX_HARD_DEBUG_Q*/
+#  define ETHR_MTX_HARD_DEBUG_WSQ
+
+#  if !defined(ETHR_MTX_HARD_DEBUG_WSQ) && defined(ETHR_MTX_HARD_DEBUG_Q)
+#    define ETHR_MTX_HARD_DEBUG_WSQ
+#  endif
+#endif
+
+#ifndef ETHR_INLINE_MTX_FUNC_NAME_
+#  define ETHR_INLINE_MTX_FUNC_NAME_(X) X
+#endif
+
+#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__)
+
+#ifdef ETHR_DEBUG
+#  ifndef ETHR_MTX_CHK_EXCL
+#    define ETHR_MTX_CHK_EXCL
+#  endif
+#  ifndef ETHR_MTX_CHK_NON_EXCL
+#    define ETHR_MTX_CHK_NON_EXCL
+#  endif
+#endif
+
+#if 0
+#  define ETHR_MTX_Q_LOCK_SPINLOCK__
+#  define ETHR_MTX_QLOCK_TYPE__ ethr_spinlock_t
+#elif defined(ETHR_PTHREADS)
+#  define ETHR_MTX_Q_LOCK_PTHREAD_MUTEX__
+#  define ETHR_MTX_QLOCK_TYPE__ pthread_mutex_t
+#elif defined(ETHR_WIN32_THREADS)
+#  define ETHR_MTX_Q_LOCK_CRITICAL_SECTION__
+#  define ETHR_MTX_QLOCK_TYPE__ CRITICAL_SECTION
+#else
+#  error Need a qlock implementation
+#endif
+
+#define ETHR_RWMTX_W_FLG__		(((ethr_sint32_t) 1) << 31)
+#define ETHR_RWMTX_W_WAIT_FLG__		(((ethr_sint32_t) 1) << 30)
+#define ETHR_RWMTX_R_WAIT_FLG__		(((ethr_sint32_t) 1) << 29)
+
+/* frequent read kind */
+#define ETHR_RWMTX_R_FLG__		(((ethr_sint32_t) 1) << 28)
+#define ETHR_RWMTX_R_ABRT_UNLCK_FLG__	(((ethr_sint32_t) 1) << 27)
+#define ETHR_RWMTX_R_PEND_UNLCK_MASK__	(ETHR_RWMTX_R_ABRT_UNLCK_FLG__ - 1)
+
+/* normal kind */
+#define ETHR_RWMTX_RS_MASK__		(ETHR_RWMTX_R_WAIT_FLG__ - 1)
+
+#define ETHR_RWMTX_WAIT_FLGS__ \
+  (ETHR_RWMTX_W_WAIT_FLG__|ETHR_RWMTX_R_WAIT_FLG__)
+
+#define ETHR_CND_WAIT_FLG__		ETHR_RWMTX_R_WAIT_FLG__
+
+#ifdef ETHR_DEBUG
+#define ETHR_DBG_CHK_UNUSED_FLG_BITS(V)			\
+  ETHR_ASSERT(!((V) & ~(ETHR_RWMTX_W_FLG__		\
+			| ETHR_RWMTX_W_WAIT_FLG__	\
+			| ETHR_RWMTX_R_WAIT_FLG__	\
+			| ETHR_RWMTX_RS_MASK__)))
+#else
+#define ETHR_DBG_CHK_UNUSED_FLG_BITS(V)
+#endif
+
+#define ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(MTX)		\
+  ETHR_DBG_CHK_UNUSED_FLG_BITS(ethr_atomic32_read(&(MTX)->mtxb.flgs))
+
+struct ethr_mutex_base_ {
+#ifdef ETHR_MTX_HARD_DEBUG_FENCE
+    long pre_fence;
+#endif
+    ethr_atomic32_t flgs;
+    short aux_scnt;
+    short main_scnt;
+    ETHR_MTX_QLOCK_TYPE__ qlck;
+    ethr_ts_event *q;
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+    int ws;
+#endif
+#ifdef ETHR_MTX_CHK_EXCL
+    ethr_atomic32_t exclusive;
+#endif
+#ifdef ETHR_MTX_CHK_NON_EXCL
+    ethr_atomic32_t non_exclusive;
+#endif
+#ifdef ETHR_MTX_HARD_DEBUG_LFS
+    ethr_atomic32_t hdbg_lfs;
+#endif
+};
+
+#endif
+
+typedef struct {
+    int main_spincount;
+    int aux_spincount;
+    int posix_compliant;
+} ethr_mutex_opt;
+
+#define ETHR_MUTEX_OPT_DEFAULT_INITER {-1, -1, 0}
+
+typedef struct {
+    int main_spincount;
+    int aux_spincount;
+    int posix_compliant;
+} ethr_cond_opt;
+
+#define ETHR_COND_OPT_DEFAULT_INITER {-1, -1, 0}
+
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+
+typedef struct ethr_mutex_ ethr_mutex;
+struct ethr_mutex_ {
+    struct ethr_mutex_base_ mtxb;
+#ifdef ETHR_MTX_HARD_DEBUG_FENCE
+    long post_fence;
+#endif
+#if ETHR_XCHK
+    int initialized;
+#endif
+};
+
+typedef struct ethr_cond_ ethr_cond;
+struct ethr_cond_ {
+#ifdef ETHR_MTX_HARD_DEBUG_FENCE
+    struct {
+	long pre_fence;
+    } mtxb; /* mtxb allows us to use same macro as for mutex and rwmutex... */
+#endif
+    ETHR_MTX_QLOCK_TYPE__ qlck;
+    ethr_ts_event *q;
+    short aux_scnt;
+    short main_scnt;
+#ifdef ETHR_MTX_HARD_DEBUG_FENCE
+    long post_fence;
+#endif
+#if ETHR_XCHK
+    int initialized;
+#endif
+};
+
+#elif defined(ETHR_PTHREADS) && !defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS)
+
+typedef struct ethr_mutex_ ethr_mutex;
+struct ethr_mutex_ {
+    pthread_mutex_t pt_mtx;
+#if ETHR_XCHK
+    int initialized;
+#endif
+};
+
+typedef struct ethr_cond_ ethr_cond;
+struct ethr_cond_ {
+    pthread_cond_t pt_cnd;
+#if ETHR_XCHK
+    int initialized;
+#endif
+};
+
+#elif defined(ETHR_WIN32_THREADS) || defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS)
+#  define ETHR_WIN_MUTEX__
+
+typedef struct ethr_mutex_ ethr_mutex;
+struct ethr_mutex_ {
+    int posix_compliant;
+    CRITICAL_SECTION cs;
+    ethr_ts_event *wakeups;
+    ethr_atomic32_t have_wakeups; /* only when posix compliant */
+    ethr_atomic32_t locked;       /* only when posix compliant */
+    ethr_spinlock_t lock;         /* only when posix compliant */
+#if ETHR_XCHK
+    int initialized;
+#endif
+};
+
+typedef struct ethr_cond_ ethr_cond;
+struct ethr_cond_ {
+    int posix_compliant;
+    CRITICAL_SECTION cs;
+    ethr_ts_event *waiters;
+    int spincount;
+#if ETHR_XCHK
+    int initialized;
+#endif
+};
+
+#else
+#  error "no mutex implementation"
+#endif
+
+int ethr_mutex_init_opt(ethr_mutex *, ethr_mutex_opt *);
+int ethr_mutex_init(ethr_mutex *);
+int ethr_mutex_destroy(ethr_mutex *);
+#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__)
+int ethr_mutex_trylock(ethr_mutex *);
+void ethr_mutex_lock(ethr_mutex *);
+void ethr_mutex_unlock(ethr_mutex *);
+#endif
+int ethr_cond_init_opt(ethr_cond *, ethr_cond_opt *);
+int ethr_cond_init(ethr_cond *);
+int ethr_cond_destroy(ethr_cond *);
+void ethr_cond_signal(ethr_cond *);
+void ethr_cond_broadcast(ethr_cond *);
+int ethr_cond_wait(ethr_cond *, ethr_mutex *);
+
+typedef enum {
+    ETHR_RWMUTEX_TYPE_NORMAL,
+    ETHR_RWMUTEX_TYPE_FREQUENT_READ,
+    ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ
+} ethr_rwmutex_type;
+
+typedef enum {
+    ETHR_RWMUTEX_LONG_LIVED,
+    ETHR_RWMUTEX_SHORT_LIVED,
+    ETHR_RWMUTEX_UNKNOWN_LIVED
+} ethr_rwmutex_lived;
+
+typedef struct {
+    ethr_rwmutex_type type;
+    ethr_rwmutex_lived lived;
+    int main_spincount;
+    int aux_spincount;
+} ethr_rwmutex_opt;
+
+#define ETHR_RWMUTEX_OPT_DEFAULT_INITER \
+  {ETHR_RWMUTEX_TYPE_NORMAL, ETHR_RWMUTEX_UNKNOWN_LIVED, -1, -1}
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+
+typedef union {
+    struct {
+	ethr_atomic32_t readers;
+	int waiting_readers;
+	int byte_offset;
+	ethr_rwmutex_lived lived;
+    } data;
+    char align__[ETHR_CACHE_LINE_SIZE];
+} ethr_rwmtx_readers_array__;
+
+typedef struct ethr_rwmutex_ ethr_rwmutex;
+struct ethr_rwmutex_ {
+    struct ethr_mutex_base_ mtxb;
+    ethr_rwmutex_type type;
+    ethr_ts_event *rq_end;
+    union {
+	ethr_rwmtx_readers_array__ *ra;
+	int rs;
+    } tdata;
+#ifdef ETHR_MTX_HARD_DEBUG_FENCE
+    long post_fence;
+#endif
+#if ETHR_XCHK
+    int initialized;
+#endif
+};
+
+#else /* pthread_rwlock */
+
+typedef struct ethr_rwmutex_ ethr_rwmutex;
+struct ethr_rwmutex_ {
+    pthread_rwlock_t pt_rwlock;
+#if ETHR_XCHK
+    int initialized;
+#endif
+};
+
+#endif /* pthread_rwlock */
+
+int ethr_rwmutex_set_reader_group(int);
+int ethr_rwmutex_init_opt(ethr_rwmutex *, ethr_rwmutex_opt *);
+int ethr_rwmutex_init(ethr_rwmutex *);
+int ethr_rwmutex_destroy(ethr_rwmutex *);
+#if defined(ETHR_USE_OWN_RWMTX_IMPL__) \
+    || !defined(ETHR_TRY_INLINE_FUNCS) \
+    || defined(ETHR_MUTEX_IMPL__)
+int ethr_rwmutex_tryrlock(ethr_rwmutex *);
+void ethr_rwmutex_rlock(ethr_rwmutex *);
+void ethr_rwmutex_runlock(ethr_rwmutex *);
+int ethr_rwmutex_tryrwlock(ethr_rwmutex *);
+void ethr_rwmutex_rwlock(ethr_rwmutex *);
+void ethr_rwmutex_rwunlock(ethr_rwmutex *);
+#endif
+
+#ifdef ETHR_MTX_HARD_DEBUG
+#define ETHR_MTX_HARD_ASSERT(A) \
+  ((void) ((A) ? 1 : ethr_assert_failed(__FILE__, __LINE__, #A)))
+#else
+#define ETHR_MTX_HARD_ASSERT(A) ((void) 1)
+#endif
+
+#ifdef ETHR_MTX_HARD_DEBUG_LFS
+#  define ETHR_MTX_HARD_DEBUG_LFS_INIT(MTXB)				\
+do {									\
+    ethr_atomic32_init(&(MTXB)->hdbg_lfs, 0);				\
+} while (0)
+#  define ETHR_MTX_HARD_DEBUG_LFS_RLOCK(MTXB)				\
+do {									\
+    ethr_sint32_t val__;						\
+    ETHR_COMPILER_BARRIER;						\
+    val__ = ethr_atomic32_inc_read(&(MTXB)->hdbg_lfs);			\
+    ETHR_MTX_HARD_ASSERT(val__ > 0);					\
+} while (0)
+#  define ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(MTXB, RES)			\
+do {									\
+    ETHR_COMPILER_BARRIER;						\
+    if ((RES) == 0)							\
+	ETHR_MTX_HARD_DEBUG_LFS_RLOCK((MTXB));				\
+    else								\
+	ETHR_MTX_HARD_ASSERT((RES) == EBUSY);				\
+} while (0)
+#  define ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(MTXB)				\
+do {									\
+    ethr_sint32_t val__ = ethr_atomic32_dec_read(&(MTXB)->hdbg_lfs);	\
+    ETHR_MTX_HARD_ASSERT(val__ >= 0);					\
+    ETHR_COMPILER_BARRIER;						\
+} while (0)
+#  define ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(MTXB)				\
+do {									\
+    ethr_sint32_t val__;						\
+    ETHR_COMPILER_BARRIER;						\
+    val__ = ethr_atomic32_dec_read(&(MTXB)->hdbg_lfs);			\
+    ETHR_MTX_HARD_ASSERT(val__ == -1);					\
+} while (0)
+#  define ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(MTXB, RES)			\
+do {									\
+    ETHR_COMPILER_BARRIER;						\
+    if ((RES) == 0)							\
+	ETHR_MTX_HARD_DEBUG_LFS_RWLOCK((MTXB));				\
+    else								\
+	ETHR_MTX_HARD_ASSERT((RES) == EBUSY);				\
+} while (0)
+#  define ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(MTXB)			\
+do {									\
+    ethr_sint32_t val__ = ethr_atomic32_inctest(&(MTXB)->hdbg_lfs);	\
+    ETHR_MTX_HARD_ASSERT(val__ == 0);					\
+    ETHR_COMPILER_BARRIER;						\
+} while (0)
+#else
+#  define ETHR_MTX_HARD_DEBUG_LFS_INIT(MTXB)
+#  define ETHR_MTX_HARD_DEBUG_LFS_RLOCK(MTXB)
+#  define ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(MTXB, RES)
+#  define ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(MTXB)
+#  define ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(MTXB)
+#  define ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(MTXB, RES)
+#  define ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(MTXB)
+#endif
+
+#ifdef ETHR_MTX_HARD_DEBUG_FENCE
+
+#if ETHR_SIZEOF_PTR == 8
+#  define ETHR_MTX_HARD_DEBUG_PRE_FENCE		0xdeadbeefdeadbeefL
+#  define ETHR_MTX_HARD_DEBUG_POST_FENCE	0xdeaddeaddeaddeadL
+#else
+#  define ETHR_MTX_HARD_DEBUG_PRE_FENCE		0xdeaddeadL
+#  define ETHR_MTX_HARD_DEBUG_POST_FENCE	0xdeaddeadL
+#endif
+
+#define ETHR_MTX_HARD_DEBUG_FENCE_CHK(X) \
+do { \
+    ETHR_COMPILER_BARRIER; \
+    ETHR_MTX_HARD_ASSERT((X)->mtxb.pre_fence == ETHR_MTX_HARD_DEBUG_PRE_FENCE);\
+    ETHR_MTX_HARD_ASSERT((X)->post_fence == ETHR_MTX_HARD_DEBUG_POST_FENCE); \
+    ETHR_COMPILER_BARRIER; \
+} while (0)
+#define ETHR_MTX_HARD_DEBUG_FENCE_INIT(X) \
+do { \
+    (X)->mtxb.pre_fence = ETHR_MTX_HARD_DEBUG_PRE_FENCE; \
+    (X)->post_fence = ETHR_MTX_HARD_DEBUG_POST_FENCE; \
+} while (0)
+#else
+#define ETHR_MTX_HARD_DEBUG_FENCE_CHK(X)
+#define ETHR_MTX_HARD_DEBUG_FENCE_INIT(X)
+#endif
+
+#ifdef ETHR_MTX_CHK_EXCL
+
+#if !defined(ETHR_DEBUG) && defined(__GNUC__)
+#warning "check exclusive is enabled"
+#endif
+
+#  define ETHR_MTX_CHK_EXCL_INIT__(MTXB)		\
+    ethr_atomic32_init(&(MTXB)->exclusive, 0)
+
+#  define ETHR_MTX_CHK_EXCL_IS_EXCL(MTXB)		\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    if (!ethr_atomic32_read(&(MTXB)->exclusive))	\
+	ethr_assert_failed(__FILE__, __LINE__, __func__,\
+			   "is exclusive");		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#  define ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(MTXB)		\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    if (ethr_atomic32_read(&(MTXB)->exclusive))		\
+	ethr_assert_failed(__FILE__, __LINE__, __func__,\
+			   "is not exclusive");		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#  define ETHR_MTX_CHK_EXCL_SET_EXCL(MTXB)		\
+do {							\
+    ETHR_MTX_CHK_EXCL_IS_NOT_EXCL((MTXB));		\
+    ethr_atomic32_set(&(MTXB)->exclusive, 1);		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#  define ETHR_MTX_CHK_EXCL_UNSET_EXCL(MTXB)		\
+do {							\
+    ETHR_MTX_CHK_EXCL_IS_EXCL((MTXB));			\
+    ethr_atomic32_set(&(MTXB)->exclusive, 0);		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+
+#ifdef ETHR_MTX_CHK_NON_EXCL
+
+#if !defined(ETHR_DEBUG) && defined(__GNUC__)
+#warning "check non-exclusive is enabled"
+#endif
+
+#    define ETHR_MTX_CHK_NON_EXCL_INIT__(MTXB)		\
+    ethr_atomic32_init(&(MTXB)->non_exclusive, 0)
+#    define ETHR_MTX_CHK_EXCL_IS_NON_EXCL(MTXB)		\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    if (!ethr_atomic32_read(&(MTXB)->non_exclusive))	\
+	ethr_assert_failed(__FILE__, __LINE__, __func__,\
+			   "is non-exclusive");		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#    define ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(MTXB)	\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    if (ethr_atomic32_read(&(MTXB)->non_exclusive))	\
+	ethr_assert_failed(__FILE__, __LINE__, __func__,\
+			   "is not non-exclusive");	\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#    define ETHR_MTX_CHK_EXCL_SET_NON_EXCL(MTXB)	\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    ethr_atomic32_inc(&(MTXB)->non_exclusive);		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#    define ETHR_MTX_CHK_EXCL_SET_NON_EXCL_NO(MTXB, NO)	\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    ethr_atomic32_add(&(MTXB)->non_exclusive, (NO));	\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#    define ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(MTXB)	\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    ethr_atomic32_dec(&(MTXB)->non_exclusive);		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#else
+#    define ETHR_MTX_CHK_NON_EXCL_INIT__(MTXB)
+#    define ETHR_MTX_CHK_EXCL_IS_NON_EXCL(MTXB)
+#    define ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(MTXB)
+#    define ETHR_MTX_CHK_EXCL_SET_NON_EXCL_NO(MTXB, NO)
+#    define ETHR_MTX_CHK_EXCL_SET_NON_EXCL(MTXB)
+#    define ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(MTXB)
+#endif
+
+#else
+#  define ETHR_MTX_CHK_EXCL_INIT__(MTXB)
+#  define ETHR_MTX_CHK_EXCL_IS_EXCL(MTXB)
+#  define ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(MTXB)
+#  define ETHR_MTX_CHK_EXCL_SET_EXCL(MTXB)
+#  define ETHR_MTX_CHK_EXCL_UNSET_EXCL(MTXB)
+#  define ETHR_MTX_CHK_NON_EXCL_INIT__(MTXB)
+#  define ETHR_MTX_CHK_EXCL_IS_NON_EXCL(MTXB)
+#  define ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(MTXB)
+#  define ETHR_MTX_CHK_EXCL_SET_NON_EXCL_NO(MTXB, NO)
+#  define ETHR_MTX_CHK_EXCL_SET_NON_EXCL(MTXB)
+#  define ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(MTXB)
+#endif
+
+#  define ETHR_MTX_CHK_EXCL_INIT(MTXB)			\
+do {							\
+    ETHR_MTX_CHK_EXCL_INIT__((MTXB));			\
+    ETHR_MTX_CHK_NON_EXCL_INIT__((MTXB));		\
+} while (0)
+
+
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+
+#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX 2000
+#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_BASE 800
+#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_INC 50
+#define ETHR_MTX_DEFAULT_AUX_SPINCOUNT 50
+
+#define ETHR_CND_DEFAULT_MAIN_SPINCOUNT 0
+#define ETHR_CND_DEFAULT_AUX_SPINCOUNT 0
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__)
+
+void ethr_mutex_lock_wait__(ethr_mutex *, ethr_sint32_t);
+void ethr_mutex_unlock_wake__(ethr_mutex *, ethr_sint32_t);
+
+static ETHR_INLINE int
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
+{
+    ethr_sint32_t act;
+    int res;
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx);
+
+    act = ethr_atomic32_cmpxchg_acqb(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__, 0);
+    res = (act == 0) ? 0 : EBUSY;
+
+#ifdef ETHR_MTX_CHK_EXCL
+    if (res == 0)
+	ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb);
+#endif
+
+    ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&mtx->mtxb, res);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx);
+
+    ETHR_COMPILER_BARRIER;
+    return res;
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
+{
+    ethr_sint32_t act;
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx);
+
+    act = ethr_atomic32_cmpxchg_acqb(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__, 0);
+    if (act != 0)
+	ethr_mutex_lock_wait__(mtx, act);
+
+    ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb);
+
+    ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx);
+
+    ETHR_COMPILER_BARRIER;
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
+{
+    ethr_sint32_t act;
+    ETHR_COMPILER_BARRIER;
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+    ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&mtx->mtxb);
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx);
+
+    ETHR_MTX_CHK_EXCL_UNSET_EXCL(&mtx->mtxb);
+
+    act = ethr_atomic32_cmpxchg_relb(&mtx->mtxb.flgs, 0, ETHR_RWMTX_W_FLG__);
+    if (act != ETHR_RWMTX_W_FLG__)
+	ethr_mutex_unlock_wake__(mtx, act);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx);
+}
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#elif defined(ETHR_PTHREADS) && !defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS)
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__)
+
+static ETHR_INLINE int
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
+{
+    int res;
+    res = pthread_mutex_trylock(&mtx->pt_mtx);
+    if (res != 0 && res != EBUSY)
+	ETHR_FATAL_ERROR__(res);
+    return res;
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
+{
+    int res = pthread_mutex_lock(&mtx->pt_mtx);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
+{
+    int res = pthread_mutex_unlock(&mtx->pt_mtx);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#elif defined(ETHR_WIN32_THREADS) || defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS)
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__)
+
+static ETHR_INLINE int
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
+{
+    if (!TryEnterCriticalSection(&mtx->cs))
+	return EBUSY;
+    if (mtx->posix_compliant)
+	ethr_atomic32_set(&mtx->locked, 1);
+    return 0;
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
+{
+    EnterCriticalSection(&mtx->cs);
+    if (mtx->posix_compliant)
+	ethr_atomic32_set(&mtx->locked, 1);
+}
+
+void ethr_mutex_cond_wakeup__(ethr_mutex *mtx);
+
+static ETHR_INLINE void
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
+{
+    if (mtx->posix_compliant) {
+	ethr_atomic32_set_mb(&mtx->locked, 0);
+	if (ethr_atomic32_read_acqb(&mtx->have_wakeups))
+	    goto cond_wakeup;
+	else
+	    goto leave_cs;
+    }
+
+    if (mtx->wakeups) {
+    cond_wakeup:
+	ethr_mutex_cond_wakeup__(mtx);
+    }
+    else {
+    leave_cs:
+	LeaveCriticalSection(&mtx->cs);
+    }
+}
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#endif
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+
+#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX 2000
+#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_BASE 800
+#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_INC 50
+#define ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT 50
+
+#else /* pthread_rwlock */
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__)
+
+static ETHR_INLINE int
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx)
+{
+    int res = pthread_rwlock_tryrdlock(&rwmtx->pt_rwlock);
+    if (res != 0 && res != EBUSY)
+	ETHR_FATAL_ERROR__(res);
+    return res;
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx)
+{
+    int res = pthread_rwlock_rdlock(&rwmtx->pt_rwlock);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx)
+{
+    int res = pthread_rwlock_unlock(&rwmtx->pt_rwlock);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+static ETHR_INLINE int
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx)
+{
+    int res = pthread_rwlock_trywrlock(&rwmtx->pt_rwlock);
+    if (res != 0 && res != EBUSY)
+	ETHR_FATAL_ERROR__(res);
+    return res;
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx)
+{
+    int res = pthread_rwlock_wrlock(&rwmtx->pt_rwlock);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx)
+{
+    int res = pthread_rwlock_unlock(&rwmtx->pt_rwlock);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#endif /* pthread_rwlock */
+
+int ethr_mutex_lib_init(int);
+int ethr_mutex_lib_late_init(int, int);
+
+#endif /* #ifndef ETHR_MUTEX_H__ */
diff --git a/erts/include/internal/ethr_optimized_fallbacks.h b/erts/include/internal/ethr_optimized_fallbacks.h
new file mode 100644
index 0000000000..45399d18e6
--- /dev/null
+++ b/erts/include/internal/ethr_optimized_fallbacks.h
@@ -0,0 +1,230 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Optimized fallbacks used when native ops are missing
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_OPTIMIZED_FALLBACKS_H__
+#define ETHR_OPTIMIZED_FALLBACKS_H__
+
+#if defined(ETHR_HAVE_NATIVE_SPINLOCKS)
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+
+static ETHR_INLINE int
+ethr_native_spinlock_destroy(ethr_native_spinlock_t *lock)
+{
+    return 0;
+}
+
+#endif
+
+#elif defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
+/* --- Native spinlocks using pthread spinlocks -------------------------- */
+#define ETHR_HAVE_NATIVE_SPINLOCKS 1
+
+#define ETHR_NATIVE_SPINLOCK_IMPL "pthread"
+
+typedef pthread_spinlock_t ethr_native_spinlock_t;
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+
+static ETHR_INLINE void
+ethr_native_spinlock_init(ethr_native_spinlock_t *lock)
+{
+    int err = pthread_spin_init((pthread_spinlock_t *) lock, 0);
+    if (err)
+	ETHR_FATAL_ERROR__(err);
+}
+
+static ETHR_INLINE int
+ethr_native_spinlock_destroy(ethr_native_spinlock_t *lock)
+{
+    return pthread_spin_destroy((pthread_spinlock_t *) lock);
+}
+
+static ETHR_INLINE void
+ethr_native_spin_unlock(ethr_native_spinlock_t *lock)
+{
+    int err = pthread_spin_unlock((pthread_spinlock_t *) lock);
+    if (err)
+	ETHR_FATAL_ERROR__(err);
+}
+
+static ETHR_INLINE void
+ethr_native_spin_lock(ethr_native_spinlock_t *lock)
+{
+    int err = pthread_spin_lock((pthread_spinlock_t *) lock);
+    if (err)
+	ETHR_FATAL_ERROR__(err);
+}
+
+#endif
+
+#elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+/* --- Native spinlocks using native atomics -------------------------------- */
+#define ETHR_HAVE_NATIVE_SPINLOCKS 1
+
+#define ETHR_NATIVE_SPINLOCK_IMPL "native-atomics"
+
+typedef ethr_atomic32_t ethr_native_spinlock_t;
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+
+#undef ETHR_NSPN_AOP__
+#define ETHR_NSPN_AOP__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
+
+static ETHR_INLINE void
+ethr_native_spinlock_init(ethr_native_spinlock_t *lock)
+{
+    ETHR_NSPN_AOP__(init)(lock, 0);
+}
+
+static ETHR_INLINE int
+ethr_native_spinlock_destroy(ethr_native_spinlock_t *lock)
+{
+    return ETHR_NSPN_AOP__(read)(lock) == 0 ? 0 : EBUSY;
+}
+
+static ETHR_INLINE void
+ethr_native_spin_unlock(ethr_native_spinlock_t *lock)
+{
+    ETHR_ASSERT(ETHR_NSPN_AOP__(read)(lock) == 1);
+    ETHR_NSPN_AOP__(set_relb)(lock, 0);
+}
+
+static ETHR_INLINE void
+ethr_native_spin_lock(ethr_native_spinlock_t *lock)
+{
+    while (ETHR_NSPN_AOP__(cmpxchg_acqb)(lock, 1, 0) != 0) {
+	while (ETHR_NSPN_AOP__(read)(lock) != 0)
+	    ETHR_SPIN_BODY;
+    }
+    ETHR_COMPILER_BARRIER;
+}
+
+#undef ETHR_NSPN_AOP__
+
+#endif
+
+#endif
+
+
+#if defined(ETHR_HAVE_NATIVE_RWSPINLOCKS)
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+
+static ETHR_INLINE int
+ethr_native_rwlock_destroy(ethr_native_rwlock_t *lock)
+{
+    return 0;
+}
+
+#endif
+
+#elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+/* --- Native rwspinlocks using native atomics ------------------------------ */
+#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1
+#define ETHR_NATIVE_RWSPINLOCK_IMPL "native-atomics"
+
+typedef ethr_atomic32_t ethr_native_rwlock_t;
+#  define ETHR_WLOCK_FLAG__ (((ethr_sint32_t) 1) << 30)
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+
+#undef ETHR_NRWSPN_AOP__
+#define ETHR_NRWSPN_AOP__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
+
+static ETHR_INLINE void
+ethr_native_rwlock_init(ethr_native_rwlock_t *lock)
+{
+    ETHR_NRWSPN_AOP__(init)(lock, 0);
+}
+
+static ETHR_INLINE int
+ethr_native_rwlock_destroy(ethr_native_rwlock_t *lock)
+{
+    return ETHR_NRWSPN_AOP__(read)(lock) == 0 ? 0 : EBUSY;
+}
+
+static ETHR_INLINE void
+ethr_native_read_unlock(ethr_native_rwlock_t *lock)
+{
+    ETHR_ASSERT(ETHR_NRWSPN_AOP__(read)(lock) >= 0);
+    ETHR_NRWSPN_AOP__(dec_relb)(lock);
+}
+
+static ETHR_INLINE void
+ethr_native_read_lock(ethr_native_rwlock_t *lock)
+{
+    ethr_sint32_t act, exp = 0;
+    while (1) {
+	act = ETHR_NRWSPN_AOP__(cmpxchg_acqb)(lock, exp+1, exp);
+	if (act == exp)
+	    break;
+	/* Wait for writer to leave */
+	while (act & ETHR_WLOCK_FLAG__) {
+	    ETHR_SPIN_BODY;
+	    act = ETHR_NRWSPN_AOP__(read)(lock);
+	}
+	exp = act;
+    }
+    ETHR_COMPILER_BARRIER;
+}
+
+static ETHR_INLINE void
+ethr_native_write_unlock(ethr_native_rwlock_t *lock)
+{
+    ETHR_ASSERT(ETHR_NRWSPN_AOP__(read)(lock) == ETHR_WLOCK_FLAG__);
+    ETHR_NRWSPN_AOP__(set_relb)(lock, 0);
+}
+
+static ETHR_INLINE void
+ethr_native_write_lock(ethr_native_rwlock_t *lock)
+{
+    ethr_sint32_t act, exp = 0;
+    while (1) {
+	act = ETHR_NRWSPN_AOP__(cmpxchg_acqb)(lock, exp|ETHR_WLOCK_FLAG__, exp);
+	if (act == exp)
+	    break;
+	/* Wait for writer to leave */
+	while (act & ETHR_WLOCK_FLAG__) {
+	    ETHR_SPIN_BODY;
+	    act = ETHR_NRWSPN_AOP__(read)(lock);
+	}
+	exp = act;
+    }
+    act |= ETHR_WLOCK_FLAG__;
+    /* Wait for readers to leave */
+    while (act != ETHR_WLOCK_FLAG__) {
+	ETHR_SPIN_BODY;
+	act = ETHR_NRWSPN_AOP__(read_acqb)(lock);
+    }
+    ETHR_COMPILER_BARRIER;
+}
+
+#undef ETHR_NRWSPN_AOP__
+
+#endif
+
+#endif
+
+#endif
diff --git a/erts/include/internal/ethread.h b/erts/include/internal/ethread.h
index 934a79c6f9..142c26c0ca 100644
--- a/erts/include/internal/ethread.h
+++ b/erts/include/internal/ethread.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -33,27 +33,11 @@
 #include <stdlib.h>
 #include "erl_errno.h"
 
-/*
- * Extra memory barrier requirements:
- * - ethr_atomic_or_old() needs to enforce a memory barrier sufficient
- *   for a lock operation.
- * - ethr_atomic_and_old() needs to enforce a memory barrier sufficient
- *   for an unlock operation.
- * - ethr_atomic_cmpxchg() needs to enforce a memory barrier sufficient
- *   for a lock and unlock operation.
- */
-
-
-#undef ETHR_USE_RWMTX_FALLBACK
-#undef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
-#undef ETHR_HAVE_OPTIMIZED_LOCKS
-
-typedef struct {
-    long tv_sec;
-    long tv_nsec;
-} ethr_timeval;
-
 #if defined(DEBUG)
+#  define ETHR_DEBUG
+#endif
+
+#if defined(ETHR_DEBUG)
 #  undef ETHR_XCHK
 #  define  ETHR_XCHK 1
 #else
@@ -68,47 +52,57 @@ typedef struct {
 #elif defined(__WIN32__)
 #  define ETHR_INLINE __forceinline
 #endif
-#if defined(DEBUG) || !defined(ETHR_INLINE) || ETHR_XCHK \
+#if defined(ETHR_DEBUG) || !defined(ETHR_INLINE) || ETHR_XCHK \
     || (defined(__GNUC__) && defined(ERTS_MIXED_CYGWIN_VC))
 #  undef ETHR_INLINE
 #  define ETHR_INLINE 
 #  undef ETHR_TRY_INLINE_FUNCS
 #endif
-#ifdef ETHR_FORCE_INLINE_FUNCS
-#  define ETHR_TRY_INLINE_FUNCS
-#endif
 
-#if !defined(ETHR_DISABLE_NATIVE_IMPLS) \
-    && (defined(PURIFY) || defined(VALGRIND) || defined(ERTS_MIXED_CYGWIN_VC))
+#if !defined(ETHR_DISABLE_NATIVE_IMPLS) && (defined(PURIFY)||defined(VALGRIND))
 #  define ETHR_DISABLE_NATIVE_IMPLS
 #endif
 
-#define ETHR_RWMUTEX_INITIALIZED 	0x99999999
-#define ETHR_MUTEX_INITIALIZED		0x77777777
-#define ETHR_COND_INITIALIZED		0x55555555
-
+/* Assume 64-byte cache line size */
 #define ETHR_CACHE_LINE_SIZE 64
+#define ETHR_CACHE_LINE_MASK (ETHR_CACHE_LINE_SIZE - 1)
 
-#ifdef ETHR_INLINE_FUNC_NAME_
-#  define ETHR_CUSTOM_INLINE_FUNC_NAME_
-#else
+#define ETHR_CACHE_LINE_ALIGN_SIZE(SZ) \
+  (((((SZ) - 1) / ETHR_CACHE_LINE_SIZE) + 1) * ETHR_CACHE_LINE_SIZE)
+
+#ifndef ETHR_INLINE_FUNC_NAME_
 #  define ETHR_INLINE_FUNC_NAME_(X) X
 #endif
 
-#define ETHR_COMPILER_BARRIER ethr_compiler_barrier()
-#ifdef __GNUC__
-#  undef ETHR_COMPILER_BARRIER
-#  define ETHR_COMPILER_BARRIER __asm__ __volatile__("":::"memory")
+#if !defined(__func__)
+#  if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L
+#    if !defined(__GNUC__) ||  __GNUC__ < 2
+#      define __func__ "[unknown_function]"
+#    else
+#      define __func__ __FUNCTION__
+#    endif
+#  endif
 #endif
 
-#ifdef DEBUG
+int ethr_assert_failed(const char *file, int line, const char *func, char *a);
+#ifdef ETHR_DEBUG
 #define ETHR_ASSERT(A) \
-  ((void) ((A) ? 1 : ethr_assert_failed(__FILE__, __LINE__, #A)))
-int ethr_assert_failed(char *f, int l, char *a);
+  ((void) ((A) ? 1 : ethr_assert_failed(__FILE__, __LINE__, __func__, #A)))
 #else
 #define ETHR_ASSERT(A) ((void) 1)
 #endif
 
+#if defined(__GNUC__)
+#  define ETHR_PROTO_NORETURN__ void __attribute__((noreturn))
+#  define ETHR_IMPL_NORETURN__ void
+#elif defined(__WIN32__) && defined(_MSC_VER)
+#  define ETHR_PROTO_NORETURN__ __declspec(noreturn) void
+#  define ETHR_IMPL_NORETURN__ __declspec(noreturn) void
+#else
+#  define ETHR_PROTO_NORETURN__ void
+#  define ETHR_IMPL_NORETURN__ void
+#endif
+
 #if defined(ETHR_PTHREADS)
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
  * The pthread implementation                                                *
@@ -118,7 +112,9 @@ int ethr_assert_failed(char *f, int l, char *a);
 #error "_GNU_SOURCE not defined. Please, compile all files with -D_GNU_SOURCE."
 #endif
 
-#if defined(ETHR_HAVE_MIT_PTHREAD_H)
+#if defined(ETHR_NEED_NPTL_PTHREAD_H)
+#include <nptl/pthread.h>
+#elif defined(ETHR_HAVE_MIT_PTHREAD_H)
 #include <pthread/mit/pthread.h>
 #elif defined(ETHR_HAVE_PTHREAD_H)
 #include <pthread.h>
@@ -128,130 +124,23 @@ int ethr_assert_failed(char *f, int l, char *a);
 
 typedef pthread_t ethr_tid;
 
-typedef struct ethr_mutex_ ethr_mutex;
-struct ethr_mutex_ {
-    pthread_mutex_t pt_mtx;
-    int is_rec_mtx;
-    ethr_mutex *prev;
-    ethr_mutex *next;
-#if ETHR_XCHK
-    int initialized;
-#endif
-};
-
-typedef struct ethr_cond_ ethr_cond;
-struct ethr_cond_ {
-    pthread_cond_t pt_cnd;
-#if ETHR_XCHK
-    int initialized;
-#endif
-};
+typedef pthread_key_t ethr_tsd_key;
 
-#ifndef ETHR_HAVE_PTHREAD_RWLOCK_INIT
-#define ETHR_USE_RWMTX_FALLBACK
-#else
-typedef struct ethr_rwmutex_ ethr_rwmutex;
-struct ethr_rwmutex_ {
-    pthread_rwlock_t pt_rwlock;
-#if ETHR_XCHK
-    int initialized;
-#endif
-};
-#endif
+#define ETHR_HAVE_ETHR_SIG_FUNCS 1
 
-/* Static initializers */
-#if ETHR_XCHK
-#define ETHR_MUTEX_XCHK_INITER	, ETHR_MUTEX_INITIALIZED
-#define ETHR_COND_XCHK_INITER	, ETHR_COND_INITIALIZED
-#else
-#define ETHR_MUTEX_XCHK_INITER
-#define ETHR_COND_XCHK_INITER
+#if defined(PURIFY) || defined(VALGRIND)
+#  define ETHR_FORCE_PTHREAD_RWLOCK
+#  define ETHR_FORCE_PTHREAD_MUTEX
 #endif
 
-#define ETHR_MUTEX_INITER {PTHREAD_MUTEX_INITIALIZER, 0, NULL, NULL ETHR_MUTEX_XCHK_INITER}
-#define ETHR_COND_INITER {PTHREAD_COND_INITIALIZER ETHR_COND_XCHK_INITER}
-
-#if defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE) \
-    || defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP)
-#  define ETHR_HAVE_ETHR_REC_MUTEX_INIT 1
-#  ifdef PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP
-#    define ETHR_REC_MUTEX_INITER \
-            {PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP, 1, NULL, NULL ETHR_MUTEX_XCHK_INITER}
-#  endif
-#else
-#  undef ETHR_HAVE_ETHR_REC_MUTEX_INIT
+#if !defined(ETHR_FORCE_PTHREAD_RWLOCK)
+#  define ETHR_USE_OWN_RWMTX_IMPL__
 #endif
 
-#ifndef ETHR_HAVE_PTHREAD_ATFORK
-#  define ETHR_NO_FORKSAFETY 1
+#if !defined(ETHR_FORCE_PTHREAD_MUTEX) && 0
+#  define ETHR_USE_OWN_MTX_IMPL__
 #endif
 
-typedef pthread_key_t ethr_tsd_key;
-
-#define ETHR_HAVE_ETHR_SIG_FUNCS 1
-
-#ifdef ETHR_TRY_INLINE_FUNCS
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
-{
-    return pthread_mutex_trylock(&mtx->pt_mtx);
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
-{
-    return pthread_mutex_lock(&mtx->pt_mtx);
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
-{
-    return pthread_mutex_unlock(&mtx->pt_mtx);
-}
-
-#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx)
-{
-    return pthread_rwlock_tryrdlock(&rwmtx->pt_rwlock);
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx)
-{
-    return pthread_rwlock_rdlock(&rwmtx->pt_rwlock);
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx)
-{
-    return pthread_rwlock_unlock(&rwmtx->pt_rwlock);
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx)
-{
-    return pthread_rwlock_trywrlock(&rwmtx->pt_rwlock);
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx)
-{
-    return pthread_rwlock_wrlock(&rwmtx->pt_rwlock);
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx)
-{
-    return pthread_rwlock_unlock(&rwmtx->pt_rwlock);
-}
-
-#endif /* ETHR_HAVE_PTHREAD_RWLOCK_INIT */
-
-#endif /* ETHR_TRY_INLINE_FUNCS */
-
 #elif defined(ETHR_WIN32_THREADS)
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
  * The native win32 threads implementation                                   *
@@ -273,546 +162,296 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx)
 #  undef WIN32_LEAN_AND_MEAN
 #endif
 
-/* Types */
-typedef long ethr_tid; /* thread id type */
-typedef struct {
-    volatile int initialized;
-    CRITICAL_SECTION cs;
-#if ETHR_XCHK
-    int is_rec_mtx;
-#endif
-} ethr_mutex;
-
-typedef struct cnd_wait_event__ cnd_wait_event_;
-
-typedef struct {
-    volatile int initialized;
-    CRITICAL_SECTION cs;
-    cnd_wait_event_ *queue;
-    cnd_wait_event_ *queue_end;
-} ethr_cond;
+#if defined(_MSC_VER)
 
-#define ETHR_USE_RWMTX_FALLBACK
+#if ETHR_SIZEOF_LONG == 4
+#define ETHR_HAVE_INT32_T 1
+typedef long ethr_sint32_t;
+typedef unsigned long ethr_uint32_t;
+#endif
 
-/* Static initializers */
+#if ETHR_SIZEOF___INT64 == 8
+#define ETHR_HAVE_INT64_T 1
+typedef __int64 ethr_sint64_t;
+typedef unsigned __int64 ethr_uint64_t;
+#endif
 
-#define ETHR_MUTEX_INITER {0}
-#define ETHR_COND_INITER {0}
+#endif
 
-#define ETHR_REC_MUTEX_INITER ETHR_MUTEX_INITER
+struct ethr_join_data_;
 
-#define ETHR_HAVE_ETHR_REC_MUTEX_INIT 1
+/* Types */
+typedef struct {
+    long id;
+    struct ethr_join_data_ *jdata;
+} ethr_tid; /* thread id type */
 
 typedef DWORD ethr_tsd_key;
 
 #undef ETHR_HAVE_ETHR_SIG_FUNCS
 
-#ifdef ETHR_TRY_INLINE_FUNCS
-int ethr_fake_static_mutex_init(ethr_mutex *mtx);
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
-{
-    if (!mtx->initialized) {
-	int res = ethr_fake_static_mutex_init(mtx);
-	if (res != 0)
-	    return res;
-    }
-    return TryEnterCriticalSection(&mtx->cs) ? 0 : EBUSY;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
-{
-    if (!mtx->initialized) {
-	int res = ethr_fake_static_mutex_init(mtx);
-	if (res != 0)
-	    return res;
-    }
-    EnterCriticalSection(&mtx->cs);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
-{
-    LeaveCriticalSection(&mtx->cs);
-    return 0;
-}
-
-#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */
+#define ETHR_USE_OWN_RWMTX_IMPL__
 
-#ifdef ERTS_MIXED_CYGWIN_VC
+#define ETHR_YIELD() (Sleep(0), 0)
 
-/* atomics */
+#else /* No supported thread lib found */
 
-#ifdef _MSC_VER
-#  if _MSC_VER < 1300
-#    define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 /* Dont trust really old compilers */
-#  else
-#    if defined(_M_IX86)
-#      define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1
-#    else /* I.e. IA64 */
-#      if _MSC_VER >= 1400 
-#        define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1
-#      else
-#        define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0
-#      endif
-#    endif
-#  endif
-#  if _MSC_VER >= 1400
-#    include <intrin.h>
-#    undef ETHR_COMPILER_BARRIER
-#    define ETHR_COMPILER_BARRIER _ReadWriteBarrier()
-#  endif
-#pragma intrinsic(_ReadWriteBarrier)
-#pragma intrinsic(_InterlockedAnd)
-#pragma intrinsic(_InterlockedOr)
+#ifdef ETHR_NO_SUPP_THR_LIB_NOT_FATAL
+#define ETHR_NO_THREAD_LIB
 #else
-#    define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 
+#error "No supported thread lib found"
 #endif
 
-#define ETHR_HAVE_OPTIMIZED_ATOMIC_OPS 1
-#define ETHR_HAVE_OPTIMIZED_LOCKS 1
-
-typedef struct {
-    volatile LONG value;
-} ethr_atomic_t;
-
-typedef struct {
-    volatile LONG locked;
-} ethr_spinlock_t;
-
-typedef struct {
-    volatile LONG counter;
-} ethr_rwlock_t;
-#define ETHR_WLOCK_FLAG__ (((LONG) 1) << 30)
-
-#ifdef ETHR_TRY_INLINE_FUNCS
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, long i)
-{
-#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__
-    var->value = (LONG) i;
-#else
-    (void) InterlockedExchange(&var->value, (LONG) i);
 #endif
-    return 0;
-}
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, long i)
-{
-#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__
-    var->value = (LONG) i;
-#else
-    (void) InterlockedExchange(&var->value, (LONG) i);
+#ifndef ETHR_HAVE_INT32_T
+#if ETHR_SIZEOF_INT == 4
+#define ETHR_HAVE_INT32_T 1
+typedef int ethr_sint32_t;
+typedef unsigned int ethr_uint32_t;
+#elif ETHR_SIZEOF_LONG == 4
+#define ETHR_HAVE_INT32_T 1
+typedef long ethr_sint32_t;
+typedef unsigned long ethr_uint32_t;
 #endif
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var, long *i)
-{
-#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__
-    *i = var->value;
-#else
-    *i = InterlockedExchangeAdd(&var->value, (LONG) 0);
 #endif
-    return 0;
-}
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, long incr)
-{
-    (void) InterlockedExchangeAdd(&var->value, (LONG) incr);
-    return 0;
-}   
-    
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_addtest)(ethr_atomic_t *var,
-					    long i,
-					    long *testp)
-{
-    *testp = InterlockedExchangeAdd(&var->value, (LONG) i);
-    *testp += i;
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var)
-{
-    (void) InterlockedIncrement(&var->value);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var)
-{
-    (void) InterlockedDecrement(&var->value);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inctest)(ethr_atomic_t *var, long *testp)
-{
-    *testp = (long) InterlockedIncrement(&var->value);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dectest)(ethr_atomic_t *var, long *testp)
-{
-    *testp = (long) InterlockedDecrement(&var->value);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_and_old)(ethr_atomic_t *var,
-					    long mask,
-					    long *old)
-{
-    /*
-     * See "Extra memory barrier requirements" note at the top
-     * of the file.
-     *
-     * According to msdn _InterlockedAnd() provides a full
-     * memory barrier.
-     */
-    *old = (long) _InterlockedAnd(&var->value, mask);
-    return 0;
-}
+#ifndef ETHR_HAVE_INT64_T
+#if ETHR_SIZEOF_INT == 8
+#define ETHR_HAVE_INT64_T 1
+typedef int ethr_sint64_t;
+typedef unsigned int ethr_uint64_t;
+#elif ETHR_SIZEOF_LONG == 8
+#define ETHR_HAVE_INT64_T 1
+typedef long ethr_sint64_t;
+typedef unsigned long ethr_uint64_t;
+#elif ETHR_SIZEOF_LONG_LONG == 8
+#define ETHR_HAVE_INT64_T 1
+typedef long long ethr_sint64_t;
+typedef unsigned long long ethr_uint64_t;
+#endif
+#endif
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_or_old)(ethr_atomic_t *var,
-					   long mask,
-					   long *old)
-{
-    /*
-     * See "Extra memory barrier requirements" note at the top
-     * of the file.
-     *
-     * According to msdn _InterlockedOr() provides a full
-     * memory barrier.
-     */
-    *old = (long) _InterlockedOr(&var->value, mask);
-    return 0;
-}
+#if ETHR_SIZEOF_PTR == 4
+#ifndef ETHR_HAVE_INT32_T
+#error "No 32-bit integer type found"
+#endif
+typedef ethr_sint32_t ethr_sint_t;
+typedef ethr_uint32_t ethr_uint_t;
+#elif ETHR_SIZEOF_PTR == 8
+#ifndef ETHR_HAVE_INT64_T
+#error "No 64-bit integer type found"
+#endif
+typedef ethr_sint64_t ethr_sint_t;
+typedef ethr_uint64_t ethr_uint_t;
+#endif
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var,
-					    long new,
-                                            long expected,
-					    long *old)
-{
-    /*
-     * See "Extra memory barrier requirements" note at the top
-     * of the file.
-     *
-     * According to msdn _InterlockedCompareExchange() provides a full
-     * memory barrier.
-     */
-    *old = _InterlockedCompareExchange(&var->value, (LONG) new, (LONG) expected);
-    return 0;
-}
+#if defined(ETHR_SIZEOF___INT128_T) && ETHR_SIZEOF___INT128_T == 16
+#define ETHR_HAVE_INT128_T
+typedef __int128_t ethr_sint128_t;
+typedef __uint128_t ethr_uint128_t;
+#endif
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var,
-					 long new,
-					 long *old)
-{
-    *old = (long) InterlockedExchange(&var->value, (LONG) new);
-    return 0;
-}
+#define ETHR_FATAL_ERROR__(ERR) \
+  ethr_fatal_error__(__FILE__, __LINE__, __func__, (ERR))
 
-/*
- * According to msdn InterlockedExchange() provides a full
- * memory barrier. 
- */
+ETHR_PROTO_NORETURN__ ethr_fatal_error__(const char *file,
+					 int line,
+					 const char *func,
+					 int err);
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock)
-{
-#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__
-    lock->locked = (LONG) 0;
+#if !defined(__GNUC__)
+#  define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0
+#elif !defined(__GNUC_MINOR__)
+#  define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+  ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
+#elif !defined(__GNUC_PATCHLEVEL__)
+#  define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
 #else
-    (void) InterlockedExchange(&lock->locked, (LONG) 0);
+#  define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
 #endif
-    return 0;
-}
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock)
-{
-    return 0;
-}
-
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock)
-{
-    ETHR_COMPILER_BARRIER;
-    {
-#ifdef DEBUG
-	LONG old =
-#endif
-	    InterlockedExchange(&lock->locked, (LONG) 0);
-#ifdef DEBUG
-	ETHR_ASSERT(old == 1);
+#if !ETHR_AT_LEAST_GCC_VSN__(2, 96, 0)
+#define __builtin_expect(X, Y) (X)
 #endif
-    }
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock)
-{
-    LONG old;
-    do {
-	old = InterlockedExchange(&lock->locked, (LONG) 1);
-    } while (old != (LONG) 0);
-    ETHR_COMPILER_BARRIER;
-    return 0;
-}
 
-/*
- * According to msdn InterlockedIncrement, InterlockedDecrement,
- * and InterlockedExchangeAdd(), _InterlockedAnd, and _InterlockedOr
- * provides full memory barriers.
- */
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwlock_init)(ethr_rwlock_t *lock)
-{
-#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__
-    lock->counter = (LONG) 0;
+#if ETHR_AT_LEAST_GCC_VSN__(3, 1, 1)
+#  define ETHR_CHOOSE_EXPR __builtin_choose_expr
 #else
-    (void) InterlockedExchange(&lock->counter, (LONG) 0);
+#  define ETHR_CHOOSE_EXPR(B, E1, E2) ((B) ? (E1) : (E2))
 #endif
-    return 0;
-}
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwlock_destroy)(ethr_rwlock_t *lock)
-{
-    return 0;
-}
+#if ((defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))) \
+     || (defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_AMD64))))
+#  define ETHR_X86_RUNTIME_CONF__
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_read_unlock)(ethr_rwlock_t *lock)
-{
-    ETHR_COMPILER_BARRIER;
-    {
-#ifdef DEBUG
-	LONG old =
+#  define ETHR_X86_RUNTIME_CONF_HAVE_DW_CMPXCHG__ \
+  (__builtin_expect(ethr_runtime__.conf.have_dw_cmpxchg != 0, 1))
+#  define ETHR_X86_RUNTIME_CONF_HAVE_NO_DW_CMPXCHG__ \
+  (__builtin_expect(ethr_runtime__.conf.have_dw_cmpxchg == 0, 0))
+#  define ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ \
+  (__builtin_expect(ethr_runtime__.conf.have_sse2 != 0, 1))
+#  define ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__ \
+  (__builtin_expect(ethr_runtime__.conf.have_sse2 == 0, 0))
 #endif
-	    InterlockedDecrement(&lock->counter);
-	ETHR_ASSERT(old != 0);
-    }
-    return 0;
-}
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_read_lock)(ethr_rwlock_t *lock)
-{
-    while (1) {
-	LONG old = InterlockedIncrement(&lock->counter);
-	if ((old & ETHR_WLOCK_FLAG__) == 0)
-	    break; /* Got read lock */
-	/* Restore and wait for writers to unlock */
-	old = InterlockedDecrement(&lock->counter);
-	while (old & ETHR_WLOCK_FLAG__) {
-#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__
-	    old = lock->counter;
-#else
-	    old = InterlockedExchangeAdd(&lock->counter, (LONG) 0);
-#endif
-	}
-    }
-    ETHR_COMPILER_BARRIER;
-    return 0;
-}
+#if (defined(__GNUC__) \
+     && !defined(ETHR_PPC_HAVE_LWSYNC) \
+     && !defined(ETHR_PPC_HAVE_NO_LWSYNC) \
+     && (defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)))
+#  define ETHR_PPC_RUNTIME_CONF__
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_write_unlock)(ethr_rwlock_t *lock)
-{
-    ETHR_COMPILER_BARRIER;
-    {
-#ifdef DEBUG
-	LONG old =
+#  define ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ \
+  (__builtin_expect(ethr_runtime__.conf.have_lwsync != 0, 1))  
+#  define ETHR_PPC_RUNTIME_CONF_HAVE_NO_LWSYNC__ \
+  (__builtin_expect(ethr_runtime__.conf.have_lwsync == 0, 0))
 #endif
-	    _InterlockedAnd(&lock->counter, ~ETHR_WLOCK_FLAG__);
-	ETHR_ASSERT(old & ETHR_WLOCK_FLAG__);
-    }
-    return 0;
-}
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock)
-{
-    LONG old;
-    do {
-	old = _InterlockedOr(&lock->counter, ETHR_WLOCK_FLAG__);
-    } while (old & ETHR_WLOCK_FLAG__);
-    /* We got the write part of the lock; wait for readers to unlock */
-    while ((old & ~ETHR_WLOCK_FLAG__) != 0) {
-#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__
-	old = lock->counter;
-#else
-	old = InterlockedExchangeAdd(&lock->counter, (LONG) 0);
+typedef struct {
+#if defined(ETHR_X86_RUNTIME_CONF__)
+    int have_dw_cmpxchg;
+    int have_sse2;
 #endif
-	ETHR_ASSERT(old & ETHR_WLOCK_FLAG__);
-    }
-    ETHR_COMPILER_BARRIER;
-    return 0;
-}
+#if defined(ETHR_PPC_RUNTIME_CONF__)
+    int have_lwsync;
+#endif
+    int dummy;
+} ethr_runtime_conf_t;
 
-#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */
 
-#endif /* #ifdef ERTS_MIXED_CYGWIN_VC */
+typedef union {
+    ethr_runtime_conf_t conf;
+    char pad__[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_runtime_conf_t))+ETHR_CACHE_LINE_SIZE];
+} ethr_runtime_t;
 
-#else /* No supported thread lib found */
 
-#ifdef ETHR_NO_SUPP_THR_LIB_NOT_FATAL
-#define ETHR_NO_THREAD_LIB
-#else
-#error "No supported thread lib found"
-#endif
+extern ethr_runtime_t ethr_runtime__;
 
+/* For native CPU-optimised atomics, spinlocks, and rwlocks. */
+#if !defined(ETHR_DISABLE_NATIVE_IMPLS)
+#  if defined(__GNUC__)
+#    if defined(ETHR_PREFER_GCC_NATIVE_IMPLS)
+#      include "gcc/ethread.h"
+#    elif defined(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS)
+#      include "libatomic_ops/ethread.h"
+#    endif
+#    if !defined(ETHR_HAVE_NATIVE_ATOMIC32) && !defined(ETHR_HAVE_NATIVE_ATOMIC64)
+#      if ETHR_SIZEOF_PTR == 4
+#        if defined(__i386__)
+#          include "i386/ethread.h"
+#        elif (defined(__powerpc__)||defined(__ppc__))&&!defined(__powerpc64__)
+#          include "ppc32/ethread.h"
+#        elif defined(__sparc__)
+#          include "sparc32/ethread.h"
+#        elif defined(__tile__)
+#          include "tile/ethread.h"
+#        endif
+#      elif ETHR_SIZEOF_PTR == 8
+#        if defined(__x86_64__)
+#          include "x86_64/ethread.h"
+#        elif defined(__sparc__) && defined(__arch64__)
+#          include "sparc64/ethread.h"
+#        endif
+#      endif
+#if 0
+#      include "gcc/ethread.h"
+#      include "libatomic_ops/ethread.h"
 #endif
+#    endif
+#  elif defined(ETHR_HAVE_LIBATOMIC_OPS)
+#    include "libatomic_ops/ethread.h"
+#  elif defined(ETHR_WIN32_THREADS)
+#    include "win/ethread.h"
+#  endif
+#endif /* !ETHR_DISABLE_NATIVE_IMPLS */
 
-/* __builtin_expect() is needed by both native atomics code 
- * and the fallback code */
-#if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96)
-#define __builtin_expect(X, Y) (X)
-#endif
+#include "ethr_atomics.h" /* The atomics API */
 
-/* For CPU-optimised atomics, spinlocks, and rwlocks. */
-#if !defined(ETHR_DISABLE_NATIVE_IMPLS) && defined(__GNUC__)
-#  if ETHR_SIZEOF_PTR == 4
-#    if defined(__i386__)
-#      include "i386/ethread.h"
-#    elif (defined(__powerpc__) || defined(__ppc__)) && !defined(__powerpc64__)
-#      include "ppc32/ethread.h"
+#if defined(__GNUC__)
+#  ifndef ETHR_SPIN_BODY
+#    if defined(__i386__) || defined(__x86_64__)
+#      define ETHR_SPIN_BODY __asm__ __volatile__("rep;nop" : : : "memory")
+#    elif defined(__ia64__)
+#      define ETHR_SPIN_BODY __asm__ __volatile__("hint @pause" : : : "memory")
 #    elif defined(__sparc__)
-#      include "sparc32/ethread.h"
-#    elif defined(__tile__)
-#      include "tile/ethread.h"
-#    endif
-#  elif ETHR_SIZEOF_PTR == 8
-#    if defined(__x86_64__)
-#      include "x86_64/ethread.h"
-#    elif defined(__sparc__) && defined(__arch64__)
-#      include "sparc64/ethread.h"
+#      define ETHR_SPIN_BODY __asm__ __volatile__("membar #LoadLoad")
+#    else
+#      define ETHR_SPIN_BODY ETHR_COMPILER_BARRIER
 #    endif
 #  endif
-#endif /* !defined(ETHR_DISABLE_NATIVE_IMPLS) && defined(__GNUC__) */
-
-#ifdef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
-#  undef ETHR_HAVE_NATIVE_ATOMICS
-#endif
-#ifdef ETHR_HAVE_OPTIMIZED_LOCKS
-#  undef ETHR_HAVE_NATIVE_LOCKS
-#endif
-
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-#define ETHR_HAVE_OPTIMIZED_ATOMIC_OPS 1
-#endif
-#ifdef ETHR_HAVE_NATIVE_LOCKS
-#define ETHR_HAVE_OPTIMIZED_LOCKS 1
-#endif
-
-typedef struct {
-    unsigned open;
-    ethr_mutex mtx;
-    ethr_cond cnd;
-} ethr_gate;
-
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-/*
- * Map ethread native atomics to ethread API atomics.
- */
-typedef ethr_native_atomic_t ethr_atomic_t;
+#elif defined(ETHR_WIN32_THREADS)
+#  ifndef ETHR_SPIN_BODY
+#    define ETHR_SPIN_BODY do {YieldProcessor();ETHR_COMPILER_BARRIER;} while(0)
+#  endif
 #endif
 
-#ifdef ETHR_HAVE_NATIVE_LOCKS
-/*
- * Map ethread native spinlocks to ethread API spinlocks.
- */
-typedef ethr_native_spinlock_t ethr_spinlock_t;
-/*
- * Map ethread native rwlocks to ethread API rwlocks.
- */
-typedef ethr_native_rwlock_t ethr_rwlock_t;
-#endif
+#define ETHR_YIELD_AFTER_BUSY_LOOPS 50
 
-#ifdef ETHR_USE_RWMTX_FALLBACK
-typedef struct {
-    ethr_mutex mtx;
-    ethr_cond rcnd;
-    ethr_cond wcnd;
-    unsigned readers;
-    unsigned waiting_readers;
-    unsigned waiting_writers;
-#if ETHR_XCHK
-    int initialized;
-#endif
-} ethr_rwmutex;
+#ifndef ETHR_SPIN_BODY
+#  define ETHR_SPIN_BODY ETHR_COMPILER_BARRIER
 #endif
 
-#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
-typedef long ethr_atomic_t;
+#ifndef ETHR_YIELD
+#  if defined(ETHR_HAVE_SCHED_YIELD)
+#    ifdef ETHR_HAVE_SCHED_H
+#      include <sched.h>
+#    endif
+#    include <errno.h>
+#    if defined(ETHR_SCHED_YIELD_RET_INT)
+#      define ETHR_YIELD() (sched_yield() < 0 ? errno : 0)
+#    else
+#      define ETHR_YIELD() (sched_yield(), 0)
+#    endif
+#  elif defined(ETHR_HAVE_PTHREAD_YIELD)
+#    if defined(ETHR_PTHREAD_YIELD_RET_INT)
+#      define ETHR_YIELD() pthread_yield()
+#    else
+#      define ETHR_YIELD() (pthread_yield(), 0)
+#    endif
+#  else
+#    define ETHR_YIELD() (ethr_compiler_barrier(), 0)
+#  endif
 #endif
 
-#ifndef ETHR_HAVE_OPTIMIZED_LOCKS
+#include "ethr_optimized_fallbacks.h"
 
-#if defined(ETHR_WIN32_THREADS)
-typedef struct {
-    CRITICAL_SECTION cs;
-} ethr_spinlock_t;
 typedef struct {
-    CRITICAL_SECTION cs;
-    unsigned counter;
-} ethr_rwlock_t;
-
-int ethr_do_spinlock_init(ethr_spinlock_t *lock);
-int ethr_do_rwlock_init(ethr_rwlock_t *lock);
+    void *(*thread_create_prepare_func)(void);
+    void (*thread_create_parent_func)(void *);
+    void (*thread_create_child_func)(void *);
+} ethr_init_data;
 
-#define ETHR_RWLOCK_WRITERS (((unsigned) 1) << 31)
+#define ETHR_INIT_DATA_DEFAULT_INITER {NULL, NULL, NULL}
 
-#elif defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-typedef struct {
-    pthread_spinlock_t spnlck;
-} ethr_spinlock_t;
 typedef struct {
-    pthread_spinlock_t spnlck;
-    unsigned counter;
-} ethr_rwlock_t;
-#define ETHR_RWLOCK_WRITERS (((unsigned) 1) << 31)
-
-#else /* ethr mutex/rwmutex */
+    void *(*alloc)(size_t);
+    void *(*realloc)(void *, size_t);
+    void (*free)(void *);
+} ethr_memory_allocator;
 
-typedef struct {
-    ethr_mutex mtx;
-} ethr_spinlock_t;
+#define ETHR_MEM_ALLOC_DEF_INITER__ {NULL, NULL, NULL}
 
 typedef struct {
-    ethr_rwmutex rwmtx;
-} ethr_rwlock_t;
+    ethr_memory_allocator std;
+    ethr_memory_allocator sl;
+    ethr_memory_allocator ll;
+} ethr_memory_allocators;
 
-#endif /* end mutex/rwmutex */
-#endif /* ETHR_HAVE_OPTIMIZED_LOCKS */
+#define ETHR_MEM_ALLOCS_DEF_INITER__					\
+  {ETHR_MEM_ALLOC_DEF_INITER__,						\
+   ETHR_MEM_ALLOC_DEF_INITER__,						\
+   ETHR_MEM_ALLOC_DEF_INITER__}
 
 typedef struct {
-    void *(*alloc)(size_t);
-    void *(*realloc)(void *, size_t);
-    void (*free)(void *);
-    void *(*thread_create_prepare_func)(void);
-    void (*thread_create_parent_func)(void *);
-    void (*thread_create_child_func)(void *);
-} ethr_init_data;
+    ethr_memory_allocators mem;
+    int reader_groups;
+    int main_threads;
+} ethr_late_init_data;
 
-#define ETHR_INIT_DATA_DEFAULT_INITER {malloc, realloc, free, NULL, NULL, NULL}
+#define ETHR_LATE_INIT_DATA_DEFAULT_INITER				\
+  {ETHR_MEM_ALLOCS_DEF_INITER__, 0, 0}
 
 typedef struct {
     int detached;			/* boolean (default false) */
@@ -821,18 +460,14 @@ typedef struct {
 
 #define ETHR_THR_OPTS_DEFAULT_INITER {0, -1}
 
-#if defined(ETHR_CUSTOM_INLINE_FUNC_NAME_) || !defined(ETHR_TRY_INLINE_FUNCS)
-#  define ETHR_NEED_MTX_PROTOTYPES__
-#  define ETHR_NEED_RWMTX_PROTOTYPES__
-#  define ETHR_NEED_SPINLOCK_PROTOTYPES__
-#  define ETHR_NEED_ATOMIC_PROTOTYPES__
-#endif
 
-#if !defined(ETHR_NEED_RWMTX_PROTOTYPES__) && defined(ETHR_USE_RWMTX_FALLBACK)
-#  define ETHR_NEED_RWMTX_PROTOTYPES__
+#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+#  define ETHR_NEED_SPINLOCK_PROTOTYPES__
+#  define ETHR_NEED_RWSPINLOCK_PROTOTYPES__
 #endif
 
 int ethr_init(ethr_init_data *);
+int ethr_late_init(ethr_late_init_data *);
 int ethr_install_exit_handler(void (*funcp)(void));
 int ethr_thr_create(ethr_tid *, void * (*)(void *), void *, ethr_thr_opts *);
 int ethr_thr_join(ethr_tid, void **);
@@ -840,79 +475,12 @@ int ethr_thr_detach(ethr_tid);
 void ethr_thr_exit(void *);
 ethr_tid ethr_self(void);
 int ethr_equal_tids(ethr_tid, ethr_tid);
-int ethr_mutex_init(ethr_mutex *);
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-int ethr_rec_mutex_init(ethr_mutex *);
-#endif
-int ethr_mutex_destroy(ethr_mutex *);
-int ethr_mutex_set_forksafe(ethr_mutex *);
-int ethr_mutex_unset_forksafe(ethr_mutex *);
-#ifdef ETHR_NEED_MTX_PROTOTYPES__
-int ethr_mutex_trylock(ethr_mutex *);
-int ethr_mutex_lock(ethr_mutex *);
-int ethr_mutex_unlock(ethr_mutex *);
-#endif
-int ethr_cond_init(ethr_cond *);
-int ethr_cond_destroy(ethr_cond *);
-int ethr_cond_signal(ethr_cond *);
-int ethr_cond_broadcast(ethr_cond *);
-int ethr_cond_wait(ethr_cond *, ethr_mutex *);
-int ethr_cond_timedwait(ethr_cond *, ethr_mutex *, ethr_timeval *);
-
-int ethr_rwmutex_init(ethr_rwmutex *);
-int ethr_rwmutex_destroy(ethr_rwmutex *);
-#ifdef ETHR_NEED_RWMTX_PROTOTYPES__
-int ethr_rwmutex_tryrlock(ethr_rwmutex *);
-int ethr_rwmutex_rlock(ethr_rwmutex *);
-int ethr_rwmutex_runlock(ethr_rwmutex *);
-int ethr_rwmutex_tryrwlock(ethr_rwmutex *);
-int ethr_rwmutex_rwlock(ethr_rwmutex *);
-int ethr_rwmutex_rwunlock(ethr_rwmutex *);
-#endif
-
-#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__
-int ethr_atomic_init(ethr_atomic_t *, long);
-int ethr_atomic_set(ethr_atomic_t *, long);
-int ethr_atomic_read(ethr_atomic_t *, long *);
-int ethr_atomic_inctest(ethr_atomic_t *, long *);
-int ethr_atomic_dectest(ethr_atomic_t *, long *);
-int ethr_atomic_inc(ethr_atomic_t *);
-int ethr_atomic_dec(ethr_atomic_t *);
-int ethr_atomic_addtest(ethr_atomic_t *, long, long *);
-int ethr_atomic_add(ethr_atomic_t *, long);
-int ethr_atomic_and_old(ethr_atomic_t *, long, long *);
-int ethr_atomic_or_old(ethr_atomic_t *, long, long *);
-int ethr_atomic_xchg(ethr_atomic_t *, long, long *);
-int ethr_atomic_cmpxchg(ethr_atomic_t *, long, long, long *);
-#endif
-
-#ifdef ETHR_NEED_SPINLOCK_PROTOTYPES__
-int ethr_spinlock_init(ethr_spinlock_t *);
-int ethr_spinlock_destroy(ethr_spinlock_t *);
-int ethr_spin_unlock(ethr_spinlock_t *);
-int ethr_spin_lock(ethr_spinlock_t *);
-
-int ethr_rwlock_init(ethr_rwlock_t *);
-int ethr_rwlock_destroy(ethr_rwlock_t *);
-int ethr_read_unlock(ethr_rwlock_t *);
-int ethr_read_lock(ethr_rwlock_t *);
-int ethr_write_unlock(ethr_rwlock_t *);
-int ethr_write_lock(ethr_rwlock_t *);
-#endif
 
-int ethr_time_now(ethr_timeval *);
 int ethr_tsd_key_create(ethr_tsd_key *);
 int ethr_tsd_key_delete(ethr_tsd_key);
 int ethr_tsd_set(ethr_tsd_key, void *);
 void *ethr_tsd_get(ethr_tsd_key);
 
-int ethr_gate_init(ethr_gate *);
-int ethr_gate_destroy(ethr_gate *);
-int ethr_gate_close(ethr_gate *);
-int ethr_gate_let_through(ethr_gate *, unsigned);
-int ethr_gate_wait(ethr_gate *);
-int ethr_gate_swait(ethr_gate *, int);
-
 #ifdef ETHR_HAVE_ETHR_SIG_FUNCS
 #include <signal.h>
 int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset);
@@ -921,528 +489,258 @@ int ethr_sigwait(const sigset_t *set, int *sig);
 
 void ethr_compiler_barrier(void);
 
-#ifdef ETHR_TRY_INLINE_FUNCS
-
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, long i)
-{
-    ethr_native_atomic_init(var, i);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, long i)
-{
-    ethr_native_atomic_set(var, i);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var, long *i)
-{
-    *i = ethr_native_atomic_read(var);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, long incr)
-{
-    ethr_native_atomic_add(var, incr);
-    return 0;
-}   
-    
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_addtest)(ethr_atomic_t *var,
-					    long i,
-					    long *testp)
-{
-    *testp = ethr_native_atomic_add_return(var, i);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var)
-{
-    ethr_native_atomic_inc(var);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var)
-{
-    ethr_native_atomic_dec(var);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inctest)(ethr_atomic_t *var, long *testp)
-{
-    *testp = ethr_native_atomic_inc_return(var);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dectest)(ethr_atomic_t *var, long *testp)
-{
-    *testp = ethr_native_atomic_dec_return(var);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_and_old)(ethr_atomic_t *var,
-					    long mask,
-					    long *old)
-{
-    /*
-     * See "Extra memory barrier requirements" note at the top
-     * of the file.
-     */
-    *old = ethr_native_atomic_and_retold(var, mask);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_or_old)(ethr_atomic_t *var,
-					   long mask,
-					   long *old)
-{
-    /*
-     * See "Extra memory barrier requirements" note at the top
-     * of the file.
-     */
-    *old = ethr_native_atomic_or_retold(var, mask);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var,
-					 long new,
-					 long *old)
-{
-    *old = ethr_native_atomic_xchg(var, new);
-    return 0;
-}   
-
-/*
- * If *var == *old, replace *old with new, else do nothing.
- * In any case return the original value of *var in *old.
- */
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var,
-					    long new,
-                                            long expected,
-					    long *old)
-{
-    /*
-     * See "Extra memory barrier requirements" note at the top
-     * of the file.
-     */
-    *old = ethr_native_atomic_cmpxchg(var, new, expected);
-    return 0;
-}
+#if defined(ETHR_HAVE_NATIVE_SPINLOCKS)
+typedef ethr_native_spinlock_t ethr_spinlock_t;
+#elif defined(__WIN32__)
+typedef CRITICAL_SECTION ethr_spinlock_t;
+#else
+typedef pthread_mutex_t ethr_spinlock_t;
+#endif
 
-#endif /* ETHR_HAVE_NATIVE_ATOMICS */
+#ifdef ETHR_NEED_SPINLOCK_PROTOTYPES__
+int ethr_spinlock_init(ethr_spinlock_t *);
+int ethr_spinlock_destroy(ethr_spinlock_t *);
+void ethr_spin_unlock(ethr_spinlock_t *);
+void ethr_spin_lock(ethr_spinlock_t *);
+#endif
 
-#ifdef ETHR_HAVE_NATIVE_LOCKS
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
 
 static ETHR_INLINE int
 ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock)
 {
+#ifdef ETHR_HAVE_NATIVE_SPINLOCKS
     ethr_native_spinlock_init(lock);
     return 0;
+#elif defined(__WIN32__)
+    if (!InitializeCriticalSectionAndSpinCount((CRITICAL_SECTION *) lock, INT_MAX))
+	return ethr_win_get_errno__();
+    return 0;
+#else
+    return pthread_mutex_init((pthread_mutex_t *) lock, NULL);
+#endif
 }
 
 static ETHR_INLINE int
 ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock)
 {
+#ifdef ETHR_HAVE_NATIVE_SPINLOCKS
+    return ethr_native_spinlock_destroy(lock);
+#elif defined(__WIN32__)
+    DeleteCriticalSection((CRITICAL_SECTION *) lock);
     return 0;
+#else
+    return pthread_mutex_destroy((pthread_mutex_t *) lock);
+#endif
 }
 
-static ETHR_INLINE int
+static ETHR_INLINE void
 ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock)
 {
+#ifdef ETHR_HAVE_NATIVE_SPINLOCKS
     ethr_native_spin_unlock(lock);
-    return 0;
+#elif defined(__WIN32__)
+    LeaveCriticalSection((CRITICAL_SECTION *) lock);
+#else
+    int err = pthread_mutex_unlock((pthread_mutex_t *) lock);
+    if (err)
+	ETHR_FATAL_ERROR__(err);
+#endif
 }
 
-static ETHR_INLINE int
+static ETHR_INLINE void
 ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock)
 {
+#ifdef ETHR_HAVE_NATIVE_SPINLOCKS
     ethr_native_spin_lock(lock);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwlock_init)(ethr_rwlock_t *lock)
-{
-    ethr_native_rwlock_init(lock);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_rwlock_destroy)(ethr_rwlock_t *lock)
-{
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_read_unlock)(ethr_rwlock_t *lock)
-{
-    ethr_native_read_unlock(lock);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_read_lock)(ethr_rwlock_t *lock)
-{
-    ethr_native_read_lock(lock);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_write_unlock)(ethr_rwlock_t *lock)
-{
-    ethr_native_write_unlock(lock);
-    return 0;
-}
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock)
-{
-    ethr_native_write_lock(lock);
-    return 0;
+#elif defined(__WIN32__)
+    EnterCriticalSection((CRITICAL_SECTION *) lock);
+#else
+    int err = pthread_mutex_lock((pthread_mutex_t *) lock);
+    if (err)
+	ETHR_FATAL_ERROR__(err);
+#endif
 }
 
-#endif /* ETHR_HAVE_NATIVE_LOCKS */
-
 #endif /* ETHR_TRY_INLINE_FUNCS */
 
-/*
- * Fallbacks for atomics used in absence of optimized implementation.
- */
-#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
+typedef struct ethr_ts_event_ ethr_ts_event; /* Needed by ethr_mutex.h */
 
-#define ETHR_ATOMIC_ADDR_BITS 4
-#define ETHR_ATOMIC_ADDR_SHIFT 3
-
-typedef struct {
-    union {
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-	pthread_spinlock_t spnlck;
+#if defined(ETHR_WIN32_THREADS)
+#  include "win/ethr_event.h"
 #else
-	ethr_mutex mtx;
-#endif
-	char buf[ETHR_CACHE_LINE_SIZE];
-    } u;
-} ethr_atomic_protection_t;
-
-extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS];
-
-
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-
-#define ETHR_ATOMIC_PTR2LCK__(PTR) \
-(&ethr_atomic_protection__[((((unsigned long) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \
-			& ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.spnlck)
-
-
-#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS)			\
-do {									\
-    pthread_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP));		\
-    int res__ = pthread_spin_lock(slp__);				\
-    if (res__ != 0)							\
-	return res__;							\
-    { EXPS; }								\
-    return pthread_spin_unlock(slp__);					\
-} while (0)
-
-#else /* ethread mutex */
+#  include "pthread/ethr_event.h"
+#endif
+
+int ethr_set_main_thr_status(int, int);
+int ethr_get_main_thr_status(int *);
+
+struct ethr_ts_event_ {
+    ethr_ts_event *next;
+    ethr_ts_event *prev;
+    ethr_event event;
+    void *udata;
+    ethr_atomic32_t uaflgs;
+    unsigned uflgs;
+    unsigned iflgs;		/* for ethr lib only */
+    short rgix;			/* for ethr lib only */
+    short mtix;			/* for ethr lib only */
+};
 
-#define ETHR_ATOMIC_PTR2LCK__(PTR) \
-(&ethr_atomic_protection__[((((unsigned long) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \
-		     & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.mtx)
+#define ETHR_TS_EV_ETHREAD	(((unsigned) 1) << 0)
+#define ETHR_TS_EV_INITED	(((unsigned) 1) << 1)
+#define ETHR_TS_EV_TMP		(((unsigned) 1) << 2)
+#define ETHR_TS_EV_MAIN_THR	(((unsigned) 1) << 3)
 
-#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS)			\
-do {									\
-    ethr_mutex *mtxp__ = ETHR_ATOMIC_PTR2LCK__((AP));			\
-    int res__ = ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(mtxp__);	\
-    if (res__ != 0)							\
-	return res__;							\
-    { EXPS; }								\
-    return ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(mtxp__);		\
-} while (0)
+int ethr_get_tmp_ts_event__(ethr_ts_event **tsepp);
+int ethr_free_ts_event__(ethr_ts_event *tsep);
+int ethr_make_ts_event__(ethr_ts_event **tsepp);
 
-#endif /* end ethread mutex */
+#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__)
+ethr_ts_event *ethr_get_ts_event(void);
+void ethr_leave_ts_event(ethr_ts_event *);
+#endif
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+#if defined(ETHR_PTHREADS)
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, long i)
-{
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = (ethr_atomic_t) i);
-}
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__)
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, long i)
-{
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = (ethr_atomic_t) i);
-}
+extern pthread_key_t ethr_ts_event_key__;
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var, long *i)
+static ETHR_INLINE ethr_ts_event *
+ETHR_INLINE_FUNC_NAME_(ethr_get_ts_event)(void)
 {
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *i = (long) *var);
+    ethr_ts_event *tsep = pthread_getspecific(ethr_ts_event_key__);
+    if (!tsep) {
+	int res = ethr_make_ts_event__(&tsep);
+	if (res != 0)
+	    ETHR_FATAL_ERROR__(res);
+	ETHR_ASSERT(tsep);
+    }
+    return tsep;
 }
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inctest)(ethr_atomic_t *incp, long *testp)
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_leave_ts_event)(ethr_ts_event *tsep)
 {
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(incp, *testp = (long) ++(*incp));
-}
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dectest)(ethr_atomic_t *decp, long *testp)
-{
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(decp, *testp = (long) --(*decp));
 }
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, long incr)
-{
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr);
-}   
-    
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_addtest)(ethr_atomic_t *incp,
-					    long i,
-					    long *testp)
-{
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(incp, *incp += i; *testp = *incp);
-}
+#endif
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *incp)
-{
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(incp, ++(*incp));
-}
+#elif defined(ETHR_WIN32_THREADS)
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *decp)
-{
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(decp, --(*decp));
-}
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__)
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_and_old)(ethr_atomic_t *var,
-					    long mask,
-					    long *old)
-{
-    /*
-     * See "Extra memory barrier requirements" note at the top
-     * of the file.
-     */
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *old = *var; *var &= mask);
-}
+extern DWORD ethr_ts_event_key__;
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_or_old)(ethr_atomic_t *var,
-					   long mask,
-					   long *old)
+static ETHR_INLINE ethr_ts_event *
+ETHR_INLINE_FUNC_NAME_(ethr_get_ts_event)(void)
 {
-    /*
-     * See "Extra memory barrier requirements" note at the top
-     * of the file.
-     */
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *old = *var; *var |= mask);
+    ethr_ts_event *tsep = TlsGetValue(ethr_ts_event_key__);
+    if (!tsep) {
+	int res = ethr_get_tmp_ts_event__(&tsep);
+	if (res != 0)
+	    ETHR_FATAL_ERROR__(res);
+	ETHR_ASSERT(tsep);
+    }
+    return tsep;
 }
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var,
-					 long new,
-					 long *old)
-{
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *old = *var; *var = new);
-}   
-
-/*
- * If *var == *old, replace *old with new, else do nothing.
- * In any case return the original value of *var in *old.
- */
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var,
-					    long new,
-                                            long expected,
-					    long *old)
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_leave_ts_event)(ethr_ts_event *tsep)
 {
-    /*
-     * See "Extra memory barrier requirements" note at the top
-     * of the file.
-     */
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(
-      var,
-      long old_val = *var;
-      *old = old_val;
-      if (__builtin_expect(old_val == expected, 1))
-          *var = new;
-      );
-    return 0;
+    if (tsep->iflgs & ETHR_TS_EV_TMP) {
+	int res = ethr_free_ts_event__(tsep);
+	if (res != 0)
+	    ETHR_FATAL_ERROR__(res);
+    }
 }
 
-#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */
-#endif /* #ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS */
-
-/*
- * Fallbacks for spin locks, and rw spin locks used in absence of
- * optimized implementation.
- */
-#ifndef ETHR_HAVE_OPTIMIZED_LOCKS
-
-#ifdef ETHR_TRY_INLINE_FUNCS
-
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock)
-{
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    return pthread_spin_init(&lock->spnlck, 0);
-#else
-    return ethr_mutex_init(&lock->mtx);
 #endif
-}
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock)
-{
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    return pthread_spin_destroy(&lock->spnlck);
-#else
-    return ethr_mutex_destroy(&lock->mtx);
 #endif
-}
 
+#include "ethr_mutex.h" /* Need atomic declarations and tse */
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock)
-{
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    return pthread_spin_unlock(&lock->spnlck);
+#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
+typedef ethr_native_rwlock_t ethr_rwlock_t;
 #else
-    return ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(&lock->mtx);
+typedef ethr_rwmutex ethr_rwlock_t;
 #endif
-}
 
-static ETHR_INLINE int
-ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock)
-{
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    return pthread_spin_lock(&lock->spnlck);
-#else
-    return ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(&lock->mtx);
+#ifdef ETHR_NEED_RWSPINLOCK_PROTOTYPES__
+int ethr_rwlock_init(ethr_rwlock_t *);
+int ethr_rwlock_destroy(ethr_rwlock_t *);
+void ethr_read_unlock(ethr_rwlock_t *);
+void ethr_read_lock(ethr_rwlock_t *);
+void ethr_write_unlock(ethr_rwlock_t *);
+void ethr_write_lock(ethr_rwlock_t *);
 #endif
-}
 
-#ifdef ETHR_USE_RWMTX_FALLBACK
-#define ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(X) X
-#else
-#define ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(X) ETHR_INLINE_FUNC_NAME_(X)
-#endif
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
 
 static ETHR_INLINE int
 ETHR_INLINE_FUNC_NAME_(ethr_rwlock_init)(ethr_rwlock_t *lock)
 {
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    lock->counter = 0;
-    return pthread_spin_init(&lock->spnlck, 0);
+#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
+    ethr_native_rwlock_init(lock);
+    return 0;
 #else
-    return ethr_rwmutex_init(&lock->rwmtx);
+    return ethr_rwmutex_init_opt((ethr_rwmutex *) lock, NULL);
 #endif
 }
 
 static ETHR_INLINE int
 ETHR_INLINE_FUNC_NAME_(ethr_rwlock_destroy)(ethr_rwlock_t *lock)
 {
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    return pthread_spin_destroy(&lock->spnlck);
+#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
+    return 0;
 #else
-    return ethr_rwmutex_destroy(&lock->rwmtx);
+    return ethr_rwmutex_destroy((ethr_rwmutex *) lock);
 #endif
 }
 
-static ETHR_INLINE int
+static ETHR_INLINE void
 ETHR_INLINE_FUNC_NAME_(ethr_read_unlock)(ethr_rwlock_t *lock)
 {
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    int res = pthread_spin_lock(&lock->spnlck);
-    if (res != 0)
-	return res;
-    lock->counter--;
-    return pthread_spin_unlock(&lock->spnlck);
+#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
+    ethr_native_read_unlock(lock);
 #else
-    return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_runlock)(&lock->rwmtx);
+    ethr_rwmutex_runlock((ethr_rwmutex *) lock);
 #endif
 }
 
-static ETHR_INLINE int
+static ETHR_INLINE void
 ETHR_INLINE_FUNC_NAME_(ethr_read_lock)(ethr_rwlock_t *lock)
 {
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    int locked = 0;
-    do {
-	int res = pthread_spin_lock(&lock->spnlck);
-	if (res != 0)
-	    return res;
-	if ((lock->counter & ETHR_RWLOCK_WRITERS) == 0) {
-	    lock->counter++;
-	    locked = 1;
-	}
-	res = pthread_spin_unlock(&lock->spnlck);
-	if (res != 0)
-	    return res;
-    } while (!locked);
-    return 0;
+#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
+    ethr_native_read_lock(lock);
 #else
-    return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_rlock)(&lock->rwmtx);
+    ethr_rwmutex_rlock((ethr_rwmutex *) lock);
 #endif
 }
 
-static ETHR_INLINE int
+static ETHR_INLINE void
 ETHR_INLINE_FUNC_NAME_(ethr_write_unlock)(ethr_rwlock_t *lock)
 {
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    lock->counter = 0;
-    return pthread_spin_unlock(&lock->spnlck);
+#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
+    ethr_native_write_unlock(lock);
 #else
-    return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_rwunlock)(&lock->rwmtx);
+    ethr_rwmutex_rwunlock((ethr_rwmutex *) lock);
 #endif
 }
 
-static ETHR_INLINE int
+static ETHR_INLINE void
 ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock)
 {
-#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-    while (1) {
-	int res = pthread_spin_lock(&lock->spnlck);
-	if (res != 0)
-	    return res;
-	lock->counter |= ETHR_RWLOCK_WRITERS;
-	if (lock->counter == ETHR_RWLOCK_WRITERS)
-	    return 0;
-	res = pthread_spin_unlock(&lock->spnlck);
-	if (res != 0)
-	    return res;
-    }
+#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
+    ethr_native_write_lock(lock);
 #else
-    return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_rwlock)(&lock->rwmtx);
+    ethr_rwmutex_rwlock((ethr_rwmutex *) lock);
 #endif
 }
 
-#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */
-
-#endif /* ETHR_HAVE_OPTIMIZED_LOCKS */
-
-#if defined(ETHR_HAVE_OPTIMIZED_LOCKS) || defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
-# define ETHR_HAVE_OPTIMIZED_SPINLOCK
-#endif
+#endif /* ETHR_TRY_INLINE_FUNCS */
 
 #endif /* #ifndef ETHREAD_H__ */
diff --git a/erts/include/internal/ethread_header_config.h.in b/erts/include/internal/ethread_header_config.h.in
index e5b4946a53..dd3599f86d 100644
--- a/erts/include/internal/ethread_header_config.h.in
+++ b/erts/include/internal/ethread_header_config.h.in
@@ -1,25 +1,43 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
 /* Define to the size of pointers */
 #undef ETHR_SIZEOF_PTR
 
+/* Define to the size of int */
+#undef ETHR_SIZEOF_INT
+
+/* Define to the size of long */
+#undef ETHR_SIZEOF_LONG
+
+/* Define to the size of long long */
+#undef ETHR_SIZEOF_LONG_LONG
+
+/* Define to the size of __int64 */
+#undef ETHR_SIZEOF___INT64
+
+/* Define to the size of __int128_t */
+#undef ETHR_SIZEOF___INT128_T
+
+/* Define if bigendian */
+#undef ETHR_BIGENDIAN
+
 /* Define if you want to disable native ethread implementations */
 #undef ETHR_DISABLE_NATIVE_IMPLS
 
@@ -29,26 +47,190 @@
 /* Define if you have pthreads */
 #undef ETHR_PTHREADS
 
+/* Define if you need the <nptl/pthread.h> header file. */
+#undef ETHR_NEED_NPTL_PTHREAD_H
+
 /* Define if you have the <pthread.h> header file. */
 #undef ETHR_HAVE_PTHREAD_H
 
 /* Define if the pthread.h header file is in pthread/mit directory. */
 #undef ETHR_HAVE_MIT_PTHREAD_H
 
-/* Define if you have the pthread_mutexattr_settype function. */
-#undef ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE
+/* Define if you have the pthread_spin_lock function. */
+#undef ETHR_HAVE_PTHREAD_SPIN_LOCK
+
+/* Define if you want to force usage of pthread rwlocks */
+#undef ETHR_FORCE_PTHREAD_RWLOCK
 
-/* Define if you have the pthread_mutexattr_setkind_np function. */
-#undef ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP
+/* Define if you have the pthread_rwlockattr_setkind_np() function. */
+#undef ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP
 
-/* Define if you have the pthread_atfork function. */
-#undef ETHR_HAVE_PTHREAD_ATFORK
+/* Define if you have the PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP rwlock
+   attribute. */
+#undef ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP
 
-/* Define if you have the pthread_spin_lock function. */
-#undef ETHR_HAVE_PTHREAD_SPIN_LOCK
+/* Define if you have a linux futex implementation. */
+#undef ETHR_HAVE_LINUX_FUTEX
+
+/* Define if x86/x86_64 out of order instructions should be synchronized */
+#undef ETHR_X86_OUT_OF_ORDER
+
+/* Define if only run in Sparc TSO mode */
+#undef ETHR_SPARC_TSO
+
+/* Define if only run in Sparc PSO, or TSO mode */
+#undef ETHR_SPARC_PSO
+
+/* Define if run in Sparc RMO, PSO, or TSO mode */
+#undef ETHR_SPARC_RMO
+
+/* Define if you have __sync_add_and_fetch() for 32-bit integers */
+#undef ETHR_HAVE___SYNC_ADD_AND_FETCH32
+
+/* Define if you have __sync_add_and_fetch() for 64-bit integers */
+#undef ETHR_HAVE___SYNC_ADD_AND_FETCH64
+
+/* Define if you have __sync_fetch_and_and() for 32-bit integers */
+#undef ETHR_HAVE___SYNC_FETCH_AND_AND32
+
+/* Define if you have __sync_fetch_and_and() for 64-bit integers */
+#undef ETHR_HAVE___SYNC_FETCH_AND_AND64
+
+/* Define if you have __sync_fetch_and_or() for 32-bit integers */
+#undef ETHR_HAVE___SYNC_FETCH_AND_OR32
+
+/* Define if you have __sync_fetch_and_or() for 64-bit integers */
+#undef ETHR_HAVE___SYNC_FETCH_AND_OR64
+
+/* Define if you have __sync_val_compare_and_swap() for 32-bit integers */
+#undef ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32
+
+/* Define if you have __sync_val_compare_and_swap() for 64-bit integers */
+#undef ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64
+
+/* Define if you have __sync_val_compare_and_swap() for 128-bit integers */
+#undef ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP128
+
+/* Define if you prefer gcc native ethread implementations */
+#undef ETHR_PREFER_GCC_NATIVE_IMPLS
+
+/* Define if you have the <sched.h> header file. */
+#undef ETHR_HAVE_SCHED_H
+
+/* Define if you have the sched_yield() function. */
+#undef ETHR_HAVE_SCHED_YIELD
+
+/* Define if you have the pthread_yield() function. */
+#undef ETHR_HAVE_PTHREAD_YIELD
+
+/* Define if pthread_yield() returns an int. */
+#undef ETHR_PTHREAD_YIELD_RET_INT
+
+/* Define if sched_yield() returns an int. */
+#undef ETHR_SCHED_YIELD_RET_INT
+
+/* Define if you use a gcc that supports -msse2 and understand sse2 specific asm statements */
+#undef ETHR_GCC_HAVE_SSE2_ASM_SUPPORT
+
+/* Define if you use a gcc that supports the double word cmpxchg instruction */
+#undef ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT
+
+/* Define if you get a register shortage with cmpxchg8b and position independent code */
+#undef ETHR_CMPXCHG8B_REGISTER_SHORTAGE
+
+/* Define if you have the pthread_rwlockattr_setkind_np() function. */
+#undef ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP
+
+/* Define if you have the PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP rwlock
+   attribute. */
+#undef ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP
+
+/* Define if you have gcc atomic operations */
+#undef ETHR_HAVE_GCC_ATOMIC_OPS
+
+/* Define if you prefer gcc native ethread implementations */
+#undef ETHR_PREFER_GCC_NATIVE_IMPLS
+
+/* Define if you have libatomic_ops atomic operations */
+#undef ETHR_HAVE_LIBATOMIC_OPS
+
+/* Define if you prefer libatomic_ops native ethread implementations */
+#undef ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS
+
+/* Define to the size of AO_t if libatomic_ops is used */
+#undef ETHR_SIZEOF_AO_T
+
+/* Define if you have _InterlockedAnd() */
+#undef ETHR_HAVE__INTERLOCKEDAND
+
+/* Define if you have _InterlockedAnd64() */
+#undef ETHR_HAVE__INTERLOCKEDAND64
+
+/* Define if you have _InterlockedCompareExchange() */
+#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE
+
+/* Define if you have _InterlockedCompareExchange64() */
+#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64
+
+/* Define if you have _InterlockedCompareExchange64_acq() */
+#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ
+
+/* Define if you have _InterlockedCompareExchange64_rel() */
+#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL
+
+/* Define if you have _InterlockedCompareExchange_acq() */
+#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ
+
+/* Define if you have _InterlockedCompareExchange_rel() */
+#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL
+
+/* Define if you have _InterlockedDecrement() */
+#undef ETHR_HAVE__INTERLOCKEDDECREMENT
+
+/* Define if you have _InterlockedDecrement64() */
+#undef ETHR_HAVE__INTERLOCKEDDECREMENT64
+
+/* Define if you have _InterlockedDecrement64_rel() */
+#undef ETHR_HAVE__INTERLOCKEDDECREMENT64_REL
+
+/* Define if you have _InterlockedDecrement_rel() */
+#undef ETHR_HAVE__INTERLOCKEDDECREMENT_REL
+
+/* Define if you have _InterlockedExchange() */
+#undef ETHR_HAVE__INTERLOCKEDEXCHANGE
+
+/* Define if you have _InterlockedExchange64() */
+#undef ETHR_HAVE__INTERLOCKEDEXCHANGE64
+
+/* Define if you have _InterlockedExchangeAdd() */
+#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD
+
+/* Define if you have _InterlockedExchangeAdd64() */
+#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64
+
+/* Define if you have _InterlockedExchangeAdd64_acq() */
+#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ
+
+/* Define if you have _InterlockedExchangeAdd_acq() */
+#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ
+
+/* Define if you have _InterlockedIncrement() */
+#undef ETHR_HAVE__INTERLOCKEDINCREMENT
+
+/* Define if you have _InterlockedIncrement64() */
+#undef ETHR_HAVE__INTERLOCKEDINCREMENT64
+
+/* Define if you have _InterlockedIncrement64_acq() */
+#undef ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ
+
+/* Define if you have _InterlockedIncrement_acq() */
+#undef ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ
+
+/* Define if you have _InterlockedOr() */
+#undef ETHR_HAVE__INTERLOCKEDOR
 
-/* Define if you have a pthread_rwlock implementation that can be used */
-#undef ETHR_HAVE_PTHREAD_RWLOCK_INIT
+/* Define if you have _InterlockedOr64() */
+#undef ETHR_HAVE__INTERLOCKEDOR64
 
 /* Define if you want to turn on extra sanity checking in the ethread library */
 #undef ETHR_XCHK
diff --git a/erts/include/internal/gcc/ethr_atomic.h b/erts/include/internal/gcc/ethr_atomic.h
new file mode 100644
index 0000000000..f598f8537b
--- /dev/null
+++ b/erts/include/internal/gcc/ethr_atomic.h
@@ -0,0 +1,211 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Native atomics ethread support using gcc's builtins
+ * Author: Rickard Green
+ */
+
+#undef ETHR_INCLUDE_ATOMIC_IMPL__
+#if !defined(ETHR_GCC_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__)
+#define ETHR_GCC_ATOMIC32_H__
+#if defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32)
+#  define ETHR_INCLUDE_ATOMIC_IMPL__ 4
+#endif
+#undef ETHR_ATOMIC_WANT_32BIT_IMPL__
+#elif !defined(ETHR_GCC_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__)
+#define ETHR_GCC_ATOMIC64_H__
+#if defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64)
+#  define ETHR_INCLUDE_ATOMIC_IMPL__ 8
+#endif
+#undef ETHR_ATOMIC_WANT_64BIT_IMPL__
+#endif
+
+#ifdef ETHR_INCLUDE_ATOMIC_IMPL__
+
+#ifndef ETHR_GCC_ATOMIC_COMMON__
+#define ETHR_GCC_ATOMIC_COMMON__
+
+#define ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ 0
+#if defined(__i386__) || defined(__x86_64__) || defined(__sparc__) \
+    || defined(__powerpc__) || defined(__ppc__) || defined(__mips__)
+#  undef ETHR_READ_AND_SET_WITHOUT_SYNC_OP__
+#  define ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ 1
+#endif
+
+#endif /* ETHR_GCC_ATOMIC_COMMON__ */
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#define ETHR_HAVE_NATIVE_ATOMIC32 1
+#define ETHR_NATIVE_ATOMIC32_IMPL "gcc"
+#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+#define ETHR_ATMC_T__ ethr_native_atomic32_t
+#define ETHR_AINT_T__ ethr_sint32_t
+#if defined(ETHR_HAVE___SYNC_ADD_AND_FETCH32)
+#  define ETHR_HAVE___SYNC_ADD_AND_FETCH
+#endif
+#if defined(ETHR_HAVE___SYNC_FETCH_AND_AND32)
+#  define ETHR_HAVE___SYNC_FETCH_AND_AND
+#endif 
+#if defined(ETHR_HAVE___SYNC_FETCH_AND_OR32)
+#  define ETHR_HAVE___SYNC_FETCH_AND_OR
+#endif
+#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8
+#define ETHR_HAVE_NATIVE_ATOMIC64 1
+#define ETHR_NATIVE_ATOMIC64_IMPL "gcc"
+#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#define ETHR_ATMC_T__ ethr_native_atomic64_t
+#define ETHR_AINT_T__ ethr_sint64_t
+#if defined(ETHR_HAVE___SYNC_ADD_AND_FETCH64)
+#  define ETHR_HAVE___SYNC_ADD_AND_FETCH
+#endif
+#if defined(ETHR_HAVE___SYNC_FETCH_AND_AND64)
+#  define ETHR_HAVE___SYNC_FETCH_AND_AND
+#endif 
+#if defined(ETHR_HAVE___SYNC_FETCH_AND_OR64)
+#  define ETHR_HAVE___SYNC_FETCH_AND_OR
+#endif
+#else
+#error "Unsupported integer size"
+#endif
+
+typedef struct {
+    volatile ETHR_AINT_T__ counter;
+} ETHR_ATMC_T__;
+
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__ *
+ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
+{
+    return (ETHR_AINT_T__ *) &var->counter;
+}
+
+#if ETHR_READ_AND_SET_WITHOUT_SYNC_OP__
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1
+#endif
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
+{
+    var->counter = value;
+}
+
+#endif /* ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ */
+
+#if ETHR_READ_AND_SET_WITHOUT_SYNC_OP__
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
+{
+    return var->counter;
+}
+
+#endif /* ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ */
+
+#if defined(ETHR_HAVE___SYNC_ADD_AND_FETCH)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(add_return_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
+{
+    return __sync_add_and_fetch(&var->counter, incr);
+}
+
+#endif
+
+#if defined(ETHR_HAVE___SYNC_FETCH_AND_AND)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(and_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
+{
+    return __sync_fetch_and_and(&var->counter, mask);
+}
+
+#endif
+
+#if defined(ETHR_HAVE___SYNC_FETCH_AND_OR)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(or_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
+{
+    return (ETHR_AINT_T__) __sync_fetch_and_or(&var->counter, mask);
+}
+
+#endif
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_mb)(ETHR_ATMC_T__ *var,
+			      ETHR_AINT_T__ new,
+			      ETHR_AINT_T__ old)
+{
+    return __sync_val_compare_and_swap(&var->counter, old, new);
+}
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#undef ETHR_NATMC_FUNC__
+#undef ETHR_ATMC_T__
+#undef ETHR_AINT_T__
+#undef ETHR_AINT_SUFFIX__
+#undef ETHR_HAVE___SYNC_ADD_AND_FETCH
+#undef ETHR_HAVE___SYNC_FETCH_AND_AND
+#undef ETHR_HAVE___SYNC_FETCH_AND_OR
+
+#endif
diff --git a/erts/include/internal/gcc/ethr_dw_atomic.h b/erts/include/internal/gcc/ethr_dw_atomic.h
new file mode 100644
index 0000000000..6736f9c547
--- /dev/null
+++ b/erts/include/internal/gcc/ethr_dw_atomic.h
@@ -0,0 +1,115 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Native double word atomics using gcc's builtins
+ * Author: Rickard Green
+ */
+
+#undef ETHR_INCLUDE_DW_ATOMIC_IMPL__
+#ifndef ETHR_GCC_DW_ATOMIC_H__
+#  define ETHR_GCC_DW_ATOMIC_H__
+#  if ((ETHR_SIZEOF_PTR == 4 \
+        && defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64)) \
+       || (ETHR_SIZEOF_PTR == 8 \
+           && defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP128) \
+	   && defined(ETHR_HAVE_INT128_T)))
+#    define ETHR_INCLUDE_DW_ATOMIC_IMPL__
+#  endif
+#endif
+
+#ifdef ETHR_INCLUDE_DW_ATOMIC_IMPL__
+#  define ETHR_HAVE_NATIVE_SU_DW_ATOMIC
+#  define ETHR_NATIVE_DW_ATOMIC_IMPL "gcc"
+
+#  if defined(__i386__) || defined(__x86_64__)
+/*
+ * If ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ is defined, it will be used
+ * at runtime in order to determine if native or fallback implementation
+ * should be used.
+ */
+#    define ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ \
+       ETHR_X86_RUNTIME_CONF_HAVE_DW_CMPXCHG__
+#  endif
+
+#  if ETHR_SIZEOF_PTR == 4
+#    define ETHR_DW_NATMC_ALIGN_MASK__ 0x7
+#    define ETHR_NATIVE_SU_DW_SINT_T ethr_sint64_t
+#  elif ETHR_SIZEOF_PTR == 8
+#    define ETHR_DW_NATMC_ALIGN_MASK__ 0xf
+#    define ETHR_NATIVE_SU_DW_SINT_T ethr_sint128_t
+#  endif
+
+typedef volatile ETHR_NATIVE_SU_DW_SINT_T * ethr_native_dw_ptr_t;
+
+/*
+ * We need 16 byte aligned memory in 64-bit mode, and 8 byte aligned
+ * memory in 32-bit mode. 16 byte aligned malloc in 64-bit mode is
+ * not common, and at least some glibc malloc implementations
+ * only 4 byte align in 32-bit mode.
+ *
+ * This code assumes 8 byte aligned memory in 64-bit mode, and 4 byte
+ * aligned memory in 32-bit mode. A malloc implementation that does
+ * not adhere to these alignment requirements is seriously broken,
+ * and we wont bother trying to work around it.
+ *
+ * Since memory alignment may be off by one word we need to align at
+ * runtime. We, therefore, need an extra word allocated.
+ */
+#define ETHR_DW_NATMC_MEM__(VAR) \
+   (&var->c[(int) ((ethr_uint_t) &(VAR)->c[0]) & ETHR_DW_NATMC_ALIGN_MASK__])
+typedef union {
+    volatile ETHR_NATIVE_SU_DW_SINT_T dw_sint;
+    volatile ethr_sint_t sint[3];
+    volatile char c[ETHR_SIZEOF_PTR*3];
+} ethr_native_dw_atomic_t;
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+#  ifdef ETHR_DEBUG
+#    define ETHR_DW_DBG_ALIGNED__(PTR) \
+       ETHR_ASSERT((((ethr_uint_t) (PTR)) & ETHR_DW_NATMC_ALIGN_MASK__) == 0);
+#  else
+#    define ETHR_DW_DBG_ALIGNED__(PTR)
+#  endif
+
+#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR
+static ETHR_INLINE ethr_sint_t *
+ethr_native_dw_atomic_addr(ethr_native_dw_atomic_t *var)
+{
+    return (ethr_sint_t *) ETHR_DW_NATMC_MEM__(var);
+}
+
+
+#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB
+
+static ETHR_INLINE ETHR_NATIVE_SU_DW_SINT_T
+ethr_native_su_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var,
+				    ETHR_NATIVE_SU_DW_SINT_T new,
+				    ETHR_NATIVE_SU_DW_SINT_T old)
+{
+    ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var);
+    ETHR_DW_DBG_ALIGNED__(p);
+    return __sync_val_compare_and_swap(p, old, new);
+}
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#endif /* ETHR_GCC_DW_ATOMIC_H__ */
+
diff --git a/erts/include/internal/gcc/ethr_membar.h b/erts/include/internal/gcc/ethr_membar.h
new file mode 100644
index 0000000000..7d428fc68e
--- /dev/null
+++ b/erts/include/internal/gcc/ethr_membar.h
@@ -0,0 +1,73 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Memory barriers when using gcc's builtins
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_GCC_MEMBAR_H__
+#define ETHR_GCC_MEMBAR_H__
+
+#define ETHR_LoadLoad	(1 << 0)
+#define ETHR_LoadStore	(1 << 1)
+#define ETHR_StoreLoad	(1 << 2)
+#define ETHR_StoreStore	(1 << 3)
+
+/*
+ * According to the documentation __sync_synchronize() will
+ * issue a full memory barrier. However, __sync_synchronize()
+ * is known to erroneously be a noop on at least some
+ * platforms with some gcc versions. This has suposedly been
+ * fixed in some gcc version, but we don't know from which
+ * version. Therefore, we only use it when it has been
+ * verified to work. Otherwise we use the workaround
+ * below.
+ */
+
+#if defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32)
+#  define ETHR_MB_T__ ethr_sint32_t
+#elif defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64)
+#  define ETHR_MB_T__ ethr_sint64_t
+#else
+#  error "No __sync_val_compare_and_swap"
+#endif
+#define ETHR_SYNC_SYNCHRONIZE_WORKAROUND__ \
+do { \
+    volatile ETHR_MB_T__ x___ = 0; \
+    (void) __sync_val_compare_and_swap(&x___, (ETHR_MB_T__) 0, (ETHR_MB_T__) 1); \
+} while (0)
+
+#define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory")
+
+#if defined(__mips__) && ETHR_AT_LEAST_GCC_VSN__(4, 2, 0)
+#  define ETHR_MEMBAR(B) __sync_synchronize()
+#  define ETHR_READ_DEPEND_MEMORY_BARRIER __sync_synchronize()
+#elif ((defined(__powerpc__) || defined(__ppc__)) \
+       && ETHR_AT_LEAST_GCC_VSN__(4, 1, 2))
+#  define ETHR_MEMBAR(B) __sync_synchronize()
+#else /* Use workaround */
+#  define ETHR_MEMBAR(B) \
+     ETHR_SYNC_SYNCHRONIZE_WORKAROUND__
+#  define ETHR_READ_DEPEND_MEMORY_BARRIER \
+     ETHR_SYNC_SYNCHRONIZE_WORKAROUND__
+#endif
+
+
+#endif /* ETHR_GCC_MEMBAR_H__ */
diff --git a/erts/include/internal/gcc/ethread.h b/erts/include/internal/gcc/ethread.h
new file mode 100644
index 0000000000..fcfdc39441
--- /dev/null
+++ b/erts/include/internal/gcc/ethread.h
@@ -0,0 +1,48 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Native atomic ethread support when using gcc
+ * Author: Rickard Green
+ */
+
+#ifndef ETHREAD_GCC_H__
+#define ETHREAD_GCC_H__
+
+#ifndef ETHR_MEMBAR
+#  include "ethr_membar.h"
+#endif
+
+#if !defined(ETHR_HAVE_NATIVE_ATOMIC32)
+#  define ETHR_ATOMIC_WANT_32BIT_IMPL__
+#  include "ethr_atomic.h"
+#endif
+
+#if ETHR_SIZEOF_PTR == 8 && !defined(ETHR_HAVE_NATIVE_ATOMIC64)
+#  define ETHR_ATOMIC_WANT_64BIT_IMPL__
+#  include "ethr_atomic.h"
+#endif
+
+#if (!defined(ETHR_HAVE_NATIVE_DW_ATOMIC) \
+     && !(ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64)) \
+     && !(ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_NATIVE_ATOMIC128)))
+#  include "ethr_dw_atomic.h"
+#endif
+
+#endif
diff --git a/erts/include/internal/i386/atomic.h b/erts/include/internal/i386/atomic.h
index 3291ad38e5..fc1b619935 100644
--- a/erts/include/internal/i386/atomic.h
+++ b/erts/include/internal/i386/atomic.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -23,133 +23,233 @@
  *
  * This code requires a 486 or newer processor.
  */
-#ifndef ETHREAD_I386_ATOMIC_H
-#define ETHREAD_I386_ATOMIC_H
 
-/* An atomic is an aligned long accessed via locked operations.
+#undef ETHR_INCLUDE_ATOMIC_IMPL__
+#if !defined(ETHR_X86_ATOMIC32_H__) \
+    && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__)
+#  define ETHR_X86_ATOMIC32_H__
+#  define ETHR_INCLUDE_ATOMIC_IMPL__ 4
+#  undef ETHR_ATOMIC_WANT_32BIT_IMPL__
+#elif !defined(ETHR_X86_ATOMIC64_H__) \
+      && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__)
+#  define ETHR_X86_ATOMIC64_H__
+#  define ETHR_INCLUDE_ATOMIC_IMPL__ 8
+#  undef ETHR_ATOMIC_WANT_64BIT_IMPL__
+#endif
+
+#ifdef ETHR_INCLUDE_ATOMIC_IMPL__
+
+#  ifndef ETHR_X86_ATOMIC_COMMON__
+#    define ETHR_X86_ATOMIC_COMMON__
+#    define ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS 1
+#  endif /* ETHR_X86_ATOMIC_COMMON__ */
+
+#  if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#    define ETHR_HAVE_NATIVE_ATOMIC32 1
+#    define ETHR_NATIVE_ATOMIC32_IMPL "ethread"
+#    define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+#    define ETHR_ATMC_T__ ethr_native_atomic32_t
+#    define ETHR_AINT_T__ ethr_sint32_t
+#    define ETHR_AINT_SUFFIX__ "l"
+#  elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8
+#    define ETHR_HAVE_NATIVE_ATOMIC64 1
+#    define ETHR_NATIVE_ATOMIC64_IMPL "ethread"
+#    define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#    define ETHR_ATMC_T__ ethr_native_atomic64_t
+#    define ETHR_AINT_T__ ethr_sint64_t
+#    define ETHR_AINT_SUFFIX__ "q"
+#  else
+#    error "Unsupported integer size"
+#  endif
+
+/* An atomic is an aligned ETHR_AINT_T__ accessed via locked operations.
  */
 typedef struct {
-    volatile long counter;
-} ethr_native_atomic_t;
+    volatile ETHR_AINT_T__ counter;
+} ETHR_ATMC_T__;
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__ *
+ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
+{
+    return (ETHR_AINT_T__ *) &var->counter;
+}
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_mb)(ETHR_ATMC_T__ *var,
+			      ETHR_AINT_T__ new,
+			      ETHR_AINT_T__ old)
+{
+    __asm__ __volatile__(
+      "lock; cmpxchg" ETHR_AINT_SUFFIX__ " %2, %3"
+      : "=a"(old), "=m"(var->counter)
+      : "r"(new), "m"(var->counter), "0"(old)
+      : "cc", "memory"); /* full memory clobber to make this a compiler barrier */
+    return old;
+}
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB 1
+#endif
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(xchg_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val)
+{   
+    ETHR_AINT_T__ tmp = val;
+    __asm__ __volatile__(
+	"xchg" ETHR_AINT_SUFFIX__ " %0, %1"
+	: "=r"(tmp)
+	: "m"(var->counter), "0"(tmp)
+	: "memory");
+    /* now tmp is the atomic's previous value */ 
+    return tmp;
+} 
 
-#ifdef __x86_64__
-#define LONG_SUFFIX "q"
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
 #else
-#define LONG_SUFFIX "l"
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1
 #endif
 
 static ETHR_INLINE void
-ethr_native_atomic_init(ethr_native_atomic_t *var, long i)
+ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
 {
     var->counter = i;
 }
-#define ethr_native_atomic_set(v, i)	ethr_native_atomic_init((v), (i))
 
-static ETHR_INLINE long
-ethr_native_atomic_read(ethr_native_atomic_t *var)
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1
+#endif
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+#if defined(_M_IX86)
+    if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__)
+	(void) ETHR_NATMC_FUNC__(xchg_mb)(var, i);
+    else
+#endif /* _M_IX86 */
+    {
+	ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+	var->counter = i;
+    }
+}
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB 1
+#endif
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+    (void) ETHR_NATMC_FUNC__(xchg_mb)(var, i);
+}
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
 {
     return var->counter;
 }
 
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB 1
+#endif
+
 static ETHR_INLINE void
-ethr_native_atomic_add(ethr_native_atomic_t *var, long incr)
+ETHR_NATMC_FUNC__(add_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
 {
     __asm__ __volatile__(
-       "lock; add" LONG_SUFFIX " %1, %0"
+       "lock; add" ETHR_AINT_SUFFIX__ " %1, %0"
        : "=m"(var->counter)
-       : "ir"(incr), "m"(var->counter));
+       : "ir"(incr), "m"(var->counter)
+       : "memory");
 }      
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB 1
+#endif
        
 static ETHR_INLINE void
-ethr_native_atomic_inc(ethr_native_atomic_t *var)
+ETHR_NATMC_FUNC__(inc_mb)(ETHR_ATMC_T__ *var)
 {
     __asm__ __volatile__(
-	"lock; inc" LONG_SUFFIX " %0"
+	"lock; inc" ETHR_AINT_SUFFIX__ " %0"
 	: "=m"(var->counter)
-	: "m"(var->counter));
+	: "m"(var->counter)
+	: "memory");
 }
 
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB 1
+#endif
+
 static ETHR_INLINE void
-ethr_native_atomic_dec(ethr_native_atomic_t *var)
+ETHR_NATMC_FUNC__(dec_mb)(ETHR_ATMC_T__ *var)
 {
     __asm__ __volatile__(
-	"lock; dec" LONG_SUFFIX " %0"
+	"lock; dec" ETHR_AINT_SUFFIX__ " %0"
 	: "=m"(var->counter)
-	: "m"(var->counter));
+	: "m"(var->counter)
+	: "memory");
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr)
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(add_return_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
 {
-    long tmp;
+    ETHR_AINT_T__ tmp;
 
     tmp = incr;
     __asm__ __volatile__(
-	"lock; xadd" LONG_SUFFIX " %0, %1" /* xadd didn't exist prior to the 486 */
+	"lock; xadd" ETHR_AINT_SUFFIX__ " %0, %1" /* xadd didn't exist prior to the 486 */
 	: "=r"(tmp)
-	: "m"(var->counter), "0"(tmp));
+	: "m"(var->counter), "0"(tmp)
+	: "memory");
     /* now tmp is the atomic's previous value */
     return tmp + incr;
 }
 
-#define ethr_native_atomic_inc_return(var) ethr_native_atomic_add_return((var), 1)
-#define ethr_native_atomic_dec_return(var) ethr_native_atomic_add_return((var), -1)
-
-static ETHR_INLINE long
-ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old)
-{
-    __asm__ __volatile__(
-      "lock; cmpxchg" LONG_SUFFIX " %2, %3"
-      : "=a"(old), "=m"(var->counter)
-      : "r"(new), "m"(var->counter), "0"(old)
-      : "cc", "memory"); /* full memory clobber to make this a compiler barrier */
-    return old;
-}
-
-static ETHR_INLINE long
-ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask)
-{
-    long tmp, old;
-
-    tmp = var->counter;
-    do {
-	old = tmp;
-        tmp = ethr_native_atomic_cmpxchg(var, tmp & mask, tmp);
-    } while (__builtin_expect(tmp != old, 0));
-    /* now tmp is the atomic's previous value */
-    return tmp;
-}
-
-static ETHR_INLINE long
-ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask)
-{
-    long tmp, old;
-
-    tmp = var->counter;
-    do {
-	old = tmp;
-        tmp = ethr_native_atomic_cmpxchg(var, tmp | mask, tmp);
-    } while (__builtin_expect(tmp != old, 0));
-    /* now tmp is the atomic's previous value */
-    return tmp;
-}
-
-static ETHR_INLINE long
-ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val)
-{   
-    long tmp = val;
-    __asm__ __volatile__(
-	"xchg" LONG_SUFFIX " %0, %1"
-	: "=r"(tmp)
-	: "m"(var->counter), "0"(tmp));
-    /* now tmp is the atomic's previous value */ 
-    return tmp;
-} 
-
-#undef LONG_SUFFIX
-
 #endif /* ETHR_TRY_INLINE_FUNCS */
 
-#endif /* ETHREAD_I386_ATOMIC_H */
+#undef ETHR_NATMC_FUNC__
+#undef ETHR_ATMC_T__
+#undef ETHR_AINT_T__
+#undef ETHR_AINT_SUFFIX__
+
+#endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */
diff --git a/erts/include/internal/i386/ethr_dw_atomic.h b/erts/include/internal/i386/ethr_dw_atomic.h
new file mode 100644
index 0000000000..9fb89bbe43
--- /dev/null
+++ b/erts/include/internal/i386/ethr_dw_atomic.h
@@ -0,0 +1,278 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Native double word atomics for x86/x86_64
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_X86_DW_ATOMIC_H__
+#define ETHR_X86_DW_ATOMIC_H__
+
+#ifdef ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT
+
+#define ETHR_HAVE_NATIVE_DW_ATOMIC
+#define ETHR_NATIVE_DW_ATOMIC_IMPL "ethread"
+
+/*
+ * If ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ is defined, it will be used
+ * at runtime in order to determine if native or fallback implementation
+ * should be used.
+ */
+#define ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ \
+  ETHR_X86_RUNTIME_CONF_HAVE_DW_CMPXCHG__
+
+#if ETHR_SIZEOF_PTR == 4
+typedef volatile ethr_sint64_t * ethr_native_dw_ptr_t;
+#  define ETHR_DW_NATMC_ALIGN_MASK__ 0x7
+#  define ETHR_DW_CMPXCHG_SFX__ "8b"
+#  define ETHR_NATIVE_SU_DW_SINT_T ethr_sint64_t
+#else
+#ifdef ETHR_HAVE_INT128_T
+#  define ETHR_NATIVE_SU_DW_SINT_T ethr_sint128_t
+typedef volatile ethr_sint128_t * ethr_native_dw_ptr_t;
+#else
+typedef struct {
+    ethr_sint64_t sint64[2];
+} ethr_native_sint128_t__;
+typedef volatile ethr_native_sint128_t__ * ethr_native_dw_ptr_t;
+#endif
+#  define ETHR_DW_NATMC_ALIGN_MASK__ 0xf
+#  define ETHR_DW_CMPXCHG_SFX__ "16b"
+#endif
+
+/*
+ * We need 16 byte aligned memory in 64-bit mode, and 8 byte aligned
+ * memory in 32-bit mode. 16 byte aligned malloc in 64-bit mode is
+ * not common, and at least some glibc malloc implementations
+ * only 4 byte align in 32-bit mode.
+ *
+ * This code assumes 8 byte aligned memory in 64-bit mode, and 4 byte
+ * aligned memory in 32-bit mode. A malloc implementation that does
+ * not adhere to these alignment requirements is seriously broken,
+ * and we wont bother trying to work around it.
+ *
+ * Since memory alignment may be off by one word we need to align at
+ * runtime. We, therefore, need an extra word allocated.
+ */
+#define ETHR_DW_NATMC_MEM__(VAR) \
+   (&var->c[(int) ((ethr_uint_t) &(VAR)->c[0]) & ETHR_DW_NATMC_ALIGN_MASK__])
+typedef union {
+#ifdef ETHR_NATIVE_SU_DW_SINT_T
+    volatile ETHR_NATIVE_SU_DW_SINT_T dw_sint;
+#endif
+    volatile ethr_sint_t sint[3];
+    volatile char c[ETHR_SIZEOF_PTR*3];
+} ethr_native_dw_atomic_t;
+
+
+#if (defined(ETHR_TRY_INLINE_FUNCS) \
+     || defined(ETHR_ATOMIC_IMPL__) \
+     || defined(ETHR_X86_SSE2_ASM_C__)) \
+    && ETHR_SIZEOF_PTR == 4 \
+    && defined(ETHR_GCC_HAVE_SSE2_ASM_SUPPORT)
+ethr_sint64_t
+ethr_sse2_native_su_dw_atomic_read(ethr_native_dw_atomic_t *var);
+void
+ethr_sse2_native_su_dw_atomic_set(ethr_native_dw_atomic_t *var,
+				  ethr_sint64_t val);
+#endif
+
+#if (defined(ETHR_TRY_INLINE_FUNCS) \
+     || defined(ETHR_ATOMIC_IMPL__) \
+     || defined(ETHR_X86_SSE2_ASM_C__))
+#  ifdef ETHR_DEBUG
+#    define ETHR_DW_DBG_ALIGNED__(PTR) \
+       ETHR_ASSERT((((ethr_uint_t) (PTR)) & ETHR_DW_NATMC_ALIGN_MASK__) == 0);
+#  else
+#    define ETHR_DW_DBG_ALIGNED__(PTR)
+#  endif
+#endif
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR
+static ETHR_INLINE ethr_sint_t *
+ethr_native_dw_atomic_addr(ethr_native_dw_atomic_t *var)
+{
+    return (ethr_sint_t *) ETHR_DW_NATMC_MEM__(var);
+}
+
+#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__
+/*
+ * When position independent code is used in 32-bit mode, the EBX register
+ * is used for storage of global offset table address, and we may not
+ * use it as input or output in an asm. We need to save and restore the
+ * EBX register explicitly (for some reason gcc doesn't provide this
+ * service to us).
+ */
+#  define ETHR_NO_CLOBBER_EBX__ 1
+#else
+#  define ETHR_NO_CLOBBER_EBX__ 0
+#endif
+
+#if ETHR_NO_CLOBBER_EBX__ && !defined(ETHR_CMPXCHG8B_REGISTER_SHORTAGE)
+/* When no optimization is on, we'll run into a register shortage */
+#  if defined(ETHR_DEBUG) || defined(DEBUG) || defined(VALGRIND) \
+      || defined(GCOV) || defined(PURIFY) || defined(PURECOV)
+#    define ETHR_CMPXCHG8B_REGISTER_SHORTAGE 1
+#  else
+#    define ETHR_CMPXCHG8B_REGISTER_SHORTAGE 0
+#  endif
+#endif
+
+
+#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB
+
+static ETHR_INLINE int
+ethr_native_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var,
+				 ethr_sint_t *new,
+				 ethr_sint_t *xchg)
+{
+    ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var);
+    char xchgd;
+
+    ETHR_DW_DBG_ALIGNED__(p);
+
+    __asm__ __volatile__(
+#if ETHR_NO_CLOBBER_EBX__
+	"pushl %%ebx\n\t"
+#  if ETHR_CMPXCHG8B_REGISTER_SHORTAGE
+	"movl (%7), %%ebx\n\t"
+	"movl 4(%7), %%ecx\n\t"
+#  else
+	"movl %8, %%ebx\n\t"
+#  endif
+#endif
+	"lock; cmpxchg" ETHR_DW_CMPXCHG_SFX__ " %0\n\t"
+	"setz %3\n\t"
+#if ETHR_NO_CLOBBER_EBX__
+	"popl %%ebx\n\t"
+#endif
+	: "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=c"(xchgd)
+	: "m"(*p), "1"(xchg[1]), "2"(xchg[0]),
+#if ETHR_NO_CLOBBER_EBX__
+#  if ETHR_CMPXCHG8B_REGISTER_SHORTAGE
+	  "3"(new)
+#  else
+	  "3"(new[1]),
+	  "r"(new[0])
+#  endif
+#else
+	  "3"(new[1]),
+	  "b"(new[0])
+#endif
+	: "cc", "memory");
+
+    return (int) xchgd;
+}
+
+#undef ETHR_NO_CLOBBER_EBX__
+
+#if ETHR_SIZEOF_PTR == 4 && defined(ETHR_GCC_HAVE_SSE2_ASM_SUPPORT)
+
+typedef union {
+    ethr_sint64_t sint64;
+    ethr_sint_t sint[2];
+} ethr_dw_atomic_no_sse2_convert_t;
+
+#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ
+
+static ETHR_INLINE ethr_sint64_t
+ethr_native_su_dw_atomic_read(ethr_native_dw_atomic_t *var)
+{
+    if (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__)
+	return ethr_sse2_native_su_dw_atomic_read(var);
+    else {
+	ethr_sint_t new[2];
+	ethr_dw_atomic_no_sse2_convert_t xchg;
+	new[0] = new[1] = xchg.sint[0] = xchg.sint[1] = 0x83838383;
+	(void) ethr_native_dw_atomic_cmpxchg_mb(var, new, xchg.sint);
+	return xchg.sint64;
+    }
+}
+
+#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET
+
+static ETHR_INLINE void
+ethr_native_su_dw_atomic_set(ethr_native_dw_atomic_t *var,
+			     ethr_sint64_t val)
+{
+    if (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__)
+	ethr_sse2_native_su_dw_atomic_set(var, val);
+    else {
+	ethr_sint_t xchg[2] = {0, 0};
+	ethr_dw_atomic_no_sse2_convert_t new;
+	new.sint64 = val;
+	while (!ethr_native_dw_atomic_cmpxchg_mb(var, new.sint, xchg));
+    }
+}
+
+#endif /* ETHR_SIZEOF_PTR == 4 */
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#if defined(ETHR_X86_SSE2_ASM_C__) \
+    && ETHR_SIZEOF_PTR == 4 \
+    && defined(ETHR_GCC_HAVE_SSE2_ASM_SUPPORT)
+
+/*
+ * 8-byte aligned loads and stores of 64-bit values are atomic from
+ * pentium and forward. An ordinary volatile load or store in 32-bit
+ * mode generates two 32-bit operations (at least with gcc-4.1.2 using
+ * -msse2). In order to guarantee one 64-bit load/store operation
+ * from/to memory we load/store via an xmm register using movq.
+ *
+ * Load/store can be achieved using cmpxchg8b, however, using movq is
+ * much faster. Unfortunately we cannot do the same thing in 64-bit
+ * mode; instead, we have to do loads and stores via cmpxchg16b.
+ *
+ * We do not inline these, but instead compile these into a separate
+ * object file using -msse2. This since we don't want to use -msse2 for
+ * the whole system. If we detect sse2 support (pentium4 and forward)
+ * at runtime, we use them; otherwise, we fall back to using cmpxchg8b
+ * for loads and stores. This way the binary can be moved between
+ * processors with and without sse2 support.
+ */
+
+ethr_sint64_t
+ethr_sse2_native_su_dw_atomic_read(ethr_native_dw_atomic_t *var)
+{
+    ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var);
+    ethr_sint64_t val;
+    ETHR_DW_DBG_ALIGNED__(p);
+    __asm__ __volatile__("movq %1, %0\n\t" : "=x"(val) : "m"(*p) : "memory");
+    return val;
+}
+
+void
+ethr_sse2_native_su_dw_atomic_set(ethr_native_dw_atomic_t *var,
+				  ethr_sint64_t val)
+{
+    ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var);
+    ETHR_DW_DBG_ALIGNED__(p);
+    __asm__ __volatile__("movq %1, %0\n\t" : "=m"(*p) : "x"(val) : "memory");
+}
+
+#endif /* ETHR_X86_SSE2_ASM_C__ */
+
+#endif /* ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT */
+
+#endif /* ETHR_X86_DW_ATOMIC_H__ */
+
diff --git a/erts/include/internal/i386/ethr_membar.h b/erts/include/internal/i386/ethr_membar.h
new file mode 100644
index 0000000000..92d9de7f3f
--- /dev/null
+++ b/erts/include/internal/i386/ethr_membar.h
@@ -0,0 +1,114 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Memory barriers for x86/x86-64
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_X86_MEMBAR_H__
+#define ETHR_X86_MEMBAR_H__
+
+#define ETHR_LoadLoad	(1 << 0)
+#define ETHR_LoadStore	(1 << 1)
+#define ETHR_StoreLoad	(1 << 2)
+#define ETHR_StoreStore	(1 << 3)
+
+#define ETHR_NO_SSE2_MEMORY_BARRIER__			\
+do {							\
+    volatile ethr_sint32_t x__ = 0;			\
+    __asm__ __volatile__ ("lock; orl $0x0, %0\n\t"	\
+			  : "=m"(x__)			\
+			  : "m"(x__)			\
+			  : "memory");			\
+} while (0)
+
+static __inline__ void
+ethr_cfence__(void)
+{
+    __asm__ __volatile__ ("" : : : "memory");
+}
+
+static __inline__ void
+ethr_mfence__(void)
+{
+#if ETHR_SIZEOF_PTR == 4
+    if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__)
+	ETHR_NO_SSE2_MEMORY_BARRIER__;
+    else
+#endif
+	__asm__ __volatile__ ("mfence\n\t" : : : "memory");
+}
+
+static __inline__ void
+ethr_sfence__(void)
+{
+#if ETHR_SIZEOF_PTR == 4
+    if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__)
+	ETHR_NO_SSE2_MEMORY_BARRIER__;
+    else
+#endif
+	__asm__ __volatile__ ("sfence\n\t" : : : "memory");
+}
+
+static __inline__ void
+ethr_lfence__(void)
+{
+#if ETHR_SIZEOF_PTR == 4
+    if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__)
+	ETHR_NO_SSE2_MEMORY_BARRIER__;
+    else
+#endif
+	__asm__ __volatile__ ("lfence\n\t" : : : "memory");
+}
+
+#define ETHR_X86_OUT_OF_ORDER_MEMBAR(B)				\
+  ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore,			\
+		   ethr_sfence__(),				\
+		   ETHR_CHOOSE_EXPR((B) == ETHR_LoadLoad,	\
+				    ethr_lfence__(),		\
+				    ethr_mfence__()))
+
+#ifdef ETHR_X86_OUT_OF_ORDER
+
+#define ETHR_MEMBAR(B) \
+  ETHR_X86_OUT_OF_ORDER_MEMBAR((B))
+
+#else /* !ETHR_X86_OUT_OF_ORDER (the default) */
+
+/*
+ * We assume that only stores before loads may be reordered. That is,
+ * we assume that *no* instructions like these are used:
+ * - CLFLUSH,
+ * - streaming stores executed with non-temporal move,
+ * - string operations, or
+ * - other instructions which aren't LoadLoad, LoadStore, and StoreStore
+ *   ordered by themselves
+ * If such instructions are used, either insert memory barriers
+ * using ETHR_X86_OUT_OF_ORDER_MEMBAR() at appropriate places, or
+ * define ETHR_X86_OUT_OF_ORDER. For more info see Intel 64 and IA-32
+ * Architectures Software Developer's Manual; Vol 3A; Chapter 8.2.2.
+ */
+
+#define ETHR_MEMBAR(B) \
+  ETHR_CHOOSE_EXPR((B) & ETHR_StoreLoad, ethr_mfence__(), ethr_cfence__())
+
+#endif /* !ETHR_X86_OUT_OF_ORDER */
+
+#endif /* ETHR_X86_MEMBAR_H__ */
diff --git a/erts/include/internal/i386/ethread.h b/erts/include/internal/i386/ethread.h
index fad8b108fa..80e4dc7b99 100644
--- a/erts/include/internal/i386/ethread.h
+++ b/erts/include/internal/i386/ethread.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -24,11 +24,15 @@
 #ifndef ETHREAD_I386_ETHREAD_H
 #define ETHREAD_I386_ETHREAD_H
 
+#include "ethr_membar.h"
+#define ETHR_ATOMIC_WANT_32BIT_IMPL__
 #include "atomic.h"
+#if ETHR_SIZEOF_PTR == 8
+#  define ETHR_ATOMIC_WANT_64BIT_IMPL__
+#  include "atomic.h"
+#endif
+#include "ethr_dw_atomic.h"
 #include "spinlock.h"
 #include "rwlock.h"
 
-#define ETHR_HAVE_NATIVE_ATOMICS 1
-#define ETHR_HAVE_NATIVE_LOCKS 1
-
 #endif /* ETHREAD_I386_ETHREAD_H */
diff --git a/erts/include/internal/i386/rwlock.h b/erts/include/internal/i386/rwlock.h
index c009be8ef1..1a8cd7da0c 100644
--- a/erts/include/internal/i386/rwlock.h
+++ b/erts/include/internal/i386/rwlock.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -26,12 +26,15 @@
 #ifndef ETHREAD_I386_RWLOCK_H
 #define ETHREAD_I386_RWLOCK_H
 
+#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1
+#define ETHR_NATIVE_RWSPINLOCK_IMPL "ethread"
+
 /* XXX: describe the algorithm */
 typedef struct {
     volatile int lock;
 } ethr_native_rwlock_t;
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
 
 #define ETHR_RWLOCK_OFFSET	(1<<24)
 
diff --git a/erts/include/internal/i386/spinlock.h b/erts/include/internal/i386/spinlock.h
index 2b4832e26a..a84fba91b1 100644
--- a/erts/include/internal/i386/spinlock.h
+++ b/erts/include/internal/i386/spinlock.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -24,6 +24,9 @@
 #ifndef ETHREAD_I386_SPINLOCK_H
 #define ETHREAD_I386_SPINLOCK_H
 
+#define ETHR_HAVE_NATIVE_SPINLOCKS 1
+#define ETHR_NATIVE_SPINLOCK_IMPL "ethread"
+
 /* A spinlock is the low byte of an aligned 32-bit integer.
  * A non-zero value means that the lock is locked.
  */
@@ -31,7 +34,7 @@ typedef struct {
     volatile unsigned int lock;
 } ethr_native_spinlock_t;
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
 
 static ETHR_INLINE void
 ethr_native_spinlock_init(ethr_native_spinlock_t *lock)
@@ -46,16 +49,20 @@ ethr_native_spin_unlock(ethr_native_spinlock_t *lock)
      * On i386 this needs to be a locked operation
      * to avoid Pentium Pro errata 66 and 92.
      */
-#if defined(__x86_64__)
-    __asm__ __volatile__("" : : : "memory");
-    *(unsigned char*)&lock->lock = 0;
-#else
-    char tmp = 0;
-    __asm__ __volatile__(
-	"xchgb %b0, %1"
-	: "=q"(tmp), "=m"(lock->lock)
-	: "0"(tmp) : "memory");
+#if !defined(__x86_64__)
+    if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) {
+	char tmp = 0;
+	__asm__ __volatile__(
+	    "xchgb %b0, %1"
+	    : "=q"(tmp), "=m"(lock->lock)
+	    : "0"(tmp) : "memory");
+    }
+    else
 #endif
+    {
+	ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+	*(unsigned char*)&lock->lock = 0;
+    }
 }
 
 static ETHR_INLINE int
diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h
new file mode 100644
index 0000000000..fb1288c330
--- /dev/null
+++ b/erts/include/internal/libatomic_ops/ethr_atomic.h
@@ -0,0 +1,298 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Native atomics ethread support using libatomic_ops
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_LIBATOMIC_OPS_ATOMIC_H__
+#define ETHR_LIBATOMIC_OPS_ATOMIC_H__
+
+/*
+ * libatomic_ops can be downloaded from:
+ *   http://www.hpl.hp.com/research/linux/atomic_ops/
+ *
+ * These operations need to be defined by libatomic_ops;
+ * otherwise, we won't compile:
+ * - AO_nop_full()
+ * - AO_load()
+ * - AO_store()
+ * - AO_compare_and_swap()
+ *
+ */
+
+#if ETHR_SIZEOF_AO_T == 4
+#define ETHR_HAVE_NATIVE_ATOMIC32 1
+#define ETHR_NATIVE_ATOMIC32_IMPL "libatomic_ops"
+#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+#define ETHR_ATMC_T__ ethr_native_atomic32_t
+#define ETHR_AINT_T__ ethr_sint32_t
+#define ETHR_AINT_SUFFIX__ "l"
+#elif ETHR_SIZEOF_AO_T == 8
+#define ETHR_HAVE_NATIVE_ATOMIC64 1
+#define ETHR_NATIVE_ATOMIC64_IMPL "libatomic_ops"
+#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#define ETHR_ATMC_T__ ethr_native_atomic64_t
+#define ETHR_AINT_T__ ethr_sint64_t
+#define ETHR_AINT_SUFFIX__ "q"
+#else
+#error "Unsupported integer size"
+#endif
+
+typedef struct {
+    volatile AO_t counter;
+} ETHR_ATMC_T__;
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__ *
+ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
+{
+    return (ETHR_AINT_T__ *) &var->counter;
+}
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1
+#endif
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
+{
+    AO_store(&var->counter, (AO_t) value);
+}
+
+#ifdef AO_HAVE_store_release
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1
+#endif
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
+{
+    AO_store_release(&var->counter, (AO_t) value);
+}
+
+#endif
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
+{
+    return (ETHR_AINT_T__) AO_load(&var->counter);
+}
+
+#ifdef AO_HAVE_load_acquire
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
+{
+    return (ETHR_AINT_T__) AO_load_acquire(&var->counter);
+}
+
+#endif
+
+#ifdef AO_HAVE_fetch_and_add
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
+{
+    return ((ETHR_AINT_T__) AO_fetch_and_add(&var->counter, (AO_t) incr)) + incr;
+}
+
+#endif
+
+#ifdef AO_HAVE_fetch_and_add1
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
+{
+    return ((ETHR_AINT_T__) AO_fetch_and_add1(&var->counter)) + 1;
+}
+
+#endif
+
+#ifdef AO_HAVE_fetch_and_add1_acquire
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
+{
+    return ((ETHR_AINT_T__) AO_fetch_and_add1_acquire(&var->counter)) + 1;
+}
+
+#endif
+
+#ifdef AO_HAVE_fetch_and_sub1
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
+{
+    return ((ETHR_AINT_T__) AO_fetch_and_sub1(&var->counter)) - 1;
+}
+
+#endif
+
+#ifdef AO_HAVE_fetch_and_sub1_release
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
+{
+    return ((ETHR_AINT_T__) AO_fetch_and_sub1_release(&var->counter)) - 1;
+}
+
+#endif
+
+#ifdef AO_HAVE_compare_and_swap
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
+			   ETHR_AINT_T__ new,
+			   ETHR_AINT_T__ exp)
+{
+    ETHR_AINT_T__ act;
+    do {
+	if (AO_compare_and_swap(&var->counter, (AO_t) exp, (AO_t) new))
+	    return exp;
+	act = (ETHR_AINT_T__) AO_load(&var->counter);
+    } while (act == exp);
+    return act;
+}
+
+#endif
+
+#ifdef AO_HAVE_compare_and_swap_acquire
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
+				ETHR_AINT_T__ new,
+				ETHR_AINT_T__ exp)
+{
+    ETHR_AINT_T__ act;
+    do {
+	if (AO_compare_and_swap_acquire(&var->counter, (AO_t) exp, (AO_t) new))
+	    return exp;
+#ifdef AO_HAVE_load_acquire
+	act = (ETHR_AINT_T__) AO_load_acquire(&var->counter);
+#else
+	act = (ETHR_AINT_T__) AO_load(&var->counter);
+#endif
+    } while (act == exp);
+#ifndef AO_HAVE_load_acquire
+    AO_nop_full();
+#endif
+    return act;
+}
+
+#endif
+
+#ifdef AO_HAVE_compare_and_swap_release
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
+				ETHR_AINT_T__ new,
+				ETHR_AINT_T__ exp)
+{
+    ETHR_AINT_T__ act;
+    do {
+	if (AO_compare_and_swap_release(&var->counter, (AO_t) exp, (AO_t) new))
+	    return exp;
+	act = (ETHR_AINT_T__) AO_load(&var->counter);
+    } while (act == exp);
+    return act;
+}
+
+#endif
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#undef ETHR_NATMC_FUNC__
+#undef ETHR_ATMC_T__
+#undef ETHR_AINT_T__
+
+#endif /* ETHR_LIBATOMIC_OPS_ATOMIC_H__ */
diff --git a/erts/include/internal/libatomic_ops/ethr_membar.h b/erts/include/internal/libatomic_ops/ethr_membar.h
new file mode 100644
index 0000000000..b8530a0094
--- /dev/null
+++ b/erts/include/internal/libatomic_ops/ethr_membar.h
@@ -0,0 +1,75 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Memory barriers when using libatomic_ops
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_LIBATOMIC_OPS_MEMBAR_H__
+#define ETHR_LIBATOMIC_OPS_MEMBAR_H__
+
+#define ETHR_LoadLoad	(1 << 0)
+#define ETHR_LoadStore	(1 << 1)
+#define ETHR_StoreLoad	(1 << 2)
+#define ETHR_StoreStore	(1 << 3)
+
+#ifndef AO_HAVE_nop_full
+#  error "No AO_nop_full()"
+#endif
+
+static __inline__ void
+ethr_mb__(void)
+{
+    AO_nop_full();
+}
+
+static __inline__ void
+ethr_rb__(void)
+{
+#ifdef AO_HAVE_nop_read
+    AO_nop_read();
+#else
+    AO_nop_full();
+#endif
+}
+
+static __inline__ void
+ethr_wb__(void)
+{
+#ifdef AO_HAVE_nop_write
+    AO_nop_write();
+#else
+    AO_nop_full();
+#endif
+}
+
+#define ETHR_MEMBAR(B)						\
+  ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore,			\
+		   ethr_wb__(),					\
+		   ETHR_CHOOSE_EXPR((B) == ETHR_LoadLoad,	\
+				    ethr_rb__(),		\
+				    ethr_mb__()))
+
+#define ETHR_COMPILER_BARRIER AO_compiler_barrier()
+#ifdef AO_NO_DD_ORDERING
+#  define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER
+#endif
+
+#endif /* ETHR_LIBATOMIC_OPS_MEMBAR_H__ */
diff --git a/erts/include/internal/libatomic_ops/ethread.h b/erts/include/internal/libatomic_ops/ethread.h
new file mode 100644
index 0000000000..e1fdd588bb
--- /dev/null
+++ b/erts/include/internal/libatomic_ops/ethread.h
@@ -0,0 +1,42 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Native atomics ethread support using libatomic_ops
+ * Author: Rickard Green
+ */
+
+#ifndef ETHREAD_LIBATOMIC_OPS_H__
+#define ETHREAD_LIBATOMIC_OPS_H__
+
+#if (defined(ETHR_HAVE_LIBATOMIC_OPS) \
+     && ((ETHR_SIZEOF_AO_T == 4 && !defined(ETHR_HAVE_NATIVE_ATOMIC32)) \
+	 || (ETHR_SIZEOF_AO_T == 8 && !defined(ETHR_HAVE_NATIVE_ATOMIC64))))
+
+#if defined(__x86_64__)
+#define AO_USE_PENTIUM4_INSTRS
+#endif
+
+#include "atomic_ops.h"
+#include "ethr_membar.h"
+#include "ethr_atomic.h"
+
+#endif
+
+#endif
diff --git a/erts/include/internal/ppc32/atomic.h b/erts/include/internal/ppc32/atomic.h
index fa701c6a92..6001620677 100644
--- a/erts/include/internal/ppc32/atomic.h
+++ b/erts/include/internal/ppc32/atomic.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -28,167 +28,254 @@
 #ifndef ETHREAD_PPC_ATOMIC_H
 #define ETHREAD_PPC_ATOMIC_H
 
+#define ETHR_HAVE_NATIVE_ATOMIC32 1
+#define ETHR_NATIVE_ATOMIC32_IMPL "ethread"
+
 typedef struct {
-    volatile int counter;
-} ethr_native_atomic_t;
+    volatile ethr_sint32_t counter;
+} ethr_native_atomic32_t;
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
 
+static ETHR_INLINE ethr_sint32_t *
+ethr_native_atomic32_addr(ethr_native_atomic32_t *var)
+{
+    return (ethr_sint32_t *) &var->counter;
+}
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
 
 static ETHR_INLINE void
-ethr_native_atomic_init(ethr_native_atomic_t *var, int i)
+ethr_native_atomic32_set(ethr_native_atomic32_t *var, ethr_sint32_t i)
 {
     var->counter = i;
 }
-#define ethr_native_atomic_set(v, i)	ethr_native_atomic_init((v), (i))
 
-static ETHR_INLINE int
-ethr_native_atomic_read(ethr_native_atomic_t *var)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_read(ethr_native_atomic32_t *var)
 {
     return var->counter;
 }
 
-static ETHR_INLINE int
-ethr_native_atomic_add_return(ethr_native_atomic_t *var, int incr)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_add_return(ethr_native_atomic32_t *var, ethr_sint32_t incr)
 {
-    int tmp;
+    ethr_sint32_t tmp;
 
     __asm__ __volatile__(
-	"eieio\n\t"
 	"1:\t"
 	"lwarx	%0,0,%1\n\t"
 	"add	%0,%2,%0\n\t"
 	"stwcx.	%0,0,%1\n\t"
 	"bne-	1b\n\t"
-	"isync"
 	: "=&r"(tmp)
 	: "r"(&var->counter), "r"(incr)
 	: "cc", "memory");
     return tmp;
 }
 
-static ETHR_INLINE void
-ethr_native_atomic_add(ethr_native_atomic_t *var, int incr)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_add_return_acqb(ethr_native_atomic32_t *var, ethr_sint32_t incr)
 {
-    /* XXX: could use weaker version here w/o eieio+isync */
-    (void)ethr_native_atomic_add_return(var, incr);
+    ethr_sint32_t res;
+    res = ethr_native_atomic32_add_return(var, incr);
+    __asm__ __volatile("isync\n\t" : : : "memory");
+    return res;
 }
 
-static ETHR_INLINE int
-ethr_native_atomic_inc_return(ethr_native_atomic_t *var)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_inc_return(ethr_native_atomic32_t *var)
 {
-    int tmp;
+    ethr_sint32_t tmp;
 
     __asm__ __volatile__(
-	"eieio\n\t"
 	"1:\t"
 	"lwarx	%0,0,%1\n\t"
 	"addic	%0,%0,1\n\t" /* due to addi's (rA|0) behaviour */
 	"stwcx.	%0,0,%1\n\t"
 	"bne-	1b\n\t"
-	"isync"
 	: "=&r"(tmp)
 	: "r"(&var->counter)
 	: "cc", "memory");
     return tmp;
 }
 
-static ETHR_INLINE void
-ethr_native_atomic_inc(ethr_native_atomic_t *var)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_inc_return_acqb(ethr_native_atomic32_t *var)
 {
-    /* XXX: could use weaker version here w/o eieio+isync */
-    (void)ethr_native_atomic_inc_return(var);
+    ethr_sint32_t res;
+    res = ethr_native_atomic32_inc_return(var);
+    __asm__ __volatile("isync\n\t" : : : "memory");
+    return res;
 }
+    
 
-static ETHR_INLINE int
-ethr_native_atomic_dec_return(ethr_native_atomic_t *var)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var)
 {
-    int tmp;
+    ethr_sint32_t tmp;
 
     __asm__ __volatile__(
-	"eieio\n\t"
 	"1:\t"
 	"lwarx	%0,0,%1\n\t"
 	"addic	%0,%0,-1\n\t"
 	"stwcx.	%0,0,%1\n\t"
 	"bne-	1b\n\t"
-	"isync"
 	: "=&r"(tmp)
 	: "r"(&var->counter)
 	: "cc", "memory");
     return tmp;
 }
 
-static ETHR_INLINE void
-ethr_native_atomic_dec(ethr_native_atomic_t *var)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_dec_return_acqb(ethr_native_atomic32_t *var)
 {
-    /* XXX: could use weaker version here w/o eieio+isync */
-    (void)ethr_native_atomic_dec_return(var);
+    ethr_sint32_t res;
+    res = ethr_native_atomic32_dec_return(var);
+    __asm__ __volatile("isync\n\t" : : : "memory");
+    return res;
 }
 
-static ETHR_INLINE int
-ethr_native_atomic_and_retold(ethr_native_atomic_t *var, int mask)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_and_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask)
 {
-    int old, new;
+    ethr_sint32_t old, new;
 
     __asm__ __volatile__(
-	"eieio\n\t"
 	"1:\t"
 	"lwarx	%0,0,%2\n\t"
 	"and	%1,%0,%3\n\t"
 	"stwcx.	%1,0,%2\n\t"
 	"bne-	1b\n\t"
-	"isync"
 	: "=&r"(old), "=&r"(new)
 	: "r"(&var->counter), "r"(mask)
 	: "cc", "memory");
     return old;
 }
 
-static ETHR_INLINE int
-ethr_native_atomic_or_retold(ethr_native_atomic_t *var, int mask)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_and_retold_acqb(ethr_native_atomic32_t *var, ethr_sint32_t mask)
 {
-    int old, new;
+    ethr_sint32_t res;
+    res = ethr_native_atomic32_and_retold(var, mask);
+    __asm__ __volatile("isync\n\t" : : : "memory");
+    return res;
+}
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_or_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask)
+{
+    ethr_sint32_t old, new;
 
     __asm__ __volatile__(
-	"eieio\n\t"
 	"1:\t"
 	"lwarx	%0,0,%2\n\t"
 	"or	%1,%0,%3\n\t"
 	"stwcx.	%1,0,%2\n\t"
 	"bne-	1b\n\t"
-	"isync"
 	: "=&r"(old), "=&r"(new)
 	: "r"(&var->counter), "r"(mask)
 	: "cc", "memory");
     return old;
 }
 
-static ETHR_INLINE int
-ethr_native_atomic_xchg(ethr_native_atomic_t *var, int val)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_or_retold_acqb(ethr_native_atomic32_t *var, ethr_sint32_t mask)
+{
+    ethr_sint32_t res;
+    res = ethr_native_atomic32_or_retold(var, mask);
+    __asm__ __volatile("isync\n\t" : : : "memory");
+    return res;
+}
+
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_xchg(ethr_native_atomic32_t *var, ethr_sint32_t val)
 {
-    int tmp;
+    ethr_sint32_t tmp;
 
     __asm__ __volatile__(
-	"eieio\n\t"
 	"1:\t"
 	"lwarx	%0,0,%1\n\t"
 	"stwcx.	%2,0,%1\n\t"
 	"bne-	1b\n\t"
-	"isync"
 	: "=&r"(tmp)
 	: "r"(&var->counter), "r"(val)
 	: "cc", "memory");
     return tmp;
 }
 
-static ETHR_INLINE int
-ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, int new, int expected)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_xchg_acqb(ethr_native_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    res = ethr_native_atomic32_xchg(var, val);
+    __asm__ __volatile("isync\n\t" : : : "memory");
+    return res;
+}
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var,
+			     ethr_sint32_t new,
+			     ethr_sint32_t expected)
+{
+  ethr_sint32_t old;
+
+  __asm__ __volatile__(
+    "1:\t"
+    "lwarx	%0,0,%2\n\t"
+    "cmpw	0,%0,%3\n\t"
+    "bne	2f\n\t"
+    "stwcx.	%1,0,%2\n\t"
+    "bne-	1b\n\t"
+    "2:"
+    : "=&r"(old)
+    : "r"(new), "r"(&var->counter), "r"(expected)
+    : "cc", "memory");
+
+    return old;
+}
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_cmpxchg_acqb(ethr_native_atomic32_t *var,
+				  ethr_sint32_t new,
+				  ethr_sint32_t expected)
 {
-  int old;
+  ethr_sint32_t old;
 
   __asm__ __volatile__(
-    "eieio\n\t"
     "1:\t"
     "lwarx	%0,0,%2\n\t"
     "cmpw	0,%0,%3\n\t"
diff --git a/erts/include/internal/ppc32/ethr_membar.h b/erts/include/internal/ppc32/ethr_membar.h
new file mode 100644
index 0000000000..ff5cc86bfb
--- /dev/null
+++ b/erts/include/internal/ppc32/ethr_membar.h
@@ -0,0 +1,63 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Memory barriers for PowerPC
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_PPC_MEMBAR_H__
+#define ETHR_PPC_MEMBAR_H__
+
+#define ETHR_LoadLoad	(1 << 0)
+#define ETHR_LoadStore	(1 << 1)
+#define ETHR_StoreLoad	(1 << 2)
+#define ETHR_StoreStore	(1 << 3)
+
+static __inline__ void
+ethr_lwsync__(void)
+{
+#ifdef ETHR_PPC_HAVE_NO_LWSYNC
+    __asm__ __volatile__ ("sync\n\t" : : : "memory");
+#else
+#ifndef ETHR_PPC_HAVE_LWSYNC
+    if (ETHR_PPC_RUNTIME_CONF_HAVE_NO_LWSYNC__)
+	__asm__ __volatile__ ("sync\n\t" : : : "memory");
+    else
+#endif
+	__asm__ __volatile__ ("lwsync\n\t" : : : "memory");
+#endif
+}
+
+static __inline__ void
+ethr_sync__(void)
+{
+    __asm__ __volatile__ ("sync\n\t" : : : "memory");
+}
+
+/*
+ * According to the "memory barrier intstructions" section of
+ * http://www.ibm.com/developerworks/systems/articles/powerpc.html
+ * we want to use sync when a StoreLoad is needed and lwsync for
+ * everything else.
+ */
+#define ETHR_MEMBAR(B) \
+  ETHR_CHOOSE_EXPR((B) & ETHR_StoreLoad, ethr_sync__(), ethr_lwsync__())
+
+#endif
diff --git a/erts/include/internal/ppc32/ethread.h b/erts/include/internal/ppc32/ethread.h
index d2a72c3dc1..e41c83c5da 100644
--- a/erts/include/internal/ppc32/ethread.h
+++ b/erts/include/internal/ppc32/ethread.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -24,11 +24,9 @@
 #ifndef ETHREAD_PPC32_ETHREAD_H
 #define ETHREAD_PPC32_ETHREAD_H
 
+#include "ethr_membar.h"
 #include "atomic.h"
 #include "spinlock.h"
 #include "rwlock.h"
 
-#define ETHR_HAVE_NATIVE_ATOMICS 1
-#define ETHR_HAVE_NATIVE_LOCKS 1
-
 #endif /* ETHREAD_PPC32_ETHREAD_H */
diff --git a/erts/include/internal/ppc32/rwlock.h b/erts/include/internal/ppc32/rwlock.h
index 9bdab12826..311f000b69 100644
--- a/erts/include/internal/ppc32/rwlock.h
+++ b/erts/include/internal/ppc32/rwlock.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -23,18 +23,20 @@
  *
  * Based on the examples in Appendix E of Motorola's
  * "Programming Environments Manual For 32-Bit Implementations
- * of the PowerPC Architecture". Uses eieio instead of sync
- * in the unlock sequence, as suggested in the manual.
+ * of the PowerPC Architecture".
  */
 #ifndef ETHREAD_PPC_RWLOCK_H
 #define ETHREAD_PPC_RWLOCK_H
 
+#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1
+#define ETHR_NATIVE_RWSPINLOCK_IMPL "ethread"
+
 /* Unlocked if zero, read-locked if negative, write-locked if +1. */
 typedef struct {
     volatile int lock;
 } ethr_native_rwlock_t;
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
 
 static ETHR_INLINE void
 ethr_native_rwlock_init(ethr_native_rwlock_t *lock)
@@ -47,9 +49,10 @@ ethr_native_read_unlock(ethr_native_rwlock_t *lock)
 {
     int tmp;
 
-    /* this is eieio + ethr_native_atomic_inc() - isync */
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+
+    /* this is ethr_native_atomic_inc() - isync */
     __asm__ __volatile__(
-	"eieio\n\t"
 	"1:\t"
 	"lwarx	%0,0,%1\n\t"
 	"addic	%0,%0,1\n\t"
@@ -105,7 +108,7 @@ ethr_native_read_lock(ethr_native_rwlock_t *lock)
 static ETHR_INLINE void
 ethr_native_write_unlock(ethr_native_rwlock_t *lock)
 {
-    __asm__ __volatile__("eieio" : : : "memory");
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
     lock->lock = 0;
 }
 
diff --git a/erts/include/internal/ppc32/spinlock.h b/erts/include/internal/ppc32/spinlock.h
index 034c20c143..4c95ec9efb 100644
--- a/erts/include/internal/ppc32/spinlock.h
+++ b/erts/include/internal/ppc32/spinlock.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -23,18 +23,20 @@
  *
  * Based on the examples in Appendix E of Motorola's
  * "Programming Environments Manual For 32-Bit Implementations
- * of the PowerPC Architecture". Uses eieio instead of sync
- * in the unlock sequence, as suggested in the manual.
+ * of the PowerPC Architecture".
  */
 #ifndef ETHREAD_PPC_SPINLOCK_H
 #define ETHREAD_PPC_SPINLOCK_H
 
+#define ETHR_HAVE_NATIVE_SPINLOCKS 1
+#define ETHR_NATIVE_SPINLOCK_IMPL "ethread"
+
 /* Unlocked if zero, locked if non-zero. */
 typedef struct {
     volatile unsigned int lock;
 } ethr_native_spinlock_t;
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
 
 static ETHR_INLINE void
 ethr_native_spinlock_init(ethr_native_spinlock_t *lock)
@@ -45,7 +47,7 @@ ethr_native_spinlock_init(ethr_native_spinlock_t *lock)
 static ETHR_INLINE void
 ethr_native_spin_unlock(ethr_native_spinlock_t *lock)
 {
-    __asm__ __volatile__("eieio" : : : "memory");
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
     lock->lock = 0;
 }
 
diff --git a/erts/include/internal/pthread/ethr_event.h b/erts/include/internal/pthread/ethr_event.h
new file mode 100644
index 0000000000..d0a77990cc
--- /dev/null
+++ b/erts/include/internal/pthread/ethr_event.h
@@ -0,0 +1,133 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2009-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Author: Rickard Green
+ */
+
+#if defined(ETHR_HAVE_LINUX_FUTEX) && defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+/* --- Linux futex implementation of ethread events ------------------------- */
+#define ETHR_LINUX_FUTEX_IMPL__
+
+#include <sys/syscall.h>
+#include <unistd.h>
+#include <linux/futex.h>
+#include <sys/time.h>
+
+#define ETHR_EVENT_OFF_WAITER__		((ethr_sint32_t) -1)
+#define ETHR_EVENT_OFF__		((ethr_sint32_t) 1)
+#define ETHR_EVENT_ON__ 		((ethr_sint32_t) 0)
+
+#if defined(FUTEX_WAIT_PRIVATE) && defined(FUTEX_WAKE_PRIVATE)
+#  define ETHR_FUTEX_WAIT__ FUTEX_WAIT_PRIVATE
+#  define ETHR_FUTEX_WAKE__ FUTEX_WAKE_PRIVATE
+#else
+#  define ETHR_FUTEX_WAIT__ FUTEX_WAIT
+#  define ETHR_FUTEX_WAKE__ FUTEX_WAKE
+#endif
+
+typedef struct {
+    ethr_atomic32_t futex;
+} ethr_event;
+
+#define ETHR_FUTEX__(FTX, OP, VAL)			\
+  (-1 == syscall(__NR_futex,				\
+		 (void *) ethr_atomic32_addr((FTX)),	\
+		 (OP),					\
+		 (int) (VAL),				\
+		 NULL,					\
+		 NULL,					\
+		 0)					\
+   ? errno : 0)
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__)
+
+static void ETHR_INLINE
+ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e)
+{
+    ethr_sint32_t val;
+    val = ethr_atomic32_xchg_mb(&e->futex, ETHR_EVENT_ON__);
+    if (val == ETHR_EVENT_OFF_WAITER__) {
+	int res = ETHR_FUTEX__(&e->futex, ETHR_FUTEX_WAKE__, 1);
+	if (res != 0)
+	    ETHR_FATAL_ERROR__(res);
+    }
+}
+
+static void ETHR_INLINE
+ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e)
+{
+    ethr_atomic32_set(&e->futex, ETHR_EVENT_OFF__);
+    ETHR_MEMORY_BARRIER;
+}
+
+#endif
+
+#elif defined(ETHR_PTHREADS)
+/* --- Posix mutex/cond implementation of events ---------------------------- */
+
+typedef struct {
+    ethr_atomic32_t state;
+    pthread_mutex_t mtx;
+    pthread_cond_t cnd;
+} ethr_event;
+
+#define ETHR_EVENT_OFF_WAITER__		-1L
+#define ETHR_EVENT_OFF__		1L
+#define ETHR_EVENT_ON__ 		0L
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__)
+
+static void ETHR_INLINE
+ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e)
+{
+    ethr_sint32_t val;
+    val = ethr_atomic32_xchg_mb(&e->state, ETHR_EVENT_ON__);
+    if (val == ETHR_EVENT_OFF_WAITER__) {
+	int res = pthread_mutex_lock(&e->mtx);
+	if (res != 0)
+	    ETHR_FATAL_ERROR__(res);
+	res = pthread_cond_signal(&e->cnd);
+	if (res != 0)
+	    ETHR_FATAL_ERROR__(res);
+	res = pthread_mutex_unlock(&e->mtx);
+	if (res != 0)
+	    ETHR_FATAL_ERROR__(res);
+    }
+}
+
+static void ETHR_INLINE
+ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e)
+{
+    ethr_atomic32_set(&e->state, ETHR_EVENT_OFF__);
+    ETHR_MEMORY_BARRIER;
+}
+
+#endif
+
+#endif
+
+int ethr_event_init(ethr_event *e);
+int ethr_event_destroy(ethr_event *e);
+int ethr_event_wait(ethr_event *e);
+int ethr_event_swait(ethr_event *e, int spincount);
+#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__)
+void ethr_event_set(ethr_event *e);
+void ethr_event_reset(ethr_event *e);
+#endif
diff --git a/erts/include/internal/sparc32/atomic.h b/erts/include/internal/sparc32/atomic.h
index d6fdc6b2a4..fe1daaa9cf 100644
--- a/erts/include/internal/sparc32/atomic.h
+++ b/erts/include/internal/sparc32/atomic.h
@@ -1,19 +1,19 @@
 /* 
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -21,153 +21,116 @@
  * Native ethread atomics on SPARC V9.
  * Author: Mikael Pettersson.
  */
-#ifndef ETHR_SPARC32_ATOMIC_H
-#define ETHR_SPARC32_ATOMIC_H
 
-typedef struct {
-    volatile long counter;
-} ethr_native_atomic_t;
-
-#ifdef ETHR_TRY_INLINE_FUNCS
+#undef ETHR_INCLUDE_ATOMIC_IMPL__
+#if !defined(ETHR_SPARC_V9_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__)
+#define ETHR_SPARC_V9_ATOMIC32_H__
+#define ETHR_INCLUDE_ATOMIC_IMPL__ 4
+#undef ETHR_ATOMIC_WANT_32BIT_IMPL__
+#elif !defined(ETHR_SPARC_V9_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__)
+#define ETHR_SPARC_V9_ATOMIC64_H__
+#define ETHR_INCLUDE_ATOMIC_IMPL__ 8
+#undef ETHR_ATOMIC_WANT_64BIT_IMPL__
+#endif
 
-#if defined(__arch64__)
-#define CASX "casx"
+#ifdef ETHR_INCLUDE_ATOMIC_IMPL__
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#define ETHR_HAVE_NATIVE_ATOMIC32 1
+#define ETHR_NATIVE_ATOMIC32_IMPL "ethread"
+#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+#define ETHR_ATMC_T__ ethr_native_atomic32_t
+#define ETHR_AINT_T__ ethr_sint32_t
+#define ETHR_CAS__ "cas"
+#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8
+#define ETHR_HAVE_NATIVE_ATOMIC64 1
+#define ETHR_NATIVE_ATOMIC64_IMPL "ethread"
+#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#define ETHR_ATMC_T__ ethr_native_atomic64_t
+#define ETHR_AINT_T__ ethr_sint64_t
+#define ETHR_CAS__ "casx"
 #else
-#define CASX "cas"
+#error "Unsupported integer size"
 #endif
 
-static ETHR_INLINE void
-ethr_native_atomic_init(ethr_native_atomic_t *var, long i)
-{
-    var->counter = i;
-}
-#define ethr_native_atomic_set(v, i)	ethr_native_atomic_init((v), (i))
-
-static ETHR_INLINE long
-ethr_native_atomic_read(ethr_native_atomic_t *var)
-{
-    return var->counter;
-}
-
-static ETHR_INLINE long
-ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr)
-{
-    long old, tmp;
-
-    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n");
-    do {
-	old = var->counter;
-	tmp = old+incr;
-	__asm__ __volatile__(
-	    CASX " [%2], %1, %0"
-	    : "=&r"(tmp)
-	    : "r"(old), "r"(&var->counter), "0"(tmp)
-	    : "memory");
-    } while (__builtin_expect(old != tmp, 0));
-    __asm__ __volatile__("membar #StoreLoad|#StoreStore");
-    return old+incr;
-}
+typedef struct {
+    volatile ETHR_AINT_T__ counter;
+} ETHR_ATMC_T__;
 
-static ETHR_INLINE void
-ethr_native_atomic_add(ethr_native_atomic_t *var, long incr)
-{
-    (void)ethr_native_atomic_add_return(var, incr);
-}
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
 
-static ETHR_INLINE long
-ethr_native_atomic_inc_return(ethr_native_atomic_t *var)
-{
-    return ethr_native_atomic_add_return(var, 1);
-}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1
+#endif
 
-static ETHR_INLINE void
-ethr_native_atomic_inc(ethr_native_atomic_t *var)
+static ETHR_INLINE ETHR_AINT_T__ *
+ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
 {
-    (void)ethr_native_atomic_add_return(var, 1);
+    return (ETHR_AINT_T__ *) &var->counter;
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_dec_return(ethr_native_atomic_t *var)
-{
-    return ethr_native_atomic_add_return(var, -1);
-}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1
+#endif
 
 static ETHR_INLINE void
-ethr_native_atomic_dec(ethr_native_atomic_t *var)
+ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
 {
-    (void)ethr_native_atomic_add_return(var, -1);
+    var->counter = i;
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask)
-{
-    long old, tmp;
-
-    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n");
-    do {
-	old = var->counter;
-	tmp = old & mask;
-	__asm__ __volatile__(
-	    CASX " [%2], %1, %0"
-	    : "=&r"(tmp)
-	    : "r"(old), "r"(&var->counter), "0"(tmp)
-	    : "memory");
-    } while (__builtin_expect(old != tmp, 0));
-    __asm__ __volatile__("membar #StoreLoad|#StoreStore");
-    return old;
-}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1
+#endif
 
-static ETHR_INLINE long
-ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
 {
-    long old, tmp;
-
-    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n");
-    do {
-	old = var->counter;
-	tmp = old | mask;
-	__asm__ __volatile__(
-	    CASX " [%2], %1, %0"
-	    : "=&r"(tmp)
-	    : "r"(old), "r"(&var->counter), "0"(tmp)
-	    : "memory");
-    } while (__builtin_expect(old != tmp, 0));
-    __asm__ __volatile__("membar #StoreLoad|#StoreStore");
-    return old;
+    return var->counter;
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val)
-{
-    long old, new;
-
-    __asm__ __volatile__("membar #LoadLoad|#StoreLoad");
-    do {
-	old = var->counter;
-	new = val;
-	__asm__ __volatile__(
-	    CASX " [%2], %1, %0"
-	    : "=&r"(new)
-	    : "r"(old), "r"(&var->counter), "0"(new)
-	    : "memory");
-    } while (__builtin_expect(old != new, 0));
-    __asm__ __volatile__("membar #StoreLoad|#StoreStore");
-    return old;
-}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG 1
+#endif
 
-static ETHR_INLINE long
-ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old)
 {
-    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n");
     __asm__ __volatile__(
-      CASX " [%2], %1, %0"
+      ETHR_CAS__ " [%2], %1, %0"
       : "=&r"(new)
       : "r"(old), "r"(&var->counter), "0"(new)
       : "memory");
-    __asm__ __volatile__("membar #StoreLoad|#StoreStore");
     return new;
 }
 
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old)
+{
+    ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(cmpxchg)(var, new, old);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+    return res;
+}
+
 #endif /* ETHR_TRY_INLINE_FUNCS */
 
-#endif /* ETHR_SPARC32_ATOMIC_H */
+#undef ETHR_NATMC_FUNC__
+#undef ETHR_ATMC_T__
+#undef ETHR_AINT_T__
+#undef ETHR_CAS__
+
+#endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */
diff --git a/erts/include/internal/sparc32/ethr_membar.h b/erts/include/internal/sparc32/ethr_membar.h
new file mode 100644
index 0000000000..6eb0c5a1d6
--- /dev/null
+++ b/erts/include/internal/sparc32/ethr_membar.h
@@ -0,0 +1,115 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Memory barriers for sparc-v9
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_SPARC_V9_MEMBAR_H__
+#define ETHR_SPARC_V9_MEMBAR_H__
+
+#if defined(ETHR_SPARC_TSO)
+/* --- Total Store Order ------------------------------------------------ */
+
+#define ETHR_LoadLoad	0
+#define ETHR_LoadStore	0
+#define ETHR_StoreLoad	1
+#define ETHR_StoreStore	0
+
+static __inline__ void
+ethr_cb__(void)
+{
+    __asm__ __volatile__ ("" : : : "memory");
+}
+
+static __inline__ void
+ethr_StoreLoad__(void)
+{
+    __asm__ __volatile__ ("membar #StoreLoad\n\t" : : : "memory");
+}
+
+
+#define ETHR_MEMBAR(B) \
+  ETHR_CHOOSE_EXPR((B), ethr_StoreLoad__(), ethr_cb__())
+
+#elif defined(ETHR_SPARC_PSO)
+/* --- Partial Store Order ---------------------------------------------- */
+
+#define ETHR_LoadLoad	0
+#define ETHR_LoadStore	0
+#define ETHR_StoreLoad	(1 << 0)
+#define ETHR_StoreStore	(1 << 1)
+
+static __inline__ void
+ethr_cb__(void)
+{
+    __asm__ __volatile__ ("" : : : "memory");
+}
+
+static __inline__ void
+ethr_StoreLoad__(void)
+{
+    __asm__ __volatile__ ("membar #StoreLoad\n\t" : : : "memory");
+}
+
+static __inline__ void
+ethr_StoreStore__(void)
+{
+    __asm__ __volatile__ ("membar #StoreStore\n\t" : : : "memory");
+}
+
+static __inline__ void
+ethr_StoreLoad_StoreStore__(void)
+{
+    __asm__ __volatile__ ("membar #StoreLoad|StoreStore\n\t" : : : "memory");
+}
+
+#define ETHR_MEMBAR(B)					\
+  ETHR_CHOOSE_EXPR(					\
+      (B) == ETHR_StoreLoad,				\
+      ethr_StoreLoad__(),				\
+      ETHR_CHOOSE_EXPR(					\
+	  (B) == ETHR_StoreStore,			\
+	  ethr_StoreStore__(),				\
+	  ETHR_CHOOSE_EXPR(				\
+	      (B) == (ETHR_StoreLoad|ETHR_StoreStore),	\
+	      ethr_StoreLoad_StoreStore__(),		\
+	      ethr_cb__())))
+
+#elif defined(ETHR_SPARC_RMO)
+/* --- Relaxed Memory Order --------------------------------------------- */
+
+#  define ETHR_LoadLoad #LoadLoad
+#  define ETHR_LoadStore #LoadStore
+#  define ETHR_StoreLoad #StoreLoad
+#  define ETHR_StoreStore #StoreStore
+
+#  define ETHR_MEMBAR_AUX__(B) \
+     __asm__ __volatile__("membar " #B "\n\t" : : : "memory")
+
+#  define ETHR_MEMBAR(B) ETHR_MEMBAR_AUX__(B)
+
+#else
+
+#  error "No memory order defined"
+
+#endif
+
+#endif
diff --git a/erts/include/internal/sparc32/ethread.h b/erts/include/internal/sparc32/ethread.h
index 1d55399640..5ad92d3da7 100644
--- a/erts/include/internal/sparc32/ethread.h
+++ b/erts/include/internal/sparc32/ethread.h
@@ -1,7 +1,7 @@
 /* 
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -24,11 +24,14 @@
 #ifndef ETHREAD_SPARC32_ETHREAD_H
 #define ETHREAD_SPARC32_ETHREAD_H
 
+#include "ethr_membar.h"
+#define ETHR_ATOMIC_WANT_32BIT_IMPL__
 #include "atomic.h"
+#if ETHR_SIZEOF_PTR == 8
+#  define ETHR_ATOMIC_WANT_64BIT_IMPL__
+#  include "atomic.h"
+#endif
 #include "spinlock.h"
 #include "rwlock.h"
 
-#define ETHR_HAVE_NATIVE_ATOMICS 1
-#define ETHR_HAVE_NATIVE_LOCKS 1
-
 #endif /* ETHREAD_SPARC32_ETHREAD_H */
diff --git a/erts/include/internal/sparc32/rwlock.h b/erts/include/internal/sparc32/rwlock.h
index 12448e0b06..8b6f2e9c57 100644
--- a/erts/include/internal/sparc32/rwlock.h
+++ b/erts/include/internal/sparc32/rwlock.h
@@ -1,19 +1,19 @@
 /* 
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -24,12 +24,15 @@
 #ifndef ETHREAD_SPARC32_RWLOCK_H
 #define ETHREAD_SPARC32_RWLOCK_H
 
+#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1
+#define ETHR_NATIVE_RWSPINLOCK_IMPL "ethread"
+
 /* Unlocked if zero, read-locked if positive, write-locked if -1. */
 typedef struct {
     volatile int lock;
 } ethr_native_rwlock_t;
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
 
 static ETHR_INLINE void
 ethr_native_rwlock_init(ethr_native_rwlock_t *lock)
@@ -42,7 +45,7 @@ ethr_native_read_unlock(ethr_native_rwlock_t *lock)
 {
     unsigned int old, new;
 
-    __asm__ __volatile__("membar #LoadLoad|#StoreLoad");
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
     do {
 	old = lock->lock;
 	new = old-1;
@@ -70,7 +73,7 @@ ethr_native_read_trylock(ethr_native_rwlock_t *lock)
 	    : "r"(old), "r"(&lock->lock), "0"(new)
 	    : "memory");
     } while (__builtin_expect(old != new, 0));
-    __asm__ __volatile__("membar #StoreLoad|#StoreStore");
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
     return 1;
 }
 
@@ -87,7 +90,7 @@ ethr_native_read_lock(ethr_native_rwlock_t *lock)
 	if (__builtin_expect(ethr_native_read_trylock(lock) != 0, 1))
 	    break;
 	do {
-	    __asm__ __volatile__("membar #LoadLoad");
+	    ETHR_MEMBAR(ETHR_LoadLoad);
 	} while (ethr_native_read_is_locked(lock));
    }
 }
@@ -95,7 +98,7 @@ ethr_native_read_lock(ethr_native_rwlock_t *lock)
 static ETHR_INLINE void
 ethr_native_write_unlock(ethr_native_rwlock_t *lock)
 {
-    __asm__ __volatile__("membar #LoadStore|#StoreStore");
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
     lock->lock = 0;
 }
 
@@ -115,7 +118,7 @@ ethr_native_write_trylock(ethr_native_rwlock_t *lock)
 	    : "r"(old), "r"(&lock->lock), "0"(new)
 	    : "memory");
     } while (__builtin_expect(old != new, 0));
-    __asm__ __volatile__("membar #StoreLoad|#StoreStore");
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
     return 1;
 }
 
@@ -132,7 +135,7 @@ ethr_native_write_lock(ethr_native_rwlock_t *lock)
 	if (__builtin_expect(ethr_native_write_trylock(lock) != 0, 1))
 	    break;
 	do {
-	    __asm__ __volatile__("membar #LoadLoad");
+	    ETHR_MEMBAR(ETHR_LoadLoad);
 	} while (ethr_native_write_is_locked(lock));
    }
 }
diff --git a/erts/include/internal/sparc32/spinlock.h b/erts/include/internal/sparc32/spinlock.h
index b4fe48b714..d4e36e09cf 100644
--- a/erts/include/internal/sparc32/spinlock.h
+++ b/erts/include/internal/sparc32/spinlock.h
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -24,12 +24,15 @@
 #ifndef ETHR_SPARC32_SPINLOCK_H
 #define ETHR_SPARC32_SPINLOCK_H
 
+#define ETHR_HAVE_NATIVE_SPINLOCKS 1
+#define ETHR_NATIVE_SPINLOCK_IMPL "ethread"
+
 /* Locked with ldstub, so unlocked when 0 and locked when non-zero. */
 typedef struct {
     volatile unsigned char lock;
 } ethr_native_spinlock_t;
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
 
 static ETHR_INLINE void
 ethr_native_spinlock_init(ethr_native_spinlock_t *lock)
@@ -40,7 +43,7 @@ ethr_native_spinlock_init(ethr_native_spinlock_t *lock)
 static ETHR_INLINE void
 ethr_native_spin_unlock(ethr_native_spinlock_t *lock)
 {
-    __asm__ __volatile__("membar #LoadStore|#StoreStore");
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
     lock->lock = 0;
 }
 
@@ -51,10 +54,10 @@ ethr_native_spin_trylock(ethr_native_spinlock_t *lock)
 
     __asm__ __volatile__(
 	"ldstub [%1], %0\n\t"
-	"membar #StoreLoad|#StoreStore"
 	: "=r"(prev)
 	: "r"(&lock->lock)
 	: "memory");
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
     return prev == 0;
 }
 
@@ -71,7 +74,7 @@ ethr_native_spin_lock(ethr_native_spinlock_t *lock)
 	if (__builtin_expect(ethr_native_spin_trylock(lock) != 0, 1))
 	    break;
 	do {
-	    __asm__ __volatile__("membar #LoadLoad");
+	    ETHR_MEMBAR(ETHR_LoadLoad);
 	} while (ethr_native_spin_is_locked(lock));
     }
 }
diff --git a/erts/include/internal/tile/atomic.h b/erts/include/internal/tile/atomic.h
index 59a9250e7c..1f1553c346 100644
--- a/erts/include/internal/tile/atomic.h
+++ b/erts/include/internal/tile/atomic.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -24,102 +24,116 @@
 #ifndef ETHREAD_TILE_ATOMIC_H
 #define ETHREAD_TILE_ATOMIC_H
 
+#define ETHR_HAVE_NATIVE_ATOMIC32 1
+#define ETHR_NATIVE_ATOMIC32_IMPL "tilera"
+
 #include <atomic.h>
 
 /* An atomic is an aligned int accessed via locked operations.
  */
 typedef struct {
-    volatile long counter;
-} ethr_native_atomic_t;
+    volatile ethr_sint32_t counter;
+} ethr_native_atomic32_t;
 
-#ifdef ETHR_TRY_INLINE_FUNCS
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
 
-static ETHR_INLINE void
-ethr_native_atomic_init(ethr_native_atomic_t *var, long i)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
+
+static ETHR_INLINE ethr_sint32_t *
+ethr_native_atomic32_addr(ethr_native_atomic32_t *var)
 {
-    var->counter = i;
+    return (ethr_sint32_t *) &var->counter;
 }
 
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT 1
+
 static ETHR_INLINE void
-ethr_native_atomic_set(ethr_native_atomic_t *var, long i)
+ethr_native_atomic32_init(ethr_native_atomic32_t *var, ethr_sint32_t i)
 {
-    __insn_mf();
-    atomic_exchange_acq(&var->counter, i);
+    var->counter = i;
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_read(ethr_native_atomic_t *var)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_read(ethr_native_atomic32_t *var)
 {
     return var->counter;
 }
 
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var)
+{
+    return atomic_compare_and_exchange_val_acq(&var->counter,
+					       0x81818181,
+					       0x81818181);
+}
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD 1
+
 static ETHR_INLINE void
-ethr_native_atomic_add(ethr_native_atomic_t *var, long incr)
+ethr_native_atomic32_add(ethr_native_atomic32_t *var, ethr_sint32_t incr)
 {
-    __insn_mf();
     atomic_add(&var->counter, incr);
-}      
-       
+}
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC 1
+
 static ETHR_INLINE void
-ethr_native_atomic_inc(ethr_native_atomic_t *var)
+ethr_native_atomic32_inc(ethr_native_atomic32_t *var)
 {
-    __insn_mf();
     atomic_increment(&var->counter);
 }
 
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC 1
+
 static ETHR_INLINE void
-ethr_native_atomic_dec(ethr_native_atomic_t *var)
+ethr_native_atomic32_dec(ethr_native_atomic32_t *var)
 {
-    __insn_mf();
     atomic_decrement(&var->counter);
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr)
-{
-    __insn_mf();
-    return atomic_exchange_and_add(&var->counter, incr) + incr;
-}
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN 1
 
-static ETHR_INLINE long
-ethr_native_atomic_inc_return(ethr_native_atomic_t *var)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_add_return(ethr_native_atomic32_t *var, ethr_sint32_t incr)
 {
-    return ethr_native_atomic_add_return(var, 1);
+    return atomic_exchange_and_add(&var->counter, incr) + incr;
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_dec_return(ethr_native_atomic_t *var)
-{
-    return ethr_native_atomic_add_return(var, -1);
-}
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD 1
 
-static ETHR_INLINE long
-ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_and_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask)
 {
-    /* Implement a barrier suitable for a mutex unlock. */
-    __insn_mf();
     return atomic_and_val(&var->counter, mask);
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_or_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask)
 {
-    __insn_mf();
     return atomic_or_val(&var->counter, mask);
 }
 
-static ETHR_INLINE long
-ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_xchg_acqb(ethr_native_atomic32_t *var, ethr_sint32_t val)
 {   
-    __insn_mf();
     return atomic_exchange_acq(&var->counter, val);
 } 
 
-static ETHR_INLINE long
-ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long expected)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_cmpxchg_acqb(ethr_native_atomic32_t *var,
+				  ethr_sint32_t new,
+				  ethr_sint32_t expected)
 {
-    /* Implement a barrier suitable for a mutex unlock. */
-    __insn_mf();
     return atomic_compare_and_exchange_val_acq(&var->counter, new, expected);
 }
 
diff --git a/erts/include/internal/tile/ethr_membar.h b/erts/include/internal/tile/ethr_membar.h
new file mode 100644
index 0000000000..7cb4f3cf9a
--- /dev/null
+++ b/erts/include/internal/tile/ethr_membar.h
@@ -0,0 +1,35 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Memory barriers for TILE64/TILEPro
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_TILE_MEMBAR_H__
+#define ETHR_TILE_MEMBAR_H__
+
+#define ETHR_LoadLoad	(1 << 0)
+#define ETHR_LoadStore	(1 << 1)
+#define ETHR_StoreLoad	(1 << 2)
+#define ETHR_StoreStore	(1 << 3)
+
+#define ETHR_MEMBAR(B) __insn_mf()
+
+#endif
diff --git a/erts/include/internal/tile/ethread.h b/erts/include/internal/tile/ethread.h
index 2de4d42bc6..7f579b50e7 100644
--- a/erts/include/internal/tile/ethread.h
+++ b/erts/include/internal/tile/ethread.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -23,8 +23,7 @@
 #ifndef ETHREAD_TILE_ETHREAD_H
 #define ETHREAD_TILE_ETHREAD_H
 
+#include "ethr_membar.h"
 #include "atomic.h"
 
-#define ETHR_HAVE_NATIVE_ATOMICS 1
-
 #endif /* ETHREAD_TILE_ETHREAD_H */
diff --git a/erts/include/internal/win/ethr_atomic.h b/erts/include/internal/win/ethr_atomic.h
new file mode 100644
index 0000000000..e11f1abf47
--- /dev/null
+++ b/erts/include/internal/win/ethr_atomic.h
@@ -0,0 +1,480 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Native atomics ethread support when using VC++
+ * Author: Rickard Green
+ */
+
+#undef ETHR_INCLUDE_ATOMIC_IMPL__
+#if !defined(ETHR_WIN_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__)
+#  define ETHR_WIN_ATOMIC32_H__
+#  if (defined(ETHR_MEMBAR) \
+       && defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE) \
+       && defined(ETHR_HAVE__INTERLOCKEDEXCHANGE))
+#    define ETHR_INCLUDE_ATOMIC_IMPL__ 4
+#  endif
+#  undef ETHR_ATOMIC_WANT_32BIT_IMPL__
+#elif !defined(ETHR_WIN_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__)
+#  define ETHR_WIN_ATOMIC64_H__
+#  if (defined(ETHR_MEMBAR) \
+     && (defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64) \
+	 || defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ) \
+	 || defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL)))
+#    define ETHR_INCLUDE_ATOMIC_IMPL__ 8
+#  endif
+#  undef ETHR_ATOMIC_WANT_64BIT_IMPL__
+#endif
+
+#if !defined(_MSC_VER) || _MSC_VER < 1400
+#  undef ETHR_INCLUDE_ATOMIC_IMPL__
+#endif
+
+#ifdef ETHR_INCLUDE_ATOMIC_IMPL__
+
+#  ifndef ETHR_WIN_ATOMIC_COMMON__
+#    define ETHR_WIN_ATOMIC_COMMON__
+
+#    if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_IA64)
+#      define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 1
+#    else
+#      define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 0
+#    endif
+
+#  endif /* ETHR_WIN_ATOMIC_COMMON__ */
+
+#  if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+
+#    define ETHR_HAVE_NATIVE_ATOMIC32 1
+#    define ETHR_NATIVE_ATOMIC32_IMPL "windows-interlocked"
+
+#    ifdef ETHR_HAVE__INTERLOCKEDDECREMENT
+#      define ETHR_WIN_HAVE_DEC
+#      pragma intrinsic(_InterlockedDecrement)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDDECREMENT_REL
+#      define ETHR_WIN_HAVE_DEC_REL
+#      pragma intrinsic(_InterlockedDecrement_rel)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDINCREMENT
+#      define ETHR_WIN_HAVE_INC
+#      pragma intrinsic(_InterlockedIncrement)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ
+#      define ETHR_WIN_HAVE_INC_ACQ
+#      pragma intrinsic(_InterlockedIncrement_acq)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD
+#      define ETHR_WIN_HAVE_XCHG_ADD
+#      pragma intrinsic(_InterlockedExchangeAdd)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ
+#      define ETHR_WIN_HAVE_XCHG_ADD_ACQ
+#      pragma intrinsic(_InterlockedExchangeAdd_acq)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE
+#      define ETHR_WIN_HAVE_XCHG
+#      pragma intrinsic(_InterlockedExchange)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDAND
+#      define ETHR_WIN_HAVE_AND
+#      pragma intrinsic(_InterlockedAnd)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDOR
+#      define ETHR_WIN_HAVE_OR
+#      pragma intrinsic(_InterlockedOr)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE
+#      define ETHR_WIN_HAVE_CMPXCHG
+#        pragma intrinsic(_InterlockedCompareExchange)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ
+#      define ETHR_WIN_HAVE_CMPXCHG_ACQ
+#      pragma intrinsic(_InterlockedCompareExchange_acq)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL
+#      define ETHR_WIN_HAVE_CMPXCHG_REL
+#      pragma intrinsic(_InterlockedCompareExchange_rel)
+#    endif
+
+#    define ETHR_ILCKD__(X) _Interlocked ## X
+#    define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## _acq
+#    define ETHR_ILCKD_REL__(X) _Interlocked ## X ## _rel
+
+#    define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+#    define ETHR_ATMC_T__ ethr_native_atomic32_t
+#    define ETHR_AINT_T__ ethr_sint32_t
+
+#  elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8
+
+#    define ETHR_HAVE_NATIVE_ATOMIC64 1
+#    define ETHR_NATIVE_ATOMIC64_IMPL "windows-interlocked"
+
+#    ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64
+#      define ETHR_WIN_HAVE_DEC
+#      pragma intrinsic(_InterlockedDecrement64)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64_REL
+#      define ETHR_WIN_HAVE_DEC_REL
+#      pragma intrinsic(_InterlockedDecrement64_rel)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ
+#      define ETHR_WIN_HAVE_INC_ACQ
+#      pragma intrinsic(_InterlockedIncrement64_acq)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64
+#      define ETHR_WIN_HAVE_INC
+#      pragma intrinsic(_InterlockedIncrement64)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64
+#      define ETHR_WIN_HAVE_XCHG_ADD
+#      pragma intrinsic(_InterlockedExchangeAdd64)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ
+#      define ETHR_WIN_HAVE_XCHG_ADD_ACQ
+#      pragma intrinsic(_InterlockedExchangeAdd64_acq)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE64
+#      define ETHR_WIN_HAVE_XCHG
+#      pragma intrinsic(_InterlockedExchange64)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDAND64
+#      define ETHR_WIN_HAVE_AND
+#      pragma intrinsic(_InterlockedAnd64)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDOR64
+#      define ETHR_WIN_HAVE_OR
+#      pragma intrinsic(_InterlockedOr64)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64
+#      define ETHR_WIN_HAVE_CMPXCHG
+#      pragma intrinsic(_InterlockedCompareExchange64)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ
+#      define ETHR_WIN_HAVE_CMPXCHG_ACQ
+#      pragma intrinsic(_InterlockedCompareExchange64_acq)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL
+#      define ETHR_WIN_HAVE_CMPXCHG_REL
+#      pragma intrinsic(_InterlockedCompareExchange64_rel)
+#    endif
+
+#    define ETHR_ILCKD__(X) _Interlocked ## X ## 64
+#    define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64_acq
+#    define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64_rel
+
+#    define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#    define ETHR_ATMC_T__ ethr_native_atomic64_t
+#    define ETHR_AINT_T__ ethr_sint64_t
+
+#  else
+#    error "Unsupported integer size"
+#  endif
+
+typedef struct {
+    volatile ETHR_AINT_T__ value;
+} ETHR_ATMC_T__;
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__ *
+ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
+{
+    return (ETHR_AINT_T__ *) &var->value;
+}
+
+#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1
+#endif
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+    var->value = i;
+}
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1
+#endif
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+#if defined(_M_IX86)
+    if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__)
+	(void) ETHR_ILCKD__(Exchange)(&var->value, i);
+    else
+#endif /* _M_IX86 */
+    {
+	ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+	var->value = i;
+    }
+}
+
+#endif /* ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ */
+
+#if defined(ETHR_WIN_HAVE_XCHG)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB 1
+#endif
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+    (void) ETHR_ILCKD__(Exchange)(&var->value, i);
+}
+
+#endif
+
+#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
+{
+    return var->value;
+}
+
+#endif /* ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ */
+
+#if defined(ETHR_WIN_HAVE_XCHG_ADD)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(add_return_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+    return ETHR_ILCKD__(ExchangeAdd)(&var->value, i) + i;
+}
+
+#endif
+
+#if defined(ETHR_WIN_HAVE_INC)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(inc_return_mb)(ETHR_ATMC_T__ *var)
+{
+    return ETHR_ILCKD__(Increment)(&var->value);
+}
+
+#endif
+
+#if defined(ETHR_WIN_HAVE_INC_ACQ)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
+{
+    return ETHR_ILCKD_ACQ__(Increment)(&var->value);
+}
+
+#endif
+
+#if defined(ETHR_WIN_HAVE_DEC)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(dec_return_mb)(ETHR_ATMC_T__ *var)
+{
+    return ETHR_ILCKD__(Decrement)(&var->value);
+}
+
+#endif
+
+#if defined(ETHR_WIN_HAVE_DEC_REL)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
+{
+    return ETHR_ILCKD_REL__(Decrement)(&var->value);
+}
+
+#endif
+
+#if defined(ETHR_WIN_HAVE_AND)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(and_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
+{
+    return ETHR_ILCKD__(And)(&var->value, mask);
+}
+
+#endif
+
+#if defined(ETHR_WIN_HAVE_OR)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(or_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
+{
+    return ETHR_ILCKD__(Or)(&var->value, mask);
+}
+
+#endif
+
+#if defined(ETHR_WIN_HAVE_XCHG)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(xchg_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new)
+{
+    return ETHR_ILCKD__(Exchange)(&var->value, new);
+}
+
+#endif
+
+#if defined(ETHR_WIN_HAVE_CMPXCHG)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_mb)(ETHR_ATMC_T__ *var,
+			      ETHR_AINT_T__ new,
+			      ETHR_AINT_T__ old)
+{
+    return ETHR_ILCKD__(CompareExchange)(&var->value, new, old);
+}
+
+#endif
+
+#if defined(ETHR_WIN_HAVE_CMPXCHG_ACQ)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
+				ETHR_AINT_T__ new,
+				ETHR_AINT_T__ old)
+{
+    return ETHR_ILCKD_ACQ__(CompareExchange)(&var->value, new, old);
+}
+
+#endif
+
+#if defined(ETHR_WIN_HAVE_CMPXCHG_REL)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
+				ETHR_AINT_T__ new,
+				ETHR_AINT_T__ old)
+{
+    return ETHR_ILCKD_REL__(CompareExchange)(&var->value, new, old);
+}
+
+#endif
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#undef ETHR_ILCKD__
+#undef ETHR_ILCKD_ACQ__
+#undef ETHR_ILCKD_REL__
+#undef ETHR_NATMC_FUNC__
+#undef ETHR_ATMC_T__
+#undef ETHR_AINT_T__
+#undef ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
+#undef ETHR_WIN_HAVE_CMPXCHG
+#undef ETHR_WIN_HAVE_DEC
+#undef ETHR_WIN_HAVE_INC
+#undef ETHR_WIN_HAVE_XCHG_ADD
+#undef ETHR_WIN_HAVE_XCHG
+#undef ETHR_WIN_HAVE_AND
+#undef ETHR_WIN_HAVE_OR
+#undef ETHR_WIN_HAVE_XCHG_ADD_ACQ
+#undef ETHR_WIN_HAVE_INC_ACQ
+#undef ETHR_WIN_HAVE_DEC_REL
+#undef ETHR_WIN_HAVE_CMPXCHG_ACQ
+#undef ETHR_WIN_HAVE_CMPXCHG_REL
+
+#endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */
diff --git a/erts/include/internal/win/ethr_dw_atomic.h b/erts/include/internal/win/ethr_dw_atomic.h
new file mode 100644
index 0000000000..a3e7ffc3aa
--- /dev/null
+++ b/erts/include/internal/win/ethr_dw_atomic.h
@@ -0,0 +1,154 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Native double word atomics for windows
+ * Author: Rickard Green
+ */
+
+#undef ETHR_INCLUDE_DW_ATOMIC_IMPL__
+#ifndef ETHR_X86_DW_ATOMIC_H__
+#  define ETHR_X86_DW_ATOMIC_H__
+#  if ((ETHR_SIZEOF_PTR == 4 \
+        && defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64)) \
+       || (ETHR_SIZEOF_PTR == 8 \
+           && defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE128)))
+#    define ETHR_INCLUDE_DW_ATOMIC_IMPL__
+#  endif
+#endif
+
+#ifdef ETHR_INCLUDE_DW_ATOMIC_IMPL__
+
+#  if ETHR_SIZEOF_PTR == 4
+#    define ETHR_HAVE_NATIVE_SU_DW_ATOMIC
+#  else
+#    define ETHR_HAVE_NATIVE_DW_ATOMIC
+#  endif
+#  define ETHR_NATIVE_DW_ATOMIC_IMPL "windows-interlocked"
+
+#  if defined(_M_IX86) || defined(_M_AMD64)
+/*
+ * If ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ is defined, it will be used
+ * at runtime in order to determine if native or fallback implementation
+ * should be used.
+ */
+#    define ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ \
+       ETHR_X86_RUNTIME_CONF_HAVE_DW_CMPXCHG__
+#  endif
+
+#  include <intrin.h>
+#  if ETHR_SIZEOF_PTR == 4
+#    pragma intrinsic(_InterlockedCompareExchange64)
+#    define ETHR_DW_NATMC_ALIGN_MASK__ 0x7
+#    define ETHR_NATIVE_SU_DW_SINT_T ethr_sint64_t
+#  else
+#    pragma intrinsic(_InterlockedCompareExchange128)
+#    define ETHR_DW_NATMC_ALIGN_MASK__ 0xf
+#  endif
+
+typedef volatile __int64 * ethr_native_dw_ptr_t;
+
+/*
+ * We need 16 byte aligned memory in 64-bit mode, and 8 byte aligned
+ * memory in 32-bit mode. 16 byte aligned malloc in 64-bit mode is
+ * not common, and at least some glibc malloc implementations
+ * only 4 byte align in 32-bit mode.
+ *
+ * This code assumes 8 byte aligned memory in 64-bit mode, and 4 byte
+ * aligned memory in 32-bit mode. A malloc implementation that does
+ * not adhere to these alignment requirements is seriously broken,
+ * and we wont bother trying to work around it.
+ *
+ * Since memory alignment may be off by one word we need to align at
+ * runtime. We, therefore, need an extra word allocated.
+ */
+#define ETHR_DW_NATMC_MEM__(VAR) \
+   (&var->c[(int) ((ethr_uint_t) &(VAR)->c[0]) & ETHR_DW_NATMC_ALIGN_MASK__])
+typedef union {
+#ifdef ETHR_NATIVE_SU_DW_SINT_T
+    volatile ETHR_NATIVE_SU_DW_SINT_T dw_sint;
+#endif
+    volatile ethr_sint_t sint[3];
+    volatile char c[ETHR_SIZEOF_PTR*3];
+} ethr_native_dw_atomic_t;
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+#ifdef ETHR_DEBUG
+#  define ETHR_DW_DBG_ALIGNED__(PTR) \
+     ETHR_ASSERT((((ethr_uint_t) (PTR)) & ETHR_DW_NATMC_ALIGN_MASK__) == 0);
+#else
+#  define ETHR_DW_DBG_ALIGNED__(PTR)
+#endif
+
+#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR
+
+static ETHR_INLINE ethr_sint_t *
+ethr_native_dw_atomic_addr(ethr_native_dw_atomic_t *var)
+{
+    ethr_sint_t *p = (ethr_sint_t *) ETHR_DW_NATMC_MEM__(var);
+    ETHR_DW_DBG_ALIGNED__(p);
+    return p;
+}
+
+
+#if ETHR_SIZEOF_PTR == 4
+
+#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB
+
+static ETHR_INLINE ethr_sint64_t
+ethr_native_su_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var,
+				    ethr_sint64_t new,
+				    ethr_sint64_t exp)
+{
+    ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var);
+    ETHR_DW_DBG_ALIGNED__(p);
+    return (ethr_sint64_t) _InterlockedCompareExchange64(p, new, exp);
+}
+
+#elif ETHR_SIZEOF_PTR == 8
+
+#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB
+
+#ifdef ETHR_BIGENDIAN
+#  define ETHR_WIN_LOW_WORD__ 1
+#  define ETHR_WIN_HIGH_WORD__ 0
+#else
+#  define ETHR_WIN_LOW_WORD__ 0
+#  define ETHR_WIN_HIGH_WORD__ 1
+#endif
+
+static ETHR_INLINE int
+ethr_native_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var,
+				 ethr_sint_t *new,
+				 ethr_sint_t *xchg)
+{
+    ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var);
+    ETHR_DW_DBG_ALIGNED__(p);
+    return (int) _InterlockedCompareExchange128(p,
+						new[ETHR_WIN_HIGH_WORD__],
+						new[ETHR_WIN_LOW_WORD__],
+						xchg);
+}
+
+#endif
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#endif /* ETHR_INCLUDE_DW_ATOMIC_IMPL__ */
diff --git a/erts/include/internal/win/ethr_event.h b/erts/include/internal/win/ethr_event.h
new file mode 100644
index 0000000000..6363174a74
--- /dev/null
+++ b/erts/include/internal/win/ethr_event.h
@@ -0,0 +1,64 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2009-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Author: Rickard Green
+ */
+
+#define ETHR_EVENT_OFF_WAITER__		((ethr_sint32_t) -1)
+#define ETHR_EVENT_OFF__		((ethr_sint32_t) 1)
+#define ETHR_EVENT_ON__ 		((ethr_sint32_t) 0)
+
+typedef struct {
+    ethr_atomic32_t state;
+    HANDLE handle;
+} ethr_event;
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__)
+
+#pragma intrinsic(_InterlockedExchange)
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e)
+{
+    /* _InterlockedExchange() imply a full memory barrier which is important */
+    ethr_sint32_t state = ethr_atomic32_xchg_wb(&e->state, ETHR_EVENT_ON__);
+    if (state == ETHR_EVENT_OFF_WAITER__) {
+	if (!SetEvent(e->handle))
+	    ETHR_FATAL_ERROR__(ethr_win_get_errno__());
+    }
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e)
+{
+    ethr_atomic32_set(&e->state, ETHR_EVENT_OFF__);
+    ETHR_MEMORY_BARRIER;
+}
+
+#endif
+
+int ethr_event_init(ethr_event *e);
+int ethr_event_destroy(ethr_event *e);
+int ethr_event_wait(ethr_event *e);
+int ethr_event_swait(ethr_event *e, int spincount);
+#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__)
+void ethr_event_set(ethr_event *e);
+void ethr_event_reset(ethr_event *e);
+#endif
diff --git a/erts/include/internal/win/ethr_membar.h b/erts/include/internal/win/ethr_membar.h
new file mode 100644
index 0000000000..8237660b2c
--- /dev/null
+++ b/erts/include/internal/win/ethr_membar.h
@@ -0,0 +1,145 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Memory barriers for Windows
+ * Author: Rickard Green
+ */
+
+#if (!defined(ETHR_WIN_MEMBAR_H__) \
+     && (defined(_MSC_VER) && _MSC_VER >= 1400) \
+     && (defined(_M_AMD64) \
+	 || defined(_M_IA64) \
+	 || defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE)))
+#define ETHR_WIN_MEMBAR_H__
+
+#define ETHR_LoadLoad	(1 << 0)
+#define ETHR_LoadStore	(1 << 1)
+#define ETHR_StoreLoad	(1 << 2)
+#define ETHR_StoreStore	(1 << 3)
+
+#include <intrin.h>
+#undef ETHR_COMPILER_BARRIER
+#define ETHR_COMPILER_BARRIER _ReadWriteBarrier()
+#pragma intrinsic(_ReadWriteBarrier)
+
+#pragma intrinsic(_InterlockedCompareExchange)
+
+#define ETHR_MB_USING_INTERLOCKED__			\
+do {							\
+    volatile long x___ = 0;				\
+    (void) _InterlockedCompareExchange(&x___, 2, 1);	\
+} while (0)
+
+#if defined(_M_IA64)
+
+#define ETHR_MEMBAR(B) __mf()
+
+#elif defined(_M_AMD64) || defined(_M_IX86)
+
+#include <emmintrin.h>
+#include <mmintrin.h>
+
+#if ETHR_SIZEOF_PTR == 4
+#  define ETHR_NO_SSE2_MB__ ETHR_MB_USING_INTERLOCKED__
+#endif
+#pragma intrinsic(_mm_mfence)
+#pragma intrinsic(_mm_sfence)
+#pragma intrinsic(_mm_lfence)
+
+static __forceinline void
+ethr_cfence__(void)
+{
+    _ReadWriteBarrier();
+}
+
+static __forceinline void
+ethr_mfence__(void)
+{
+#if ETHR_SIZEOF_PTR == 4
+    if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__)
+	ETHR_NO_SSE2_MB__;
+    else
+#endif
+	_mm_mfence();
+}
+
+static __forceinline void
+ethr_sfence__(void)
+{
+#if ETHR_SIZEOF_PTR == 4
+    if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__)
+	ETHR_NO_SSE2_MB__;
+    else
+#endif
+	_mm_sfence();
+}
+
+static __forceinline void
+ethr_lfence__(void)
+{
+#if ETHR_SIZEOF_PTR == 4
+    if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__)
+	ETHR_NO_SSE2_MB__;
+    else
+#endif
+	_mm_lfence();
+}
+
+#define ETHR_X86_OUT_OF_ORDER_MEMBAR(B)				\
+  ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore,			\
+		   ethr_sfence__(),				\
+		   ETHR_CHOOSE_EXPR((B) == ETHR_LoadLoad,	\
+				    ethr_lfence__(),		\
+				    ethr_mfence__()))
+
+#ifdef ETHR_X86_OUT_OF_ORDER
+
+#define ETHR_MEMBAR(B) \
+  ETHR_X86_OUT_OF_ORDER_MEMBAR((B))
+
+#else /* !ETHR_X86_OUT_OF_ORDER (the default) */
+
+/*
+ * We assume that only stores before loads may be reordered. That is,
+ * we assume that *no* instructions like these are used:
+ * - CLFLUSH,
+ * - streaming stores executed with non-temporal move,
+ * - string operations, or
+ * - other instructions which aren't LoadLoad, LoadStore, and StoreStore
+ *   ordered by themselves
+ * If such instructions are used, either insert memory barriers
+ * using ETHR_X86_OUT_OF_ORDER_MEMBAR() at appropriate places, or
+ * define ETHR_X86_OUT_OF_ORDER. For more info see Intel 64 and IA-32
+ * Architectures Software Developer's Manual; Vol 3A; Chapter 8.2.2.
+ */
+
+#define ETHR_MEMBAR(B) \
+  ETHR_CHOOSE_EXPR((B) & ETHR_StoreLoad, ethr_mfence__(), ethr_cfence__())
+
+#endif
+
+#else /* No knowledge about platform; use interlocked fallback */
+
+#define ETHR_MEMBAR(B) ETHR_MB_USING_INTERLOCKED__
+#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MB_USING_INTERLOCKED__
+
+#endif
+
+#endif /* ETHR_WIN_MEMBAR_H__ */
diff --git a/erts/include/internal/win/ethread.h b/erts/include/internal/win/ethread.h
new file mode 100644
index 0000000000..8be35e810e
--- /dev/null
+++ b/erts/include/internal/win/ethread.h
@@ -0,0 +1,37 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Native atomic and spinlock ethread support when using VC++
+ * Author: Rickard Green
+ */
+
+#ifndef ETHREAD_WIN_H__
+#define ETHREAD_WIN_H__
+
+#include "ethr_membar.h"
+#define ETHR_ATOMIC_WANT_32BIT_IMPL__
+#include "ethr_atomic.h"
+#if ETHR_SIZEOF_PTR == 8
+#  define ETHR_ATOMIC_WANT_64BIT_IMPL__
+#  include "ethr_atomic.h"
+#endif
+#include "ethr_dw_atomic.h"
+
+#endif
diff --git a/erts/lib_src/Makefile.in b/erts/lib_src/Makefile.in
index 49f5b1f048..aed889eaef 100644
--- a/erts/lib_src/Makefile.in
+++ b/erts/lib_src/Makefile.in
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 2004-2010. All Rights Reserved.
+# Copyright Ericsson AB 2004-2012. All Rights Reserved.
 #
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -65,7 +65,7 @@ TYPE_SUFFIX=.purecov
 PRE_LD=purecov $(PURECOV_BUILD_OPTIONS)
 else
 ifeq ($(TYPE),gcov)
-CFLAGS=@DEBUG_CFLAGS@ -fprofile-arcs -ftest-coverage -O0
+CFLAGS=@DEBUG_CFLAGS@ -DGCOV -fprofile-arcs -ftest-coverage -O0
 TYPE_SUFFIX=.gcov
 PRE_LD=
 else
@@ -138,7 +138,7 @@ INCLUDES=-I$(ERTS_INCL) -I$(ERTS_INCL)/$(TARGET) -I$(ERTS_INCL_INT) -I$(ERTS_INC
 INCLUDES += -I../emulator/beam -I../emulator/sys/$(ERLANG_OSTYPE)
 
 USING_MINGW=@MIXED_CYGWIN_MINGW@
-USING_VC=@MIXED_CYGWIN_VC@
+USING_VC=@MIXED_VC@
 
 ifeq ($(USING_VC),yes)
 LIB_SUFFIX=.lib
@@ -280,8 +280,18 @@ endif
 #
 # ethread library
 #
+ETHR_THR_LIB_BASE_DIR=@ETHR_THR_LIB_BASE_DIR@
 ifneq ($(strip $(ETHR_LIB_NAME)),)
-ETHREAD_LIB_SRC=common/ethread.c
+ETHREAD_LIB_SRC=common/ethr_aux.c \
+		common/ethr_atomics.c \
+		common/ethr_mutex.c \
+		common/ethr_cbf.c \
+		$(ETHR_THR_LIB_BASE_DIR)/ethread.c \
+		$(ETHR_THR_LIB_BASE_DIR)/ethr_event.c
+ETHR_X86_SSE2_ASM=@ETHR_X86_SSE2_ASM@
+ifeq ($(ETHR_X86_SSE2_ASM),yes)
+ETHREAD_LIB_SRC += pthread/ethr_x86_sse2_asm.c
+endif
 ETHREAD_LIB_NAME=ethread$(TYPE_SUFFIX)
 
 ifeq ($(USING_VC),yes)
@@ -308,10 +318,14 @@ ETHREAD_LIB=
 
 endif
 
+_create_dirs := $(shell mkdir -p $(CREATE_DIRS))
 #
 # Everything to build
 #
-all: $(CREATE_DIRS) $(ETHREAD_LIB) $(ERTS_LIBS) $(ERTS_INTERNAL_LIBS)
+.PHONY: all
+all: $(OBJ_DIR)/MADE
+
+$(OBJ_DIR)/MADE: $(ETHREAD_LIB) $(ERTS_LIBS) $(ERTS_INTERNAL_LIBS)
 ifeq ($(OMIT_OMIT_FP),yes)
 	@echo '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *'
 	@echo '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *'
@@ -321,6 +335,8 @@ ifeq ($(OMIT_OMIT_FP),yes)
 	@echo '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *'
 	@echo '* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *'
 endif
+	echo  $? > $(OBJ_DIR)/MADE
+
 #
 # The libs ...
 #
@@ -376,16 +392,19 @@ $(ERTS_LIB): $(ERTS_LIB_OBJS)
 # Object files
 #
 
+$(r_OBJ_DIR)/ethr_x86_sse2_asm.o: pthread/ethr_x86_sse2_asm.c
+	$(CC) -msse2 $(THR_DEFS) $(CFLAGS) $(INCLUDES) -c $< -o $@
+
 $(r_OBJ_DIR)/%.o: common/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) $(INCLUDES) -c $< -o $@
 
-$(r_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c
+$(r_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) $(INCLUDES) -c $< -o $@
 
 $(OBJ_DIR)/%.o: common/%.c
 	$(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@
 
-$(OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c
+$(OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c
 	$(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@
 
 # Win32 specific
@@ -393,35 +412,28 @@ $(OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c
 $(MD_OBJ_DIR)/%.o: common/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) -MD $(INCLUDES) -c $< -o $@
 
-$(MD_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c
+$(MD_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) -MD $(INCLUDES) -c $< -o $@
 
 $(MDd_OBJ_DIR)/%.o: common/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) -MDd $(INCLUDES) -c $< -o $@
 
-$(MDd_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c
+$(MDd_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) -MDd $(INCLUDES) -c $< -o $@
 
 $(MT_OBJ_DIR)/%.o: common/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) -MT $(INCLUDES) -c $< -o $@
 
-$(MT_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c
+$(MT_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) -MT $(INCLUDES) -c $< -o $@
 
 $(MTd_OBJ_DIR)/%.o: common/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) -MTd $(INCLUDES) -c $< -o $@
 
-$(MTd_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c
+$(MTd_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c
 	$(CC) $(THR_DEFS) $(CFLAGS) -MTd $(INCLUDES) -c $< -o $@
 
 #
-# Create directories
-#
-
-$(CREATE_DIRS):
-	$(MKDIR) -p $@
-
-#
 # Install
 #
 
@@ -438,6 +450,9 @@ RELEASE_LIBS=$(ERTS_LIBS)
 INTERNAL_RELEASE_INCLUDES= \
 	$(ERTS_INCL_INT)/README \
 	$(ERTS_INCL_INT)/ethread.h \
+	$(ERTS_INCL_INT)/ethr_mutex.h \
+	$(ERTS_INCL_INT)/ethr_optimized_fallbacks.h \
+	$(ERTS_INCL_INT)/ethr_atomics.h \
 	$(ERTS_INCL_INT)/$(TARGET)/ethread.mk \
 	$(ERTS_INCL_INT)/$(TARGET)/erts_internal.mk \
 	$(ERTS_INCL_INT)/$(TARGET)/ethread_header_config.h \
@@ -447,55 +462,58 @@ INTERNAL_RELEASE_INCLUDES= \
 	$(ERTS_INCL_INT)/erl_misc_utils.h \
 	$(ERTS_INCL_INT)/erl_errno.h
 
-INTERNAL_X_RELEASE_INCLUDE_DIRS= i386 x86_64 ppc32 sparc32 sparc64 tile
+INTERNAL_X_RELEASE_INCLUDE_DIRS= \
+        i386 x86_64 ppc32 sparc32 sparc64 tile gcc pthread win libatomic_ops
 
 INTERNAL_RELEASE_LIBS= \
 	../lib/internal/README \
 	$(ETHREAD_LIB) \
 	$(ERTS_INTERNAL_LIBS)
 
+.PHONY: release_spec
 release_spec: all
 ifneq ($(strip $(RELEASE_INCLUDES)),)
-	$(INSTALL_DIR) $(RELSYSDIR)/include
-	$(INSTALL_DIR) $(RELEASE_PATH)/usr/include
-	$(INSTALL_DATA) $(RELEASE_INCLUDES) $(RELSYSDIR)/include
-	$(INSTALL_DATA) $(RELEASE_INCLUDES) $(RELEASE_PATH)/usr/include
+	$(INSTALL_DIR) "$(RELSYSDIR)/include"
+	$(INSTALL_DIR) "$(RELEASE_PATH)/usr/include"
+	$(INSTALL_DATA) $(RELEASE_INCLUDES) "$(RELSYSDIR)/include"
+	$(INSTALL_DATA) $(RELEASE_INCLUDES) "$(RELEASE_PATH)/usr/include"
 endif
 ifneq ($(strip $(INTERNAL_RELEASE_INCLUDES)),)
-	$(INSTALL_DIR) $(RELSYSDIR)/include/internal
-	$(INSTALL_DATA) $(INTERNAL_RELEASE_INCLUDES) $(RELSYSDIR)/include/internal
+	$(INSTALL_DIR) "$(RELSYSDIR)/include/internal"
+	$(INSTALL_DATA) $(INTERNAL_RELEASE_INCLUDES) "$(RELSYSDIR)/include/internal"
 endif
 ifneq ($(strip $(INTERNAL_X_RELEASE_INCLUDE_DIRS)),)
 	for xdir in $(INTERNAL_X_RELEASE_INCLUDE_DIRS); do \
-		$(INSTALL_DIR) $(RELSYSDIR)/include/internal/$$xdir; \
+		$(INSTALL_DIR) "$(RELSYSDIR)/include/internal/$$xdir"; \
 		$(INSTALL_DATA) $(ERTS_INCL_INT)/$$xdir/*.h \
-			$(RELSYSDIR)/include/internal/$$xdir; \
+			"$(RELSYSDIR)/include/internal/$$xdir"; \
 	done
 endif
 ifneq ($(strip $(RELEASE_LIBS)),)
-	$(INSTALL_DIR) $(RELSYSDIR)/lib
-	$(INSTALL_DIR) $(RELEASE_PATH)/usr/lib
-	$(INSTALL_DATA) $(RELEASE_LIBS) $(RELSYSDIR)/lib
-	$(INSTALL_DATA) $(RELEASE_LIBS) $(RELEASE_PATH)/usr/lib
+	$(INSTALL_DIR) "$(RELSYSDIR)/lib"
+	$(INSTALL_DIR) "$(RELEASE_PATH)/usr/lib"
+	$(INSTALL_DATA) $(RELEASE_LIBS) "$(RELSYSDIR)/lib"
+	$(INSTALL_DATA) $(RELEASE_LIBS) "$(RELEASE_PATH)/usr/lib"
 endif
 ifneq ($(strip $(INTERNAL_RELEASE_LIBS)),)
-	$(INSTALL_DIR) $(RELSYSDIR)/lib/internal
-	$(INSTALL_DATA) $(INTERNAL_RELEASE_LIBS) $(RELSYSDIR)/lib/internal
+	$(INSTALL_DIR) "$(RELSYSDIR)/lib/internal"
+	$(INSTALL_DATA) $(INTERNAL_RELEASE_LIBS) "$(RELSYSDIR)/lib/internal"
 endif
 
+.PHONY: docs
 docs: 
 
+.PHONY: release_docs_spec
 release_docs_spec:
 
-
 #
 # Cleanup
 #
+.PHONY: clean
 clean:
 	$(RM) -rf ../lib/internal/$(TARGET)/*
 	$(RM) -rf ../lib/$(TARGET)/*
 	$(RM) -rf obj/$(TARGET)/*
-	$(RM) -f $(TARGET)/depend.mk
 
 #
 # Make dependencies
@@ -537,9 +555,11 @@ SED_MDd_DEPEND=sed '$(SED_PREFIX)$(SED_REPL_MDd_O);$(SED_REPL_TT_DIR);$(SED_REPL
 SED_MT_DEPEND=sed '$(SED_PREFIX)$(SED_REPL_MT_O);$(SED_REPL_TT_DIR);$(SED_REPL_TARGET)'
 SED_MTd_DEPEND=sed '$(SED_PREFIX)$(SED_REPL_MTd_O);$(SED_REPL_TT_DIR);$(SED_REPL_TARGET)'
 
-DEPEND_MK=$(TARGET)/depend.mk
+DEPEND_MK=$(OBJ_DIR)/depend.mk
 
-depend:
+.PHONY: depend
+depend: $(DEPEND_MK)
+$(DEPEND_MK):
 	@echo "Generating dependency file $(DEPEND_MK)..."
 	@echo "# Generated dependency rules" > $(DEPEND_MK);
 	@echo "# " >> $(DEPEND_MK);
@@ -612,6 +632,8 @@ endif
 endif
 	@echo "# EOF" >> $(DEPEND_MK);
 
+ifneq ($(MAKECMDGOALS),clean)
 -include $(DEPEND_MK)
+endif
 
 # eof
diff --git a/erts/lib_src/common/erl_memory_trace_parser.c b/erts/lib_src/common/erl_memory_trace_parser.c
index 54c3dfadec..625c140cf9 100644
--- a/erts/lib_src/common/erl_memory_trace_parser.c
+++ b/erts/lib_src/common/erl_memory_trace_parser.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -368,12 +368,16 @@ emtp_state_destroy(emtp_state *statep)
 
 
 #define GET_UI8(UI, BP) ((UI) = *((BP)++))
+#define SKIP_UI8(BP) ((BP)++)
+
 #define GET_UI16(UI, BP)						\
  do {									\
      (UI) = ((( (usgnd_int_16) (BP)[0]) << 8)				\
 	     | ((usgnd_int_16) (BP)[1]));				\
      (BP) += UI16_SZ;							\
 } while(0)
+#define SKIP_UI16(BP) ((BP) += UI16_SZ)
+
 
 #define GET_UI32(UI, BP)						\
  do {									\
@@ -383,6 +387,7 @@ emtp_state_destroy(emtp_state *statep)
 	     | ( (usgnd_int_32) (BP)[3]));				\
      (BP) += UI32_SZ;							\
 } while(0)
+#define SKIP_UI32(BP) ((BP) += UI32_SZ)
 
 #define GET_UI64(UI, BP)						\
  do {									\
@@ -396,6 +401,7 @@ emtp_state_destroy(emtp_state *statep)
 	     | ( (usgnd_int_64) (BP)[7]));				\
      (BP) += UI64_SZ;							\
 } while(0)
+#define SKIP_UI64(BP) ((BP) += UI64_SZ)
 
 #define GET_VSZ_UI16(UI, BP, MSB)					\
 do {									\
@@ -1267,11 +1273,10 @@ parse_header(emtp_state *statep,
 
 	switch (statep->version.major) {
 	case 1: {
-	    usgnd_int_32 hdr_sz;
 	    NEED(2*UI32_SZ + 2*UI16_SZ, trace_size);
 
 	    GET_UI32(statep->flags, tracep);
-	    GET_UI32(hdr_sz, tracep); /* ignore this; may contain garbage! */
+	    SKIP_UI32(tracep); /* ignore this; may contain garbage! */
 	    GET_UI16(statep->max_allocator_ix, tracep);
 	    GET_UI16(statep->max_block_type_ix, tracep);
 
@@ -1564,13 +1569,13 @@ parse_header(emtp_state *statep,
 		}
 
 		case ERTS_MT_BLOCK_TYPE_HDR_TAG: {
-		    usgnd_int_16 bt_ix, a_ix, btflgs;
+		    usgnd_int_16 bt_ix, a_ix;
 
 		    if (entry_sz
 			< UI8_SZ + 3*UI16_SZ + UI8_SZ + 0 + UI16_SZ)
 			ERROR(EMTP_PARSE_ERROR);
 
-		    GET_UI16(btflgs, c_p);
+		    SKIP_UI16(c_p); /* bitflags */
 		    GET_UI16(bt_ix, c_p);
 		    if (bt_ix > statep->max_block_type_ix)
 			ERROR(EMTP_PARSE_ERROR);
diff --git a/erts/lib_src/common/erl_misc_utils.c b/erts/lib_src/common/erl_misc_utils.c
index 9c25d33a3c..3b123063fa 100644
--- a/erts/lib_src/common/erl_misc_utils.c
+++ b/erts/lib_src/common/erl_misc_utils.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2006-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2006-2012. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -21,10 +21,13 @@
 #include "config.h"
 #endif
 
+#if defined(__WIN32__)
+#  include <windows.h>
+#endif
+
 #include "erl_misc_utils.h"
 
 #if defined(__WIN32__)
-#  include <windows.h>
 #elif defined(VXWORKS)
 #  include <selectLib.h>
 #else /* UNIX */
@@ -52,6 +55,12 @@
 #  ifdef HAVE_UNISTD_H
 #    include <unistd.h>
 #  endif
+#  if defined(_SC_NPROC_CONF) && !defined(_SC_NPROCESSORS_CONF)
+#    define _SC_NPROCESSORS_CONF _SC_NPROC_CONF
+#  endif
+#  if defined(_SC_NPROC_ONLN) && !defined(_SC_NPROCESSORS_ONLN)
+#    define _SC_NPROCESSORS_ONLN _SC_NPROC_ONLN
+#  endif
 #  if (defined(NO_SYSCONF) || !defined(_SC_NPROCESSORS_CONF))
 #    ifdef HAVE_SYS_SYSCTL_H
 #      include <sys/sysctl.h>
@@ -59,8 +68,38 @@
 #  endif
 #endif
 
-#ifdef HAVE_SCHED_xETAFFINITY
+#if defined(HAVE_SCHED_xETAFFINITY)
 #  include <sched.h>
+#  define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__
+#define ERTS_MU_GET_PROC_AFFINITY__(CPUINFOP, CPUSET)			\
+     (sched_getaffinity((CPUINFOP)->pid,				\
+			sizeof(cpu_set_t),				\
+			(CPUSET)) != 0 ? -errno : 0)
+#define ERTS_MU_SET_THR_AFFINITY__(SETP)				\
+     (sched_setaffinity(0, sizeof(cpu_set_t), (SETP)) != 0 ? -errno : 0)
+#elif defined(HAVE_CPUSET_xETAFFINITY)
+#  include <sys/param.h>
+#  include <sys/cpuset.h>
+#  define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__
+#define ERTS_MU_GET_PROC_AFFINITY__(CPUINFOP, CPUSET)			\
+     (cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, 	    \
+			sizeof(cpuset_t),				\
+			(CPUSET)) != 0 ? -errno : 0)
+#define ERTS_MU_SET_THR_AFFINITY__(CPUSETP)				\
+     (cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1,    \
+            sizeof(cpuset_t),               \
+            (CPUSETP)) != 0 ? -errno : 0)
+#  define cpu_set_t cpuset_t
+#elif defined(__WIN32__)
+#  define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__
+#  define cpu_set_t DWORD
+#  define CPU_SETSIZE (sizeof(DWORD)*8)
+#  define CPU_ZERO(SETP) (*(SETP) = (DWORD) 0)
+#  define CPU_SET(CPU, SETP) (*(SETP) |= (((DWORD) 1) << (CPU)))
+#  define CPU_CLR(CPU, SETP) (*(SETP) &= ~(((DWORD) 1) << (CPU)))
+#  define CPU_ISSET(CPU, SETP) ((*(SETP) & (((DWORD) 1) << (CPU))) != (DWORD) 0)
+#define ERTS_MU_GET_PROC_AFFINITY__ get_proc_affinity
+#define ERTS_MU_SET_THR_AFFINITY__ set_thr_affinity
 #endif
 #ifdef HAVE_PSET_INFO
 #  include <sys/pset.h>
@@ -80,8 +119,33 @@
 #  define ERTS_SYS_CPU_PATH	"/sys/devices/system/cpu"
 #endif
 
+#ifdef __FreeBSD__
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#endif
+
 static int read_topology(erts_cpu_info_t *cpuinfo);
 
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
+static int
+cpu_sets_are_eq(cpu_set_t *x, cpu_set_t *y)
+{
+    int i;
+    for (i = 0; i < CPU_SETSIZE; i++) {
+	if (CPU_ISSET(i, x)) {
+	    if (!CPU_ISSET(i, y))
+		return 0;
+	}
+	else {
+	    if (CPU_ISSET(i, y))
+		return 0;
+	}
+    }
+    return 1;
+}
+
+#endif
+
 int
 erts_milli_sleep(long ms)
 {
@@ -105,30 +169,66 @@ struct erts_cpu_info_t_ {
     int available;
     int topology_size;
     erts_cpu_topology_t *topology;
-#if defined(HAVE_SCHED_xETAFFINITY)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     char *affinity_str;
     char affinity_str_buf[CPU_SETSIZE/4+2];
     cpu_set_t cpuset;
+#if defined(HAVE_SCHED_xETAFFINITY)
     pid_t pid;
+#endif
 #elif defined(HAVE_PSET_INFO)
     processorid_t *cpuids;
 #endif
 };
 
+#if defined(__WIN32__)
+
+static __forceinline int
+get_proc_affinity(erts_cpu_info_t *cpuinfo, cpu_set_t *cpuset)
+{
+    DWORD pamask, samask;
+    if (GetProcessAffinityMask(GetCurrentProcess(), &pamask, &samask)) {
+	*cpuset = (cpu_set_t) pamask;
+	return 0;
+    }
+    else {
+	*cpuset = (cpu_set_t) 0;
+	return -erts_get_last_win_errno();
+    }
+}
+
+static __forceinline int
+set_thr_affinity(cpu_set_t *set)
+{
+    if (*set == (cpu_set_t) 0)
+	return -ENOTSUP;
+    if (SetThreadAffinityMask(GetCurrentThread(), *set) == 0)
+	return -erts_get_last_win_errno();
+    else
+	return 0;
+}
+
+#endif
+
 erts_cpu_info_t *
 erts_cpu_info_create(void)
 {
     erts_cpu_info_t *cpuinfo = malloc(sizeof(erts_cpu_info_t));
     if (!cpuinfo)
 	return NULL;
-#if defined(HAVE_SCHED_xETAFFINITY)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     cpuinfo->affinity_str = NULL;
+#if defined(HAVE_SCHED_xETAFFINITY)
     cpuinfo->pid = getpid();
+#endif
 #elif defined(HAVE_PSET_INFO)
     cpuinfo->cpuids = NULL;
 #endif
     cpuinfo->topology_size = 0;
     cpuinfo->topology = NULL;
+    cpuinfo->configured = -1;
+    cpuinfo->online = -1;
+    cpuinfo->available = -1;
     erts_cpu_info_update(cpuinfo);
     return cpuinfo;
 }
@@ -153,31 +253,40 @@ erts_cpu_info_destroy(erts_cpu_info_t *cpuinfo)
     }
 }
 
-void
+int
 erts_cpu_info_update(erts_cpu_info_t *cpuinfo)
 {
-    cpuinfo->configured = 0;
-    cpuinfo->online = 0;
-    cpuinfo->available = 0;
+    int changed = 0;
+    int configured = 0;
+    int online = 0;
+    int available = 0;
+    erts_cpu_topology_t *old_topology;
+    int old_topology_size;
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
+    cpu_set_t cpuset;
+#endif
 
 #ifdef __WIN32__
     {
+	int i;
 	SYSTEM_INFO sys_info;
 	GetSystemInfo(&sys_info);
-	cpuinfo->configured = (int) sys_info.dwNumberOfProcessors;
-	
+	configured = (int) sys_info.dwNumberOfProcessors;
+	for (i = 0; i < sizeof(DWORD)*8; i++)
+	    if (sys_info.dwActiveProcessorMask & (((DWORD) 1) << i))
+		online++;
     }
 #elif !defined(NO_SYSCONF) && (defined(_SC_NPROCESSORS_CONF) \
 			       || defined(_SC_NPROCESSORS_ONLN))
 #ifdef _SC_NPROCESSORS_CONF
-    cpuinfo->configured = (int) sysconf(_SC_NPROCESSORS_CONF);
-    if (cpuinfo->configured < 0)
-	cpuinfo->configured = 0;
+    configured = (int) sysconf(_SC_NPROCESSORS_CONF);
+    if (configured < 0)
+	configured = 0;
 #endif
 #ifdef _SC_NPROCESSORS_ONLN
-    cpuinfo->online = (int) sysconf(_SC_NPROCESSORS_ONLN);
-    if (cpuinfo->online < 0)
-	cpuinfo->online = 0;
+    online = (int) sysconf(_SC_NPROCESSORS_ONLN);
+    if (online < 0)
+	online = 0;
 #endif
 #elif defined(HAVE_SYS_SYSCTL_H) && defined(CTL_HW) && (defined(HW_NCPU) \
 							|| defined(HW_AVAILCPU))
@@ -189,71 +298,138 @@ erts_cpu_info_update(erts_cpu_info_t *cpuinfo)
 	len = sizeof(int);
 	mib[0] = CTL_HW;
 	mib[1] = HW_NCPU;
-	if (sysctl(&mib[0], 2, &cpuinfo->configured, &len, NULL, 0) < 0)
-	    cpuinfo->configured = 0;
+	if (sysctl(&mib[0], 2, &configured, &len, NULL, 0) < 0)
+	    configured = 0;
 #endif
 #ifdef HW_AVAILCPU
 	len = sizeof(int);
 	mib[0] = CTL_HW;
 	mib[1] = HW_AVAILCPU;
-	if (sysctl(&mib[0], 2, &cpuinfo->online, &len, NULL, 0) < 0)
-	    cpuinfo->online = 0;
+	if (sysctl(&mib[0], 2, &online, &len, NULL, 0) < 0)
+	    online = 0;
 #endif
     }
 #endif
 
-    if (cpuinfo->online > cpuinfo->configured)
-	cpuinfo->online = cpuinfo->configured;
+    if (online > configured)
+	online = configured;
+
+    if (cpuinfo->configured != configured)
+	changed = 1;
+    if (cpuinfo->online != online)
+	changed = 1;
+
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
+    if (ERTS_MU_GET_PROC_AFFINITY__(cpuinfo, &cpuset) == 0) {
+	if (!changed && !cpu_sets_are_eq(&cpuset, &cpuinfo->cpuset))
+	    changed = 1;
 
-#ifdef HAVE_SCHED_xETAFFINITY
-    if (sched_getaffinity(cpuinfo->pid, sizeof(cpu_set_t), &cpuinfo->cpuset) == 0) {
-	int i, c, cn, si;
-	c = cn = 0;
-	si = sizeof(cpuinfo->affinity_str_buf) - 1;
-	cpuinfo->affinity_str_buf[si] = '\0';
-	for (i = 0; i < CPU_SETSIZE; i++) {
-	    if (CPU_ISSET(i, &cpuinfo->cpuset)) {
-		c |= 1 << cn;
-		cpuinfo->available++;
+	if (!changed)
+	    available = cpuinfo->available;
+	else {
+	    int i, c, cn, si;
+
+	    memcpy((void *) &cpuinfo->cpuset,
+		   (void *) &cpuset,
+		   sizeof(cpu_set_t));
+
+	    c = cn = 0;
+	    si = sizeof(cpuinfo->affinity_str_buf) - 1;
+	    cpuinfo->affinity_str_buf[si] = '\0';
+	    for (i = 0; i < CPU_SETSIZE; i++) {
+		if (CPU_ISSET(i, &cpuinfo->cpuset)) {
+		    c |= 1 << cn;
+		    available++;
+		}
+		cn++;
+		if (cn == 4) {
+		    cpuinfo->affinity_str_buf[--si] = (c < 10
+						       ? '0' + c
+						       : 'A' + c - 10);
+		    c = cn = 0;
+		}
 	    }
-	    cn++;
-	    if (cn == 4) {
+	    if (c)
 		cpuinfo->affinity_str_buf[--si] = (c < 10
 						   ? '0' + c
 						   : 'A' + c - 10);
-		c = cn = 0;
-	    }
+	    while (cpuinfo->affinity_str_buf[si] == '0')
+		si++;
+	    cpuinfo->affinity_str = &cpuinfo->affinity_str_buf[si];
 	}
-	if (c)
-	    cpuinfo->affinity_str_buf[--si] = (c < 10
-					       ? '0' + c
-					       : 'A' + c - 10);
-	while (cpuinfo->affinity_str_buf[si] == '0')
-	    si++;
-	cpuinfo->affinity_str = &cpuinfo->affinity_str_buf[si];
     }
 #elif defined(HAVE_PSET_INFO)
     {
-	uint_t numcpus = cpuinfo->configured;
-	if (cpuinfo->cpuids)
-	    free(cpuinfo->cpuids);
-	cpuinfo->cpuids = malloc(sizeof(processorid_t)*numcpus);
-	if (cpuinfo->cpuids) {
-	    if (pset_info(PS_MYID, NULL, &numcpus, &cpuinfo->cpuids) == 0)
-		cpuinfo->available = (int) numcpus;
-	    if (cpuinfo->available < 0) {
-		free(cpuinfo->cpuid);
-		cpuinfo->available = 0;
+	processorid_t *cpuids;
+	uint_t numcpus = configured;
+	cpuids = malloc(sizeof(processorid_t)*numcpus);
+	if (cpuids) {
+	    if (pset_info(PS_MYID, NULL, &numcpus, &cpuids) == 0)
+		available = (int) numcpus;
+	    if (available < 0) {
+		free(cpuids);
+		cpuids = NULL;
+		available = 0;
 	    }
 	}
+	if (!cpuids) {
+	    if (cpuinfo->cpuids)
+		changed = 1;
+	}
+	else {
+	    if (cpuinfo->cpuids)
+		changed = 1;
+	    if (memcmp((void *) cpuinfo->cpuids,
+		       (void *) cpuids,
+		       sizeof(processorid_t)*numcpus) != 0)
+		changed = 1;
+
+	}
+	if (!changed) {
+	    if (cpuids)
+		free(cpuids);
+	}
+	else {
+	    if (cpuinfo->cpuids)
+		free(cpuinfo->cpuids);
+	    cpuinfo->cpuids = cpuids;
+	}
     }
 #endif
 
-    if (cpuinfo->available > cpuinfo->online)
-	cpuinfo->available = cpuinfo->online;
+    if (available > online)
+	available = online;
+
+    if (cpuinfo->available != available)
+	changed = 1;
+
+    cpuinfo->configured = configured;
+    cpuinfo->online = online;
+    cpuinfo->available = available;
+
+    old_topology = cpuinfo->topology;
+    old_topology_size = cpuinfo->topology_size;
+    cpuinfo->topology = NULL;
 
     read_topology(cpuinfo);
 
+    if (cpuinfo->topology_size != old_topology_size
+	|| (old_topology_size != 0
+	    && memcmp((void *) cpuinfo->topology,
+		      (void *) old_topology,
+		      (sizeof(erts_cpu_topology_t)
+		       * old_topology_size)) != 0)) {
+	changed = 1;
+	if (old_topology)
+	    free(old_topology);
+    }
+    else {
+	if (cpuinfo->topology)
+	    free(cpuinfo->topology);
+	cpuinfo->topology = old_topology;
+    }
+
+    return changed;
 }
 
 int
@@ -289,7 +465,7 @@ erts_get_cpu_available(erts_cpu_info_t *cpuinfo)
 char *
 erts_get_unbind_from_cpu_str(erts_cpu_info_t *cpuinfo)
 {
-#if defined(HAVE_SCHED_xETAFFINITY)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     if (!cpuinfo)
 	return "false";
     return cpuinfo->affinity_str;
@@ -303,7 +479,7 @@ erts_get_available_cpu(erts_cpu_info_t *cpuinfo, int no)
 {
     if (!cpuinfo || no < 1 || cpuinfo->available < no)
 	return -EINVAL;
-#ifdef HAVE_SCHED_xETAFFINITY
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     {
 	cpu_set_t *allowed = &cpuinfo->cpuset;
 	int ix, n;
@@ -335,8 +511,8 @@ int
 erts_is_cpu_available(erts_cpu_info_t *cpuinfo, int id)
 {
     if (cpuinfo && 0 <= id) {
-#ifdef HAVE_SCHED_xETAFFINITY
-	if (id <= CPU_SETSIZE)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
+	if (id < CPU_SETSIZE)
 	    return CPU_ISSET(id, &cpuinfo->cpuset);
 #elif defined(HAVE_PROCESSOR_BIND)
 	int no;
@@ -373,8 +549,8 @@ erts_get_cpu_topology(erts_cpu_info_t *cpuinfo,
 	return 0;
     memcpy((void *) topology,
 	   (void *) cpuinfo->topology,
-	   cpuinfo->configured*sizeof(erts_cpu_topology_t));
-    return cpuinfo->configured;
+	   cpuinfo->topology_size*sizeof(erts_cpu_topology_t));
+    return cpuinfo->topology_size;
 }
 
 int
@@ -388,7 +564,7 @@ erts_bind_to_cpu(erts_cpu_info_t *cpuinfo, int cpu)
      */
     if (!cpuinfo)
 	return -EINVAL;
-#ifdef HAVE_SCHED_xETAFFINITY
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     {
 	cpu_set_t bind_set;
 	if (cpu < 0)
@@ -398,9 +574,7 @@ erts_bind_to_cpu(erts_cpu_info_t *cpuinfo, int cpu)
 
 	CPU_ZERO(&bind_set);
 	CPU_SET(cpu, &bind_set);
-	if (sched_setaffinity(0, sizeof(cpu_set_t), &bind_set) != 0)
-	    return -errno;
-	return 0;
+	return ERTS_MU_SET_THR_AFFINITY__(&bind_set);
     }
 #elif defined(HAVE_PROCESSOR_BIND)
     if (cpu < 0)
@@ -418,10 +592,8 @@ erts_unbind_from_cpu(erts_cpu_info_t *cpuinfo)
 {
     if (!cpuinfo)
 	return -EINVAL;
-#if defined(HAVE_SCHED_xETAFFINITY)
-    if (sched_setaffinity(0, sizeof(cpu_set_t), &cpuinfo->cpuset) != 0)
-	return -errno;
-    return 0;
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
+    return ERTS_MU_SET_THR_AFFINITY__(&cpuinfo->cpuset);
 #elif defined(HAVE_PROCESSOR_BIND)
     if (processor_bind(P_LWPID, P_MYID, PBIND_NONE, NULL) != 0)
 	return -errno;
@@ -434,7 +606,7 @@ erts_unbind_from_cpu(erts_cpu_info_t *cpuinfo)
 int
 erts_unbind_from_cpu_str(char *str)
 {
-#if defined(HAVE_SCHED_xETAFFINITY)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     char *c = str;
     int cpus = 0;
     int shft = 0;
@@ -486,9 +658,7 @@ erts_unbind_from_cpu_str(char *str)
     if (!cpus)
 	return -EINVAL;
 
-    if (sched_setaffinity(0, sizeof(cpu_set_t), &cpuset) != 0)
-	return -errno;
-    return 0;
+    return ERTS_MU_SET_THR_AFFINITY__(&cpuset);
 #elif defined(HAVE_PROCESSOR_BIND)
     if (processor_bind(P_LWPID, P_MYID, PBIND_NONE, NULL) != 0)
 	return -errno;
@@ -541,6 +711,56 @@ cpu_cmp(const void *vx, const void *vy)
     return 0;
 }
 
+static void
+adjust_processor_nodes(erts_cpu_info_t *cpuinfo, int no_nodes)
+{
+    erts_cpu_topology_t *prev, *this, *last;
+    if (no_nodes > 1) {
+	int processor = -1;
+	int processor_node = 0;
+	int node = -1;
+
+	qsort(cpuinfo->topology,
+	      cpuinfo->topology_size,
+	      sizeof(erts_cpu_topology_t),
+	      pn_cmp);
+
+	prev = NULL;
+	this = &cpuinfo->topology[0];
+	last = &cpuinfo->topology[cpuinfo->topology_size-1];
+	while (1) {
+	    if (processor == this->processor) {
+		if (node != this->node)
+		    processor_node = 1;
+	    }
+	    else {
+		if (processor_node) {
+		make_processor_node:
+		    while (prev->processor == processor) {
+			prev->processor_node = prev->node;
+			prev->node = -1;
+			if (prev == &cpuinfo->topology[0])
+			    break;
+			prev--;
+		    }
+		    processor_node = 0;
+		}
+		processor = this->processor;
+		node = this->node;
+	    }
+	    if (this == last) {
+		if (processor_node) {
+		    prev = this;
+		    goto make_processor_node;
+		}
+		break;
+	    }
+	    prev = this++;
+	}
+    }
+}
+
+
 #ifdef __linux__
 
 static int
@@ -594,15 +814,12 @@ read_topology(erts_cpu_info_t *cpuinfo)
 
     errno = 0;
 
-    if (cpuinfo->topology)
-	free(cpuinfo->topology);
-
     if (cpuinfo->configured < 1)
 	goto error;
 
     cpuinfo->topology = malloc(sizeof(erts_cpu_topology_t)
 			       * cpuinfo->configured);
-    if (!cpuinfo)
+    if (!cpuinfo->topology)
 	goto error;
 
     for (ix = 0; ix < cpuinfo->configured; ix++) {
@@ -617,8 +834,8 @@ read_topology(erts_cpu_info_t *cpuinfo)
     ix = -1;
 
     if (realpath(ERTS_SYS_NODE_PATH, npath)) {
-	got_nodes = 1;
 	ndir = opendir(npath);
+	got_nodes = (ndir != NULL);
     }
 
     do {
@@ -710,49 +927,7 @@ read_topology(erts_cpu_info_t *cpuinfo)
 		cpuinfo->topology = t;
 	}
 
-	if (no_nodes > 1) {
-	    int processor = -1;
-	    int processor_node = 0;
-	    int node = -1;
-
-	    qsort(cpuinfo->topology,
-		  cpuinfo->topology_size,
-		  sizeof(erts_cpu_topology_t),
-		  pn_cmp);
-
-	    prev = NULL;
-	    this = &cpuinfo->topology[0];
-	    last = &cpuinfo->topology[cpuinfo->configured-1];
-	    while (1) {
-		if (processor == this->processor) {
-		    if (node != this->node)
-			processor_node = 1;
-		}
-		else {
-		    if (processor_node) {
-		    make_processor_node:
-			while (prev->processor == processor) {
-			    prev->processor_node = prev->node;
-			    prev->node = -1;
-			    if (prev == &cpuinfo->topology[0])
-				break;
-			    prev--;
-			}
-			processor_node = 0;
-		    }
-		    processor = this->processor;
-		    node = this->node;
-		}
-		if (this == last) {
-		    if (processor_node) {
-			prev = this;
-			goto make_processor_node;
-		    }
-		    break;
-		}
-		prev = this++;
-	    }
-	}
+	adjust_processor_nodes(cpuinfo, no_nodes);
 
 	qsort(cpuinfo->topology,
 	      cpuinfo->topology_size,
@@ -764,7 +939,7 @@ read_topology(erts_cpu_info_t *cpuinfo)
 
 	if (res > 1) {
 	    prev = this++;
-	    last = &cpuinfo->topology[cpuinfo->configured-1];
+	    last = &cpuinfo->topology[cpuinfo->topology_size-1];
 
 	    while (1) {
 		this->thread = ((this->node == prev->node
@@ -849,15 +1024,12 @@ read_topology(erts_cpu_info_t *cpuinfo)
 
     errno = 0;
 
-    if (cpuinfo->topology)
-	free(cpuinfo->topology);
-
     if (cpuinfo->configured < 1)
 	goto error;
 
     cpuinfo->topology = malloc(sizeof(erts_cpu_topology_t)
 			       * cpuinfo->configured);
-    if (!cpuinfo)
+    if (!cpuinfo->topology)
 	goto error;
 
     for (ix = 0; ix < cpuinfo->configured; ix++) {
@@ -922,7 +1094,7 @@ read_topology(erts_cpu_info_t *cpuinfo)
 
 	if (res > 1) {
 	    prev = this++;
-	    last = &cpuinfo->topology[cpuinfo->configured-1];
+	    last = &cpuinfo->topology[cpuinfo->topology_size-1];
 
 	    while (1) {
 		this->thread = ((this->node == prev->node
@@ -938,6 +1110,8 @@ read_topology(erts_cpu_info_t *cpuinfo)
 	}
     }
 
+    adjust_processor_nodes(cpuinfo, 1);
+
  error:
 
     if (res == 0) {
@@ -956,6 +1130,519 @@ read_topology(erts_cpu_info_t *cpuinfo)
 
 }
 
+#elif defined(__WIN32__)
+
+/*
+ * We cannot use Relation* out of the box since all of them are not
+ * always part of the LOGICAL_PROCESSOR_RELATIONSHIP enum. They are
+ * however documented as follows...
+ */
+#define ERTS_MU_RELATION_PROCESSOR_CORE       0 /* RelationProcessorCore */
+#define ERTS_MU_RELATION_NUMA_NODE            1 /* RelationNumaNode */
+#define ERTS_MU_RELATION_CACHE                2 /* RelationCache */
+#define ERTS_MU_RELATION_PROCESSOR_PACKAGE    3 /* RelationProcessorPackage */
+
+static __forceinline int
+rel_cmp_val(int r)
+{
+    switch (r) {
+    case ERTS_MU_RELATION_NUMA_NODE:         return 0;
+    case ERTS_MU_RELATION_PROCESSOR_PACKAGE: return 1;
+    case ERTS_MU_RELATION_PROCESSOR_CORE:    return 2;
+    default: /* currently not used */        return 3;
+    }
+}
+
+static int
+slpi_cmp(const void *vx, const void *vy)
+{
+    PSYSTEM_LOGICAL_PROCESSOR_INFORMATION x, y;
+    x = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION) vx;
+    y = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION) vy;
+
+    if ((int) x->Relationship != (int) y->Relationship)
+	return (rel_cmp_val((int) x->Relationship)
+		- rel_cmp_val((int) y->Relationship));
+
+    switch ((int) x->Relationship) {
+    case ERTS_MU_RELATION_NUMA_NODE:
+	if (x->NumaNode.NodeNumber == y->NumaNode.NodeNumber)
+	    break;
+	return ((int) x->NumaNode.NodeNumber) - ((int) y->NumaNode.NodeNumber);
+    case ERTS_MU_RELATION_PROCESSOR_CORE:
+    case ERTS_MU_RELATION_PROCESSOR_PACKAGE:
+    default:
+	break;
+    }
+
+    if (x->ProcessorMask == y->ProcessorMask)
+	return 0;
+    return x->ProcessorMask < y->ProcessorMask ? -1 : 1;
+}
+
+typedef BOOL (WINAPI *glpi_t)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD);
+
+static int
+read_topology(erts_cpu_info_t *cpuinfo)
+{
+    int res = 0;
+    glpi_t glpi;
+    int *core_id = NULL;
+    PSYSTEM_LOGICAL_PROCESSOR_INFORMATION slpip = NULL;
+    int wix, rix, max_l, l, packages, nodes, no_slpi;
+    DWORD slpi_size = 0;
+
+
+    glpi = (glpi_t) GetProcAddress(GetModuleHandle("kernel32"),
+				   "GetLogicalProcessorInformation");
+    if (!glpi)
+	return -ENOTSUP;
+
+    cpuinfo->topology = NULL;
+
+    if (cpuinfo->configured < 1 || sizeof(ULONG_PTR)*8 < cpuinfo->configured)
+	goto error;
+
+    while (1) {
+	DWORD werr;
+	if (TRUE == glpi(slpip, &slpi_size))
+	    break;
+	werr = GetLastError();
+	if (werr != ERROR_INSUFFICIENT_BUFFER) {
+	    res = -erts_map_win_error_to_errno(werr);
+	    goto error;
+	}
+	if (slpip)
+	    free(slpip);
+	slpip = malloc(slpi_size);
+	if (!slpip) {
+	    res = -ENOMEM;
+	    goto error;
+	}
+    }
+
+    no_slpi = (int) slpi_size/sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
+
+    qsort(slpip,
+	  no_slpi,
+	  sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION),
+	  slpi_cmp);
+
+    /*
+     * Now numa node relations appear before package relations which
+     * appear before core relations which appear before relations
+     * we aren't interested in...
+     */
+
+    max_l = 0;
+    packages = 0;
+    nodes = 0;
+    for (rix = 0; rix < no_slpi; rix++) {
+	PSYSTEM_LOGICAL_PROCESSOR_INFORMATION this = &slpip[rix];
+	for (l = sizeof(ULONG_PTR)*8 - 1; l > 0; l--) {
+	    if (slpip[rix].ProcessorMask & (((ULONG_PTR) 1) << l)) {
+		if (max_l < l)
+		    max_l = l;
+		break;
+	    }
+	}
+	if ((int) slpip[rix].Relationship == ERTS_MU_RELATION_PROCESSOR_PACKAGE)
+	    packages++;
+	if ((int) slpip[rix].Relationship == ERTS_MU_RELATION_NUMA_NODE)
+	    nodes++;
+    }
+
+    if (!packages) {
+      packages = 1;
+    }
+    core_id = malloc(sizeof(int)*packages);
+    if (!core_id) {
+	res = -ENOMEM;
+	goto error;
+    }
+
+    for (rix = 0; rix < packages; rix++)
+	core_id[rix] = 0;
+
+    cpuinfo->topology_size = max_l + 1;
+    cpuinfo->topology = malloc(sizeof(erts_cpu_topology_t)
+			       * cpuinfo->topology_size);
+    if (!cpuinfo->topology) {
+	res = -ENOMEM;
+	goto error;
+    }
+
+    for (wix = 0; wix < cpuinfo->topology_size; wix++) {
+	cpuinfo->topology[wix].node = -1;
+	cpuinfo->topology[wix].processor = -1;
+	cpuinfo->topology[wix].processor_node = -1;
+	cpuinfo->topology[wix].core = -1;
+	cpuinfo->topology[wix].thread = -1;
+	cpuinfo->topology[wix].logical = -1;
+    }
+
+    nodes = 0;
+    packages = 0;
+
+    for (rix = 0; rix < no_slpi; rix++) {
+
+        switch ((int) slpip[rix].Relationship) {
+        case ERTS_MU_RELATION_NUMA_NODE:
+	    for (l = 0; l < sizeof(ULONG_PTR)*8; l++) {
+		if (slpip[rix].ProcessorMask & (((ULONG_PTR) 1) << l)) {
+		    cpuinfo->topology[l].logical = l;
+		    cpuinfo->topology[l].node = slpip[rix].NumaNode.NodeNumber;
+		}
+	    }
+	    nodes++;
+            break;
+        case ERTS_MU_RELATION_PROCESSOR_PACKAGE:
+	    for (l = 0; l < sizeof(ULONG_PTR)*8; l++) {
+		if (slpip[rix].ProcessorMask & (((ULONG_PTR) 1) << l)) {
+		    cpuinfo->topology[l].logical = l;
+		    cpuinfo->topology[l].processor = packages;
+		}
+	    }
+	    packages++;
+            break;
+        case ERTS_MU_RELATION_PROCESSOR_CORE: {
+	    int thread = 0;
+	    int processor = -1;
+	    for (l = 0; l < sizeof(ULONG_PTR)*8; l++) {
+		/*
+		 * Nodes and packages may not be supported; pretend
+		 * that there are one if this is the case...
+		 */
+		if (slpip[rix].ProcessorMask & (((ULONG_PTR) 1) << l)) {
+		    if (!nodes) {
+		      cpuinfo->topology[l].node = 0;
+		    }
+		    if (!packages) {
+		      cpuinfo->topology[l].processor = 0;
+		    }
+		    if (processor < 0) {
+			processor = cpuinfo->topology[l].processor;
+			if (processor < 0) {
+			    res = -EINVAL;
+			    goto error;
+			}
+		    }
+		    else if (processor != cpuinfo->topology[l].processor) {
+			res = -EINVAL;
+			goto error;
+		    }
+		    cpuinfo->topology[l].logical = l;
+		    cpuinfo->topology[l].thread = thread;
+		    cpuinfo->topology[l].core = core_id[processor];
+		    thread++;
+		}
+	    }
+	    core_id[processor]++;
+            break;
+	}
+        default:
+	    /*
+	     * We have reached the end of the relationships
+	     * that we (currently) are interested in...
+	     */
+	    goto relationships_done;
+        }
+    }
+
+ relationships_done:
+
+    /*
+     * There may be unused entries; remove them...
+     */
+    for (rix = wix = 0; rix < cpuinfo->topology_size; rix++) {
+	if (cpuinfo->topology[rix].logical >= 0) {
+	    if (wix != rix)
+		cpuinfo->topology[wix] = cpuinfo->topology[rix];
+	    wix++;
+	}
+    }
+
+    if (cpuinfo->topology_size != wix) {
+	erts_cpu_topology_t *new = cpuinfo->topology;
+	new = realloc(cpuinfo->topology,
+		      sizeof(erts_cpu_topology_t)*wix);
+	if (!new) {
+	    res = -ENOMEM;
+	    goto error;
+	}
+	cpuinfo->topology = new;
+	cpuinfo->topology_size = wix;
+    }
+
+    res = wix;
+
+    adjust_processor_nodes(cpuinfo, nodes);
+
+    qsort(cpuinfo->topology,
+	  cpuinfo->topology_size,
+	  sizeof(erts_cpu_topology_t),
+	  cpu_cmp);
+
+    if (res < cpuinfo->online)
+	res = -EINVAL;
+
+ error:
+
+    if (res <= 0) {
+	cpuinfo->topology_size = 0;
+	if (cpuinfo->topology) {
+	    free(cpuinfo->topology);
+	    cpuinfo->topology = NULL;
+	}
+    }
+
+    if (slpip)
+	free(slpip);
+    if (core_id)
+	free(core_id);
+
+    return res;
+}
+
+#elif defined(__FreeBSD__)
+
+/**
+ * FreeBSD topology detection is based on kern.sched.topology_spec XML as
+ * exposed by the ULE scheduler and described in SMP(4). It is available in
+ * 8.0 and higher.
+ *
+ * Threads are identified in this XML chunk with a THREAD flag. The function
+ * (simplistically) distinguishes cores and processors by the amount of cache
+ * they share (0 => processor, otherwise => core). Nodes are not identified
+ * (ULE doesn't handle NUMA yet, I believe).
+ */
+
+/**
+ * Recursively parse a topology_spec <group> tag.
+ */
+static
+const char* parse_topology_spec_group(erts_cpu_info_t *cpuinfo, const char* xml, int parentCacheLevel, int* processor_p, int* core_p, int* index_procs_p) {
+    int error = 0;
+    int cacheLevel = parentCacheLevel;
+    const char* next_group_start = strstr(xml + 1, "<group");
+    int is_thread_group = 0;
+    const char* next_cache_level;
+    const char* next_thread_flag;
+    const char* next_group_end;
+    const char* next_children;
+    const char* next_children_end;
+
+    /* parse the cache level */
+    next_cache_level = strstr(xml, "cache-level=\"");
+    if (next_cache_level && (next_group_start == NULL || next_cache_level < next_group_start)) {
+	sscanf(next_cache_level, "cache-level=\"%i\"", &cacheLevel);
+    }
+
+    /* parse the threads flag */
+    next_thread_flag = strstr(xml, "THREAD");
+    if (next_thread_flag && (next_group_start == NULL || next_thread_flag < next_group_start))
+	is_thread_group = 1;
+
+    /* Determine if it's a leaf with the position of the next children tag */
+    next_group_end = strstr(xml, "</group>");
+    next_children = strstr(xml, "<children>");
+    next_children_end = strstr(xml, "</children>");
+    if (next_children == NULL || next_group_end < next_children) {
+	do {
+	    const char* next_cpu_start;
+	    const char* next_cpu_cdata;
+	    const char* next_cpu_end;
+	    int cpu_str_size;
+	    char* cpu_str;
+	    char* cpu_crsr;
+	    char* brkb;
+	    int thread = 0;
+	    int index_procs = *index_procs_p;
+
+	    next_cpu_start = strstr(xml, "<cpu");
+	    if (!next_cpu_start) {
+		error = 1;
+		break;
+	    }
+	    next_cpu_cdata = strstr(next_cpu_start, ">") + 1;
+	    if (!next_cpu_cdata) {
+		error = 1;
+		break;
+	    }
+	    next_cpu_end = strstr(next_cpu_cdata, "</cpu>");
+	    if (!next_cpu_end) {
+		error = 1;
+		break;
+	    }
+	    cpu_str_size = next_cpu_end - next_cpu_cdata;
+	    cpu_str = (char*) malloc(cpu_str_size + 1);
+	    memcpy(cpu_str, (const char*) next_cpu_cdata, cpu_str_size);
+	    cpu_str[cpu_str_size] = 0;
+	    for (cpu_crsr = strtok_r(cpu_str, " \t,", &brkb); cpu_crsr; cpu_crsr = strtok_r(NULL, " \t,", &brkb)) {
+		int cpu_id;
+		if (index_procs >= cpuinfo->configured) {
+		    void* t = realloc(cpuinfo->topology, (sizeof(erts_cpu_topology_t) * (index_procs + 1)));
+		    if (t) {
+			cpuinfo->topology = t;
+		    } else {
+			error = 1;
+			break;
+		    }
+		}
+		cpu_id = atoi(cpu_crsr);
+		cpuinfo->topology[index_procs].node = -1;
+		cpuinfo->topology[index_procs].processor = *processor_p;
+		cpuinfo->topology[index_procs].processor_node = -1;
+		cpuinfo->topology[index_procs].core = *core_p;
+		cpuinfo->topology[index_procs].thread = thread;
+		cpuinfo->topology[index_procs].logical = cpu_id;
+		if (is_thread_group) {
+		    thread++;
+		} else {
+		    *core_p = (*core_p)++;
+		}
+		index_procs++;
+	    }
+	    *index_procs_p = index_procs;
+	    free(cpu_str);
+	} while (0);
+	xml = next_group_end;
+    } else {
+	while (next_group_start != NULL && next_group_start < next_children_end) {
+	    xml = parse_topology_spec_group(cpuinfo, next_group_start, cacheLevel, processor_p, core_p, index_procs_p);
+	    if (!xml)
+		break;
+	    next_group_start = strstr(xml, "<group");
+	    next_children_end = strstr(xml, "</children>");
+	}
+    }
+
+    if (parentCacheLevel == 0) {
+	*core_p = 0;
+	*processor_p = (*processor_p)++;
+    } else {
+	*core_p = (*core_p)++;
+    }
+
+    if (error)
+	xml = NULL;
+
+    return xml;
+}
+
+/**
+ * Parse the topology_spec. Return the number of CPUs or 0 if parsing failed.
+ */
+static
+int parse_topology_spec(erts_cpu_info_t *cpuinfo, const char* xml) {
+    int res = 1;
+    int index_procs = 0;
+    int core = 0;
+    int processor = 0;
+    xml = strstr(xml, "<groups");
+    if (!xml)
+	return -1;
+
+    xml += 7;
+    xml = strstr(xml, "<group");
+    while (xml) {
+	xml = parse_topology_spec_group(cpuinfo, xml, 0, &processor, &core, &index_procs);
+	if (!xml) {
+	    res = 0;
+	    break;
+	}
+	xml = strstr(xml, "<group");
+    }
+
+    if (res)
+	res = index_procs;
+
+    return res;
+}
+
+static int
+read_topology(erts_cpu_info_t *cpuinfo)
+{
+    int ix;
+    int res = 0;
+    size_t topology_spec_size = 0;
+    void* topology_spec = NULL;
+
+    errno = 0;
+
+    if (cpuinfo->configured < 1)
+	goto error;
+
+    cpuinfo->topology_size = cpuinfo->configured;
+    cpuinfo->topology = malloc(sizeof(erts_cpu_topology_t)
+			       * cpuinfo->configured);
+    if (!cpuinfo->topology) {
+	res = -ENOMEM;
+	goto error;
+    }
+
+    for (ix = 0; ix < cpuinfo->configured; ix++) {
+	cpuinfo->topology[ix].node = -1;
+	cpuinfo->topology[ix].processor = -1;
+	cpuinfo->topology[ix].processor_node = -1;
+	cpuinfo->topology[ix].core = -1;
+	cpuinfo->topology[ix].thread = -1;
+	cpuinfo->topology[ix].logical = -1;
+    }
+
+    if (!sysctlbyname("kern.sched.topology_spec", NULL, &topology_spec_size, NULL, 0)) {
+	topology_spec = malloc(topology_spec_size);
+	if (!topology_spec) {
+	    res = -ENOMEM;
+	    goto error;
+	}
+
+	if (sysctlbyname("kern.sched.topology_spec", topology_spec, &topology_spec_size, NULL, 0)) {
+	    goto error;
+	}
+
+	res = parse_topology_spec(cpuinfo, topology_spec);
+	if (!res || res < cpuinfo->online)
+	    res = 0;
+	else {
+	    cpuinfo->topology_size = res;
+
+	    if (cpuinfo->topology_size != cpuinfo->configured) {
+		void *t = realloc(cpuinfo->topology, (sizeof(erts_cpu_topology_t)
+						  * cpuinfo->topology_size));
+		if (t)
+		    cpuinfo->topology = t;
+	    }
+
+	    adjust_processor_nodes(cpuinfo, 1);
+
+	    qsort(cpuinfo->topology,
+	        cpuinfo->topology_size,
+	        sizeof(erts_cpu_topology_t),
+	        cpu_cmp);
+	}
+    }
+
+error:
+
+    if (res == 0) {
+	cpuinfo->topology_size = 0;
+	if (cpuinfo->topology) {
+	    free(cpuinfo->topology);
+	    cpuinfo->topology = NULL;
+	}
+	if (errno)
+	    res = -errno;
+	else
+	    res = -EINVAL;
+    }
+
+    if (topology_spec)
+	free(topology_spec);
+
+    return res;
+}
+
 #else
 
 static int
@@ -965,3 +1652,98 @@ read_topology(erts_cpu_info_t *cpuinfo)
 }
 
 #endif
+
+#if defined(__WIN32__)
+
+int
+erts_map_win_error_to_errno(DWORD win_error)
+{
+    switch (win_error) {
+    case ERROR_INVALID_FUNCTION:		return EINVAL;	/* 1	*/
+    case ERROR_FILE_NOT_FOUND:			return ENOENT;	/* 2	*/
+    case ERROR_PATH_NOT_FOUND:			return ENOENT;	/* 3	*/
+    case ERROR_TOO_MANY_OPEN_FILES:		return EMFILE;	/* 4	*/
+    case ERROR_ACCESS_DENIED:			return EACCES;	/* 5	*/
+    case ERROR_INVALID_HANDLE:			return EBADF;	/* 6	*/
+    case ERROR_ARENA_TRASHED:			return ENOMEM;	/* 7	*/
+    case ERROR_NOT_ENOUGH_MEMORY:		return ENOMEM;	/* 8	*/
+    case ERROR_INVALID_BLOCK:			return ENOMEM;	/* 9	*/
+    case ERROR_BAD_ENVIRONMENT:			return E2BIG;	/* 10	*/
+    case ERROR_BAD_FORMAT:			return ENOEXEC;	/* 11	*/
+    case ERROR_INVALID_ACCESS:			return EINVAL;	/* 12	*/
+    case ERROR_INVALID_DATA:			return EINVAL;	/* 13	*/
+    case ERROR_OUTOFMEMORY:			return ENOMEM;	/* 14	*/
+    case ERROR_INVALID_DRIVE:			return ENOENT;	/* 15	*/
+    case ERROR_CURRENT_DIRECTORY:		return EACCES;	/* 16	*/
+    case ERROR_NOT_SAME_DEVICE:			return EXDEV;	/* 17	*/
+    case ERROR_NO_MORE_FILES:			return ENOENT;	/* 18	*/
+    case ERROR_WRITE_PROTECT:			return EACCES;	/* 19	*/
+    case ERROR_BAD_UNIT:			return EACCES;	/* 20	*/
+    case ERROR_NOT_READY:			return EACCES;	/* 21	*/
+    case ERROR_BAD_COMMAND:			return EACCES;	/* 22	*/
+    case ERROR_CRC:				return EACCES;	/* 23	*/
+    case ERROR_BAD_LENGTH:			return EACCES;	/* 24	*/
+    case ERROR_SEEK:				return EACCES;	/* 25	*/
+    case ERROR_NOT_DOS_DISK:			return EACCES;	/* 26	*/
+    case ERROR_SECTOR_NOT_FOUND:		return EACCES;	/* 27	*/
+    case ERROR_OUT_OF_PAPER:			return EACCES;	/* 28	*/
+    case ERROR_WRITE_FAULT:			return EACCES;	/* 29	*/
+    case ERROR_READ_FAULT:			return EACCES;	/* 30	*/
+    case ERROR_GEN_FAILURE:			return EACCES;	/* 31	*/
+    case ERROR_SHARING_VIOLATION:		return EACCES;	/* 32	*/
+    case ERROR_LOCK_VIOLATION:			return EACCES;	/* 33	*/
+    case ERROR_WRONG_DISK:			return EACCES;	/* 34	*/
+    case ERROR_SHARING_BUFFER_EXCEEDED:		return EACCES;	/* 36	*/
+    case ERROR_BAD_NETPATH:			return ENOENT;	/* 53	*/
+    case ERROR_NETWORK_ACCESS_DENIED:		return EACCES;	/* 65	*/
+    case ERROR_BAD_NET_NAME:			return ENOENT;	/* 67	*/
+    case ERROR_FILE_EXISTS:			return EEXIST;	/* 80	*/
+    case ERROR_CANNOT_MAKE:			return EACCES;	/* 82	*/
+    case ERROR_FAIL_I24:			return EACCES;	/* 83	*/
+    case ERROR_INVALID_PARAMETER:		return EINVAL;	/* 87	*/
+    case ERROR_NO_PROC_SLOTS:			return EAGAIN;	/* 89	*/
+    case ERROR_DRIVE_LOCKED:			return EACCES;	/* 108	*/
+    case ERROR_BROKEN_PIPE:			return EPIPE;	/* 109	*/
+    case ERROR_DISK_FULL:			return ENOSPC;	/* 112	*/
+    case ERROR_INVALID_TARGET_HANDLE:		return EBADF;	/* 114	*/
+    case ERROR_WAIT_NO_CHILDREN:		return ECHILD;	/* 128	*/
+    case ERROR_CHILD_NOT_COMPLETE:		return ECHILD;	/* 129	*/
+    case ERROR_DIRECT_ACCESS_HANDLE:		return EBADF;	/* 130	*/
+    case ERROR_NEGATIVE_SEEK:			return EINVAL;	/* 131	*/
+    case ERROR_SEEK_ON_DEVICE:			return EACCES;	/* 132	*/
+    case ERROR_DIR_NOT_EMPTY:			return ENOTEMPTY;/* 145	*/
+    case ERROR_NOT_LOCKED:			return EACCES;	/* 158	*/
+    case ERROR_BAD_PATHNAME:			return ENOENT;	/* 161	*/
+    case ERROR_MAX_THRDS_REACHED:		return EAGAIN;	/* 164	*/
+    case ERROR_LOCK_FAILED:			return EACCES;	/* 167	*/
+    case ERROR_ALREADY_EXISTS:			return EEXIST;	/* 183	*/
+    case ERROR_INVALID_STARTING_CODESEG:	return ENOEXEC;	/* 188	*/
+    case ERROR_INVALID_STACKSEG:		return ENOEXEC;	/* 189	*/
+    case ERROR_INVALID_MODULETYPE:		return ENOEXEC;	/* 190	*/
+    case ERROR_INVALID_EXE_SIGNATURE:		return ENOEXEC;	/* 191	*/
+    case ERROR_EXE_MARKED_INVALID:		return ENOEXEC;	/* 192	*/
+    case ERROR_BAD_EXE_FORMAT:			return ENOEXEC;	/* 193	*/
+    case ERROR_ITERATED_DATA_EXCEEDS_64k:	return ENOEXEC;	/* 194	*/
+    case ERROR_INVALID_MINALLOCSIZE:		return ENOEXEC;	/* 195	*/
+    case ERROR_DYNLINK_FROM_INVALID_RING:	return ENOEXEC;	/* 196	*/
+    case ERROR_IOPL_NOT_ENABLED:		return ENOEXEC;	/* 197	*/
+    case ERROR_INVALID_SEGDPL:			return ENOEXEC;	/* 198	*/
+    case ERROR_AUTODATASEG_EXCEEDS_64k:		return ENOEXEC;	/* 199	*/
+    case ERROR_RING2SEG_MUST_BE_MOVABLE:	return ENOEXEC;	/* 200	*/
+    case ERROR_RELOC_CHAIN_XEEDS_SEGLIM:	return ENOEXEC;	/* 201	*/
+    case ERROR_INFLOOP_IN_RELOC_CHAIN:		return ENOEXEC;	/* 202	*/
+    case ERROR_FILENAME_EXCED_RANGE:		return ENOENT;	/* 206	*/
+    case ERROR_NESTING_NOT_ALLOWED:		return EAGAIN;	/* 215	*/
+    case ERROR_NOT_ENOUGH_QUOTA:		return ENOMEM;	/* 1816	*/
+    default:					return EINVAL;
+    }
+}
+
+int
+erts_get_last_win_errno(void)
+{
+    return erts_map_win_error_to_errno(GetLastError());
+}
+
+
+#endif
diff --git a/erts/lib_src/common/erl_printf.c b/erts/lib_src/common/erl_printf.c
index 72d18ab6f1..2c177ee5ac 100644
--- a/erts/lib_src/common/erl_printf.c
+++ b/erts/lib_src/common/erl_printf.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2012. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -108,7 +108,7 @@ write_f_add_cr(void *vfp, char* buf, size_t len)
 	if (PUTC(buf[i], (FILE *) vfp) == EOF)
 	    return get_error_result();
     }
-    return 0;
+    return len;
 }
 
 static int
@@ -126,13 +126,14 @@ write_f(void *vfp, char* buf, size_t len)
 #endif
     if (FWRITE((void *) buf, sizeof(char), len, (FILE *) vfp) != len)
 	return get_error_result();
-    return 0;
+    return len;
 }
 
 static int
 write_fd(void *vfdp, char* buf, size_t len)
 {
     ssize_t size;
+    size_t res = len;
     ASSERT(vfdp);
 
     while (len) {
@@ -149,7 +150,7 @@ write_fd(void *vfdp, char* buf, size_t len)
 	len -= size;
     }
 
-    return 0;
+    return res;
 }
 
 static int
@@ -160,7 +161,7 @@ write_s(void *vwbufpp, char* bufp, size_t len)
     ASSERT(len > 0);
     memcpy((void *) *wbufpp, (void *) bufp, len);
     *wbufpp += len;
-    return 0;
+    return len;
 }
 
 
@@ -172,6 +173,7 @@ typedef struct {
 static int
 write_sn(void *vwsnap, char* buf, size_t len)
 {
+    int rv = 0;
     write_sn_arg_t *wsnap = (write_sn_arg_t *) vwsnap;
     ASSERT(wsnap);
     ASSERT(len > 0);
@@ -179,11 +181,13 @@ write_sn(void *vwsnap, char* buf, size_t len)
 	size_t sz = len;
 	if (sz >= wsnap->len)
 	    sz = wsnap->len;
+	rv = (int)sz;
 	memcpy((void *) wsnap->buf, (void *) buf, sz);
 	wsnap->buf += sz;
 	wsnap->len -= sz;
+	return sz;
     }
-    return 0;
+    return rv;
 }
 
 static int
@@ -201,7 +205,7 @@ write_ds(void *vdsbufp, char* buf, size_t len)
     }
     memcpy((void *) (dsbufp->str + dsbufp->str_len), (void *) buf, len);
     dsbufp->str_len += len;
-    return 0;
+    return len;
 }
 
 int
diff --git a/erts/lib_src/common/erl_printf_format.c b/erts/lib_src/common/erl_printf_format.c
index bd3d38e649..473791dce4 100644
--- a/erts/lib_src/common/erl_printf_format.c
+++ b/erts/lib_src/common/erl_printf_format.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -25,9 +25,9 @@
  *  width:      [0-9]+ | '*' 
  *  precision:  [0-9]+ | '*' 
  *  length: hh | h | l | ll | L | j | t | b<sz>
- *  conversion: d,i | o,u,x,X | e,E | f,F | g,G | a,A | c | s | T |
+ *  conversion: d,i | o,u,x,X | e,E | f,F | g,G | a,A | c | s | T | R |
  *              p | n | %
- *  sz: 8 | 16 | 32 | 64 | p
+ *  sz: 8 | 16 | 32 | 64 | p | e
  */
 
 /* Without this, variable argument lists break on VxWorks */
@@ -76,6 +76,18 @@
 #endif
 #endif
 
+#ifndef ERTS_SIZEOF_ETERM
+#  ifdef HALFWORD_HEAP_EMULATOR
+#    if SIZEOF_VOID_P == 8
+#      define ERTS_SIZEOF_ETERM 4
+#    else
+#      error "HALFWORD_HEAP_EMULATOR only allowed on 64-bit architecture"
+#    endif
+#  else
+#    define ERTS_SIZEOF_ETERM SIZEOF_VOID_P
+#  endif
+#endif
+
 #if defined(__GNUC__)
 #  undef inline
 #  define inline __inline__
@@ -89,7 +101,7 @@
 #endif
 
 #define FMTC_d    0x0000
-#define FMTC_i    0x0001
+#define FMTC_R    0x0001
 #define FMTC_o    0x0002
 #define FMTC_u    0x0003
 #define FMTC_x    0x0004
@@ -153,7 +165,7 @@ static char heX[] = "0123456789ABCDEF";
 #define SIGN(X) ((X) > 0 ? 1 : ((X) < 0 ? -1 : 0)) 
 #define USIGN(X) ((X) == 0 ? 0 : 1)
 
-int (*erts_printf_eterm_func)(fmtfn_t, void*, unsigned long, long) = NULL;
+int (*erts_printf_eterm_func)(fmtfn_t, void*, unsigned long, long, unsigned long*) = NULL;
 
 static int
 noop_fn(void *vfp, char* buf, size_t len)
@@ -171,8 +183,8 @@ static int fmt_fld(fmtfn_t fn,void* arg,
     int len;
 
     /* format the prefix */
-    if ((sign || (fmt & (FMTF_sgn|FMTF_blk))) &&
-	(((fmt & FMTC_MASK) == FMTC_d) || ((fmt & FMTC_MASK) == FMTC_i))) {
+    if ((sign || (fmt & (FMTF_sgn|FMTF_blk)))
+	&& (fmt & FMTC_MASK) == FMTC_d) {
 	if (sign < 0)
 	    *pp++ = '-';
 	else if ((fmt & FMTF_sgn))
@@ -233,7 +245,6 @@ static int fmt_long(fmtfn_t fn,void* arg,int sign,unsigned long uval,
 
     switch(fmt & FMTC_MASK) {
     case FMTC_d:
-    case FMTC_i:
     case FMTC_u:
 	break;
     case FMTC_o: 
@@ -286,7 +297,6 @@ static int fmt_long_long(fmtfn_t fn,void* arg,int sign,
 
     switch(fmt & FMTC_MASK) {
     case FMTC_d:
-    case FMTC_i:
     case FMTC_u:
 	break;
     case FMTC_o: 
@@ -378,7 +388,7 @@ static int fmt_double(fmtfn_t fn,void*arg,double val,
 	max_size++;
 	if (precision)
 	    max_size += precision;
-	else if (fmt && FMTF_alt)
+	else if (fmt & FMTF_alt)
 	    max_size++;
 	break;
     case FMTC_E:
@@ -392,7 +402,7 @@ static int fmt_double(fmtfn_t fn,void*arg,double val,
 	max_size += 4;
 	if (precision)
 	    max_size += precision;
-	else if (fmt && FMTF_alt)
+	else if (fmt & FMTF_alt)
 	    max_size++;
 	aexp = exp >= 0 ? exp : -exp;
 	if (aexp < 100)
@@ -520,6 +530,17 @@ int erts_printf_format(fmtfn_t fn, void* arg, char* fmt, va_list ap)
 #error No integer datatype with the same size as 'void *' found
 #endif
 		}
+		else if (*ptr == 'e') {
+		    ptr++;
+#if SIZEOF_INT == ERTS_SIZEOF_ETERM
+#elif SIZEOF_LONG == ERTS_SIZEOF_ETERM
+		    fmt |= FMTL_l;
+#elif SIZEOF_LONG_LONG == ERTS_SIZEOF_ETERM
+		    fmt |= FMTL_ll;
+#else
+#error No integer datatype with the same size as Eterm found
+#endif
+		}
 		else {
 		    int bits = 0;
 		    while(isdigit((int) *ptr))
@@ -599,7 +620,7 @@ int erts_printf_format(fmtfn_t fn, void* arg, char* fmt, va_list ap)
 	    /* specifier */
 	    switch(*ptr) {
 	    case 'd': ptr++; fmt |= FMTC_d; break;
-	    case 'i': ptr++; fmt |= FMTC_i; break;
+	    case 'i': ptr++; fmt |= FMTC_d; break;
 	    case 'o': ptr++; fmt |= FMTC_o; break;
 	    case 'u': ptr++; fmt |= FMTC_u; break;
 	    case 'x': ptr++; fmt |= FMTC_x; break;
@@ -614,6 +635,7 @@ int erts_printf_format(fmtfn_t fn, void* arg, char* fmt, va_list ap)
 	    case 'p': ptr++; fmt |= FMTC_p; break;
 	    case 'n': ptr++; fmt |= FMTC_n; break;
 	    case 'T': ptr++; fmt |= FMTC_T; break;
+	    case 'R': ptr++; fmt |= FMTC_R; break;
 	    case '%':
 		FMT(fn,arg,ptr,1,count);
 		ptr++;
@@ -627,7 +649,6 @@ int erts_printf_format(fmtfn_t fn, void* arg, char* fmt, va_list ap)
 
 	    switch(fmt & FMTC_MASK) {
 	    case FMTC_d:
-	    case FMTC_i:
 		switch(fmt & FMTL_MASK) {
 		case FMTL_hh: {
 		    signed char tval = (signed char) va_arg(ap,int);
@@ -791,9 +812,12 @@ int erts_printf_format(fmtfn_t fn, void* arg, char* fmt, va_list ap)
 		default: *va_arg(ap,int*) = count; break;
 		}
 		break;
-	    case FMTC_T: {
+	    case FMTC_T:    /* Eterm */
+	    case FMTC_R: {  /* Eterm, Eterm* base  (base ignored if !HALFWORD_HEAP) */
 		long prec;
 		unsigned long eterm;
+		unsigned long* eterm_base;
+		
 		if (!erts_printf_eterm_func)
 		    return -EINVAL;
 		if (precision < 0)
@@ -803,14 +827,16 @@ int erts_printf_format(fmtfn_t fn, void* arg, char* fmt, va_list ap)
 		else
 		    prec = (long) precision;
 		eterm = va_arg(ap, unsigned long);
+		eterm_base = ((fmt & FMTC_MASK) == FMTC_R) ?
+		    va_arg(ap, unsigned long*) : NULL;
 		if (width > 0 && !(fmt & FMTF_adj)) {
-		    res = (*erts_printf_eterm_func)(noop_fn, NULL, eterm, prec);
+		    res = (*erts_printf_eterm_func)(noop_fn, NULL, eterm, prec, eterm_base);
 		    if (res < 0)
 			return res;
 		    if (width > res)
 			BLANKS(fn, arg, width - res, count);
 		}
-		res = (*erts_printf_eterm_func)(fn, arg, eterm, prec);
+		res = (*erts_printf_eterm_func)(fn, arg, eterm, prec, eterm_base);
 		if (res < 0)
 		    return res;
 		count += res;
@@ -901,7 +927,7 @@ erts_printf_slong(fmtfn_t fn, void *arg, char conv, int pad, int width,
     unsigned long ul_val;
     switch (conv) {
     case 'd': fmt |= FMTC_d; break;
-    case 'i': fmt |= FMTC_i; break;
+    case 'i': fmt |= FMTC_d; break;
     case 'o': fmt |= FMTC_o; break;
     case 'x': fmt |= FMTC_x; break;
     case 'X': fmt |= FMTC_X; break;
diff --git a/erts/lib_src/common/ethr_atomics.c b/erts/lib_src/common/ethr_atomics.c
new file mode 100644
index 0000000000..c52166a7ec
--- /dev/null
+++ b/erts/lib_src/common/ethr_atomics.c
@@ -0,0 +1,4704 @@
+/*
+ * --------------- DO NOT EDIT THIS FILE! ---------------
+ * This file was automatically generated by the
+ * $ERL_TOP/erts/lib_src/utils/make_atomics_api script.
+ * If you need to make changes, edit the script and
+ * regenerate this file.
+ * --------------- DO NOT EDIT THIS FILE! ---------------
+ */
+
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: The ethread atomics API
+ * Author: Rickard Green
+ */
+
+/*
+ * This file maps native atomic implementations to ethread
+ * API atomics. If no native atomic implementation
+ * is available, a less efficient fallback is used instead.
+ * The API consists of 32-bit size, word size (pointer size),
+ * and double word size atomics.
+ *
+ * The following atomic operations are implemented for
+ * 32-bit size, and word size atomics:
+ * - cmpxchg
+ * - xchg
+ * - set
+ * - init
+ * - add_read
+ * - read
+ * - inc_read
+ * - dec_read
+ * - add
+ * - inc
+ * - dec
+ * - read_band
+ * - read_bor
+ *
+ * The following atomic operations are implemented for
+ * double word size atomics:
+ * - cmpxchg
+ * - set
+ * - read
+ * - init
+ *
+ * Appart from a function implementing the atomic operation
+ * with unspecified memory barrier semantics, there are
+ * functions implementing each operation with the following
+ * implied memory barrier semantics:
+ * - mb   - Full memory barrier. Orders both loads, and
+ *          stores before, and after the atomic operation.
+ *          No load or store is allowed to be reordered
+ *          over the atomic operation.
+ * - relb - Release barrier. Orders both loads, and
+ *          stores appearing *before* the atomic
+ *          operation. These are not allowed to be
+ *          reordered over the atomic operation.
+ * - acqb - Acquire barrier. Orders both loads, and stores
+ *          appearing *after* the atomic operation. These
+ *          are not allowed to be reordered over the
+ *          atomic operation.
+ * - wb   - Write barrier. Orders *only* stores. These are
+ *          not allowed to be reordered over the barrier.
+ *          Store in atomic operation is ordered *after*
+ *          the barrier.
+ * - rb   - Read barrier. Orders *only* loads. These are
+ *          not allowed to be reordered over the barrier.
+ *          Load in atomic operation is ordered *before*
+ *          the barrier. 
+ * - ddrb - Data dependency read barrier. Orders *only*
+ *          loads according to data dependency across the
+ *          barrier. Load in atomic operation is ordered
+ *          before the barrier.
+ *
+ * We implement all of these operation/barrier
+ * combinations, regardless of whether they are useful
+ * or not (some of them are useless).
+ *
+ * Double word size atomic functions are on the followning
+ * form:
+ *   ethr_dw_atomic_<OP>[_<BARRIER>]
+ *
+ * Word size atomic functions are on the followning
+ * form:
+ *   ethr_atomic_<OP>[_<BARRIER>]
+ *
+ * 32-bit size atomic functions are on the followning
+ * form:
+ *   ethr_atomic32_<OP>[_<BARRIER>]
+ *
+ * Apart from the operation/barrier functions
+ * described above also 'addr' functions are implemented
+ * which return the actual memory address used of the
+ * atomic variable. The 'addr' functions have no barrier
+ * versions.
+ *
+ * The native atomic implementation does not need to
+ * implement all operation/barrier combinations.
+ * Functions that have no native implementation will be
+ * constructed from existing native functionality. These
+ * functions will perform the wanted operation and will
+ * produce sufficient memory barriers, but may
+ * in some cases be less efficient than pure native
+ * versions.
+ *
+ * When we create ethread API operation/barrier functions by
+ * adding barriers before and after native operations it is
+ * assumed that:
+ * - A native read operation begins, and ends with a load.
+ * - A native set operation begins, and ends with a store.
+ * - An init operation begins with either a load, or a store,
+ *   and ends with either a load, or a store.
+ * - All other operations begins with a load, and ends with
+ *   either a load, or a store.
+ *
+ * This is the minimum functionality that a native
+ * implementation needs to provide:
+ *
+ * - Functions that need to be implemented:
+ *
+ *   - ethr_native_[dw_|su_dw_]atomic[BITS]_addr
+ *   - ethr_native_[dw_|su_dw_]atomic[BITS]_cmpxchg[_<BARRIER>]
+ *     (at least one cmpxchg of optional barrier)
+ *
+ * - Macros that needs to be defined:
+ *
+ *   A macro informing about the presence of the native
+ *   implementation:
+ *
+ *   - ETHR_HAVE_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]
+ *
+ *   A macro naming (a string constant) the implementation:
+ *
+ *   - ETHR_NATIVE_[DW_]ATOMIC[BITS]_IMPL
+ *
+ *   Each implemented native atomic function has to
+ *   be accompanied by a defined macro on the following
+ *   form informing about its presence:
+ *
+ *   - ETHR_HAVE_ETHR_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]_<OP>[_<BARRIER>]
+ *
+ *   A (sparc-v9 style) membar macro:
+ *
+ *   - ETHR_MEMBAR(B)
+ *
+ *     Which takes a combination of the following macros
+ *     or:ed (using |) together:
+ *
+ *     - ETHR_LoadLoad
+ *     - ETHR_LoadStore
+ *     - ETHR_StoreLoad
+ *     - ETHR_StoreStore
+ *
+ */
+
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#define ETHR_TRY_INLINE_FUNCS
+#define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X ## __
+#define ETHR_INLINE_ATMC_FUNC_NAME_(X) X ## __
+#define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X ## __
+#define ETHR_ATOMIC_IMPL__
+
+#include "ethread.h"
+#include "ethr_internal.h"
+
+#if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \
+     || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS))
+/*
+ * Spinlock based fallback for atomics used in absence of a native
+ * implementation.
+ */
+
+#define ETHR_ATMC_FLLBK_ADDR_BITS 10
+#define ETHR_ATMC_FLLBK_ADDR_SHIFT 6
+
+typedef struct {
+    union {
+	ethr_spinlock_t lck;
+	char buf[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_spinlock_t))];
+    } u;
+} ethr_atomic_protection_t;
+
+extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS];
+
+#define ETHR_ATOMIC_PTR2LCK__(PTR) \
+(&ethr_atomic_protection__[((((ethr_uint_t) (PTR)) >> ETHR_ATMC_FLLBK_ADDR_SHIFT) \
+			   & ((1 << ETHR_ATMC_FLLBK_ADDR_BITS) - 1))].u.lck)
+
+
+#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS)			\
+do {									\
+    ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP));		\
+    ethr_spin_lock(slp__);						\
+    { EXPS; }								\
+    ethr_spin_unlock(slp__);						\
+} while (0)
+
+ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS];
+
+#endif
+
+
+#if defined(ETHR_AMC_FALLBACK__)
+
+/*
+ * Fallback for large sized (word and/or double word size) atomics using
+ * an "Atomic Modification Counter" based on smaller sized native atomics.
+ *
+ * We use a 63-bit modification counter and a one bit exclusive flag.
+ * If 32-bit native atomics are used, we need two 32-bit native atomics.
+ * The exclusive flag is the least significant bit, or if multiple atomics
+ * are used, the least significant bit of the least significant atomic.
+ *
+ * When using the AMC fallback the following is true:
+ * - Reads of the same atomic variable can be done in parallel.
+ * - Uncontended reads doesn't cause any cache line invalidations,
+ *   since no modifications are done.
+ * - Assuming that the AMC atomic(s) and the integer(s) containing the
+ *   value of the implemented atomic resides in the same cache line,
+ *   modifications will only cause invalidations of one cache line.
+ *
+ * When using the spinlock based fallback none of the above is true,
+ * however, the spinlock based fallback consumes less memory.
+ */
+
+#  if ETHR_AMC_NO_ATMCS__ != 1  && ETHR_AMC_NO_ATMCS__ != 2
+#    error "Not supported"
+#  endif
+#  define ETHR_AMC_MAX_TRY_READ__ 10
+#  ifdef ETHR_DEBUG
+#    define ETHR_DBG_CHK_EXCL_STATE(ASP, S) \
+do { \
+    ETHR_AMC_SINT_T__ act = ETHR_AMC_ATMC_FUNC__(read)(&(ASP)->atomic[0]); \
+    ETHR_ASSERT(act == (S) + 1); \
+    ETHR_ASSERT(act & 1); \
+} while (0)
+#  else
+#    define ETHR_DBG_CHK_EXCL_STATE(ASP, S)
+#  endif
+
+static ETHR_INLINE void
+amc_init(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val)
+{
+    avar[0] = val[0];
+    if (dw)
+	avar[1] = val[1];
+#if ETHR_AMC_NO_ATMCS__ == 2
+    ETHR_AMC_ATMC_FUNC__(init)(&amc->atomic[1], 0);
+#endif
+    ETHR_AMC_ATMC_FUNC__(init_wb)(&amc->atomic[0], 0);
+}
+
+static ETHR_INLINE ETHR_AMC_SINT_T__
+amc_set_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ prev_state0)
+{
+    ETHR_AMC_SINT_T__ state0 = prev_state0;
+    /* Set exclusive flag. */
+    while (1) {
+	ETHR_AMC_SINT_T__ act_state0, new_state0;
+	while (state0 & 1) { /* Wait until exclusive bit has been cleared */
+	    ETHR_SPIN_BODY;
+	    state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
+	}
+	/* Try to set exclusive bit */
+	new_state0 = state0 + 1;
+	act_state0 = ETHR_AMC_ATMC_FUNC__(cmpxchg_acqb)(&amc->atomic[0],
+							new_state0,
+							state0);
+	if (state0 == act_state0)
+	    return state0; /* old state0 */
+	state0 = act_state0;
+    }
+}
+
+static ETHR_INLINE void
+amc_inc_mc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0)
+{
+    ETHR_AMC_SINT_T__ state0 = old_state0;
+
+    /* Increment modification counter and reset exclusive flag. */
+
+    ETHR_DBG_CHK_EXCL_STATE(amc, state0);
+
+    state0 += 2;
+
+    ETHR_ASSERT((state0 & 1) == 0);
+
+#if ETHR_AMC_NO_ATMCS__ == 2
+    if (state0 == 0) {
+	/*
+	 * state0 wrapped, so we need to increment state1. There is no need
+	 * for atomic inc op, since this is always done while having exclusive
+	 * flag.
+	 */
+	ETHR_AMC_SINT_T__ state1 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1]);
+	state1++;
+	ETHR_AMC_ATMC_FUNC__(set)(&amc->atomic[1], state1);
+    }
+#endif
+    ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], state0);
+}
+
+static ETHR_INLINE void
+amc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0)
+{
+    ETHR_DBG_CHK_EXCL_STATE(amc, old_state0);
+    /*
+     * Reset exclusive flag, but leave modification counter unchanged,
+     * i.e., restore state to what it was before setting exclusive
+     * flag.
+     */
+    ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], old_state0);
+}
+
+static ETHR_INLINE void
+amc_set(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val)
+{
+    ETHR_AMC_SINT_T__ state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
+
+    state0 = amc_set_excl(amc, state0);
+
+    avar[0] = val[0];
+    if (dw)
+	avar[1] = val[1];
+
+    amc_inc_mc_unset_excl(amc, state0);
+}
+
+static ETHR_INLINE int
+amc_try_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar,
+	     ethr_sint_t *val, ETHR_AMC_SINT_T__ *state0p)
+{
+    /* *state0p should contain last read value if aborting */
+    ETHR_AMC_SINT_T__ old_state0;
+#if ETHR_AMC_NO_ATMCS__ == 2
+    ETHR_AMC_SINT_T__ state1;
+    int abrt;
+#endif
+
+    *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]);
+    if ((*state0p) & 1) 
+	return 0; /* exclusive flag set; abort */
+#if ETHR_AMC_NO_ATMCS__ == 2
+    state1 = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[1]);
+#else
+    ETHR_COMPILER_BARRIER;
+#endif
+
+    val[0] = avar[0];
+    if (dw)
+	val[1] = avar[1];
+
+    ETHR_READ_MEMORY_BARRIER;
+
+    /*
+     * Abort if state has changed (i.e, either the exclusive
+     * flag is set, or modification counter changed).
+     */
+    old_state0 = *state0p;
+#if ETHR_AMC_NO_ATMCS__ == 2
+    *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]);
+    abrt = (old_state0 != *state0p);
+    abrt |= (state1 != ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1]));
+    return abrt == 0;
+#else
+    *state0p = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
+    return old_state0 == *state0p;
+#endif
+}
+
+static ETHR_INLINE void
+amc_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val)
+{
+    ETHR_AMC_SINT_T__ state0;
+    int i;
+
+#if ETHR_AMC_MAX_TRY_READ__ == 0
+    state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
+#else
+    for (i = 0; i < ETHR_AMC_MAX_TRY_READ__; i++) {
+	if (amc_try_read(amc, dw, avar, val, &state0))
+	    return; /* read success */
+	ETHR_SPIN_BODY;
+    }
+#endif
+
+    state0 = amc_set_excl(amc, state0);
+
+    val[0] = avar[0];
+    if (dw)
+	val[1] = avar[1];
+
+    amc_unset_excl(amc, state0);
+}
+
+static ETHR_INLINE int
+amc_cmpxchg(ethr_amc_t *amc, int dw, ethr_sint_t *avar,
+	    ethr_sint_t *new, ethr_sint_t *xchg)
+{
+    ethr_sint_t val[2];
+    ETHR_AMC_SINT_T__ state0;
+
+    if (amc_try_read(amc, dw, avar, val, &state0)) {
+	if (val[0] != xchg[0] || (dw && val[1] != xchg[1])) {
+	    xchg[0] = val[0];
+	    if (dw)
+		xchg[1] = val[1];
+	    return 0; /* failed */
+	}
+	/* Operation will succeed if not interrupted */
+    }
+
+    state0 = amc_set_excl(amc, state0);
+
+    if (xchg[0] != avar[0] || (dw && xchg[1] != avar[1])) {
+	xchg[0] = avar[0];
+	if (dw)
+	    xchg[1] = avar[1];
+
+	ETHR_DBG_CHK_EXCL_STATE(amc, state0);
+
+	amc_unset_excl(amc, state0);
+	return 0; /* failed */
+    }
+
+    avar[0] = new[0];
+    if (dw)
+	avar[1] = new[1];
+
+    amc_inc_mc_unset_excl(amc, state0);
+    return 1;
+}
+
+
+#define ETHR_AMC_MODIFICATION_OPS__(AMC, OPS)			\
+do {								\
+    ETHR_AMC_SINT_T__ state0__;					\
+    state0__ = ETHR_AMC_ATMC_FUNC__(read)(&(AMC)->atomic[0]);	\
+    state0__ = amc_set_excl((AMC), state0__);			\
+    { OPS; }							\
+    amc_inc_mc_unset_excl((AMC), state0__);			\
+} while (0)
+
+#endif /* amc fallback */
+
+
+int
+ethr_init_atomics(void)
+{
+#if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \
+     || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS))
+    int i;
+    for (i = 0; i < (1 << ETHR_ATMC_FLLBK_ADDR_BITS); i++) {
+	int res = ethr_spinlock_init(&ethr_atomic_protection__[i].u.lck);
+	if (res != 0)
+	    return res;
+    }
+#endif
+    return 0;
+}
+
+
+/* ---------- Double word size atomic implementation ---------- */
+
+
+
+/*
+ * Double word atomics need runtime test.
+ */
+
+int ethr_have_native_dw_atomic(void)
+{
+    return ethr_have_native_dw_atomic__();
+}
+     
+
+/* --- addr() --- */
+
+ethr_sint_t *ETHR_DW_ATOMIC_FUNC__(addr)(ethr_dw_atomic_t *var)
+{
+    ethr_sint_t *res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    res =  ethr_dw_atomic_addr__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    res = (ethr_sint_t *) ((&var->fallback))->sint;
+#else
+    res = (ethr_sint_t *) (&var->fallback);
+#endif
+    return res;
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+ethr_sint_t *ethr_dw_atomic_addr(ethr_dw_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_dw_atomic_addr__(var);
+}
+#endif
+
+
+/* -- cmpxchg() -- */
+
+
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+    int res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    res =  ethr_dw_atomic_cmpxchg__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), 
+	{
+	    res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
+	    if (res) {
+		(&var->fallback)->sint[0] = val->sint[0];
+		(&var->fallback)->sint[1] = val->sint[1];
+	    }
+	    else {
+		old_val->sint[0] = (&var->fallback)->sint[0];
+		old_val->sint[1] = (&var->fallback)->sint[1];
+	    }
+	});
+#endif
+    return res;
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+int ethr_dw_atomic_cmpxchg(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_dw_atomic_cmpxchg__(var, val, old_val);
+}
+#endif
+
+int ethr_dw_atomic_cmpxchg_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    return ethr_dw_atomic_cmpxchg_ddrb__(var, val, old_val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    return ethr_dw_atomic_cmpxchg(var, val, old_val);
+#else
+    return ethr_dw_atomic_cmpxchg_rb(var, val, old_val);
+#endif
+}
+
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+    int res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    res =  ethr_dw_atomic_cmpxchg_rb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), 
+	{
+	    res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
+	    if (res) {
+		(&var->fallback)->sint[0] = val->sint[0];
+		(&var->fallback)->sint[1] = val->sint[1];
+	    }
+	    else {
+		old_val->sint[0] = (&var->fallback)->sint[0];
+		old_val->sint[1] = (&var->fallback)->sint[1];
+	    }
+	});
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+    return res;
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+int ethr_dw_atomic_cmpxchg_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_dw_atomic_cmpxchg_rb__(var, val, old_val);
+}
+#endif
+
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+    int res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    res =  ethr_dw_atomic_cmpxchg_wb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), 
+	{
+	    res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
+	    if (res) {
+		(&var->fallback)->sint[0] = val->sint[0];
+		(&var->fallback)->sint[1] = val->sint[1];
+	    }
+	    else {
+		old_val->sint[0] = (&var->fallback)->sint[0];
+		old_val->sint[1] = (&var->fallback)->sint[1];
+	    }
+	});
+#endif
+    return res;
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+int ethr_dw_atomic_cmpxchg_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_dw_atomic_cmpxchg_wb__(var, val, old_val);
+}
+#endif
+
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+    int res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    res =  ethr_dw_atomic_cmpxchg_acqb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), 
+	{
+	    res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
+	    if (res) {
+		(&var->fallback)->sint[0] = val->sint[0];
+		(&var->fallback)->sint[1] = val->sint[1];
+	    }
+	    else {
+		old_val->sint[0] = (&var->fallback)->sint[0];
+		old_val->sint[1] = (&var->fallback)->sint[1];
+	    }
+	});
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+int ethr_dw_atomic_cmpxchg_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_dw_atomic_cmpxchg_acqb__(var, val, old_val);
+}
+#endif
+
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+    int res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    res =  ethr_dw_atomic_cmpxchg_relb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), 
+	{
+	    res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
+	    if (res) {
+		(&var->fallback)->sint[0] = val->sint[0];
+		(&var->fallback)->sint[1] = val->sint[1];
+	    }
+	    else {
+		old_val->sint[0] = (&var->fallback)->sint[0];
+		old_val->sint[1] = (&var->fallback)->sint[1];
+	    }
+	});
+#endif
+    return res;
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+int ethr_dw_atomic_cmpxchg_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_dw_atomic_cmpxchg_relb__(var, val, old_val);
+}
+#endif
+
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+    int res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    res =  ethr_dw_atomic_cmpxchg_mb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), 
+	{
+	    res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
+	    if (res) {
+		(&var->fallback)->sint[0] = val->sint[0];
+		(&var->fallback)->sint[1] = val->sint[1];
+	    }
+	    else {
+		old_val->sint[0] = (&var->fallback)->sint[0];
+		old_val->sint[1] = (&var->fallback)->sint[1];
+	    }
+	});
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+int ethr_dw_atomic_cmpxchg_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_dw_atomic_cmpxchg_mb__(var, val, old_val);
+}
+#endif
+
+
+/* -- set() -- */
+
+
+void ETHR_DW_ATOMIC_FUNC__(set)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    ethr_dw_atomic_set__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_set(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_dw_atomic_set__(var, val);
+}
+#endif
+
+void ethr_dw_atomic_set_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_dw_atomic_set_ddrb__(var, val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    ethr_dw_atomic_set(var, val);
+#else
+    ethr_dw_atomic_set_rb(var, val);
+#endif
+}
+
+void ETHR_DW_ATOMIC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    ethr_dw_atomic_set_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_set_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_dw_atomic_set_rb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(set_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    ethr_dw_atomic_set_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_set_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_dw_atomic_set_wb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(set_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    ethr_dw_atomic_set_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_set_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_dw_atomic_set_acqb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(set_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    ethr_dw_atomic_set_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_set_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_dw_atomic_set_relb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    ethr_dw_atomic_set_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_set_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_dw_atomic_set_mb__(var, val);
+}
+#endif
+
+
+/* -- read() -- */
+
+
+void ETHR_DW_ATOMIC_FUNC__(read)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    ethr_dw_atomic_read__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_read(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_dw_atomic_read__(var, val);
+}
+#endif
+
+void ethr_dw_atomic_read_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_dw_atomic_read_ddrb__(var, val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    ethr_dw_atomic_read(var, val);
+#else
+    ethr_dw_atomic_read_rb(var, val);
+#endif
+}
+
+void ETHR_DW_ATOMIC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    ethr_dw_atomic_read_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_read_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_dw_atomic_read_rb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(read_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    ethr_dw_atomic_read_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_read_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_dw_atomic_read_wb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(read_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    ethr_dw_atomic_read_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+    ETHR_MEMBAR(ETHR_LoadStore);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_read_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_dw_atomic_read_acqb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(read_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    ethr_dw_atomic_read_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_read_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_dw_atomic_read_relb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    ethr_dw_atomic_read_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+    ETHR_MEMBAR(ETHR_LoadStore);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_read_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_dw_atomic_read_mb__(var, val);
+}
+#endif
+
+
+/* -- init() -- */
+
+
+void ETHR_DW_ATOMIC_FUNC__(init)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    ethr_dw_atomic_init__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_init(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(var);
+    ethr_dw_atomic_init__(var, val);
+}
+#endif
+
+void ethr_dw_atomic_init_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_dw_atomic_init_ddrb__(var, val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    ethr_dw_atomic_init(var, val);
+#else
+    ethr_dw_atomic_init_rb(var, val);
+#endif
+}
+
+void ETHR_DW_ATOMIC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    ethr_dw_atomic_init_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_init_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(var);
+    ethr_dw_atomic_init_rb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(init_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    ethr_dw_atomic_init_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_init_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(var);
+    ethr_dw_atomic_init_wb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(init_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    ethr_dw_atomic_init_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_init_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(var);
+    ethr_dw_atomic_init_acqb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(init_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    ethr_dw_atomic_init_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_init_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(var);
+    ethr_dw_atomic_init_relb__(var, val);
+}
+#endif
+
+void ETHR_DW_ATOMIC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    ethr_dw_atomic_init_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
+void ethr_dw_atomic_init_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+    ETHR_ASSERT(var);
+    ethr_dw_atomic_init_mb__(var, val);
+}
+#endif
+
+
+/* ---------- Word size atomic implementation ---------- */
+
+
+
+
+/* --- addr() --- */
+
+ethr_sint_t *ethr_atomic_addr(ethr_atomic_t *var)
+{
+    ethr_sint_t *res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_addr__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    res = (ethr_sint_t *) (var)->sint;
+#else
+    res = (ethr_sint_t *) var;
+#endif
+    return res;
+}
+
+
+/* -- cmpxchg() -- */
+
+
+ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_cmpxchg__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    res = old_val;
+    (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_cmpxchg_ddrb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    return ethr_atomic_cmpxchg_ddrb__(var, val, old_val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    return ethr_atomic_cmpxchg(var, val, old_val);
+#else
+    return ethr_atomic_cmpxchg_rb(var, val, old_val);
+#endif
+}
+
+ethr_sint_t ethr_atomic_cmpxchg_rb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_cmpxchg_rb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    res = old_val;
+    (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_cmpxchg_wb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_cmpxchg_wb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    res = old_val;
+    (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_cmpxchg_acqb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    res = old_val;
+    (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_cmpxchg_relb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = old_val;
+    (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_cmpxchg_mb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_cmpxchg_mb__(var, val, old_val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    res = old_val;
+    (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+
+/* -- xchg() -- */
+
+
+ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_xchg__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_xchg_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    return ethr_atomic_xchg_ddrb__(var, val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    return ethr_atomic_xchg(var, val);
+#else
+    return ethr_atomic_xchg_rb(var, val);
+#endif
+}
+
+ethr_sint_t ethr_atomic_xchg_rb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_xchg_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_xchg_wb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_xchg_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_xchg_acqb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_xchg_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_xchg_relb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_xchg_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_xchg_mb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_xchg_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+
+/* -- set() -- */
+
+
+void ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_set__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    amc_set(&var->amc, 0, &var->sint, &val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic_set_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_set_ddrb__(var, val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    ethr_atomic_set(var, val);
+#else
+    ethr_atomic_set_rb(var, val);
+#endif
+}
+
+void ethr_atomic_set_rb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_set_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    amc_set(&var->amc, 0, &var->sint, &val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+void ethr_atomic_set_wb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_set_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    amc_set(&var->amc, 0, &var->sint, &val);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic_set_acqb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_set_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    amc_set(&var->amc, 0, &var->sint, &val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+void ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_set_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    amc_set(&var->amc, 0, &var->sint, &val);
+#else
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic_set_mb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_set_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    amc_set(&var->amc, 0, &var->sint, &val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+
+/* -- init() -- */
+
+
+void ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_init__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    amc_init(&var->amc, 0, &var->sint, &val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic_init_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_init_ddrb__(var, val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    ethr_atomic_init(var, val);
+#else
+    ethr_atomic_init_rb(var, val);
+#endif
+}
+
+void ethr_atomic_init_rb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_init_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    amc_init(&var->amc, 0, &var->sint, &val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+void ethr_atomic_init_wb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_init_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    amc_init(&var->amc, 0, &var->sint, &val);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic_init_acqb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_init_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    amc_init(&var->amc, 0, &var->sint, &val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+void ethr_atomic_init_relb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_init_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    amc_init(&var->amc, 0, &var->sint, &val);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic_init_mb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_init_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    amc_init(&var->amc, 0, &var->sint, &val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+
+/* -- add_read() -- */
+
+
+ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_add_read__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_add_read_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    return ethr_atomic_add_read_ddrb__(var, val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    return ethr_atomic_add_read(var, val);
+#else
+    return ethr_atomic_add_read_rb(var, val);
+#endif
+}
+
+ethr_sint_t ethr_atomic_add_read_rb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_add_read_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_add_read_wb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_add_read_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_add_read_acqb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_add_read_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_add_read_relb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_add_read_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_add_read_mb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_add_read_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+
+/* -- read() -- */
+
+
+ethr_sint_t ethr_atomic_read(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_read__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    amc_read(&var->amc, 0, &var->sint, &res);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_read_ddrb(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    return ethr_atomic_read_ddrb__(var);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    return ethr_atomic_read(var);
+#else
+    return ethr_atomic_read_rb(var);
+#endif
+}
+
+ethr_sint_t ethr_atomic_read_rb(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_read_rb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    amc_read(&var->amc, 0, &var->sint, &res);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_read_wb(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_read_wb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    amc_read(&var->amc, 0, &var->sint, &res);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_read_acqb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    amc_read(&var->amc, 0, &var->sint, &res);
+    ETHR_MEMBAR(ETHR_LoadStore);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_read_relb(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_read_relb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    amc_read(&var->amc, 0, &var->sint, &res);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_read_mb(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_read_mb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    amc_read(&var->amc, 0, &var->sint, &res);
+    ETHR_MEMBAR(ETHR_LoadStore);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#endif
+    return res;
+}
+
+
+/* -- inc_read() -- */
+
+
+ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_inc_read__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_inc_read_ddrb(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    return ethr_atomic_inc_read_ddrb__(var);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    return ethr_atomic_inc_read(var);
+#else
+    return ethr_atomic_inc_read_rb(var);
+#endif
+}
+
+ethr_sint_t ethr_atomic_inc_read_rb(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_inc_read_rb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_inc_read_wb(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_inc_read_wb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_inc_read_acqb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_inc_read_relb(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_inc_read_relb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_inc_read_mb(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_inc_read_mb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+
+/* -- dec_read() -- */
+
+
+ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_dec_read__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_dec_read_ddrb(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    return ethr_atomic_dec_read_ddrb__(var);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    return ethr_atomic_dec_read(var);
+#else
+    return ethr_atomic_dec_read_rb(var);
+#endif
+}
+
+ethr_sint_t ethr_atomic_dec_read_rb(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_dec_read_rb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_dec_read_wb(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_dec_read_wb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_dec_read_acqb(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_dec_read_acqb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_dec_read_relb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_dec_read_mb(ethr_atomic_t *var)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_dec_read_mb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+
+/* -- add() -- */
+
+
+void ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_add__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+#endif
+
+}
+
+void ethr_atomic_add_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_add_ddrb__(var, val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    ethr_atomic_add(var, val);
+#else
+    ethr_atomic_add_rb(var, val);
+#endif
+}
+
+void ethr_atomic_add_rb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_add_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+void ethr_atomic_add_wb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_add_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+#endif
+
+}
+
+void ethr_atomic_add_acqb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_add_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+void ethr_atomic_add_relb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_add_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+#endif
+
+}
+
+void ethr_atomic_add_mb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_add_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+
+/* -- inc() -- */
+
+
+void ethr_atomic_inc(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_inc__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+#endif
+
+}
+
+void ethr_atomic_inc_ddrb(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_inc_ddrb__(var);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    ethr_atomic_inc(var);
+#else
+    ethr_atomic_inc_rb(var);
+#endif
+}
+
+void ethr_atomic_inc_rb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_inc_rb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+void ethr_atomic_inc_wb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_inc_wb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+#endif
+
+}
+
+void ethr_atomic_inc_acqb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_inc_acqb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+void ethr_atomic_inc_relb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_inc_relb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+#endif
+
+}
+
+void ethr_atomic_inc_mb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_inc_mb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+
+/* -- dec() -- */
+
+
+void ethr_atomic_dec(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_dec__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+#endif
+
+}
+
+void ethr_atomic_dec_ddrb(ethr_atomic_t *var)
+{
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_dec_ddrb__(var);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    ethr_atomic_dec(var);
+#else
+    ethr_atomic_dec_rb(var);
+#endif
+}
+
+void ethr_atomic_dec_rb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_dec_rb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+void ethr_atomic_dec_wb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_dec_wb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+#endif
+
+}
+
+void ethr_atomic_dec_acqb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_dec_acqb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+void ethr_atomic_dec_relb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_dec_relb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+#endif
+
+}
+
+void ethr_atomic_dec_mb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    ethr_atomic_dec_mb__(var);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+
+/* -- read_band() -- */
+
+
+ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_read_band__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_read_band_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    return ethr_atomic_read_band_ddrb__(var, val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    return ethr_atomic_read_band(var, val);
+#else
+    return ethr_atomic_read_band_rb(var, val);
+#endif
+}
+
+ethr_sint_t ethr_atomic_read_band_rb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_read_band_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_read_band_wb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_read_band_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_read_band_acqb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_read_band_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_read_band_relb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_read_band_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_read_band_mb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_read_band_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+
+/* -- read_bor() -- */
+
+
+ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_read_bor__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_read_bor_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    return ethr_atomic_read_bor_ddrb__(var, val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    return ethr_atomic_read_bor(var, val);
+#else
+    return ethr_atomic_read_bor_rb(var, val);
+#endif
+}
+
+ethr_sint_t ethr_atomic_read_bor_rb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_read_bor_rb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_read_bor_wb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_read_bor_wb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_read_bor_acqb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_read_bor_acqb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_read_bor_relb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_read_bor_relb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+#endif
+    return res;
+}
+
+ethr_sint_t ethr_atomic_read_bor_mb(ethr_atomic_t *var, ethr_sint_t val)
+{
+    ethr_sint_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic_read_bor_mb__(var, val);
+#elif defined(ETHR_AMC_FALLBACK__)
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+
+/* ---------- 32-bit atomic implementation ---------- */
+
+
+
+
+/* --- addr() --- */
+
+ethr_sint32_t *ethr_atomic32_addr(ethr_atomic32_t *var)
+{
+    ethr_sint32_t *res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_addr__(var);
+
+#else
+    res = (ethr_sint32_t *) var;
+#endif
+    return res;
+}
+
+
+/* -- cmpxchg() -- */
+
+
+ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_cmpxchg__(var, val, old_val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_cmpxchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    return ethr_atomic32_cmpxchg_ddrb__(var, val, old_val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    return ethr_atomic32_cmpxchg(var, val, old_val);
+#else
+    return ethr_atomic32_cmpxchg_rb(var, val, old_val);
+#endif
+}
+
+ethr_sint32_t ethr_atomic32_cmpxchg_rb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_cmpxchg_rb__(var, val, old_val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_cmpxchg_wb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_cmpxchg_wb__(var, val, old_val);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_cmpxchg_acqb__(var, val, old_val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_cmpxchg_relb__(var, val, old_val);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_cmpxchg_mb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_cmpxchg_mb__(var, val, old_val);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+
+/* -- xchg() -- */
+
+
+ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_xchg__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_xchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    return ethr_atomic32_xchg_ddrb__(var, val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    return ethr_atomic32_xchg(var, val);
+#else
+    return ethr_atomic32_xchg_rb(var, val);
+#endif
+}
+
+ethr_sint32_t ethr_atomic32_xchg_rb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_xchg_rb__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_xchg_wb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_xchg_wb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_xchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_xchg_acqb__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_xchg_relb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_xchg_relb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_xchg_mb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_xchg_mb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+
+/* -- set() -- */
+
+
+void ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_set__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic32_set_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_set_ddrb__(var, val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    ethr_atomic32_set(var, val);
+#else
+    ethr_atomic32_set_rb(var, val);
+#endif
+}
+
+void ethr_atomic32_set_rb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_set_rb__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+void ethr_atomic32_set_wb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_set_wb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic32_set_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_set_acqb__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+void ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_set_relb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic32_set_mb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_set_mb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+
+/* -- init() -- */
+
+
+void ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_init__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic32_init_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_init_ddrb__(var, val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    ethr_atomic32_init(var, val);
+#else
+    ethr_atomic32_init_rb(var, val);
+#endif
+}
+
+void ethr_atomic32_init_rb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_init_rb__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+void ethr_atomic32_init_wb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_init_wb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic32_init_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_init_acqb__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+void ethr_atomic32_init_relb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_init_relb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+#endif
+
+}
+
+void ethr_atomic32_init_mb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_init_mb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+
+/* -- add_read() -- */
+
+
+ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_add_read__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_add_read_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    return ethr_atomic32_add_read_ddrb__(var, val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    return ethr_atomic32_add_read(var, val);
+#else
+    return ethr_atomic32_add_read_rb(var, val);
+#endif
+}
+
+ethr_sint32_t ethr_atomic32_add_read_rb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_add_read_rb__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_add_read_wb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_add_read_wb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_add_read_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_add_read_acqb__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_add_read_relb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_add_read_relb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_add_read_mb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_add_read_mb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+
+/* -- read() -- */
+
+
+ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_read__(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_ddrb(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    return ethr_atomic32_read_ddrb__(var);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    return ethr_atomic32_read(var);
+#else
+    return ethr_atomic32_read_rb(var);
+#endif
+}
+
+ethr_sint32_t ethr_atomic32_read_rb(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_read_rb__(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_wb(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_read_wb__(var);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_acqb(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_read_acqb__(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_relb(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_read_relb__(var);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_mb(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_read_mb__(var);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#endif
+    return res;
+}
+
+
+/* -- inc_read() -- */
+
+
+ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_inc_read__(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_inc_read_ddrb(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    return ethr_atomic32_inc_read_ddrb__(var);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    return ethr_atomic32_inc_read(var);
+#else
+    return ethr_atomic32_inc_read_rb(var);
+#endif
+}
+
+ethr_sint32_t ethr_atomic32_inc_read_rb(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_inc_read_rb__(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_inc_read_wb(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_inc_read_wb__(var);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_inc_read_acqb__(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_inc_read_relb(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_inc_read_relb__(var);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_inc_read_mb(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_inc_read_mb__(var);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+
+/* -- dec_read() -- */
+
+
+ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_dec_read__(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_dec_read_ddrb(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    return ethr_atomic32_dec_read_ddrb__(var);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    return ethr_atomic32_dec_read(var);
+#else
+    return ethr_atomic32_dec_read_rb(var);
+#endif
+}
+
+ethr_sint32_t ethr_atomic32_dec_read_rb(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_dec_read_rb__(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_dec_read_wb(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_dec_read_wb__(var);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_dec_read_acqb(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_dec_read_acqb__(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_dec_read_relb(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_dec_read_relb__(var);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_dec_read_mb(ethr_atomic32_t *var)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_dec_read_mb__(var);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+
+/* -- add() -- */
+
+
+void ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_add__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+#endif
+
+}
+
+void ethr_atomic32_add_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_add_ddrb__(var, val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    ethr_atomic32_add(var, val);
+#else
+    ethr_atomic32_add_rb(var, val);
+#endif
+}
+
+void ethr_atomic32_add_rb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_add_rb__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+void ethr_atomic32_add_wb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_add_wb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+#endif
+
+}
+
+void ethr_atomic32_add_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_add_acqb__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+void ethr_atomic32_add_relb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_add_relb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+#endif
+
+}
+
+void ethr_atomic32_add_mb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_add_mb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+
+/* -- inc() -- */
+
+
+void ethr_atomic32_inc(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_inc__(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+#endif
+
+}
+
+void ethr_atomic32_inc_ddrb(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_inc_ddrb__(var);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    ethr_atomic32_inc(var);
+#else
+    ethr_atomic32_inc_rb(var);
+#endif
+}
+
+void ethr_atomic32_inc_rb(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_inc_rb__(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+void ethr_atomic32_inc_wb(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_inc_wb__(var);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+#endif
+
+}
+
+void ethr_atomic32_inc_acqb(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_inc_acqb__(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+void ethr_atomic32_inc_relb(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_inc_relb__(var);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+#endif
+
+}
+
+void ethr_atomic32_inc_mb(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_inc_mb__(var);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+
+/* -- dec() -- */
+
+
+void ethr_atomic32_dec(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_dec__(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+#endif
+
+}
+
+void ethr_atomic32_dec_ddrb(ethr_atomic32_t *var)
+{
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_dec_ddrb__(var);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    ethr_atomic32_dec(var);
+#else
+    ethr_atomic32_dec_rb(var);
+#endif
+}
+
+void ethr_atomic32_dec_rb(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_dec_rb__(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+
+}
+
+void ethr_atomic32_dec_wb(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_dec_wb__(var);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+#endif
+
+}
+
+void ethr_atomic32_dec_acqb(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_dec_acqb__(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+void ethr_atomic32_dec_relb(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_dec_relb__(var);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+#endif
+
+}
+
+void ethr_atomic32_dec_mb(ethr_atomic32_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    ethr_atomic32_dec_mb__(var);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+
+}
+
+
+/* -- read_band() -- */
+
+
+ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_read_band__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_band_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    return ethr_atomic32_read_band_ddrb__(var, val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    return ethr_atomic32_read_band(var, val);
+#else
+    return ethr_atomic32_read_band_rb(var, val);
+#endif
+}
+
+ethr_sint32_t ethr_atomic32_read_band_rb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_read_band_rb__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_band_wb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_read_band_wb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_band_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_read_band_acqb__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_band_relb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_read_band_relb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_band_mb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_read_band_mb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+
+/* -- read_bor() -- */
+
+
+ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_read_bor__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_bor_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    return ethr_atomic32_read_bor_ddrb__(var, val);
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    return ethr_atomic32_read_bor(var, val);
+#else
+    return ethr_atomic32_read_bor_rb(var, val);
+#endif
+}
+
+ethr_sint32_t ethr_atomic32_read_bor_rb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_read_bor_rb__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_bor_wb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_read_bor_wb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_StoreStore);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_bor_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_read_bor_acqb__(var, val);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_bor_relb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_read_bor_relb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+#endif
+    return res;
+}
+
+ethr_sint32_t ethr_atomic32_read_bor_mb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+    res =  ethr_atomic32_read_bor_mb__(var, val);
+#else
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
+    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
+#endif
+    return res;
+}
+
+
+
+/* --------- Info functions --------- */
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+char *zero_ops[] = {NULL};
+#endif
+
+
+static char *native_su_dw_atomic_ops[] = {
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG
+    "cmpxchg",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RB
+    "cmpxchg_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_WB
+    "cmpxchg_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_ACQB
+    "cmpxchg_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RELB
+    "cmpxchg_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB
+    "cmpxchg_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET
+    "set",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RB
+    "set_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_WB
+    "set_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_ACQB
+    "set_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RELB
+    "set_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_MB
+    "set_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ
+    "read",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RB
+    "read_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_WB
+    "read_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_ACQB
+    "read_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RELB
+    "read_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_MB
+    "read_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT
+    "init",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RB
+    "init_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_WB
+    "init_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_ACQB
+    "init_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RELB
+    "init_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_MB
+    "init_mb",
+#endif
+    NULL
+};
+
+char **
+ethr_native_su_dw_atomic_ops(void)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (!ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+	return &zero_ops[0];
+#endif
+    return &native_su_dw_atomic_ops[0];
+}
+
+
+static char *native_dw_atomic_ops[] = {
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG
+    "cmpxchg",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RB
+    "cmpxchg_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_WB
+    "cmpxchg_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_ACQB
+    "cmpxchg_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RELB
+    "cmpxchg_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB
+    "cmpxchg_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET
+    "set",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RB
+    "set_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_WB
+    "set_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_ACQB
+    "set_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RELB
+    "set_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_MB
+    "set_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ
+    "read",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RB
+    "read_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_WB
+    "read_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_ACQB
+    "read_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RELB
+    "read_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_MB
+    "read_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT
+    "init",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RB
+    "init_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_WB
+    "init_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_ACQB
+    "init_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RELB
+    "init_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_MB
+    "init_mb",
+#endif
+    NULL
+};
+
+char **
+ethr_native_dw_atomic_ops(void)
+{
+
+#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+    if (!ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
+	return &zero_ops[0];
+#endif
+    return &native_dw_atomic_ops[0];
+}
+
+
+static char *native_atomic64_ops[] = {
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG
+    "cmpxchg",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB
+    "cmpxchg_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB
+    "cmpxchg_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB
+    "cmpxchg_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB
+    "cmpxchg_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB
+    "cmpxchg_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG
+    "xchg",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RB
+    "xchg_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_WB
+    "xchg_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_ACQB
+    "xchg_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RELB
+    "xchg_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB
+    "xchg_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET
+    "set",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB
+    "set_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB
+    "set_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB
+    "set_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB
+    "set_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB
+    "set_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT
+    "init",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB
+    "init_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB
+    "init_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB
+    "init_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB
+    "init_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB
+    "init_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN
+    "add_return",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RB
+    "add_return_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_WB
+    "add_return_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB
+    "add_return_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB
+    "add_return_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB
+    "add_return_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ
+    "read",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB
+    "read_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB
+    "read_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB
+    "read_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB
+    "read_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB
+    "read_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN
+    "inc_return",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RB
+    "inc_return_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_WB
+    "inc_return_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB
+    "inc_return_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB
+    "inc_return_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB
+    "inc_return_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN
+    "dec_return",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RB
+    "dec_return_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_WB
+    "dec_return_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB
+    "dec_return_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB
+    "dec_return_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB
+    "dec_return_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD
+    "add",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RB
+    "add_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_WB
+    "add_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_ACQB
+    "add_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RELB
+    "add_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB
+    "add_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC
+    "inc",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RB
+    "inc_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_WB
+    "inc_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_ACQB
+    "inc_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RELB
+    "inc_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB
+    "inc_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC
+    "dec",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RB
+    "dec_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_WB
+    "dec_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_ACQB
+    "dec_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RELB
+    "dec_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB
+    "dec_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD
+    "and_retold",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RB
+    "and_retold_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_WB
+    "and_retold_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_ACQB
+    "and_retold_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RELB
+    "and_retold_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB
+    "and_retold_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD
+    "or_retold",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RB
+    "or_retold_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_WB
+    "or_retold_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_ACQB
+    "or_retold_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RELB
+    "or_retold_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB
+    "or_retold_mb",
+#endif
+    NULL
+};
+
+char **
+ethr_native_atomic64_ops(void)
+{
+
+    return &native_atomic64_ops[0];
+}
+
+
+static char *native_atomic32_ops[] = {
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG
+    "cmpxchg",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB
+    "cmpxchg_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB
+    "cmpxchg_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB
+    "cmpxchg_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB
+    "cmpxchg_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB
+    "cmpxchg_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG
+    "xchg",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RB
+    "xchg_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_WB
+    "xchg_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB
+    "xchg_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RELB
+    "xchg_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB
+    "xchg_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET
+    "set",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RB
+    "set_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB
+    "set_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_ACQB
+    "set_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB
+    "set_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB
+    "set_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT
+    "init",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RB
+    "init_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_WB
+    "init_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_ACQB
+    "init_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RELB
+    "init_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_MB
+    "init_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN
+    "add_return",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RB
+    "add_return_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_WB
+    "add_return_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB
+    "add_return_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB
+    "add_return_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB
+    "add_return_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ
+    "read",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB
+    "read_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_WB
+    "read_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB
+    "read_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RELB
+    "read_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB
+    "read_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN
+    "inc_return",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RB
+    "inc_return_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_WB
+    "inc_return_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB
+    "inc_return_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB
+    "inc_return_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB
+    "inc_return_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN
+    "dec_return",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RB
+    "dec_return_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_WB
+    "dec_return_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB
+    "dec_return_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB
+    "dec_return_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB
+    "dec_return_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD
+    "add",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RB
+    "add_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_WB
+    "add_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_ACQB
+    "add_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RELB
+    "add_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB
+    "add_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC
+    "inc",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RB
+    "inc_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_WB
+    "inc_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_ACQB
+    "inc_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RELB
+    "inc_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB
+    "inc_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC
+    "dec",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RB
+    "dec_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_WB
+    "dec_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_ACQB
+    "dec_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RELB
+    "dec_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB
+    "dec_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD
+    "and_retold",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RB
+    "and_retold_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_WB
+    "and_retold_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB
+    "and_retold_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB
+    "and_retold_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB
+    "and_retold_mb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD
+    "or_retold",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RB
+    "or_retold_rb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_WB
+    "or_retold_wb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB
+    "or_retold_acqb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB
+    "or_retold_relb",
+#endif
+#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB
+    "or_retold_mb",
+#endif
+    NULL
+};
+
+char **
+ethr_native_atomic32_ops(void)
+{
+
+    return &native_atomic32_ops[0];
+}
diff --git a/erts/lib_src/common/ethr_aux.c b/erts/lib_src/common/ethr_aux.c
new file mode 100644
index 0000000000..89149b716b
--- /dev/null
+++ b/erts/lib_src/common/ethr_aux.c
@@ -0,0 +1,655 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: A Thread library for use in the ERTS and other OTP
+ *              applications.
+ * Author: Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+
+#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHR_AUX_IMPL__
+#include "ethread.h"
+#include "ethr_internal.h"
+#include <string.h>
+#include <limits.h>
+
+#ifndef __WIN32__
+#include <unistd.h>
+#endif
+
+#define ERTS_TS_EV_ALLOC_DEFAULT_POOL_SIZE 4000
+#define ERTS_TS_EV_ALLOC_POOL_SIZE 25
+
+erts_cpu_info_t *ethr_cpu_info__;
+
+int ethr_not_completely_inited__ = 1;
+int ethr_not_inited__ = 1;
+
+ethr_memory_allocators ethr_mem__ = ETHR_MEM_ALLOCS_DEF_INITER__;
+
+void *(*ethr_thr_prepare_func__)(void) = NULL;
+void (*ethr_thr_parent_func__)(void *) = NULL;
+void (*ethr_thr_child_func__)(void *) = NULL;
+
+typedef struct ethr_xhndl_list_ ethr_xhndl_list;
+struct ethr_xhndl_list_ {
+    ethr_xhndl_list *next;
+    void (*funcp)(void);
+};
+
+size_t ethr_pagesize__;
+size_t ethr_min_stack_size__; /* kilo words */
+size_t ethr_max_stack_size__; /* kilo words */
+
+ethr_rwmutex xhndl_rwmtx;
+ethr_xhndl_list *xhndl_list;
+
+static int main_threads;
+
+static int init_ts_event_alloc(void);
+
+ethr_runtime_t ethr_runtime__
+#ifdef __GNUC__
+__attribute__ ((aligned (ETHR_CACHE_LINE_SIZE)))
+#endif
+    ;
+
+#if defined(ETHR_X86_RUNTIME_CONF__)
+
+/*
+ * x86/x86_64 specifics shared between windows and
+ * pthread implementations.
+ */
+
+#define ETHR_IS_X86_VENDOR(V, B, C, D) \
+   (sizeof(V) == 13 && is_x86_vendor((V), (B), (C), (D)))
+
+static ETHR_INLINE int
+is_x86_vendor(char *str, int ebx, int ecx, int edx)
+{
+    return (*((int *) &str[0]) == ebx
+	    && *((int *) &str[sizeof(int)]) == edx
+	    && *((int *) &str[sizeof(int)*2]) == ecx);
+}
+
+static void
+x86_init(void)
+{
+    int eax, ebx, ecx, edx;
+
+    eax = ebx = ecx = edx = 0;
+
+    ethr_x86_cpuid__(&eax, &ebx, &ecx, &edx);
+
+    if (eax > 0
+	&& (ETHR_IS_X86_VENDOR("GenuineIntel", ebx, ecx, edx)
+	    || ETHR_IS_X86_VENDOR("AuthenticAMD", ebx, ecx, edx))) {
+	eax = 1;
+	ethr_x86_cpuid__(&eax, &ebx, &ecx, &edx);
+    }
+    else {
+	/*
+	 * The meaning of the feature flags for this
+	 * vendor have not been verified.
+	 */
+	eax = ebx = ecx = edx = 0;
+    }
+
+    /*
+     * The feature flags tested below have only been verified
+     * for vendors checked above. Also note that only these
+     * feature flags have been verified to have these specific
+     * meanings. If another feature flag test is introduced,
+     * it has to be verified to have the same meaning for all
+     * vendors above.
+     */
+
+#if ETHR_SIZEOF_PTR == 8
+    /* bit 13 of ecx is set if we have cmpxchg16b */
+    ethr_runtime__.conf.have_dw_cmpxchg = (ecx & (1 << 13));
+#elif ETHR_SIZEOF_PTR == 4
+    /* bit 8 of edx is set if we have cmpxchg8b */
+    ethr_runtime__.conf.have_dw_cmpxchg = (edx & (1 << 8));
+#else
+#  error "Not supported"
+#endif
+    /* bit 26 of edx is set if we have sse2 */
+    ethr_runtime__.conf.have_sse2 = (edx & (1 << 26));
+}
+
+#endif /* ETHR_X86_RUNTIME_CONF__ */
+
+
+int
+ethr_init_common__(ethr_init_data *id)
+{
+    int res;
+
+#if defined(ETHR_X86_RUNTIME_CONF__)
+    x86_init();
+#endif
+
+    if (id) {
+	ethr_thr_prepare_func__	= id->thread_create_prepare_func;
+	ethr_thr_parent_func__	= id->thread_create_parent_func;
+	ethr_thr_child_func__	= id->thread_create_child_func;
+    }
+
+    ethr_cpu_info__ = erts_cpu_info_create();
+    if (!ethr_cpu_info__)
+	return ENOMEM;
+
+#ifdef _SC_PAGESIZE
+    ethr_pagesize__ = (size_t) sysconf(_SC_PAGESIZE);
+#elif defined(HAVE_GETPAGESIZE)
+    ethr_pagesize__ = (size_t) getpagesize();
+#else
+    ethr_pagesize__ = (size_t) 4*1024; /* Guess 4 KB */
+#endif
+
+    /* User needs at least 4 KB */
+    ethr_min_stack_size__ = 4*1024;
+#if SIZEOF_VOID_P == 8
+    /* Double that on 64-bit archs */
+    ethr_min_stack_size__ *= 2;
+#endif
+    /* On some systems as much as about 4 KB is used by the system */
+    ethr_min_stack_size__ += 4*1024;
+    /* There should be room for signal handlers */
+#ifdef SIGSTKSZ
+    ethr_min_stack_size__ += SIGSTKSZ;
+#else
+    ethr_min_stack_size__ += ethr_pagesize__;
+#endif
+    /* The system may think that we need more stack */
+#if defined(PTHREAD_STACK_MIN)
+    if (ethr_min_stack_size__ < PTHREAD_STACK_MIN)
+	ethr_min_stack_size__ = PTHREAD_STACK_MIN;
+#elif defined(_SC_THREAD_STACK_MIN)
+    {
+	size_t thr_min_stk_sz = (size_t) sysconf(_SC_THREAD_STACK_MIN);
+	if (ethr_min_stack_size__ < thr_min_stk_sz)
+	    ethr_min_stack_size__ = thr_min_stk_sz;
+    }
+#endif
+    /* The guard is at least on some platforms included in the stack size
+       passed when creating threads */
+#ifdef ETHR_STACK_GUARD_SIZE
+    ethr_min_stack_size__ += ETHR_STACK_GUARD_SIZE;
+#endif
+    ethr_min_stack_size__ = ETHR_PAGE_ALIGN(ethr_min_stack_size__);
+
+    ethr_min_stack_size__ = ETHR_B2KW(ethr_min_stack_size__);
+
+    ethr_max_stack_size__ = 32*1024*1024;
+#if SIZEOF_VOID_P == 8
+    ethr_max_stack_size__ *= 2;
+#endif
+    ethr_max_stack_size__ = ETHR_B2KW(ethr_max_stack_size__);
+
+    res = ethr_init_atomics();
+    if (res != 0)
+	return res;
+
+    res = ethr_mutex_lib_init(erts_get_cpu_configured(ethr_cpu_info__));
+    if (res != 0)
+	return res;
+
+    xhndl_list = NULL;
+
+    return 0;
+}
+
+int
+ethr_late_init_common__(ethr_late_init_data *lid)
+{
+    ethr_ts_event *tsep = NULL;
+    int reader_groups;
+    int res;
+    int i;
+    ethr_memory_allocator *m[] = {&ethr_mem__.std,
+				  &ethr_mem__.sl,
+				  &ethr_mem__.ll};
+    if (lid)
+	ethr_mem__ = lid->mem;
+    if (!ethr_mem__.std.alloc
+	|| !ethr_mem__.std.realloc
+	|| !ethr_mem__.std.free) {
+	ethr_mem__.std.alloc = malloc;
+	ethr_mem__.std.realloc = realloc;
+	ethr_mem__.std.free = free;
+    }
+    for (i = 0; i < sizeof(m)/sizeof(m[0]); i++) {
+	if (!m[i]->alloc || !m[i]->realloc || !m[i]->free) {
+	    m[i]->alloc = ethr_mem__.std.alloc;
+	    m[i]->realloc = ethr_mem__.std.realloc;
+	    m[i]->free = ethr_mem__.std.free;
+	}
+
+    }
+    res = init_ts_event_alloc();
+    if (res != 0)
+	return res;
+    res = ethr_make_ts_event__(&tsep);
+    if (res == 0)
+	tsep->iflgs |= ETHR_TS_EV_ETHREAD;
+    if (!lid) {
+	main_threads = 0;
+	reader_groups = 0;
+    }
+    else {
+	if (lid->main_threads < 0 || USHRT_MAX < lid->main_threads)
+	    return res;
+	main_threads = lid->main_threads;
+	reader_groups = lid->reader_groups;
+    }
+    res = ethr_mutex_lib_late_init(reader_groups, main_threads);
+    if (res != 0)
+	return res;
+    ethr_not_completely_inited__ = 0; /* Need it for
+					 rwmutex_init */
+    res = ethr_rwmutex_init(&xhndl_rwmtx);
+    ethr_not_completely_inited__ = 1;
+    if (res != 0)
+	return res;
+    return 0;
+}
+
+int
+ethr_install_exit_handler(void (*funcp)(void))
+{
+    ethr_xhndl_list *xhp;
+
+#if ETHR_XCHK
+    if (ethr_not_completely_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+
+    if (!funcp)
+	return EINVAL;
+
+    xhp = (ethr_xhndl_list *) ethr_mem__.std.alloc(sizeof(ethr_xhndl_list));
+    if (!xhp)
+	return ENOMEM;
+
+    ethr_rwmutex_rwlock(&xhndl_rwmtx);
+
+    xhp->funcp = funcp;
+    xhp->next = xhndl_list;
+    xhndl_list = xhp;
+
+    ethr_rwmutex_rwunlock(&xhndl_rwmtx);
+
+    return 0;
+}
+
+void
+ethr_run_exit_handlers__(void)
+{
+    ethr_xhndl_list *xhp;
+
+    ethr_rwmutex_rlock(&xhndl_rwmtx);
+
+    xhp = xhndl_list;
+
+    ethr_rwmutex_runlock(&xhndl_rwmtx);
+
+    for (; xhp; xhp = xhp->next)
+	(*xhp->funcp)();
+}
+
+/*
+ * Thread specific event alloc, etc.
+ *
+ * Note that we don't know when it is safe to destroy an event, but
+ * we know when it is safe to reuse it. ts_event_free() therefore
+ * never destroys an event (but makes freed events available for
+ * reuse).
+ *
+ * We could easily keep track of the usage of events, and by this
+ * make it possible to destroy events. We would however suffer a
+ * performance penalty for this and save very little memory.
+ */
+
+typedef union {
+    ethr_ts_event ts_ev;
+    char align[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_ts_event))];
+} ethr_aligned_ts_event;
+
+static ethr_spinlock_t ts_ev_alloc_lock;
+static ethr_ts_event *free_ts_ev;
+
+static ethr_ts_event *ts_event_pool(int size, ethr_ts_event **endpp)
+{
+    int i;
+    ethr_aligned_ts_event *atsev;
+    atsev = ethr_mem__.std.alloc(sizeof(ethr_aligned_ts_event) * size
+				 + ETHR_CACHE_LINE_SIZE);
+    if (!atsev)
+	return NULL;
+    if ((((ethr_uint_t) atsev) & ETHR_CACHE_LINE_MASK) == 0)
+	atsev = ((ethr_aligned_ts_event *)
+		 ((((ethr_uint_t) atsev) & ~ETHR_CACHE_LINE_MASK)
+		  + ETHR_CACHE_LINE_SIZE));
+    for (i = 1; i < size; i++) {
+	atsev[i-1].ts_ev.next = &atsev[i].ts_ev;
+	ethr_atomic32_init(&atsev[i-1].ts_ev.uaflgs, 0);
+	atsev[i-1].ts_ev.iflgs = 0;
+    }
+    ethr_atomic32_init(&atsev[size-1].ts_ev.uaflgs, 0);
+    atsev[size-1].ts_ev.iflgs = 0;
+    atsev[size-1].ts_ev.next = NULL;
+    if (endpp)
+	*endpp = &atsev[size-1].ts_ev;
+    return &atsev[0].ts_ev;
+}
+
+static int init_ts_event_alloc(void)
+{
+    free_ts_ev = ts_event_pool(ERTS_TS_EV_ALLOC_DEFAULT_POOL_SIZE,
+			       NULL);
+    if (!free_ts_ev)
+	return ENOMEM;
+    return ethr_spinlock_init(&ts_ev_alloc_lock);
+}
+
+static ethr_ts_event *ts_event_alloc(void)
+{
+    ethr_ts_event *ts_ev;
+    ethr_spin_lock(&ts_ev_alloc_lock);
+    if (free_ts_ev) {
+	ts_ev = free_ts_ev;
+	free_ts_ev = ts_ev->next;
+	ethr_spin_unlock(&ts_ev_alloc_lock);
+    }
+    else {
+	ethr_ts_event *ts_ev_pool_end;
+	ethr_spin_unlock(&ts_ev_alloc_lock);
+
+	ts_ev = ts_event_pool(ERTS_TS_EV_ALLOC_POOL_SIZE, &ts_ev_pool_end);
+	if (!ts_ev)
+	    return NULL;
+
+	ethr_spin_lock(&ts_ev_alloc_lock);
+	ts_ev_pool_end->next = free_ts_ev;
+	free_ts_ev = ts_ev->next;
+	ethr_spin_unlock(&ts_ev_alloc_lock);
+    }
+    return ts_ev;
+}
+
+static void ts_event_free(ethr_ts_event *ts_ev)
+{
+    ETHR_ASSERT(!ts_ev->udata);
+    ethr_spin_lock(&ts_ev_alloc_lock);
+    ts_ev->next = free_ts_ev;
+    free_ts_ev = ts_ev;
+    ethr_spin_unlock(&ts_ev_alloc_lock);
+}
+
+int ethr_make_ts_event__(ethr_ts_event **tsepp)
+{
+    int res;
+    ethr_ts_event *tsep = *tsepp;
+
+    if (!tsep) {
+	tsep = ts_event_alloc();
+	if (!tsep)
+	    return ENOMEM;
+    }
+
+    if ((tsep->iflgs & ETHR_TS_EV_INITED) == 0) {
+	res = ethr_event_init(&tsep->event);
+	if (res != 0) {
+	    ts_event_free(tsep);
+	    return res;
+	}
+    }
+
+    tsep->iflgs = ETHR_TS_EV_INITED;
+    tsep->udata = NULL;
+    tsep->rgix = 0;
+    tsep->mtix = 0;
+
+    res = ethr_set_tse__(tsep);
+    if (res != 0 && tsepp && *tsepp) {
+	ts_event_free(tsep);
+	return res;
+    }
+
+    if (tsepp)
+	*tsepp = tsep;
+
+    return 0;
+}
+
+int ethr_get_tmp_ts_event__(ethr_ts_event **tsepp)
+{
+    int res;
+    ethr_ts_event *tsep = *tsepp;
+
+    if (!tsep) {
+	tsep = ts_event_alloc();
+	if (!tsep)
+	    return ENOMEM;
+    }
+
+    if ((tsep->iflgs & ETHR_TS_EV_INITED) == 0) {
+	res = ethr_event_init(&tsep->event);
+	if (res != 0) {
+	    ts_event_free(tsep);
+	    return res;
+	}
+    }
+
+    tsep->iflgs = ETHR_TS_EV_INITED|ETHR_TS_EV_TMP;
+    tsep->udata = NULL;
+
+    if (tsepp)
+	*tsepp = tsep;
+
+    return 0;
+}
+
+int ethr_free_ts_event__(ethr_ts_event *tsep)
+{
+    ts_event_free(tsep);
+    return 0;
+}
+
+void ethr_ts_event_destructor__(void *vtsep)
+{
+    if (vtsep) {
+	ethr_ts_event *tsep = (ethr_ts_event *) vtsep;
+	ts_event_free(tsep);
+	ethr_set_tse__(NULL);
+    }
+}
+
+int ethr_set_main_thr_status(int on, int no)
+{
+    ethr_ts_event *tsep = ethr_get_tse__();
+    if (!tsep)
+	return EINVAL;
+    if (on) {
+	if (no < 1 || main_threads < no)
+	    return EINVAL;
+	tsep->mtix = (unsigned short) no;
+	tsep->iflgs |= ETHR_TS_EV_MAIN_THR;
+    }
+    else {
+	tsep->iflgs &= ~ETHR_TS_EV_MAIN_THR;
+	tsep->mtix = (unsigned short) 0;
+    }
+    return 0;
+}
+
+int ethr_get_main_thr_status(int *on)
+{
+    ethr_ts_event *tsep = ethr_get_tse__();
+    if (!tsep)
+	*on = 0;
+    else {
+	if (tsep->iflgs & ETHR_TS_EV_MAIN_THR)
+	    *on = 1;
+	else
+	    *on = 0;
+    }
+    return 0;
+}
+
+/* Spinlocks and rwspinlocks */
+
+int
+ethr_spinlock_init(ethr_spinlock_t *lock)
+{
+#if ETHR_XCHK 
+    if (!lock) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    return ethr_spinlock_init__(lock);
+}
+
+int
+ethr_spinlock_destroy(ethr_spinlock_t *lock)
+{
+#if ETHR_XCHK 
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!lock) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    return ethr_spinlock_destroy__(lock);
+}
+
+void
+ethr_spin_unlock(ethr_spinlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_spin_unlock__(lock);
+}
+
+void
+ethr_spin_lock(ethr_spinlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_spin_lock__(lock);
+}
+
+int
+ethr_rwlock_init(ethr_rwlock_t *lock)
+{
+#if ETHR_XCHK 
+    if (!lock) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    return ethr_rwlock_init__(lock);
+}
+
+int
+ethr_rwlock_destroy(ethr_rwlock_t *lock)
+{
+#if ETHR_XCHK 
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!lock) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    return ethr_rwlock_destroy__(lock);
+}
+
+void
+ethr_read_unlock(ethr_rwlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_read_unlock__(lock);
+}
+
+void
+ethr_read_lock(ethr_rwlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_read_lock__(lock);
+}
+
+void
+ethr_write_unlock(ethr_rwlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_write_unlock__(lock);
+}
+
+void
+ethr_write_lock(ethr_rwlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_write_lock__(lock);
+}
+
+ETHR_IMPL_NORETURN__ ethr_fatal_error__(const char *file,
+					int line,
+					const char *func,
+					int err)
+{
+    char *errstr;
+    if (err == ENOTSUP)
+	errstr = "Operation not supported";
+    else {
+	errstr = strerror(err);
+	if (!errstr)
+	    errstr = "Unknown error";
+    }
+    fprintf(stderr, "%s:%d: Fatal error in %s(): %s (%d)\n",
+	    file, line, func, errstr, err);
+    ethr_abort__();
+}
+
+int ethr_assert_failed(const char *file, int line, const char *func, char *a)
+{
+    fprintf(stderr, "%s:%d: %s(): Assertion failed: %s\n", file, line, func, a);
+    ethr_abort__();
+    return 0;
+}
diff --git a/erts/lib_src/common/ethr_cbf.c b/erts/lib_src/common/ethr_cbf.c
new file mode 100644
index 0000000000..04feceec89
--- /dev/null
+++ b/erts/lib_src/common/ethr_cbf.c
@@ -0,0 +1,36 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+
+/*
+ * We keep this function alone in a separate file so the
+ * compiler wont optimize it away.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "ethread.h"
+
+void
+ethr_compiler_barrier_fallback(void)
+{
+
+}
diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c
new file mode 100644
index 0000000000..e363279f2e
--- /dev/null
+++ b/erts/lib_src/common/ethr_mutex.c
@@ -0,0 +1,3311 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Mutex, rwmutex and condition variable implementation
+ * Author: Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#define ETHR_INLINE_MTX_FUNC_NAME_(X) X ## __
+#define ETHR_MUTEX_IMPL__
+#define ETHR_TRY_INLINE_FUNCS
+
+#include <limits.h>
+#include "ethread.h"
+#include "ethr_internal.h"
+
+#define ETHR_SPIN_WITH_WAITERS 1
+
+#define ETHR_MTX_MAX_FLGS_SPIN 10
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+static int default_rwmtx_main_spincount;
+static int default_rwmtx_aux_spincount;
+#endif
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+static int default_mtx_main_spincount;
+static int default_mtx_aux_spincount;
+static int default_cnd_main_spincount;
+static int default_cnd_aux_spincount;
+#endif
+
+static int no_spin;
+
+#ifndef ETHR_USE_OWN_RWMTX_IMPL__
+static pthread_rwlockattr_t write_pref_attr_data;
+static pthread_rwlockattr_t *write_pref_attr;
+#endif
+
+#if defined(ETHR_MTX_Q_LOCK_SPINLOCK__)
+#  define ETHR_MTX_QLOCK_INIT ethr_spinlock_init
+#  define ETHR_MTX_QLOCK_DESTROY ethr_spinlock_destroy
+#  define ETHR_MTX_Q_LOCK ethr_spin_lock
+#  define ETHR_MTX_Q_UNLOCK ethr_spin_unlock
+#elif defined(ETHR_MTX_Q_LOCK_PTHREAD_MUTEX__)
+#  define ETHR_MTX_QLOCK_INIT(QL) pthread_mutex_init((QL), NULL)
+#  define ETHR_MTX_QLOCK_DESTROY pthread_mutex_destroy
+#  define ETHR_MTX_Q_LOCK(L)					\
+do {								\
+    int res__ = pthread_mutex_lock(L);				\
+    if (res__ != 0)						\
+	ETHR_FATAL_ERROR__(res__);				\
+} while (0)
+#  define ETHR_MTX_Q_UNLOCK(L)					\
+do {								\
+    int res__ = pthread_mutex_unlock(L);			\
+    if (res__ != 0)						\
+	ETHR_FATAL_ERROR__(res__);				\
+} while (0)
+#elif defined(ETHR_MTX_Q_LOCK_CRITICAL_SECTION__)
+#  define ETHR_MTX_QLOCK_INIT(QL) (InitializeCriticalSection((QL)), 0)
+#  define ETHR_MTX_QLOCK_DESTROY(QL) (DeleteCriticalSection((QL)), 0)
+#  define ETHR_MTX_Q_LOCK(QL) EnterCriticalSection((QL))
+#  define ETHR_MTX_Q_UNLOCK(QL) LeaveCriticalSection((QL))
+#endif
+
+int
+ethr_mutex_lib_init(int cpu_conf)
+{
+    int res = 0;
+
+    no_spin = cpu_conf == 1;
+
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+    default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_BASE;
+    default_mtx_aux_spincount = ETHR_MTX_DEFAULT_AUX_SPINCOUNT;
+    default_cnd_main_spincount = ETHR_CND_DEFAULT_MAIN_SPINCOUNT;
+    default_cnd_aux_spincount = ETHR_CND_DEFAULT_AUX_SPINCOUNT;
+#endif
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+
+    default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_BASE;
+    default_rwmtx_aux_spincount = ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT;
+
+#else
+
+#if defined(ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP) \
+    && defined(ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP)
+    res = pthread_rwlockattr_init(&write_pref_attr_data);
+    if (res != 0)
+	return res;
+    res = pthread_rwlockattr_setkind_np(
+	&write_pref_attr_data,
+	PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
+    write_pref_attr = &write_pref_attr_data;
+#else
+    write_pref_attr = NULL;
+#endif
+
+#endif
+
+    return res;
+}
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+
+#ifdef ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS
+#if 0 /*
+       * When inc and dec are real atomic instructions as on x86, the
+       * ETHR_RLOCK_WITH_INC_DEC implementations performs better with
+       * lots of read locks compared to the cmpxchg based implementation.
+       * It, however, performs worse with lots of mixed reads and writes.
+       * It could be used for rwlocks that are known to be read locked
+       * much, but the readers array based implementation outperforms it
+       * by far. Therefore, it has been disabled, and will probably be
+       * removed some time in the future.
+       */
+#  define ETHR_RLOCK_WITH_INC_DEC
+#endif
+#endif
+
+static int reader_groups_array_size = 0;
+static int main_threads_array_size = 0;
+
+#endif
+
+int
+ethr_mutex_lib_late_init(int no_reader_groups, int no_main_threads)
+{
+
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+    default_mtx_main_spincount += (no_main_threads
+				   * ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_INC);
+    if (default_mtx_main_spincount > ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX)
+	default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX;
+#endif
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+
+    default_rwmtx_main_spincount += (no_main_threads
+				     * ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_INC);
+    if (default_rwmtx_main_spincount > ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX)
+	default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX;
+
+    reader_groups_array_size = (no_reader_groups <= 1
+				? 1
+				: no_reader_groups + 1);
+    main_threads_array_size = (no_main_threads <= 1
+			       ? 1
+			       : no_main_threads + 1);
+#endif
+    return 0;
+}
+
+int
+ethr_rwmutex_set_reader_group(int ix)
+{
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+    ethr_ts_event *tse;
+
+    if (ix < 0 || reader_groups_array_size <= ix)
+	return EINVAL;
+
+    tse = ethr_get_ts_event();
+
+    if ((tse->iflgs & ETHR_TS_EV_ETHREAD) == 0) {
+	ethr_leave_ts_event(tse);
+	return EINVAL;
+    }
+
+    tse->rgix = ix;
+    
+    ethr_leave_ts_event(tse);
+#endif
+    return 0;
+}
+
+#if defined(ETHR_MTX_HARD_DEBUG_Q) || defined(ETHR_MTX_HARD_DEBUG_WSQ)
+static void hard_debug_chk_q__(struct ethr_mutex_base_ *, int);
+#define ETHR_RWMTX_HARD_DEBUG_CHK_Q(RWMTX) hard_debug_chk_q__(&(RWMTX)->mtxb,1)
+#define ETHR_MTX_HARD_DEBUG_CHK_Q(MTX) hard_debug_chk_q__(&(MTX)->mtxb, 0)
+#else
+#define ETHR_RWMTX_HARD_DEBUG_CHK_Q(RWMTX)
+#define ETHR_MTX_HARD_DEBUG_CHK_Q(MTX)
+#endif
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+static void
+rwmutex_transfer_read_lock(ethr_rwmutex *rwmtx,
+			   ethr_sint32_t initial,
+			   int q_locked);
+static void
+rwmutex_unlock_wake(ethr_rwmutex *rwmtx,
+		    int have_w,
+		    ethr_sint32_t initial,
+		    int transfer_read_lock);
+static int
+rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
+			     ethr_sint32_t initial,
+			     ethr_ts_event *tse,
+			     int start_next_ix,
+			     int check_before_try,
+			     int try_write_lock);
+#endif
+
+/* -- Utilities used by multiple implementations -- */
+
+#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__) \
+    || defined(ETHR_WIN32_THREADS)
+
+static ETHR_INLINE void
+enqueue(ethr_ts_event **queue,
+	ethr_ts_event *tse_start,
+	ethr_ts_event *tse_end)
+{
+    if (!*queue) {
+	*queue = tse_start;
+	tse_start->prev = tse_end;
+	tse_end->next = tse_start;
+    }
+    else {
+	tse_end->next = *queue;
+	tse_start->prev = (*queue)->prev;
+	(*queue)->prev->next = tse_start;
+	(*queue)->prev = tse_end;
+    }
+}
+
+
+static ETHR_INLINE void
+dequeue(ethr_ts_event **queue,
+	ethr_ts_event *tse_start,
+	ethr_ts_event *tse_end)
+{
+    if (tse_start->prev == tse_end) {
+	ETHR_ASSERT(*queue == tse_start && tse_end->next == tse_start);
+	*queue = NULL;
+    }
+    else {
+	if (*queue == tse_start)
+	    *queue = tse_end->next;
+	tse_end->next->prev = tse_start->prev;
+	tse_start->prev->next = tse_end->next;
+    }
+}
+
+#endif
+
+#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__)
+
+static ETHR_INLINE void
+insert(ethr_ts_event *tse_pred, ethr_ts_event *tse)
+{
+    tse->next = tse_pred->next;
+    tse->prev = tse_pred;
+    tse_pred->next->prev = tse;
+    tse_pred->next = tse;
+}
+
+static ETHR_INLINE void
+rwmutex_freqread_wtng_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix = (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ
+	      ? tse->rgix
+	      : tse->mtix);
+    rwmtx->tdata.ra[ix].data.waiting_readers++;
+}
+
+static ETHR_INLINE void
+rwmutex_freqread_rdrs_add(ethr_rwmutex *rwmtx,
+			  ethr_rwmutex_type type,
+			  int ix,
+			  int inc)
+{
+    if (type == ETHR_RWMUTEX_TYPE_FREQUENT_READ || ix == 0)
+	ethr_atomic32_add(&rwmtx->tdata.ra[ix].data.readers, inc);
+    else {
+	ETHR_ASSERT(type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 0);
+	ETHR_ASSERT(inc == 1);
+	ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 1);
+    }
+}
+
+static ETHR_INLINE void
+rwmutex_freqread_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix;
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	ix = tse->rgix;
+    atomic_inc:
+	ethr_atomic32_inc(&rwmtx->tdata.ra[ix].data.readers);
+    }
+    else {
+	ix = tse->mtix;
+	if (ix == 0)
+	    goto atomic_inc;
+	ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 0);
+	ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 1);
+    }
+}
+
+#if 0 /* Not used */
+
+static ETHR_INLINE void
+rwmutex_freqread_rdrs_dec(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix;
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	ix = tse->rgix;
+    atomic_dec:
+	ethr_atomic32_dec(&rwmtx->tdata.ra[ix].data.readers);
+    }
+    else {
+	ix = tse->mtix;
+	if (ix == 0)
+	    goto atomic_dec;
+	ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
+	ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 0);
+    }
+}
+
+#endif
+
+static ETHR_INLINE ethr_sint32_t
+rwmutex_freqread_rdrs_dec_read(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix;
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	ix = tse->rgix;
+    atomic_dec_read:
+	return ethr_atomic32_dec_read(&rwmtx->tdata.ra[ix].data.readers);
+    }
+    else {
+	ix = tse->mtix;
+	if (ix == 0)
+	    goto atomic_dec_read;
+	ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
+	ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 0);
+	return (ethr_sint32_t) 0;
+    }
+}
+
+static ETHR_INLINE ethr_sint32_t
+rwmutex_freqread_rdrs_dec_read_relb(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix;
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	ix = tse->rgix;
+    atomic_dec_read:
+	return ethr_atomic32_dec_read_relb(&rwmtx->tdata.ra[ix].data.readers);
+    }
+    else {
+	ix = tse->mtix;
+	if (ix == 0)
+	    goto atomic_dec_read;
+	ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
+	ethr_atomic32_set_relb(&rwmtx->tdata.ra[ix].data.readers,
+			     (ethr_sint32_t) 0);
+	return (ethr_sint32_t) 0;
+    }
+}
+
+static ETHR_INLINE ethr_sint32_t
+rwmutex_freqread_rdrs_read(ethr_rwmutex *rwmtx, int ix)
+{
+    ethr_sint32_t res = ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers);
+#ifdef ETHR_DEBUG
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+	ETHR_ASSERT(res >= 0);
+	break;
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
+	ETHR_ASSERT(ix == 0 ? res >= 0 : (res == 0 || res == 1));
+	break;
+    default:
+	ETHR_ASSERT(0);
+	break;
+    }
+#endif
+    return res;
+}
+
+static void
+event_wait(struct ethr_mutex_base_ *mtxb,
+	   ethr_ts_event *tse,
+	   int spincount,
+	   ethr_sint32_t type,
+	   int is_rwmtx,
+	   int is_freq_read)
+{
+    int locked = 0;
+    ethr_sint32_t act;
+    int need_try_complete_runlock = 0;
+    int transfer_read_lock = 0;
+
+    /* Need to enqueue and wait... */
+
+    tse->uflgs = type;
+    ethr_atomic32_set(&tse->uaflgs, type);
+
+    ETHR_MTX_Q_LOCK(&mtxb->qlck);
+    locked = 1;
+
+#ifdef ETHR_MTX_HARD_DEBUG_Q
+    hard_debug_chk_q__(mtxb, is_rwmtx);
+#endif
+
+    act = ethr_atomic32_read(&mtxb->flgs);
+
+    if (act & type) {
+
+	/* Wait bit already there; enqueue... */
+
+	ETHR_ASSERT(mtxb->q);
+	if (type == ETHR_RWMTX_W_WAIT_FLG__) {
+	    enqueue(&mtxb->q, tse, tse);
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+	    mtxb->ws++;
+#endif
+	}
+	else {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_ASSERT(is_rwmtx);
+	    ETHR_ASSERT(rwmtx->rq_end);
+	    insert(rwmtx->rq_end, tse);
+	    rwmtx->rq_end = tse;
+	    if (is_freq_read)
+		rwmutex_freqread_wtng_rdrs_inc(rwmtx, tse);
+	    else
+		rwmtx->tdata.rs++;
+	}
+    }
+    else {
+
+	/* Set wait bit */
+
+	while (1) {
+	    ethr_sint32_t new, exp = act;
+	    need_try_complete_runlock = 0;
+	    transfer_read_lock = 0;
+
+	    if (type == ETHR_RWMTX_W_WAIT_FLG__) {
+		if (is_freq_read && act == ETHR_RWMTX_R_FLG__)
+		    need_try_complete_runlock = 1;
+		if (act != 0)
+		    new = act | ETHR_RWMTX_W_WAIT_FLG__;
+		else
+		    new = ETHR_RWMTX_W_FLG__; /* Try to get it */
+	    }
+	    else {
+		ETHR_ASSERT(is_rwmtx);
+
+		if (!is_freq_read) {
+		    if (act & (ETHR_RWMTX_W_FLG__| ETHR_RWMTX_W_WAIT_FLG__))
+			new = act | ETHR_RWMTX_R_WAIT_FLG__;
+		    else
+			new = act + 1; /* Try to get it */
+		}
+		else {
+		    new = act | ETHR_RWMTX_R_WAIT_FLG__;
+		    if ((act & (ETHR_RWMTX_W_FLG__
+				| ETHR_RWMTX_W_WAIT_FLG__)) == 0) {
+			/* Transfer read lock to this thread. */
+			transfer_read_lock = 1;
+		    }
+		}
+	    }
+
+	    act = ethr_atomic32_cmpxchg_acqb(&mtxb->flgs, new, exp);
+	    if (exp == act) {
+		if (new & type) {
+		    act = new;
+		    break;
+		}
+		else {
+		    /* Got it */
+		    goto done;
+		}
+	    }
+	}
+
+	/* Enqueue */
+
+	if (type == ETHR_RWMTX_R_WAIT_FLG__) {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_ASSERT(is_rwmtx);
+	    ETHR_ASSERT(!rwmtx->rq_end);
+	    rwmtx->rq_end =  tse;
+	    if (is_freq_read)
+		rwmutex_freqread_wtng_rdrs_inc(rwmtx, tse);
+	    else
+		rwmtx->tdata.rs++;
+	}
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+	else {
+	    mtxb->ws++;
+	}
+#endif
+
+	enqueue(&mtxb->q, tse, tse);
+    }
+
+#ifdef ETHR_MTX_HARD_DEBUG_Q
+    hard_debug_chk_q__(mtxb, is_rwmtx);
+#endif
+
+    /* Wait */
+    locked = 0;
+
+    ETHR_ASSERT(!(transfer_read_lock && need_try_complete_runlock));
+
+    if (transfer_read_lock) {
+	ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type
+		    != ETHR_RWMUTEX_TYPE_NORMAL);
+	/*
+	 * We are the only one in the queue and we are not write
+	 * locked; rwmutex_transfer_read_lock() will:
+	 * - transfer a read lock to us (since we're first in q)
+	 * - unlock the Q-lock
+	 */
+	rwmutex_transfer_read_lock(((ethr_rwmutex *) mtxb), act, 1);
+    }
+    else {
+	ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
+
+	if (need_try_complete_runlock) {
+	    ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type
+			!= ETHR_RWMUTEX_TYPE_NORMAL);
+	    /*
+	     * We were the only one in queue when we enqueued, and it
+	     * was seemingly read locked. We need to try to complete a
+	     * runlock otherwise we might be hanging forever. If the
+	     * runlock could be completed we will be dequeued and
+	     * woken by ourselves.
+	     */
+	    rwmutex_try_complete_runlock((ethr_rwmutex *) mtxb,
+					 act, tse, 0, 1, 0);
+	}
+    }
+
+    while (1) {
+	ethr_event_reset(&tse->event);
+
+	act = ethr_atomic32_read_acqb(&tse->uaflgs);
+	if (!act)
+	    goto done; /* Got it */
+
+	ETHR_ASSERT(act == type);
+	ethr_event_swait(&tse->event, spincount);
+	/* swait result: 0 || EINTR */
+
+	act = ethr_atomic32_read_acqb(&tse->uaflgs);
+	if (!act)
+	    goto done; /* Got it */
+    }
+
+ done:
+    if (locked)
+	ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
+}
+
+static void
+wake_writer(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
+{
+    ethr_ts_event *tse;
+
+    tse = mtxb->q;
+    ETHR_ASSERT(tse);
+    dequeue(&mtxb->q, tse, tse);
+
+    ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+    ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_RWMTX_W_WAIT_FLG__);
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+    mtxb->ws--;
+#endif
+#if defined(ETHR_MTX_HARD_DEBUG_Q) || defined(ETHR_MTX_HARD_DEBUG_WSQ)
+    hard_debug_chk_q__(mtxb, is_rwmtx);
+#endif
+
+    ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
+
+    ethr_atomic32_set(&tse->uaflgs, 0);
+    ethr_event_set(&tse->event);
+}
+
+static ETHR_INLINE int
+initial_spincount(struct ethr_mutex_base_ *mtxb)
+{
+    return (mtxb->aux_scnt < ETHR_MTX_MAX_FLGS_SPIN
+	    ? mtxb->aux_scnt
+	    : ETHR_MTX_MAX_FLGS_SPIN);
+}
+
+static ETHR_INLINE int
+update_spincount(struct ethr_mutex_base_ *mtxb,
+		 ethr_ts_event *tse,
+		 int *scnt_state,
+		 int *scnt)
+{
+    int state = *scnt_state;
+    if (state <= 0) {
+	/* Here state is max spincount to do on event negated */
+	*scnt = -state;
+    }
+    else {
+	/* Here state is initial spincount made on flags */
+	*scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
+		 ? mtxb->main_scnt
+		 : mtxb->aux_scnt);
+	if (*scnt <= state)
+	    *scnt = 0;
+	else {
+	    if (*scnt <= ETHR_MTX_MAX_FLGS_SPIN)
+		*scnt_state = 0; /* No spin on event */
+	    else {
+		 /* Spin on event after... */
+		*scnt_state = -1*(*scnt - ETHR_MTX_MAX_FLGS_SPIN);
+		/* ... we have spun on flags */
+		*scnt = ETHR_MTX_MAX_FLGS_SPIN;
+	    }
+	    *scnt -= state;
+	    return 0;
+	}
+    }
+    return 1;
+}
+
+int check_readers_array(ethr_rwmutex *rwmtx,
+			int start_rix,
+			int length);
+
+static ETHR_INLINE void
+write_lock_wait(struct ethr_mutex_base_ *mtxb,
+		ethr_sint32_t initial,
+		int is_rwmtx,
+		int is_freq_read)
+{
+    ethr_sint32_t act = initial;
+    int scnt, start_scnt;
+    ethr_ts_event *tse = NULL;
+    int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+    int res;
+
+    ETHR_ASSERT(!is_freq_read || is_rwmtx);
+
+    start_scnt = scnt = initial_spincount(mtxb);
+
+    /*
+     * Spin trying to write lock for a while. If unsuccessful,
+     * wait on event.
+     */
+
+    while (1) {
+	while (act != 0) {
+
+	    if (is_freq_read && act == ETHR_RWMTX_R_FLG__) {
+		ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+		scnt--;
+		if (!tse)
+		    tse = ethr_get_ts_event();
+		res = rwmutex_try_complete_runlock(rwmtx, act,
+						   tse, 0, 0,
+						   1);
+		if (res != EBUSY)
+		    goto done; /* Got it */
+		if (scnt <= 0)
+		    goto chk_spin;
+		if (--until_yield == 0) {
+		    until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		    ETHR_YIELD();
+		}
+	    }
+
+	    if (scnt <= 0) {
+	    chk_spin:
+		scnt = 0;
+
+		if (!tse)
+		    tse = ethr_get_ts_event();
+		if (update_spincount(mtxb, tse, &start_scnt, &scnt)) {
+		    event_wait(mtxb, tse, scnt, ETHR_RWMTX_W_WAIT_FLG__,
+			       is_rwmtx, is_freq_read);
+		    goto done;
+		}
+	    }
+	    ETHR_SPIN_BODY;
+	    if (--until_yield == 0) {
+		until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		ETHR_YIELD();
+	    }
+	    act = ethr_atomic32_read(&mtxb->flgs);
+	    scnt--;
+	}
+
+	act = ethr_atomic32_cmpxchg_acqb(&mtxb->flgs,
+					 ETHR_RWMTX_W_FLG__,
+					 0);
+	if (act == 0)
+	    goto done; /* Got it */
+    }
+
+ done:
+    if (tse)
+	ethr_leave_ts_event(tse);
+}
+
+static int
+mtxb_init(struct ethr_mutex_base_ *mtxb,
+	  int def_main_scnt,
+	  int main_scnt,
+	  int def_aux_scnt,
+	  int aux_scnt)
+{
+    ETHR_MTX_HARD_DEBUG_LFS_INIT(mtxb);
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+    mtxb->ws = 0;
+#endif
+    ETHR_MTX_CHK_EXCL_INIT(mtxb);
+    if (no_spin) {
+	mtxb->main_scnt = 0;
+	mtxb->aux_scnt = 0;
+    }
+    else {
+	if (main_scnt > SHRT_MAX)
+	    mtxb->main_scnt = SHRT_MAX;
+	else if (main_scnt < 0)
+	    mtxb->main_scnt = def_main_scnt;
+	else
+	    mtxb->main_scnt = (short) main_scnt;
+	if (aux_scnt > SHRT_MAX)
+	    mtxb->aux_scnt = SHRT_MAX;
+	else if (aux_scnt < 0)
+	    mtxb->aux_scnt = def_aux_scnt;
+	else
+	    mtxb->aux_scnt = (short) aux_scnt;
+	if (mtxb->main_scnt < mtxb->aux_scnt)
+	    mtxb->main_scnt = mtxb->aux_scnt;
+
+    }
+    mtxb->q = NULL;
+    ethr_atomic32_init(&mtxb->flgs, 0);
+    return ETHR_MTX_QLOCK_INIT(&mtxb->qlck);
+}
+
+static int
+mtxb_destroy(struct ethr_mutex_base_ *mtxb)
+{
+    ethr_sint32_t act;
+    ETHR_MTX_Q_LOCK(&mtxb->qlck);
+    act = ethr_atomic32_read(&mtxb->flgs);
+    ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
+    if (act != 0)
+	return EINVAL;
+    return ETHR_MTX_QLOCK_DESTROY(&mtxb->qlck);
+}
+
+
+#endif /* ETHR_USE_OWN_RWMTX_IMPL__ || ETHR_USE_OWN_MTX_IMPL__ */
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Mutex and condition variable implementation                               *
+\*                                                                           */
+
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+
+/* -- Mutex ---------------------------------------------------------------- */
+
+int
+ethr_mutex_init_opt(ethr_mutex *mtx, ethr_mutex_opt *opt)
+{
+    int res;
+#if ETHR_XCHK
+    if (!mtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    mtx->initialized = ETHR_MUTEX_INITIALIZED;
+#endif
+    ETHR_MTX_HARD_DEBUG_FENCE_INIT(mtx);
+    res = mtxb_init(&mtx->mtxb,
+		    default_mtx_main_spincount,
+		    opt ? opt->main_spincount : -1,
+		    default_mtx_aux_spincount,
+		    opt ? opt->aux_spincount : -1);
+#if ETHR_XCHK
+    if (res != 0)
+	mtx->initialized = 0;
+#endif
+    return res;
+}
+
+int
+ethr_mutex_init(ethr_mutex *mtx)
+{
+    return ethr_mutex_init_opt(mtx, NULL);
+}
+
+int
+ethr_mutex_destroy(ethr_mutex *mtx)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!mtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    mtx->initialized = 0;
+#endif
+    return mtxb_destroy(&mtx->mtxb);
+}
+
+void
+ethr_mutex_lock_wait__(ethr_mutex *mtx, ethr_sint32_t initial)
+{
+    write_lock_wait(&mtx->mtxb, initial, 0, 0);
+}
+
+void
+ethr_mutex_unlock_wake__(ethr_mutex *mtx, ethr_sint32_t initial)
+{
+    ethr_ts_event *tse;
+
+    ETHR_MTX_Q_LOCK(&mtx->mtxb.qlck);
+    tse = mtx->mtxb.q;
+
+    ETHR_ASSERT(tse);
+    ETHR_ASSERT(ethr_atomic32_read(&mtx->mtxb.flgs)
+		== (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__));
+    ETHR_ASSERT(initial & ETHR_RWMTX_W_WAIT_FLG__);
+    ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+
+    /*
+     * If we have multiple waiters, there is no need to modify
+     * mtxb->flgs; otherwise, we need to clear the write wait bit...
+     */
+    if (tse->next == mtx->mtxb.q)
+	ethr_atomic32_set(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__);
+
+    wake_writer(&mtx->mtxb, 0);
+}
+
+/* -- Condition variables -------------------------------------------------- */
+
+static void
+enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end)
+{
+    ethr_sint32_t act;
+
+    /*
+     * `ethr_cond_signal()' and `ethr_cond_broadcast()' end up here. If `mtx'
+     * is not currently locked by current thread, we almost certainly have a
+     * hard to debug race condition. There might however be some (strange)
+     * use for it. POSIX also allow a call to `pthread_cond_signal' or
+     * `pthread_cond_broadcast' even though the the associated mutex isn't
+     * locked by the caller. Therefore, we also allow this kind of strange
+     * usage, but optimize for the case where the mutex is locked by the
+     * calling thread.
+     */
+
+    ETHR_MTX_Q_LOCK(&mtx->mtxb.qlck);
+
+    ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+    {
+	int dbg_nws__ = 0;
+	ethr_ts_event *dbg_tse__;
+	for (dbg_tse__ = tse_start;
+	     dbg_tse__ != tse_end;
+	     dbg_tse__ = dbg_tse__->next)
+	    dbg_nws__++;
+	mtx->mtxb.ws += dbg_nws__ + 1;
+    }
+#endif
+
+    act = ethr_atomic32_read(&mtx->mtxb.flgs);
+    ETHR_ASSERT(act == 0
+		|| act == ETHR_RWMTX_W_FLG__
+		|| act == (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__));
+    if (act & ETHR_RWMTX_W_FLG__) {
+	/* The normal sane case */
+	if (!(act & ETHR_RWMTX_W_WAIT_FLG__)) {
+	    ETHR_ASSERT(!mtx->mtxb.q);
+	    act = ethr_atomic32_cmpxchg(&mtx->mtxb.flgs,
+					(ETHR_RWMTX_W_FLG__
+					 | ETHR_RWMTX_W_WAIT_FLG__),
+					ETHR_RWMTX_W_FLG__);
+	    if (act != ETHR_RWMTX_W_FLG__) {
+		/*
+		 * Sigh... this wasn't so sane after all since, the mutex was
+		 * obviously not locked by the current thread....
+		 */
+		ETHR_ASSERT(act == 0);
+		goto mtx_unlocked;
+	    }
+	}
+
+#ifdef ETHR_DEBUG
+	if (act & ETHR_RWMTX_W_WAIT_FLG__)
+	    ETHR_ASSERT(mtx->mtxb.q);
+	else
+	    ETHR_ASSERT(!mtx->mtxb.q);
+#endif
+
+	enqueue(&mtx->mtxb.q, tse_start, tse_end);
+
+	ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+	ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck);
+
+    }
+    else {
+	int multi;
+    mtx_unlocked:
+	/* Sigh... mutex isn't locked... */
+
+	multi = tse_start != tse_end;
+
+	while (1) {
+	    ethr_sint32_t new, exp = act;
+
+	    if (multi || (act & ETHR_RWMTX_W_FLG__))
+		new = ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__;
+	    else
+		new = ETHR_RWMTX_W_FLG__;
+
+	    act = ethr_atomic32_cmpxchg(&mtx->mtxb.flgs, new, exp);
+	    if (exp == act) {
+		ETHR_ASSERT(!mtx->mtxb.q);
+		if (act & ETHR_RWMTX_W_FLG__) {
+		    enqueue(&mtx->mtxb.q, tse_start, tse_end);
+
+		    ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+		    ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck);
+
+		}
+		else {
+		    ETHR_ASSERT(!mtx->mtxb.q);
+		    /*
+		     * Acquired the mutex on behalf of the
+		     * first thread in the queue; wake
+		     * it and enqueue the rest...
+		     */
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+		    mtx->mtxb.ws--;
+#endif
+		    if (multi) {
+			enqueue(&mtx->mtxb.q, tse_start->next, tse_end);
+			ETHR_ASSERT(mtx->mtxb.q);
+		    }
+
+		    ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+		    ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck);
+
+		    ethr_atomic32_set(&tse_start->uaflgs, 0);
+		    ethr_event_set(&tse_start->event);
+		}
+		break;
+	    }
+	}
+    }
+}
+
+int
+ethr_cond_init_opt(ethr_cond *cnd, ethr_cond_opt *opt)
+{
+#if ETHR_XCHK
+    if (!cnd) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    cnd->initialized = ETHR_COND_INITIALIZED;
+#endif
+    ETHR_MTX_HARD_DEBUG_FENCE_INIT(cnd);
+    cnd->q = NULL;
+    if (no_spin) {
+	cnd->main_scnt = 0;
+	cnd->aux_scnt = 0;
+    }
+    else {
+	if (!opt || opt->main_spincount < 0)
+	    cnd->main_scnt = default_cnd_main_spincount;
+	else if (opt->main_spincount > SHRT_MAX)
+	    cnd->main_scnt = SHRT_MAX;
+	else
+	    cnd->main_scnt = (short) opt->main_spincount;
+	if (!opt || opt->aux_spincount < 0)
+	    cnd->aux_scnt = default_cnd_aux_spincount;
+	else if (opt->aux_spincount > SHRT_MAX)
+	    cnd->aux_scnt = SHRT_MAX;
+	else
+	    cnd->aux_scnt = (short) opt->aux_spincount;
+	if (cnd->main_scnt < cnd->aux_scnt)
+	    cnd->main_scnt = cnd->aux_scnt;
+    }
+    ETHR_MTX_QLOCK_INIT(&cnd->qlck);
+    return 0;
+}
+
+int
+ethr_cond_init(ethr_cond *cnd)
+{
+    return ethr_cond_init_opt(cnd, NULL);
+}
+
+int
+ethr_cond_destroy(ethr_cond *cnd)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    cnd->initialized = 0;
+#endif
+    return ETHR_MTX_QLOCK_DESTROY(&cnd->qlck);
+}
+
+void
+ethr_cond_signal(ethr_cond *cnd)
+{
+    ethr_ts_event *tse;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+
+    ETHR_MTX_Q_LOCK(&cnd->qlck);
+
+    tse = cnd->q;
+
+    if (!tse) {
+	ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+	ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+    }
+    else {
+	ethr_mutex *mtx = (ethr_mutex *) tse->udata;
+
+	ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+	ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+	ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_CND_WAIT_FLG__);
+
+	ethr_atomic32_set(&tse->uaflgs, ETHR_RWMTX_W_WAIT_FLG__);
+
+	dequeue(&cnd->q, tse, tse);
+
+	ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+
+	tse->next = tse->prev = NULL;
+
+	enqueue_mtx(mtx, tse, tse);
+
+	ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+	ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+    }
+}
+
+void
+ethr_cond_broadcast(ethr_cond *cnd)
+{
+    int got_all;
+    ethr_ts_event *tse;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+    do {
+	got_all = 1;
+
+	ETHR_MTX_Q_LOCK(&cnd->qlck);
+
+	tse = cnd->q;
+
+	if (!tse) {
+	    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+	    ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+	}
+	else {
+	    ethr_mutex *mtx = (ethr_mutex *) tse->udata;
+	    ethr_ts_event *tse_tmp, *tse_end;
+
+	    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+	    tse_end = cnd->q->prev;
+
+	    tse_tmp = tse;
+
+	    do {
+
+		if (mtx == (ethr_mutex *) tse_tmp->udata) {
+		    /* The normal case */
+
+		    ETHR_ASSERT(tse_tmp->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+		    ETHR_ASSERT(ethr_atomic32_read(&tse_tmp->uaflgs)
+				== ETHR_CND_WAIT_FLG__);
+
+		    ethr_atomic32_set(&tse_tmp->uaflgs,
+				      ETHR_RWMTX_W_WAIT_FLG__);
+		}
+		else {
+		    /* Should be very unusual */
+		    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+		    tse_end = tse_tmp->prev;
+		    got_all = 0;
+		    break;
+		}
+
+		tse_tmp = tse_tmp->next;
+
+	    } while (tse_tmp != cnd->q);
+
+	    dequeue(&cnd->q, tse, tse_end);
+
+	    ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+
+	    enqueue_mtx(mtx, tse, tse_end);
+	}
+
+    } while (!got_all);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+}
+
+int
+ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
+{
+    int woken;
+    int scnt;
+    void *udata = NULL;
+    ethr_ts_event *tse;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+    ETHR_ASSERT(mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    tse = ethr_get_ts_event();
+
+    scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
+	    ? cnd->main_scnt
+	    : cnd->aux_scnt);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+
+    udata = tse->udata; /* Got to restore udata before returning */
+    tse->udata = (void *) mtx;
+
+    tse->uflgs = ETHR_RWMTX_W_WAIT_FLG__; /* Prep for mutex lock op */
+    ethr_atomic32_set(&tse->uaflgs, ETHR_CND_WAIT_FLG__);
+
+    ETHR_MTX_Q_LOCK(&cnd->qlck);
+
+    enqueue(&cnd->q, tse, tse);
+
+    ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+
+    ethr_mutex_unlock(mtx);
+
+    /* Wait */
+    woken = 0;
+    while (1) {
+	ethr_sint32_t act;
+
+	ethr_event_reset(&tse->event);
+
+	act = ethr_atomic32_read_acqb(&tse->uaflgs);
+	if (!act)
+	    break; /* Mtx locked */
+
+	/* First time, got EINTR, or spurious wakeup... */
+
+	ETHR_ASSERT(act == ETHR_CND_WAIT_FLG__
+		    || act == ETHR_RWMTX_W_WAIT_FLG__);
+
+	if (woken) {
+	    /*
+	     * If act == ETHR_RWMTX_W_WAIT_FLG__, we have already been enqueued
+	     * on the mutex; continue wait until locked...
+	     */
+	    if (act == ETHR_CND_WAIT_FLG__) {
+		ETHR_MTX_Q_LOCK(&cnd->qlck);
+		act = ethr_atomic32_read(&tse->uaflgs);
+		ETHR_ASSERT(act == ETHR_CND_WAIT_FLG__
+			    || act == ETHR_RWMTX_W_WAIT_FLG__);
+		/*
+		 * If act == ETHR_RWMTX_W_WAIT_FLG__, we have already
+		 * enqueued on the mutex; continue wait until locked...
+		 */
+		if (act == ETHR_CND_WAIT_FLG__)
+		    dequeue(&cnd->q, tse, tse);
+
+		ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+
+		if (act == ETHR_CND_WAIT_FLG__) {
+		    tse->udata = udata;
+		    ethr_leave_ts_event(tse);
+		    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+		    ethr_mutex_lock(mtx);
+		    return EINTR;
+		}
+	    }
+	    ETHR_ASSERT(act == ETHR_RWMTX_W_WAIT_FLG__);
+	}
+	ethr_event_swait(&tse->event, scnt);
+	/* swait result: 0 || EINTR */
+	woken = 1;
+    }
+
+    ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+    ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb);
+    tse->udata = udata;
+    ethr_leave_ts_event(tse);
+    return 0;
+}
+
+#elif defined(ETHR_PTHREADS) && !defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS)
+/* -- pthread mutex and condition variables -------------------------------- */
+
+int
+ethr_mutex_init(ethr_mutex *mtx)
+{
+#if ETHR_XCHK
+    if (!mtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    mtx->initialized = ETHR_MUTEX_INITIALIZED;
+#endif
+    return pthread_mutex_init(&mtx->pt_mtx, NULL);
+}
+
+int
+ethr_mutex_init_opt(ethr_mutex *mtx, ethr_mutex_opt *opt)
+{
+    return ethr_mutex_init(mtx);
+}
+
+int
+ethr_mutex_destroy(ethr_mutex *mtx)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+#if ETHR_XCHK
+    mtx->initialized = 0;
+#endif
+    return pthread_mutex_destroy(&mtx->pt_mtx);
+}
+
+int
+ethr_cond_init(ethr_cond *cnd)
+{
+#if ETHR_XCHK
+    if (!cnd) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    cnd->initialized = ETHR_COND_INITIALIZED;
+#endif
+    return pthread_cond_init(&cnd->pt_cnd, NULL);
+}
+
+int
+ethr_cond_init_opt(ethr_cond *cnd, ethr_cond_opt *opt)
+{
+    return ethr_cond_init(cnd);
+}
+
+int
+ethr_cond_destroy(ethr_cond *cnd)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    cnd->initialized = 0;
+#endif
+    return pthread_cond_destroy(&cnd->pt_cnd);
+}
+
+void
+ethr_cond_signal(ethr_cond *cnd)
+{
+    int res;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+
+    res = pthread_cond_signal(&cnd->pt_cnd);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+void
+ethr_cond_broadcast(ethr_cond *cnd)
+{
+    int res;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+
+    res = pthread_cond_broadcast(&cnd->pt_cnd);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+int
+ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
+{
+    int res;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+    ETHR_ASSERT(mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    res = pthread_cond_wait(&cnd->pt_cnd, &mtx->pt_mtx);
+    if (res != 0 && res != EINTR)
+	ETHR_FATAL_ERROR__(res);
+    return res;
+}
+
+#elif defined(ETHR_WIN32_THREADS) || defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS)
+
+/*
+ * As of Vista/Server, 2008 Windows has condition variables that can be
+ * used with critical sections. However, we need to be able to run on
+ * older Windows versions too, so we need to implement condition variables
+ * ourselves.
+ */
+
+#ifdef ETHR_DBG_WIN_MTX_WITH_PTHREADS
+/*
+ * For debugging of this implementation on POSIX platforms...
+ */
+
+#define ethr_win_get_errno__() EINVAL
+#if defined(__GNUC__)
+#define __forceinline __inline__
+#else
+#define __forceinline
+#endif
+
+static int
+InitializeCriticalSectionAndSpinCount(CRITICAL_SECTION *cs, int sc)
+{
+    return 0 == pthread_mutex_init((pthread_mutex_t *) cs, NULL);
+}
+
+static void DeleteCriticalSection(CRITICAL_SECTION *cs)
+{
+    int res = pthread_mutex_destroy((pthread_mutex_t *) cs);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+int TryEnterCriticalSection(CRITICAL_SECTION *cs)
+{
+    int res;
+    res = pthread_mutex_trylock((pthread_mutex_t *) cs);
+    if (res != 0 && res != EBUSY)
+	ETHR_FATAL_ERROR__(res);
+    return res == 0;
+}
+
+void EnterCriticalSection(CRITICAL_SECTION *cs)
+{
+    int res = pthread_mutex_lock((pthread_mutex_t *) cs);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+void LeaveCriticalSection(CRITICAL_SECTION *cs)
+{
+    int res = pthread_mutex_unlock((pthread_mutex_t *) cs);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+#endif
+
+#define ETHR_CND_WAIT__ ((ethr_sint32_t) 0x11dead11)
+#define ETHR_CND_WAKEUP__ ((ethr_sint32_t) 0x11beef11)
+
+static __forceinline void
+cond_wakeup(ethr_ts_event *tse)
+{
+    ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_CND_WAIT__);
+
+    ethr_atomic32_set_relb(&tse->uaflgs, ETHR_CND_WAKEUP__);
+    ethr_event_set(&tse->event);
+}
+
+void
+ethr_mutex_cond_wakeup__(ethr_mutex *mtx)
+{
+    /*
+     * Called by ethr_mutex_unlock() when we have
+     * cond signal/broadcast wakeups waiting to
+     * be completed.
+     */
+    ethr_ts_event *tse;
+
+    if (!mtx->posix_compliant) {
+	tse = mtx->wakeups;
+	dequeue(&mtx->wakeups, tse, tse);
+    }
+    else {
+	ethr_spin_lock(&mtx->lock);
+	tse = mtx->wakeups;
+	if (tse)
+	  dequeue(&mtx->wakeups, tse, tse);
+	if (!mtx->wakeups)
+	  ethr_atomic32_set_relb(&mtx->have_wakeups, 0);
+	ethr_spin_unlock(&mtx->lock);
+    }
+
+    LeaveCriticalSection(&mtx->cs);
+
+    ETHR_ASSERT(tse || mtx->posix_compliant);
+
+    /*
+     * We delay actual condition variable wakeup until
+     * this point when we have left the critical section.
+     * This in order to avoid that the other thread is
+     * woken and then right away have to go to sleep
+     * waiting for the critical section that we are in.
+     *
+     * We also only wake one thread at a time even if
+     * there are multiple threads waiting to be woken.
+     * Otherwise all but one will be woken and then right
+     * away have to go to sleep on the critical section.
+     * Since each wakeup is guaranteed to generate at
+     * least one lock/unlock sequence on this mutex, all
+     * threads will eventually be woken.
+     */
+
+    if (tse)
+	cond_wakeup(tse);
+}
+
+int
+ethr_mutex_init_opt(ethr_mutex *mtx, ethr_mutex_opt *opt)
+{
+    int spincount;
+#if ETHR_XCHK
+    if (!mtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    mtx->initialized = ETHR_MUTEX_INITIALIZED;
+#endif
+
+    spincount = opt ? opt->aux_spincount : 0;
+    if (spincount < 0)
+	spincount = 0;
+
+    if (!InitializeCriticalSectionAndSpinCount(&mtx->cs, spincount)) {
+#if ETHR_XCHK
+	mtx->initialized = 0;
+#endif
+	return ethr_win_get_errno__();
+    }
+
+    mtx->posix_compliant = opt ? opt->posix_compliant : 0;
+    mtx->wakeups = NULL;
+    if (mtx->posix_compliant) {
+	ethr_atomic32_init(&mtx->locked, 0);
+	ethr_atomic32_init(&mtx->have_wakeups, 0);
+	ethr_spinlock_init(&mtx->lock);
+    }
+    return 0;
+}
+
+int
+ethr_mutex_init(ethr_mutex *mtx)
+{
+    return ethr_mutex_init_opt(mtx, NULL);
+}
+
+int
+ethr_mutex_destroy(ethr_mutex *mtx)
+{
+    DeleteCriticalSection(&mtx->cs);
+    if (mtx->posix_compliant)
+	return ethr_spinlock_destroy(&mtx->lock);
+    else
+	return 0;
+}
+
+int
+ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
+{
+    void *udata;
+    ethr_ts_event *tse = ethr_get_ts_event();
+    int spincount;
+
+    udata = tse->udata;
+    tse->udata = (void *) mtx;
+    ethr_atomic32_set_relb(&tse->uaflgs, ETHR_CND_WAIT__);
+
+    EnterCriticalSection(&cnd->cs);
+    enqueue(&cnd->waiters, tse, tse);
+    LeaveCriticalSection(&cnd->cs);
+
+    ethr_mutex_unlock(mtx);
+
+    spincount = cnd->spincount;
+
+    while (ethr_atomic32_read_acqb(&tse->uaflgs) != ETHR_CND_WAKEUP__) {
+	ethr_event_reset(&tse->event);
+	if (ethr_atomic32_read_acqb(&tse->uaflgs) == ETHR_CND_WAKEUP__)
+	    break;
+	ethr_event_swait(&tse->event, spincount);
+	spincount = 0;
+    }
+
+    tse->udata = udata;
+    ethr_leave_ts_event(tse);
+
+    ethr_mutex_lock(mtx);
+
+    return 0;
+}
+
+static __forceinline void
+posix_compliant_mtx_enqueue(ethr_mutex *mtx,
+			    ethr_ts_event *tse_start,
+			    ethr_ts_event *tse_end)
+{
+    ethr_ts_event *tse_wakeup = NULL; /* Avoid erroneous compiler warning... */
+    /*
+     * The associated mutex might not be locked, so we need to
+     * check if it is. If locked, enqueue for wakeup at unlock;
+     * otherwise, wakeup the first one now and enqueue the rest.
+     */
+    if (tse_start == tse_end && !ethr_atomic32_read(&mtx->locked)) {
+	tse_wakeup = tse_start;
+    wakeup:
+	cond_wakeup(tse_wakeup);
+    }
+    else {
+	int need_wakeup;
+	ethr_spin_lock(&mtx->lock);
+	if (!mtx->wakeups)
+	    ethr_atomic32_set_mb(&mtx->have_wakeups, 1);
+	need_wakeup = !ethr_atomic32_read(&mtx->locked);
+	if (need_wakeup) {
+	    if (tse_start == tse_end) {
+		if (!mtx->wakeups)
+		    ethr_atomic32_set_relb(&mtx->have_wakeups, 0);
+		ethr_spin_unlock(&mtx->lock);
+		tse_wakeup = tse_start;
+		goto wakeup;
+	    }
+	    tse_wakeup = tse_start;
+	    tse_start = tse_start->next;
+	}
+	enqueue(&mtx->wakeups, tse_start, tse_end);
+	ethr_spin_unlock(&mtx->lock);
+	if (need_wakeup)
+	    goto wakeup;
+    }
+}
+
+static __forceinline void
+enqueue_cond_wakeups(ethr_ts_event *queue, int posix_compliant)
+{
+    if (queue) {
+	int more;
+	ethr_ts_event *q = queue;
+
+	/*
+	 * Waiters may be using different mutexes...
+	 */
+
+	do {
+	    ethr_mutex *mtx;
+	    ethr_ts_event *tse, *tse_start, *tse_end;
+
+	    more = 0;
+	    tse_start = q;
+	    mtx = (ethr_mutex *) tse_start->udata;
+
+	    ETHR_ASSERT(posix_compliant
+			? mtx->posix_compliant
+			: !mtx->posix_compliant);
+
+	    ETHR_ASSERT(ethr_atomic32_read(&tse_start->uaflgs)
+			== ETHR_CND_WAIT__);
+	    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+	    tse_end = tse_start->prev;
+
+	    for (tse = tse_start->next; tse != tse_start; tse = tse->next) {
+
+		ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs)
+			    == ETHR_CND_WAIT__);
+
+		if (mtx != (ethr_mutex *) tse->udata) {
+		    tse_end = tse->prev;
+		    dequeue(&q, tse_start, tse_end);
+		    more = 1;
+		    break;
+		}
+	    }
+
+	    if (posix_compliant)
+		posix_compliant_mtx_enqueue(mtx, tse_start, tse_end);
+	    else
+		enqueue(&mtx->wakeups, tse_start, tse_end);
+
+	} while (more);
+    }
+}
+
+void
+ethr_cond_broadcast(ethr_cond *cnd)
+{
+    ethr_ts_event *waiters;
+
+    EnterCriticalSection(&cnd->cs);
+    waiters = cnd->waiters;
+    cnd->waiters = NULL;
+    LeaveCriticalSection(&cnd->cs);
+
+    if (cnd->posix_compliant)
+	enqueue_cond_wakeups(waiters, 1);
+    else
+	enqueue_cond_wakeups(waiters, 0);
+}
+
+void
+ethr_cond_signal(ethr_cond *cnd)
+{
+    ethr_mutex *mtx;
+    ethr_ts_event *tse;
+
+    EnterCriticalSection(&cnd->cs);
+    tse = cnd->waiters;
+    if (tse)
+      dequeue(&cnd->waiters, tse, tse);
+    LeaveCriticalSection(&cnd->cs);
+
+    if (tse) {
+	mtx = (ethr_mutex *) tse->udata;
+
+	ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_CND_WAIT__);
+	ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+	ETHR_ASSERT(cnd->posix_compliant
+		    ? mtx->posix_compliant
+		    : !mtx->posix_compliant);
+
+	if (cnd->posix_compliant)
+	    posix_compliant_mtx_enqueue(mtx, tse, tse);
+	else
+	    enqueue(&mtx->wakeups, tse, tse);
+    }
+}
+
+int
+ethr_cond_init_opt(ethr_cond *cnd, ethr_cond_opt *opt)
+{
+    int spincount;
+
+#if ETHR_XCHK
+    if (!cnd) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    cnd->initialized = ETHR_COND_INITIALIZED;
+#endif
+
+    spincount = opt ? opt->aux_spincount : 0;
+    if (spincount < 0)
+	spincount = 0;
+
+    if (!InitializeCriticalSectionAndSpinCount(&cnd->cs, spincount)) {
+#if ETHR_XCHK
+	cnd->initialized = 0;
+#endif
+	return ethr_win_get_errno__();
+    }
+
+    cnd->posix_compliant = opt ? opt->posix_compliant : 0;
+    cnd->waiters = NULL;
+    cnd->spincount = spincount;
+    return 0;
+}
+
+int
+ethr_cond_init(ethr_cond *cnd)
+{
+    return ethr_cond_init_opt(cnd, NULL);
+}
+
+int
+ethr_cond_destroy(ethr_cond *cnd)
+{
+    DeleteCriticalSection(&cnd->cs);
+    return 0;
+}
+
+#endif
+
+/* -- Exported symbols of inline functions --------------------------------- */
+
+int
+ethr_mutex_trylock(ethr_mutex *mtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    return ethr_mutex_trylock__(mtx);
+}
+
+void
+ethr_mutex_lock(ethr_mutex *mtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    ethr_mutex_lock__(mtx);
+}
+
+void
+ethr_mutex_unlock(ethr_mutex *mtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    ethr_mutex_unlock__(mtx);
+}
+
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Read/Write Mutex                                                          *
+\*                                                                           */
+
+static void
+wake_readers(ethr_rwmutex *rwmtx, int rs)
+{
+    ethr_ts_event *tse;
+#ifdef ETHR_DEBUG
+    int drs = 0;
+#endif
+
+    tse = rwmtx->mtxb.q;
+    ETHR_ASSERT(tse);
+    ETHR_ASSERT(rwmtx->rq_end);
+    dequeue(&rwmtx->mtxb.q, tse, rwmtx->rq_end);
+    rwmtx->rq_end->next = NULL;
+    rwmtx->rq_end = NULL;
+
+    ETHR_ASSERT(!rwmtx->mtxb.q
+		|| (ethr_atomic32_read(&rwmtx->mtxb.q->uaflgs)
+		    == ETHR_RWMTX_W_WAIT_FLG__));
+
+    ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
+    ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+
+    while (tse) {
+	ethr_ts_event *tse_next;
+
+#ifdef ETHR_DEBUG
+	ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_R_WAIT_FLG__);
+	ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs)
+		    == ETHR_RWMTX_R_WAIT_FLG__);
+	drs++;
+#endif
+
+	tse_next = tse->next; /* we aren't allowed to read tse->next
+				 after we have reset uaflgs */
+
+	ethr_atomic32_set(&tse->uaflgs, 0);
+	ethr_event_set(&tse->event);
+	tse = tse_next;
+    }
+
+    ETHR_ASSERT(rs == drs);
+}
+
+static ETHR_INLINE int
+is_w_waiter(ethr_ts_event *tse)
+{
+    ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__
+		|| tse->uflgs == ETHR_RWMTX_R_WAIT_FLG__);
+    return tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__;
+}
+
+static ETHR_INLINE int
+multiple_w_waiters(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(rwmtx->mtxb.q);
+    ETHR_ASSERT(rwmtx->mtxb.q->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+
+    if (!rwmtx->rq_end)
+	return rwmtx->mtxb.q->next != rwmtx->mtxb.q;
+    else {
+	ETHR_ASSERT(rwmtx->mtxb.q->next != rwmtx->mtxb.q);
+	if (rwmtx->mtxb.q->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__)
+	    return 1;
+	ETHR_ASSERT(rwmtx->rq_end->next == rwmtx->mtxb.q
+		    || rwmtx->rq_end->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+	return rwmtx->rq_end->next != rwmtx->mtxb.q;
+    }
+}
+
+int check_readers_array(ethr_rwmutex *rwmtx,
+			int start_rix,
+			int length)
+{
+    int ix = start_rix;
+
+    ETHR_MEMORY_BARRIER;
+
+    do {
+	ethr_sint32_t act = rwmutex_freqread_rdrs_read(rwmtx, ix);
+	if (act != 0)
+	    return EBUSY;
+	ix++;
+	if (ix == length)
+	    ix = 0;
+    } while (ix != start_rix);
+
+    return 0;
+}
+
+static void
+rwmutex_freqread_rdrs_dec_chk_wakeup(ethr_rwmutex *rwmtx,
+				     ethr_ts_event *tse,
+				     ethr_sint32_t initial)
+{
+    ethr_sint32_t act = initial;
+
+    if ((act & (ETHR_RWMTX_W_FLG__|
+		ETHR_RWMTX_R_ABRT_UNLCK_FLG__)) == 0) {
+	if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+	    if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) {
+		/*
+		 * We *need* to try to complete the runlock.
+		 * A writer that just enqueued (not seen by us
+		 * in flag field) may depend on someone else
+		 * completing the runlock. We just took over
+		 * that responsibilty since we modified reader
+		 * groups.
+		 */
+		rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
+	    }
+	}
+	else if ((act & ETHR_RWMTX_WAIT_FLGS__) == ETHR_RWMTX_R_WAIT_FLG__)
+	    rwmutex_transfer_read_lock(rwmtx, act, 0);
+	else if ((act & ETHR_RWMTX_WAIT_FLGS__) == ETHR_RWMTX_W_WAIT_FLG__)
+	    rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
+	else {
+	    /*
+	     * Don't know if we got readers or writers
+	     * first in queue; need to peek
+	     */
+	    ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
+	    if (!rwmtx->mtxb.q)
+		ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+	    else if (is_w_waiter(rwmtx->mtxb.q)) {
+		act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+		ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+		if ((act & ETHR_RWMTX_W_FLG__) == 0)
+		    rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
+	    }
+	    else {
+		/*
+		 * rwmutex_transfer_read_lock() will
+		 * unlock Q lock.
+		 */
+		act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+		if (act & ETHR_RWMTX_W_FLG__)
+		    ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+		else
+		    rwmutex_transfer_read_lock(rwmtx, act, 1);
+	    }
+	}
+    }
+}
+
+static void
+rwmutex_freqread_restore_failed_tryrlock(ethr_rwmutex *rwmtx,
+					 ethr_ts_event *tse)
+{
+    ethr_sint32_t act;
+    /*
+     * Restore failed increment
+     */
+    act = rwmutex_freqread_rdrs_dec_read(rwmtx, tse);
+
+    ETHR_MEMORY_BARRIER;
+
+    if (act == 0) {
+	act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	rwmutex_freqread_rdrs_dec_chk_wakeup(rwmtx, tse, act);
+    }
+}
+
+static int
+rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
+			     ethr_sint32_t initial,
+			     ethr_ts_event *tse,
+			     int start_next_ix,
+			     int check_before_try,
+			     int try_write_lock)
+{
+    ethr_ts_event *tse_tmp;
+    ethr_sint32_t act = initial;
+    int six, res, length;
+
+    ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
+
+    if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__)
+	return try_write_lock ? EBUSY : 0;
+
+    tse_tmp = tse;
+    if (!tse_tmp)
+	tse_tmp = ethr_get_ts_event();
+
+    if ((act & ETHR_RWMTX_WAIT_FLGS__) && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0)
+	goto check_waiters;
+
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	length = reader_groups_array_size;
+	six = tse_tmp->rgix;
+    }
+    else {
+	length = main_threads_array_size;
+	six = tse_tmp->mtix;
+    }
+    if (start_next_ix) {
+	six++;
+	if (six >= length)
+	    six = 0;
+    }
+
+    if (!tse)
+	ethr_leave_ts_event(tse_tmp);
+
+    if (check_before_try) {
+	res = check_readers_array(rwmtx, six, length);
+
+	ETHR_MEMORY_BARRIER;
+
+	if (res == EBUSY)
+	    return try_write_lock ? EBUSY : 0;
+    }
+
+ restart:
+
+    while (1) {
+	ethr_sint32_t exp = act;
+	ethr_sint32_t new = act+1;
+
+	ETHR_ASSERT((act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) == 0);
+
+	ETHR_ASSERT((act & ETHR_RWMTX_R_PEND_UNLCK_MASK__)
+		    < ETHR_RWMTX_R_PEND_UNLCK_MASK__);
+
+	act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+	if (exp == act) {
+	    act = new;
+	    break;
+	}
+
+	if (!try_write_lock) {
+	    if (act == 0 || (act & (ETHR_RWMTX_W_FLG__
+				    | ETHR_RWMTX_R_ABRT_UNLCK_FLG__)))
+		return 0;
+	    if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+		if ((act & ETHR_RWMTX_R_FLG__) == 0)
+		    return 0;
+	    }
+	    else if ((act & ETHR_RWMTX_R_FLG__) == 0) {
+		if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__)
+		    return 0;
+		goto check_waiters;
+	    }
+	}
+	else {
+	    if (act == 0)
+		goto tryrwlock;
+	    if (act & (ETHR_RWMTX_W_FLG__
+		       | ETHR_RWMTX_R_ABRT_UNLCK_FLG__))
+		return EBUSY;
+	}
+    }
+
+    res = check_readers_array(rwmtx, six, length);
+
+    ETHR_MEMORY_BARRIER;
+
+    ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
+
+    while (1) {
+	int finished_abort = 0;
+	ethr_sint32_t exp = act;
+	ethr_sint32_t new = act;
+
+	new--;
+	if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) {
+	    if ((new & ETHR_RWMTX_R_PEND_UNLCK_MASK__) == 0) {
+		new &= ~ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
+		finished_abort = 1;
+	    }
+	    ETHR_ASSERT(act & ETHR_RWMTX_R_FLG__);
+	}
+	else if ((act & ETHR_RWMTX_R_FLG__) && res != EBUSY) {
+	    new &= ~ETHR_RWMTX_R_FLG__;
+	}
+
+	ETHR_ASSERT(act & ETHR_RWMTX_R_PEND_UNLCK_MASK__);
+
+	act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+	if (exp == act) {
+	    act = new;
+	    if (act & ETHR_RWMTX_W_FLG__)
+		return try_write_lock ? EBUSY : 0;
+	    if (finished_abort && (act & ETHR_RWMTX_WAIT_FLGS__))
+		goto restart;
+	    if (act & (ETHR_RWMTX_R_FLG__
+		       | ETHR_RWMTX_R_ABRT_UNLCK_FLG__
+		       | ETHR_RWMTX_R_PEND_UNLCK_MASK__))
+		return try_write_lock ? EBUSY : 0;
+	    /* Read unlock completed */
+	    break;
+	}
+    }
+
+    /*
+     * Read unlock completed, but we have to check if
+     * threads have to be woken (or if we should try
+     * to write lock it).
+     */
+
+    if (act & ETHR_RWMTX_WAIT_FLGS__) {
+    check_waiters:
+	rwmutex_unlock_wake(rwmtx, 0, act, 0);
+	return try_write_lock ? EBUSY : 0;
+    }
+
+    if (!try_write_lock)
+	return 0;
+
+ tryrwlock:
+    /* Try to write lock it */
+
+    act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+				     ETHR_RWMTX_W_FLG__,
+				     0);
+    return act == 0 ? 0 : EBUSY;
+}
+
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+
+static ETHR_INLINE void
+rwmutex_incdec_restore_failed_tryrlock(ethr_rwmutex *rwmtx)
+{
+    ethr_sint32_t act;
+    /*
+     * Restore failed increment
+     */
+    act = ethr_atomic32_dec_read(&rwmtx->mtxb.flgs);
+    if ((act & ETHR_RWMTX_WAIT_FLGS__)
+	&& (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) {
+	rwmutex_unlock_wake(rwmtx, 0, act, 0);
+    }
+}
+
+#endif
+
+static void
+rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial)
+{
+    ethr_sint32_t act = initial, exp;
+    int scnt, start_scnt;
+    ethr_ts_event *tse = NULL;
+    int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+
+    start_scnt = scnt = initial_spincount(&rwmtx->mtxb);
+
+    /*
+     * Spin trying to read lock for a while. If unsuccessful,
+     * wait on event.
+     */
+
+    while (1) {
+
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	rwmutex_incdec_restore_failed_tryrlock(rwmtx);
+	act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+#endif
+
+	while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+	    if (scnt <= 0) {
+		tse = ethr_get_ts_event();
+		if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
+		    event_wait(&rwmtx->mtxb, tse, scnt,
+			       ETHR_RWMTX_R_WAIT_FLG__, 1, 0);
+		    goto done;
+		}
+	    }
+	    ETHR_SPIN_BODY;
+	    if (--until_yield == 0) {
+		until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		ETHR_YIELD();
+	    }
+	    act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	    scnt--;
+	}
+	exp = act;
+
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	act = ethr_atomic32_inc_read(&rwmtx->mtxb.flgs);
+	if ((act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) == 0)
+	    goto done; /* Got it */
+#else
+	act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
+	if (act == exp)
+	    goto done; /* Got it */
+#endif
+    }
+
+ done:
+    if (tse)
+	ethr_leave_ts_event(tse);
+}
+
+static void
+rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
+			    ethr_ts_event *tse);
+
+static int
+rwmutex_freqread_rlock(ethr_rwmutex *rwmtx, ethr_ts_event *tse, int trylock)
+{
+    int res = 0;
+    ethr_sint32_t act;
+
+    rwmutex_freqread_rdrs_inc(rwmtx, tse);
+
+    ETHR_MEMORY_BARRIER;
+
+    act = ethr_atomic32_read_acqb(&rwmtx->mtxb.flgs);
+
+    if (act != ETHR_RWMTX_R_FLG__) {
+	int wake_other_readers;
+
+	while (1) {
+	    ethr_sint32_t exp, new;
+
+	    wake_other_readers = 0;
+
+	    if (act == 0)
+		new = act | ETHR_RWMTX_R_FLG__;
+	    else if (act == ETHR_RWMTX_R_FLG__)
+		break; /* Got it */
+	    else if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+		rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse);
+		if (trylock)
+		    res = EBUSY;
+		else
+		    rwmutex_freqread_rlock_wait(rwmtx, tse);
+		break;
+	    }
+	    else if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) {
+		if ((act & ETHR_RWMTX_R_FLG__) == 0)
+		    ETHR_FATAL_ERROR__(EFAULT);
+		/*
+		 * An aborted runlock, not write locked, and no write
+		 * waiters, i.e., we got it...
+		 */
+		if (act & ETHR_RWMTX_R_WAIT_FLG__)
+		    wake_other_readers = 1;
+		break;
+	    }
+	    else {
+		new = act | ETHR_RWMTX_R_FLG__;
+		if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) {
+		    /*
+		     * Someone is doing tryrwlock (no writer and no
+		     * write waiters); we will try to abort that...
+		     */
+		    new |= ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
+		}
+
+		if (act & ETHR_RWMTX_R_WAIT_FLG__)
+		    wake_other_readers = 1;
+	    }
+
+	    exp = act;
+	    act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp);
+	    if (act == exp)
+		break;
+	}
+
+	if (wake_other_readers)
+	    rwmutex_transfer_read_lock(rwmtx, act, 0);
+    }
+
+    return res;
+}
+
+static void
+rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
+			    ethr_ts_event *tse)
+{
+    ethr_sint32_t act;
+    int scnt, start_scnt;
+    int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+
+    start_scnt = scnt = initial_spincount(&rwmtx->mtxb);
+
+    /*
+     * Spin trying to read lock for a while. If unsuccessful,
+     * wait on event.
+     */
+
+    while (1) {
+
+	act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+
+	while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+	    if (scnt <= 0) {
+		if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
+		    event_wait(&rwmtx->mtxb, tse, scnt,
+			       ETHR_RWMTX_R_WAIT_FLG__, 1, 1);
+		    return; /* Got it */
+		}
+	    }
+	    ETHR_SPIN_BODY;
+	    if (--until_yield == 0) {
+		until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		ETHR_YIELD();
+	    }
+	    act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	    scnt--;
+	}
+
+	if (rwmutex_freqread_rlock(rwmtx, tse, 1) != EBUSY)
+	    break; /* Got it */
+    }
+}
+
+static void
+rwmutex_normal_rwlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial)
+{
+    write_lock_wait(&rwmtx->mtxb, initial, 1, 0);
+}
+
+static void
+rwmutex_freqread_rwlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial)
+{
+    write_lock_wait(&rwmtx->mtxb, initial, 1, 1);
+}
+
+static ETHR_INLINE void
+rwlock_wake_set_flags(ethr_rwmutex *rwmtx,
+		      ethr_sint32_t new_initial,
+		      ethr_sint32_t act_initial)
+{
+    ethr_sint32_t act, act_mask;
+    int chk_abrt_flg;
+
+    ETHR_MEMORY_BARRIER;
+
+    if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) {
+	/* r pend unlock mask may vary and must be retained */
+	act_mask = ETHR_RWMTX_R_PEND_UNLCK_MASK__;
+	if (new_initial & ETHR_RWMTX_R_FLG__)
+	    chk_abrt_flg = 1;
+	else
+	    chk_abrt_flg = 0;
+    }
+    else {
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	/* rs mask may vary and must be retained */
+	act_mask = ETHR_RWMTX_RS_MASK__;
+	chk_abrt_flg = 0;
+#else
+	/* rs mask always zero */
+	ETHR_ASSERT((act_initial & ETHR_RWMTX_RS_MASK__) == 0);
+	ethr_atomic32_set(&rwmtx->mtxb.flgs, new_initial);
+	return;
+#endif
+    }
+
+    act = act_initial;
+    while (1) {
+	ethr_sint32_t exp = act;
+	ethr_sint32_t new = new_initial + (act & act_mask);
+	if (chk_abrt_flg && (act & act_mask))
+	    new |= ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
+	act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+	if (act == exp)
+	    break;
+	exp = act;
+    }
+}
+
+#ifdef ETHR_DEBUG
+
+static void
+dbg_unlock_wake(ethr_rwmutex *rwmtx,
+		int have_w,
+		ethr_ts_event *tse)
+{
+    ethr_sint32_t exp, act, imask;
+
+    exp = have_w ? ETHR_RWMTX_W_FLG__ : 0;
+
+    if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL)
+	imask = ETHR_RWMTX_R_PEND_UNLCK_MASK__|ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
+    else {
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	imask = ETHR_RWMTX_RS_MASK__;
+#else
+	imask = 0;
+#endif
+    }
+
+    ETHR_ASSERT(tse);
+
+    if (is_w_waiter(tse)) {
+
+	exp |= ETHR_RWMTX_W_WAIT_FLG__;
+	if (rwmtx->rq_end) {
+	    exp |= ETHR_RWMTX_R_WAIT_FLG__;
+	}
+	act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	ETHR_ASSERT((exp & ~imask) == (act & ~imask));
+
+	ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
+
+    }
+    else {
+
+	exp |= ETHR_RWMTX_R_WAIT_FLG__;
+	if (rwmtx->rq_end->next != rwmtx->mtxb.q)
+	    exp |= ETHR_RWMTX_W_WAIT_FLG__;
+	else if (exp == ETHR_RWMTX_R_WAIT_FLG__) {
+	    if (!have_w) {
+		if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL)
+		    imask |= ETHR_RWMTX_R_FLG__;
+		else
+		    imask |= ETHR_RWMTX_RS_MASK__;
+	    }
+	}
+	act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	ETHR_ASSERT((exp & ~imask) == (act & ~imask));
+
+	ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
+
+    }
+}
+
+#endif
+
+static void
+rwmutex_transfer_read_lock(ethr_rwmutex *rwmtx,
+			   ethr_sint32_t initial,
+			   int q_locked)
+{
+    ethr_sint32_t act = initial;
+
+    if (!q_locked) {
+	ethr_ts_event *tse;
+	ETHR_ASSERT(initial & ETHR_RWMTX_R_WAIT_FLG__);
+	ETHR_ASSERT((initial & ETHR_RWMTX_W_FLG__) == 0);
+	ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
+
+	act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	tse = rwmtx->mtxb.q;
+	if ((act & ETHR_RWMTX_W_FLG__) || !tse || is_w_waiter(tse)) {
+	    /* Someone else woke the readers up... */
+	    ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+	    return;
+	}
+    }
+
+    rwmutex_unlock_wake(rwmtx, 0, initial, 1);
+}
+
+static void
+rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, ethr_sint32_t initial,
+		    int transfer_read_lock)
+{
+    ethr_sint32_t new, act = initial;
+    ethr_ts_event *tse;
+
+    if (transfer_read_lock) {
+	/*
+	 * - Q already locked
+	 * - Got R waiters first in Q
+	 * - Not W locked
+	 */
+	tse = rwmtx->mtxb.q;
+
+	ETHR_ASSERT(act & ETHR_RWMTX_R_WAIT_FLG__);
+	ETHR_ASSERT((act & (ETHR_RWMTX_W_FLG__)) == 0);
+	ETHR_ASSERT(tse && !is_w_waiter(tse));
+    }
+    else {
+
+	if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+	    if (!have_w)
+		return;
+	    else {
+		while ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+		    ethr_sint32_t exp = act;
+		    new = exp & ~ETHR_RWMTX_W_FLG__;
+		    act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+		    if (act == exp)
+			return;
+		}
+	    }
+	}
+
+	ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
+	tse = rwmtx->mtxb.q;
+
+	if (!have_w) {
+	    if (!tse) {
+#ifdef ETHR_DEBUG
+		act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+		ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0);
+#endif
+		goto already_served;
+	    }
+	    act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	    if (act == (ETHR_RWMTX_R_WAIT_FLG__|ETHR_RWMTX_R_FLG__)) {
+		ETHR_ASSERT(tse && !is_w_waiter(tse));
+	    }
+	    else if (act & ~ETHR_RWMTX_WAIT_FLGS__) {
+	    already_served:
+		ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+		return;
+	    }
+	}
+    }
+
+#ifdef ETHR_DEBUG
+    dbg_unlock_wake(rwmtx, have_w, tse);
+#endif
+
+    if (is_w_waiter(tse)) {
+
+	if (!have_w) {
+	    act = ethr_atomic32_read_bor(&rwmtx->mtxb.flgs,
+				       ETHR_RWMTX_W_FLG__);
+	    ETHR_ASSERT((act & ~(ETHR_RWMTX_WAIT_FLGS__
+				 | (rwmtx->type == ETHR_RWMUTEX_TYPE_NORMAL
+				    ? 0
+				    : ETHR_RWMTX_R_PEND_UNLCK_MASK__))) == 0);
+	    ETHR_ASSERT(act & ETHR_RWMTX_W_WAIT_FLG__);
+	    act |= ETHR_RWMTX_W_FLG__;
+	}
+
+	/*
+	 * If we have multiple write waiters, there
+	 * is no need to modify mtxb->flgs; otherwise,
+	 * we need to clear the write wait bit...
+	 */
+	if (!multiple_w_waiters(rwmtx)) {
+	    new = ETHR_RWMTX_W_FLG__;
+	    if (tse->next != rwmtx->mtxb.q) {
+		ETHR_ASSERT(tse->next->uflgs == ETHR_RWMTX_R_WAIT_FLG__);
+		new |= ETHR_RWMTX_R_WAIT_FLG__;
+	    }
+
+	    rwlock_wake_set_flags(rwmtx, new, act);
+	}
+
+	wake_writer(&rwmtx->mtxb, 1);
+    }
+    else {
+	int rs;
+
+	if (rwmtx->type == ETHR_RWMUTEX_TYPE_NORMAL) {
+	    rs = rwmtx->tdata.rs;
+	    new = (ethr_sint32_t) rs;
+	    rwmtx->tdata.rs = 0;
+	}
+	else {
+	    ethr_rwmutex_type type = rwmtx->type;
+	    int length = (type == ETHR_RWMUTEX_TYPE_FREQUENT_READ
+			  ? reader_groups_array_size
+			  : main_threads_array_size);
+	    int ix;
+
+	    rs = 0;
+	    for (ix = 0; ix < length; ix++) {
+		int wrs = rwmtx->tdata.ra[ix].data.waiting_readers;
+		rwmtx->tdata.ra[ix].data.waiting_readers = 0;
+		ETHR_ASSERT(wrs >= 0);
+		if (wrs) {
+		    rs += wrs;
+		    rwmutex_freqread_rdrs_add(rwmtx, type, ix, wrs);
+		}
+	    }
+
+	    new = ETHR_RWMTX_R_FLG__;
+	}
+
+	if (rwmtx->rq_end->next != rwmtx->mtxb.q)
+	    new |= ETHR_RWMTX_W_WAIT_FLG__;
+
+	rwlock_wake_set_flags(rwmtx, new, act);
+
+	wake_readers(rwmtx, rs);
+    }
+}
+
+static ethr_rwmtx_readers_array__ *
+alloc_readers_array(int length, ethr_rwmutex_lived lived)
+{
+    ethr_rwmtx_readers_array__ *ra;
+    size_t sz;
+    void *mem;
+
+    sz = sizeof(ethr_rwmtx_readers_array__) * (length + 1);
+
+    switch (lived) {
+    case ETHR_RWMUTEX_LONG_LIVED:
+	mem = ethr_mem__.ll.alloc(sz);
+	break;
+    case ETHR_RWMUTEX_SHORT_LIVED:
+	mem = ethr_mem__.sl.alloc(sz);
+	break;
+    default:
+	mem = ethr_mem__.std.alloc(sz);
+	break;
+    }
+    if (!mem)
+	return NULL;
+
+    if ((((ethr_uint_t) mem) & ETHR_CACHE_LINE_MASK) == 0) {
+	ra = (ethr_rwmtx_readers_array__ *) mem;
+	ra->data.byte_offset = 0;
+    }
+    else {
+	ra = ((ethr_rwmtx_readers_array__ *)
+	      ((((ethr_uint_t) mem) & ~ETHR_CACHE_LINE_MASK)
+	       + ETHR_CACHE_LINE_SIZE));
+	ra->data.byte_offset = (int) ((ethr_uint_t) ra
+				      - (ethr_uint_t) mem);
+    }
+    ra->data.lived = lived;
+    return ra;
+}
+
+static void
+free_readers_array(ethr_rwmtx_readers_array__ *ra)
+{
+    void *ptr = (void *) (((char *) ra) - ra->data.byte_offset);
+    switch (ra->data.lived) {
+    case ETHR_RWMUTEX_LONG_LIVED:
+	ethr_mem__.ll.free(ptr);
+	break;
+    case ETHR_RWMUTEX_SHORT_LIVED:
+	ethr_mem__.sl.free(ptr);
+	break;
+    default:
+	ethr_mem__.std.free(ptr);
+	break;
+    }
+}
+
+int
+ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt)
+{
+    int res;
+    ethr_rwmtx_readers_array__ *ra = NULL;
+#if ETHR_XCHK
+    if (ethr_not_completely_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!rwmtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED;
+#endif
+    ETHR_MTX_HARD_DEBUG_FENCE_INIT(rwmtx);
+    rwmtx->rq_end = NULL;
+    rwmtx->type = opt ? opt->type : ETHR_RWMUTEX_TYPE_NORMAL;
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+	if (main_threads_array_size <= reader_groups_array_size) {
+	    /* No point using reader groups... */
+	    rwmtx->type = ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ;
+	}
+	/* Fall through */
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
+	int length;
+
+	length = (rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ
+		  ? main_threads_array_size
+		  : reader_groups_array_size);
+
+	if (length == 1) {
+	    /* No point using a frequent reader type... */
+	    rwmtx->type = ETHR_RWMUTEX_TYPE_NORMAL;
+	}
+	else {
+	    int ix;
+	    ra = alloc_readers_array(length,
+				     (opt
+				      ? opt->lived
+				      : ETHR_RWMUTEX_UNKNOWN_LIVED));
+	    if (!ra) {
+		res = ENOMEM;
+		goto error;
+	    }
+
+	    rwmtx->tdata.ra = ra;
+
+	    for (ix = 0; ix < length; ix++) {
+		ethr_atomic32_init(&rwmtx->tdata.ra[ix].data.readers, 0);
+		rwmtx->tdata.ra[ix].data.waiting_readers = 0;
+	    }
+	    break;
+	}
+    }
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	rwmtx->tdata.rs = 0;
+	break;
+    default:
+	res = EINVAL;
+	goto error;
+    }
+    res = mtxb_init(&rwmtx->mtxb,
+		    default_rwmtx_main_spincount,
+		    opt ? opt->main_spincount : -1,
+		    default_rwmtx_aux_spincount,
+		    opt ? opt->aux_spincount : -1);
+    if (res == 0)
+	return 0;
+
+ error:
+
+    if (ra)
+	free_readers_array(ra);
+
+#if ETHR_XCHK
+    rwmtx->initialized = 0;
+#endif
+    return res;
+}
+
+int
+ethr_rwmutex_init(ethr_rwmutex *rwmtx)
+{
+    return ethr_rwmutex_init_opt(rwmtx, NULL);
+}
+
+int
+ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
+{
+    int res;
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+    if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) {
+	ethr_sint32_t act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	if (act == ETHR_RWMTX_R_FLG__)
+	    rwmutex_try_complete_runlock(rwmtx, act, NULL, 0, 0, 0);
+    }
+    res = mtxb_destroy(&rwmtx->mtxb);
+    if (res != 0)
+	return res;
+    if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL)
+	free_readers_array(rwmtx->tdata.ra);
+#if ETHR_XCHK
+    rwmtx->initialized = 0;
+#endif
+    return 0;
+}
+
+#define ETHR_MAX_TRYRLOCK_TRIES 5
+
+int
+ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
+{
+    int res = 0;
+    ethr_sint32_t act;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL: {
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))
+	    res = EBUSY;
+	else {
+	    act = ethr_atomic32_inc_read_acqb(&rwmtx->mtxb.flgs);
+	    if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+		rwmutex_incdec_restore_failed_tryrlock(rwmtx);
+		res = EBUSY;
+	    }
+	}
+#else
+	ethr_sint32_t exp = 0;
+	int tries = 0;
+
+	while (1) {
+	    act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
+	    if (act == exp) {
+		res = 0;
+		break;
+	    }
+	    if (tries > ETHR_MAX_TRYRLOCK_TRIES
+		|| (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))) {
+		res = EBUSY;
+		break;
+	    }
+	    tries++;
+	    exp = act;
+	}
+#endif
+	break;
+    }
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
+	ethr_ts_event *tse = ethr_get_ts_event();
+	res = rwmutex_freqread_rlock(rwmtx, tse, 1);
+	ethr_leave_ts_event(tse);
+	break;
+    }
+    }
+
+#ifdef ETHR_MTX_CHK_EXCL
+    if (res == 0) {
+	ETHR_MTX_CHK_EXCL_SET_NON_EXCL(&rwmtx->mtxb);
+	ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb);
+    }
+#endif
+
+    ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(&rwmtx->mtxb, res);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+    return res;
+}
+
+void
+ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
+{
+    ethr_sint32_t act;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL: {
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	act = ethr_atomic32_inc_read_acqb(&rwmtx->mtxb.flgs);
+	if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))
+	    rwmutex_normal_rlock_wait(rwmtx, act);
+#else
+	ethr_sint32_t exp = 0;
+
+	while (1) {
+	    act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
+	    if (act == exp)
+		break;
+
+	    if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+		rwmutex_normal_rlock_wait(rwmtx, act);
+		break;
+	    }
+	    exp = act;
+	}
+#endif
+	break;
+    }
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
+	ethr_ts_event *tse = ethr_get_ts_event();
+	rwmutex_freqread_rlock(rwmtx, tse, 0);
+	ethr_leave_ts_event(tse);
+	break;
+    }
+    }
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+    ETHR_MTX_CHK_EXCL_SET_NON_EXCL(&rwmtx->mtxb);
+    ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb);
+    ETHR_MTX_HARD_DEBUG_LFS_RLOCK(&rwmtx->mtxb);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+}
+
+void
+ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
+{
+    ethr_sint32_t act;
+
+    ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb);
+    ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(&rwmtx->mtxb);
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+    ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(&rwmtx->mtxb);
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	act = ethr_atomic32_dec_read_relb(&rwmtx->mtxb.flgs);
+	if ((act & ETHR_RWMTX_WAIT_FLGS__)
+	    && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) {
+	    ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
+	    rwmutex_unlock_wake(rwmtx, 0, act, 0);
+	}
+	break;
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
+	ethr_ts_event *tse = ethr_get_ts_event();
+
+	act = rwmutex_freqread_rdrs_dec_read_relb(rwmtx, tse);
+
+	ETHR_ASSERT(act >= 0);
+
+	ETHR_MEMORY_BARRIER;
+
+	if (act == 0) {
+	    act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+	    if (act != ETHR_RWMTX_R_FLG__)
+		rwmutex_freqread_rdrs_dec_chk_wakeup(rwmtx, tse, act);
+	}
+
+	ethr_leave_ts_event(tse);
+	break;
+    }
+    }
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+}
+
+int
+ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
+{
+    int res = 0;
+    ethr_sint32_t act;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+					 ETHR_RWMTX_W_FLG__, 0);
+	if (act != 0)
+	    res = EBUSY;
+	break;
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
+
+	res = 0;
+	act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+
+	do {
+
+	    if (act == 0)
+		act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+						 ETHR_RWMTX_W_FLG__, 0);
+	    else if (act == ETHR_RWMTX_R_FLG__) {
+		res = rwmutex_try_complete_runlock(rwmtx, act, NULL,
+						   0, 1, 1);
+		break;
+	    }
+	    else {
+		res = EBUSY;
+		break;
+	    }
+
+	} while (act != 0);
+
+	break;
+    }
+
+#ifdef ETHR_MTX_CHK_EXCL
+    if (res == 0) {
+	ETHR_MTX_CHK_EXCL_SET_EXCL(&rwmtx->mtxb);
+	ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb);
+    }
+#endif
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+    ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&rwmtx->mtxb, res);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    return res;
+}
+
+void
+ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
+{
+    ethr_sint32_t act;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+					 ETHR_RWMTX_W_FLG__, 0);
+	if (act != 0)
+	    rwmutex_normal_rwlock_wait(rwmtx, act);
+	break;
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
+
+	act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+
+	do {
+
+	    if (act != 0) {
+		rwmutex_freqread_rwlock_wait(rwmtx, act);
+		break;
+	    }
+
+	    act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+					     ETHR_RWMTX_W_FLG__, 0);
+
+	} while (act != 0);
+
+	break;
+    }
+
+    ETHR_MTX_CHK_EXCL_SET_EXCL(&rwmtx->mtxb);
+    ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb);
+    ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&rwmtx->mtxb);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
+}
+
+void
+ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
+{
+    ethr_sint32_t act;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+    ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&rwmtx->mtxb);
+
+    ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb);
+    ETHR_MTX_CHK_EXCL_UNSET_EXCL(&rwmtx->mtxb);
+
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	act = ethr_atomic32_cmpxchg_relb(&rwmtx->mtxb.flgs,
+					 0, ETHR_RWMTX_W_FLG__);
+	if (act != ETHR_RWMTX_W_FLG__)
+	    rwmutex_unlock_wake(rwmtx, 1, act, 0);
+	break;
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
+	act = ethr_atomic32_cmpxchg_relb(&rwmtx->mtxb.flgs, 0,
+					 ETHR_RWMTX_W_FLG__);
+	if (act != ETHR_RWMTX_W_FLG__)
+	    rwmutex_unlock_wake(rwmtx, 1, act, 0);
+	break;
+    }
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+}
+
+#else
+/* -- pthread read/write mutex --------------------------------------------- */
+
+int
+ethr_rwmutex_init(ethr_rwmutex *rwmtx)
+{
+#if ETHR_XCHK
+    if (!rwmtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED;
+#endif
+    return pthread_rwlock_init(&rwmtx->pt_rwlock, write_pref_attr);
+}
+
+int
+ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt)
+{
+    return ethr_rwmutex_init(rwmtx);
+}
+
+int
+ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
+{
+    int res;
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    res = pthread_rwlock_destroy(&rwmtx->pt_rwlock);
+#if ETHR_XCHK
+    rwmtx->initialized = 0;
+#endif
+    return res;
+}
+
+/* -- Exported symbols of inline functions --------------------------------- */
+
+int
+ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    return ethr_rwmutex_tryrlock__(rwmtx);
+}
+
+void
+ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ethr_rwmutex_rlock__(rwmtx);
+}
+
+void
+ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ethr_rwmutex_runlock__(rwmtx);
+}
+
+int
+ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    return ethr_rwmutex_tryrwlock__(rwmtx);
+}
+
+void
+ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    return ethr_rwmutex_rwlock__(rwmtx);
+}
+
+void
+ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ethr_rwmutex_rwunlock__(rwmtx);
+}
+
+#endif /* pthread */
+
+
+#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__)
+
+#ifdef ETHR_MTX_HARD_DEBUG_Q
+static void
+hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
+{
+    int res;
+    ethr_sint32_t flgs = ethr_atomic32_read(&mtxb->flgs);
+
+    ETHR_MTX_HARD_ASSERT(res == 0);
+
+    ETHR_MTX_HARD_ASSERT(!(flgs & ETHR_RWMTX_R_WAIT_FLG__) || is_rwmtx);
+
+    if (!(flgs & ETHR_RWMTX_WAIT_FLGS__)) {
+	ETHR_MTX_HARD_ASSERT(!mtxb->q);
+	if (is_rwmtx) {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_MTX_HARD_ASSERT(!rwmtx->rq_end);
+	    ETHR_MTX_HARD_ASSERT(!rwmtx->rs);
+	}
+    }
+    else {
+	ethr_ts_event *tse;
+	int ws = 0, rs = 0, rsf = 0, ref = 0;
+
+	ETHR_MTX_HARD_ASSERT(mtxb->q);
+
+	tse = mtxb->q;
+
+	do {
+	    ethr_sint32_t type;
+
+	    ETHR_MTX_HARD_ASSERT(tse->next->prev == tse);
+	    ETHR_MTX_HARD_ASSERT(tse->prev->next == tse);
+
+	    type = ethr_atomic32_read(&tse->uaflgs);
+	    ETHR_MTX_HARD_ASSERT(type == tse->uflgs);
+	    switch (type) {
+	    case ETHR_RWMTX_W_WAIT_FLG__:
+		ws++;
+		break;
+	    case ETHR_RWMTX_R_WAIT_FLG__: {
+		ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+		ETHR_MTX_HARD_ASSERT(is_rwmtx);
+		if (!rsf)
+		    rsf = 1;
+		ETHR_MTX_HARD_ASSERT(!ref);
+		if (rwmtx->rq_end == tse) {
+		    ETHR_MTX_HARD_ASSERT(
+			tse->next == rwmtx->mtxb.q
+			|| tse->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+		    ref = 1;
+		}
+		rs++;
+		break;
+	    }
+	    default:
+		ETHR_MTX_HARD_ASSERT(! "invalid wait type found");
+	    }
+
+	    tse = tse->next;
+	} while (tse != mtxb->q);
+
+	if (is_rwmtx) {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_MTX_HARD_ASSERT(rs == rwmtx->rs);
+	}
+
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+	ETHR_MTX_HARD_ASSERT(ws == mtxb->ws);
+#endif
+
+	if (flgs & ETHR_RWMTX_W_WAIT_FLG__)
+	    ETHR_MTX_HARD_ASSERT(ws);
+	else
+	    ETHR_MTX_HARD_ASSERT(!ws);
+
+	if (flgs & ETHR_RWMTX_R_WAIT_FLG__) {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_MTX_HARD_ASSERT(is_rwmtx);
+	    ETHR_MTX_HARD_ASSERT(rwmtx->rq_end);
+	    ETHR_MTX_HARD_ASSERT(rsf);
+	    ETHR_MTX_HARD_ASSERT(ref);
+	    ETHR_MTX_HARD_ASSERT(rs);
+	}
+	else {
+	    if (is_rwmtx) {
+		ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+		ETHR_MTX_HARD_ASSERT(!rwmtx->rq_end);
+	    }
+	    ETHR_MTX_HARD_ASSERT(!rsf);
+	    ETHR_MTX_HARD_ASSERT(!ref);
+	    ETHR_MTX_HARD_ASSERT(!rs);
+	}
+    }
+}
+
+#elif defined(ETHR_MTX_HARD_DEBUG_WSQ)
+
+static void
+hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
+{
+    int ws = 0;
+    int rs = 0;
+
+    if (mtxb->q) {
+	ethr_ts_event *tse = mtxb->q;
+	do {
+	    switch (tse->uflgs) {
+	    case ETHR_RWMTX_W_WAIT_FLG__:
+		ws++;
+		break;
+	    case ETHR_RWMTX_R_WAIT_FLG__:
+		rs++;
+		break;
+	    default:
+		ETHR_MTX_HARD_ASSERT(0);
+		break;
+	    }
+	    tse = tse->next;
+	} while (tse != mtxb->q);
+    }
+
+    ETHR_MTX_HARD_ASSERT(mtxb->ws == ws);
+    if (is_rwmtx) {
+	ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	ETHR_MTX_HARD_ASSERT(rwmtx->rs == rs);
+    }
+}
+
+#endif
+
+#endif
diff --git a/erts/lib_src/common/ethread.c b/erts/lib_src/common/ethread.c
deleted file mode 100644
index eb4d0cad20..0000000000
--- a/erts/lib_src/common/ethread.c
+++ /dev/null
@@ -1,3346 +0,0 @@
-/*
- * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- * 
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- * 
- * %CopyrightEnd%
- */
-
-/*
- * Description: A Thread library for use in the ERTS and other OTP
- *              applications.
- * Author: Rickard Green
- */
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#undef ETHR_STACK_GUARD_SIZE 
-
-#if defined(ETHR_PTHREADS)
-
-#ifdef ETHR_TIME_WITH_SYS_TIME
-#  include <time.h>
-#  include <sys/time.h>
-#else
-#  ifdef ETHR_HAVE_SYS_TIME_H
-#    include <sys/time.h>
-#  else
-#    include <time.h>
-#  endif
-#endif
-#include <sys/types.h>
-#include <unistd.h>
-#include <signal.h>
-
-#ifdef ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE
-#  define ETHR_STACK_GUARD_SIZE (pagesize)
-#endif
-
-#elif defined(ETHR_WIN32_THREADS)
-
-#undef WIN32_LEAN_AND_MEAN
-#define WIN32_LEAN_AND_MEAN
-#include <windows.h>
-#include <process.h>
-#include <winerror.h>
-
-#else
-#error "Missing thread implementation"
-#endif
-
-#include <limits.h>
-
-#define ETHR_FORCE_INLINE_FUNCS
-#define ETHR_INLINE_FUNC_NAME_(X) X ## __
-#include "ethread.h"
-
-#ifndef ETHR_HAVE_ETHREAD_DEFINES
-#error Missing configure defines
-#endif
-
-/*
- * ----------------------------------------------------------------------------
- * Common stuff
- * ----------------------------------------------------------------------------
- */
-
-#define ETHR_MAX_THREADS 2048 /* Has to be an even power of 2 */
-
-static int ethr_not_inited = 1;
-
-#define ASSERT(A) ETHR_ASSERT((A))
-
-static void *(*allocp)(size_t) = malloc;
-static void *(*reallocp)(void *, size_t) = realloc;
-static void (*freep)(void *) = free;
-
-#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
-ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS];
-#endif
-
-void *(*thread_create_prepare_func)(void) = NULL;
-void (*thread_create_parent_func)(void *) = NULL;
-void (*thread_create_child_func)(void *) = NULL;
-
-typedef struct ethr_xhndl_list_ ethr_xhndl_list;
-struct ethr_xhndl_list_ {
-    ethr_xhndl_list *next;
-    void (*funcp)(void);
-};
-
-static size_t pagesize;
-#define ETHR_PAGE_ALIGN(SZ) (((((size_t) (SZ)) - 1)/pagesize + 1)*pagesize)
-static size_t min_stack_size; /* kilo words */
-static size_t max_stack_size; /* kilo words */
-#define ETHR_B2KW(B) ((((size_t) (B)) - 1)/(sizeof(void *)*1024) + 1)
-#define ETHR_KW2B(KW) (((size_t) (KW))*sizeof(void *)*1024)
-
-ethr_mutex xhndl_mtx;
-ethr_xhndl_list *xhndl_list;
-
-static int
-init_common(ethr_init_data *id)
-{
-    int res;
-    if (id) {
-	allocp				= id->alloc;
-	reallocp			= id->realloc;
-	freep				= id->free;
-	thread_create_prepare_func	= id->thread_create_prepare_func;
-	thread_create_parent_func	= id->thread_create_parent_func;
-	thread_create_child_func	= id->thread_create_child_func;
-    }
-    if (!allocp || !reallocp || !freep)
-	return EINVAL;
-
-#ifdef _SC_PAGESIZE
-    pagesize = (size_t) sysconf(_SC_PAGESIZE);
-#elif defined(HAVE_GETPAGESIZE)
-    pagesize = (size_t) getpagesize();
-#else
-    pagesize = (size_t) 4*1024; /* Guess 4 KB */
-#endif
-
-    /* User needs at least 4 KB */
-    min_stack_size = 4*1024;
-#if SIZEOF_VOID_P == 8
-    /* Double that on 64-bit archs */
-    min_stack_size *= 2;
-#endif
-    /* On some systems as much as about 4 KB is used by the system */
-    min_stack_size += 4*1024;
-    /* There should be room for signal handlers */
-#ifdef SIGSTKSZ
-    min_stack_size += SIGSTKSZ;
-#else
-    min_stack_size += pagesize;
-#endif
-    /* The system may think that we need more stack */
-#if defined(PTHREAD_STACK_MIN)
-    if (min_stack_size < PTHREAD_STACK_MIN)
-	min_stack_size = PTHREAD_STACK_MIN;
-#elif defined(_SC_THREAD_STACK_MIN)
-    {
-	size_t thr_min_stk_sz = (size_t) sysconf(_SC_THREAD_STACK_MIN);
-	if (min_stack_size < thr_min_stk_sz)
-	    min_stack_size = thr_min_stk_sz;
-    }
-#endif
-    /* The guard is at least on some platforms included in the stack size
-       passed when creating threads */
-#ifdef ETHR_STACK_GUARD_SIZE
-    min_stack_size += ETHR_STACK_GUARD_SIZE;
-#endif
-    min_stack_size = ETHR_PAGE_ALIGN(min_stack_size);
-
-    min_stack_size = ETHR_B2KW(min_stack_size);
-
-    max_stack_size = 32*1024*1024;
-#if SIZEOF_VOID_P == 8
-    max_stack_size *= 2;
-#endif
-    max_stack_size = ETHR_B2KW(max_stack_size);
-
-    xhndl_list = NULL;
-
-    res = ethr_mutex_init(&xhndl_mtx);
-    if (res != 0)
-	return res;
-
-    res = ethr_mutex_set_forksafe(&xhndl_mtx);
-    if (res != 0 && res != ENOTSUP)
-	return res;
-
-    return 0;
-}
-
-int
-ethr_install_exit_handler(void (*funcp)(void))
-{
-    ethr_xhndl_list *xhp;
-    int res;
-
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-
-    if (!funcp)
-	return EINVAL;
-
-    xhp = (ethr_xhndl_list *) (*allocp)(sizeof(ethr_xhndl_list));
-    if (!xhp)
-	return ENOMEM;
-
-    res = ethr_mutex_lock__(&xhndl_mtx);
-    if (res != 0) {
-	(*freep)((void *) xhp);
-	return res;
-    }
-
-    xhp->funcp = funcp;
-    xhp->next = xhndl_list;
-    xhndl_list = xhp;
-
-    res = ethr_mutex_unlock__(&xhndl_mtx);
-    if (res != 0)
-	abort();
-
-    return res;
-}
-
-static void
-run_exit_handlers(void)
-{
-    int res;
-    ethr_xhndl_list *xhp;
-
-    res = ethr_mutex_lock__(&xhndl_mtx);
-    if (res != 0)
-	abort();
-
-    xhp = xhndl_list;
-
-    res = ethr_mutex_unlock__(&xhndl_mtx);
-    if (res != 0)
-	abort();
-
-    for (; xhp; xhp = xhp->next)
-	(*xhp->funcp)();
-}
-
-#if defined(ETHR_PTHREADS)
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
- * pthread implementation                                                    *
-\*                                                                           */
-
-typedef struct {
-    pthread_mutex_t mtx;
-    pthread_cond_t cnd;
-    int initialized;
-    void *(*thr_func)(void *);
-    void *arg;
-    void *prep_func_res;
-} thr_wrap_data_;
-
-static int no_ethreads;
-static ethr_mutex no_ethrs_mtx;
-
-#ifndef ETHR_HAVE_PTHREAD_ATFORK
-#define ETHR_HAVE_PTHREAD_ATFORK 0
-#endif
-
-#if !ETHR_HAVE_PTHREAD_ATFORK
-#warning "Cannot enforce fork-safety"
-#endif
-
-/*
- * ----------------------------------------------------------------------------
- * Static functions
- * ----------------------------------------------------------------------------
- */
-
-/*
- * Functions with safe_ prefix aborts on failure. To be used when
- * we cannot recover after failure.
- */
-
-static ETHR_INLINE void
-safe_mutex_lock(pthread_mutex_t *mtxp)
-{
-    int res = pthread_mutex_lock(mtxp);
-    if (res != 0)
-	abort();
-}
-
-static ETHR_INLINE void
-safe_mutex_unlock(pthread_mutex_t *mtxp)
-{
-    int res = pthread_mutex_unlock(mtxp);
-    if (res != 0)
-	abort();
-}
-
-static ETHR_INLINE void
-safe_cond_signal(pthread_cond_t *cndp)
-{
-    int res = pthread_cond_signal(cndp);
-    if (res != 0)
-	abort();
-}
-
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-
-static volatile int rec_mtx_attr_need_init = 1;
-static pthread_mutexattr_t rec_mtx_attr;
-
-static int init_rec_mtx_attr(void);
-
-#endif
-
-#if ETHR_HAVE_PTHREAD_ATFORK
-
-static ethr_mutex forksafe_mtx = ETHR_MUTEX_INITER;
-
-static void lock_mutexes(void)
-{
-    ethr_mutex *m = &forksafe_mtx;
-    do {
-
-	safe_mutex_lock(&m->pt_mtx);
-
-	m = m->next;
-
-    } while (m != &forksafe_mtx);
-}
-
-static void unlock_mutexes(void)
-{
-    ethr_mutex *m = forksafe_mtx.prev;
-    do {
-
-	safe_mutex_unlock(&m->pt_mtx);
-
-	m = m->prev;
-
-    } while (m->next != &forksafe_mtx);
-}
-
-#if ETHR_INIT_MUTEX_IN_CHILD_AT_FORK
-
-static void reinit_mutexes(void)
-{
-    ethr_mutex *m = forksafe_mtx.prev;
-    do {
-	pthread_mutexattr_t *attrp = NULL;
-
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-	if (m->is_rec_mtx) {
-	    if (rec_mtx_attr_need_init) {
-		int res = init_rec_mtx_attr();
-		if (res != 0)
-		    abort();
-	    }
-	    attrp = &rec_mtx_attr;
-	}
-#endif
-	if (pthread_mutex_init(&m->pt_mtx, attrp) != 0)
-	    abort();
-
-	m = m->prev;
-
-    } while (m->next != &forksafe_mtx);
-}
-
-#endif
-
-static int
-init_forksafe(void)
-{
-    static int init_done = 0;
-    int res = 0;
-
-    if (init_done)
-	return res;
-
-    forksafe_mtx.prev = &forksafe_mtx;
-    forksafe_mtx.next = &forksafe_mtx;
-
-    res = pthread_atfork(lock_mutexes,
-			 unlock_mutexes,
-#if ETHR_INIT_MUTEX_IN_CHILD_AT_FORK
-			 reinit_mutexes
-#else
-			 unlock_mutexes
-#endif
-	);
-
-    init_done = 1;
-    return res;
-}
-
-#endif
-
-
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-
-#if defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE)
-
-#define SET_REC_MUTEX_ATTR(AP) \
-  pthread_mutexattr_settype((AP), PTHREAD_MUTEX_RECURSIVE);
-
-#elif defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP)
-
-#define SET_REC_MUTEX_ATTR(AP) \
-  pthread_mutexattr_setkind_np((AP), PTHREAD_MUTEX_RECURSIVE_NP);
-
-#else
-
-#error "Don't know how to set recursive mutex attributes"
-
-#endif
-
-static int
-init_rec_mtx_attr(void)
-{
-    int res, mres;
-    static pthread_mutex_t attrinit_mtx = PTHREAD_MUTEX_INITIALIZER;
-
-    mres = pthread_mutex_lock(&attrinit_mtx);
-    if (mres != 0)
-	return mres;
-    /* Got here under race conditions; check again ... */
-    if (!rec_mtx_attr_need_init)
-	res = 0;
-    else {
-	res = pthread_mutexattr_init(&rec_mtx_attr);
-	if (res == 0) {
-	    res = SET_REC_MUTEX_ATTR(&rec_mtx_attr);
-	    if (res == 0)
-		rec_mtx_attr_need_init = 0;
-	    else
-		(void) pthread_mutexattr_destroy(&rec_mtx_attr);
-	}
-    }
-
-    mres = pthread_mutex_unlock(&attrinit_mtx);
-    if (mres != 0)
-	return mres;
-    return res;
-}
-
-#endif /* #if ETHR_HAVE_ETHR_REC_MUTEX_INIT */
-
-static ETHR_INLINE void thr_exit_cleanup(void)
-{
-    run_exit_handlers();
-    safe_mutex_lock(&no_ethrs_mtx.pt_mtx);
-    ASSERT(no_ethreads > 0);
-    no_ethreads--;
-    safe_mutex_unlock(&no_ethrs_mtx.pt_mtx);
-}
-
-static void *thr_wrapper(void *vtwd)
-{
-    void *res;
-    thr_wrap_data_ *twd = (thr_wrap_data_ *) vtwd;
-    void *(*thr_func)(void *) = twd->thr_func;
-    void *arg = twd->arg;
-
-    safe_mutex_lock(&twd->mtx);
-
-    if (thread_create_child_func)
-	(*thread_create_child_func)(twd->prep_func_res);
-
-    twd->initialized = 1;
-
-    safe_cond_signal(&twd->cnd);
-    safe_mutex_unlock(&twd->mtx);
-
-    res = (*thr_func)(arg);
-    thr_exit_cleanup();
-    return res;
-}
-
-
-/*
- * ----------------------------------------------------------------------------
- * Exported functions
- * ----------------------------------------------------------------------------
- */
-
-int
-ethr_init(ethr_init_data *id)
-{
-    int res;
-
-    if (!ethr_not_inited)
-	return EINVAL;
-
-    ethr_not_inited = 0;
-
-    res = init_common(id);
-    if (res != 0)
-	goto error;
-
-#if ETHR_HAVE_PTHREAD_ATFORK
-    init_forksafe();
-#endif
-
-    no_ethreads = 1;
-    res = ethr_mutex_init(&no_ethrs_mtx);
-    if (res != 0)
-	goto error;
-    res = ethr_mutex_set_forksafe(&no_ethrs_mtx);
-    if (res != 0 && res != ENOTSUP)
-	goto error;
-
-#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
-    {
-	int i;
-	for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) {
-#ifdef ETHR_HAVE_PTHREAD_SPIN_LOCK
-	    res = pthread_spin_init(&ethr_atomic_protection__[i].u.spnlck, 0);
-#else
-	    res = ethr_mutex_init(&ethr_atomic_protection__[i].u.mtx);
-#endif
-	    if (res != 0)
-		goto error;
-	}
-    }
-#endif
-
-    return 0;
-
- error:
-    ethr_not_inited = 1;
-    return res;
-
-}
-
-int
-ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg,
-		ethr_thr_opts *opts)
-{
-    thr_wrap_data_ twd;
-    pthread_attr_t attr;
-    int res, dres;
-    int use_stack_size = (opts && opts->suggested_stack_size >= 0
-			  ? opts->suggested_stack_size
-			  : -1 /* Use system default */);
-
-#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE
-    if (use_stack_size < 0)
-	use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE;
-#endif
-
-    twd.initialized = 0;
-    twd.thr_func = func;
-    twd.arg = arg;
-
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!tid || !func) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    /* Call prepare func if it exist */
-    if (thread_create_prepare_func)
-	twd.prep_func_res = (*thread_create_prepare_func)();
-    else
-	twd.prep_func_res = NULL;
-
-    /* Set som thread attributes */
-    res = pthread_attr_init(&attr);
-    if (res != 0)
-	goto cleanup_parent_func;
-    res = pthread_mutex_init(&twd.mtx, NULL);
-    if (res != 0)
-	goto cleanup_attr_destroy;
-    res = pthread_cond_init(&twd.cnd, NULL);
-    if (res != 0)
-	goto cleanup_mutex_destroy;
-
-    /* Schedule child thread in system scope (if possible) ... */
-    res = pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM);
-    if (res != 0 && res != ENOTSUP)
-	goto cleanup_cond_destroy;
-
-    if (use_stack_size >= 0) {
-	size_t suggested_stack_size = (size_t) use_stack_size;
-	size_t stack_size;
-#ifdef DEBUG
-	suggested_stack_size /= 2; /* Make sure we got margin */
-#endif
-#ifdef ETHR_STACK_GUARD_SIZE
-	/* The guard is at least on some platforms included in the stack size
-	   passed when creating threads */
-	suggested_stack_size += ETHR_B2KW(ETHR_STACK_GUARD_SIZE);
-#endif
-	if (suggested_stack_size < min_stack_size)
-	    stack_size = ETHR_KW2B(min_stack_size);
-	else if (suggested_stack_size > max_stack_size)
-	    stack_size = ETHR_KW2B(max_stack_size);
-	else
-	    stack_size = ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size));
-	(void) pthread_attr_setstacksize(&attr, stack_size);
-    }
-
-#ifdef ETHR_STACK_GUARD_SIZE
-    (void) pthread_attr_setguardsize(&attr, ETHR_STACK_GUARD_SIZE);
-#endif
-
-    /* Detached or joinable... */
-    res = pthread_attr_setdetachstate(&attr,
-				      (opts && opts->detached
-				       ? PTHREAD_CREATE_DETACHED
-				       : PTHREAD_CREATE_JOINABLE));
-    if (res != 0)
-	goto cleanup_cond_destroy;
-    
-    res = pthread_mutex_lock(&twd.mtx);
-
-    if (res != 0)
-	goto cleanup_cond_destroy;
-
-    safe_mutex_lock(&no_ethrs_mtx.pt_mtx);
-    if (no_ethreads < ETHR_MAX_THREADS) {
-	no_ethreads++;
-	safe_mutex_unlock(&no_ethrs_mtx.pt_mtx);
-    }
-    else {
-	res = EAGAIN;
-	safe_mutex_unlock(&no_ethrs_mtx.pt_mtx);
-	goto cleanup_mutex_unlock;
-    }
-
-    res = pthread_create((pthread_t *) tid, &attr, thr_wrapper, (void *) &twd);
-
-    if (res != 0) {
-	safe_mutex_lock(&no_ethrs_mtx.pt_mtx);
-	ASSERT(no_ethreads > 0);
-	no_ethreads--;
-	safe_mutex_unlock(&no_ethrs_mtx.pt_mtx);
-    }
-    else {
-
-	/* Wait for child to initialize... */
-	while (!twd.initialized) {
-	    res = pthread_cond_wait(&twd.cnd, &twd.mtx);
-	    if (res != 0 && res != EINTR)
-		break;
-	}
-
-    }
-
-    /* Cleanup... */
- cleanup_mutex_unlock:
-    dres = pthread_mutex_unlock(&twd.mtx);
-    if (res == 0)
-	res = dres;
- cleanup_cond_destroy:
-    dres = pthread_cond_destroy(&twd.cnd);
-    if (res == 0)
-	res = dres;
- cleanup_mutex_destroy:
-    dres = pthread_mutex_destroy(&twd.mtx);
-    if (res == 0)
-	res = dres;
- cleanup_attr_destroy:
-    dres = pthread_attr_destroy(&attr);
-    if (res == 0)
-	res = dres;
- cleanup_parent_func:
-    if (thread_create_parent_func)
-	(*thread_create_parent_func)(twd.prep_func_res);
-
-    return res;
-}
-
-int
-ethr_thr_join(ethr_tid tid, void **res)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return pthread_join((pthread_t) tid, res);
-}
-
-int
-ethr_thr_detach(ethr_tid tid)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return pthread_detach((pthread_t) tid);
-}
-
-void
-ethr_thr_exit(void *res)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return;
-    }
-#endif
-    thr_exit_cleanup();
-    pthread_exit(res);
-}
-
-ethr_tid
-ethr_self(void)
-{
-    return (ethr_tid) pthread_self();
-}
-
-int
-ethr_equal_tids(ethr_tid tid1, ethr_tid tid2)
-{
-    return pthread_equal((pthread_t) tid1, (pthread_t) tid2);
-}
-
-
-/*
- * Mutex functions
- */
-
-
-int
-ethr_mutex_init(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    mtx->initialized = ETHR_MUTEX_INITIALIZED;
-#endif
-    mtx->prev = NULL;
-    mtx->next = NULL;
-    mtx->is_rec_mtx = 0;
-    return pthread_mutex_init(&mtx->pt_mtx, NULL);
-}
-
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-
-int
-ethr_rec_mutex_init(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    mtx->initialized = ETHR_MUTEX_INITIALIZED;
-#endif
-    if (rec_mtx_attr_need_init)
-	init_rec_mtx_attr();
-
-    mtx->prev = NULL;
-    mtx->next = NULL;
-    mtx->is_rec_mtx = 1;
-    return pthread_mutex_init(&mtx->pt_mtx, &rec_mtx_attr);
-}
-
-#endif /* #if ETHR_HAVE_ETHR_REC_MUTEX_INIT */
-
-int
-ethr_mutex_destroy(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (mtx->next) {
-	ASSERT(mtx->prev);
-	ethr_mutex_unset_forksafe(mtx);
-    }
-#if ETHR_XCHK
-    mtx->initialized = 0;
-#endif
-    return pthread_mutex_destroy(&mtx->pt_mtx);
-}
-
-int ethr_mutex_set_forksafe(ethr_mutex *mtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-#if ETHR_HAVE_PTHREAD_ATFORK
-    res = pthread_mutex_lock(&forksafe_mtx.pt_mtx);
-    if (res != 0)
-	return res;
-    if (!forksafe_mtx.next) {
-	ASSERT(!forksafe_mtx.prev);
-	init_forksafe();
-    }
-    if (mtx->next) {
-	/* forksafe already set for this mutex */ 
-	ASSERT(mtx->prev);
-    }
-    else {
-	mtx->next = forksafe_mtx.next;
-	mtx->prev = &forksafe_mtx;
-	forksafe_mtx.next->prev = mtx;
-	forksafe_mtx.next = mtx;
-    }
-
-    res = pthread_mutex_unlock(&forksafe_mtx.pt_mtx);
-
-#else /* #if ETHR_HAVE_PTHREAD_ATFORK */
-    res = ENOTSUP;
-#endif /* #if ETHR_HAVE_PTHREAD_ATFORK */
-    return res;
-}
-
-int ethr_mutex_unset_forksafe(ethr_mutex *mtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-#if ETHR_HAVE_PTHREAD_ATFORK
-    res = pthread_mutex_lock(&forksafe_mtx.pt_mtx);
-    if (res != 0)
-	return res;
-    if (!forksafe_mtx.next) {
-	ASSERT(!forksafe_mtx.prev);
-	init_forksafe();
-    }
-    if (!mtx->next) {
-	/* forksafe already unset for this mutex */ 
-	ASSERT(!mtx->prev);
-    }
-    else {
-	mtx->prev->next = mtx->next;
-	mtx->next->prev = mtx->prev;
-	mtx->next = NULL;
-	mtx->prev = NULL;
-    }
-    res = pthread_mutex_unlock(&forksafe_mtx.pt_mtx);
-
-#else /* #if ETHR_HAVE_PTHREAD_ATFORK */
-    res = ENOTSUP;
-#endif /* #if ETHR_HAVE_PTHREAD_ATFORK */
-    return res;
-}
-
-int
-ethr_mutex_trylock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_trylock__(mtx);
-}
-
-int
-ethr_mutex_lock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_lock__(mtx);
-}
-
-int
-ethr_mutex_unlock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_unlock__(mtx);
-}
-
-/*
- * Condition variable functions
- */
-
-int
-ethr_cond_init(ethr_cond *cnd)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    cnd->initialized = ETHR_COND_INITIALIZED;
-#endif
-    return pthread_cond_init(&cnd->pt_cnd, NULL);
-}
-
-int
-ethr_cond_destroy(ethr_cond *cnd)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    cnd->initialized = 0;
-#endif
-    return pthread_cond_destroy(&cnd->pt_cnd);
-}
-
-int
-ethr_cond_signal(ethr_cond *cnd)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return pthread_cond_signal(&cnd->pt_cnd);
-}
-
-int
-ethr_cond_broadcast(ethr_cond *cnd)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return pthread_cond_broadcast(&cnd->pt_cnd);
-}
-
-int
-ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| cnd->initialized != ETHR_COND_INITIALIZED
-	|| !mtx
-	|| mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return pthread_cond_wait(&cnd->pt_cnd, &mtx->pt_mtx);
-}
-
-int
-ethr_cond_timedwait(ethr_cond *cnd, ethr_mutex *mtx, ethr_timeval *timeout)
-{
-    struct timespec to;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| cnd->initialized != ETHR_COND_INITIALIZED
-	|| !mtx
-	|| mtx->initialized != ETHR_MUTEX_INITIALIZED
-	|| !timeout) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    to.tv_sec = timeout->tv_sec;
-    to.tv_nsec = timeout->tv_nsec;
-
-    return pthread_cond_timedwait(&cnd->pt_cnd, &mtx->pt_mtx, &to);
-}
-
-
-#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT
-
-int
-ethr_rwmutex_init(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED;
-#endif
-    return pthread_rwlock_init(&rwmtx->pt_rwlock, NULL);
-}
-
-int
-ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = pthread_rwlock_destroy(&rwmtx->pt_rwlock);
-#if ETHR_XCHK
-    rwmtx->initialized = 0;
-#endif
-    return res;
-}
-
-int
-ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_tryrlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_rlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_runlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_tryrwlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_rwlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_rwunlock__(rwmtx);
-}
-
-#endif /* #ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT */
-
-/*
- * Current time
- */
-
-int
-ethr_time_now(ethr_timeval *time)
-{
-    int res;
-    struct timeval tv;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!time) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    res = gettimeofday(&tv, NULL);
-    time->tv_sec = (long) tv.tv_sec;
-    time->tv_nsec = ((long) tv.tv_usec)*1000;
-    return res;
-}
-
-/*
- * Thread specific data
- */
-
-int
-ethr_tsd_key_create(ethr_tsd_key *keyp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!keyp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return pthread_key_create((pthread_key_t *) keyp, NULL);
-}
-
-int
-ethr_tsd_key_delete(ethr_tsd_key key)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return pthread_key_delete((pthread_key_t) key);
-}
-
-int
-ethr_tsd_set(ethr_tsd_key key, void *value)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return pthread_setspecific((pthread_key_t) key, value);
-}
-
-void *
-ethr_tsd_get(ethr_tsd_key key)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return NULL;
-    }
-#endif
-    return pthread_getspecific((pthread_key_t) key);
-}
-
-/*
- * Signal functions
- */
-
-#if ETHR_HAVE_ETHR_SIG_FUNCS
-
-int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!set && !oset) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-  return pthread_sigmask(how, set, oset);
-}
-
-int ethr_sigwait(const sigset_t *set, int *sig)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!set || !sig) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (sigwait(set, sig) < 0)
-	return errno;
-    return 0;
-}
-
-#endif /* #if ETHR_HAVE_ETHR_SIG_FUNCS */
-
-#elif defined(ETHR_WIN32_THREADS)
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
- * Native win32 threads implementation                                       *
-\*                                                                           */
-
-#define INVALID_TID -1
-
-/* The spin count values are more or less taken out of the blue */
-#define ETHR_MUTEX_SPIN_COUNT	5000
-#define ETHR_COND_SPIN_COUNT	1000
-
-ethr_tid serial_shift; /* Bits to shift serial when constructing a tid */
-ethr_tid last_serial; /* Last thread table serial used */
-ethr_tid last_ix; /* Last thread table index used */
-ethr_tid thr_ix_mask; /* Mask used to mask out thread table index from a tid */
-
-/* Event used for conditional variables. On per thread. */
-/*typedef struct cnd_wait_event__ cnd_wait_event_;*/
-struct cnd_wait_event__ {
-    HANDLE handle;
-    cnd_wait_event_ *prev;
-    cnd_wait_event_ *next;
-    int in_queue;
-};
-
-/* Thread specific data. Stored in the thread table */
-typedef struct {
-    ethr_tid thr_id;
-    HANDLE thr_handle;
-    ethr_tid joiner;
-    void *result;
-    cnd_wait_event_ wait_event;
-} thr_data_;
-
-/* Argument passed to thr_wrapper() */
-typedef struct {
-    void * (*func)(void *);
-    void * arg;
-    thr_data_ *ptd;
-    thr_data_ *td;
-    int res;
-    void *prep_func_res;
-} thr_wrap_data_;
-
-
-static CRITICAL_SECTION thr_table_cs; /* Critical section used to protect
-					 the thread table from concurrent
-					 accesses. */
-static CRITICAL_SECTION fake_static_init_cs; /* Critical section used to protect
-					initialazition of 'statically
-					initialized' mutexes */
-static thr_data_ * thr_table[ETHR_MAX_THREADS]; /* The thread table */
-
-static DWORD tls_own_thr_data;
-
-static thr_data_ main_thr_data;
-
-#define THR_IX(TID)	((TID) & thr_ix_mask)
-#define OWN_THR_DATA	((thr_data_ *) TlsGetValue(tls_own_thr_data))
-
-/*
- * ----------------------------------------------------------------------------
- * Static functions
- * ----------------------------------------------------------------------------
- */
-
-static int
-get_errno(void)
-{
-    switch (GetLastError()) {
-    case ERROR_INVALID_FUNCTION:		return EINVAL;	/* 1	*/
-    case ERROR_FILE_NOT_FOUND:			return ENOENT;	/* 2	*/
-    case ERROR_PATH_NOT_FOUND:			return ENOENT;	/* 3	*/
-    case ERROR_TOO_MANY_OPEN_FILES:		return EMFILE;	/* 4	*/
-    case ERROR_ACCESS_DENIED:			return EACCES;	/* 5	*/
-    case ERROR_INVALID_HANDLE:			return EBADF;	/* 6	*/
-    case ERROR_ARENA_TRASHED:			return ENOMEM;	/* 7	*/
-    case ERROR_NOT_ENOUGH_MEMORY:		return ENOMEM;	/* 8	*/
-    case ERROR_INVALID_BLOCK:			return ENOMEM;	/* 9	*/
-    case ERROR_BAD_ENVIRONMENT:			return E2BIG;	/* 10	*/
-    case ERROR_BAD_FORMAT:			return ENOEXEC;	/* 11	*/
-    case ERROR_INVALID_ACCESS:			return EINVAL;	/* 12	*/
-    case ERROR_INVALID_DATA:			return EINVAL;	/* 13	*/
-    case ERROR_OUTOFMEMORY:			return ENOMEM;	/* 14	*/
-    case ERROR_INVALID_DRIVE:			return ENOENT;	/* 15	*/
-    case ERROR_CURRENT_DIRECTORY:		return EACCES;	/* 16	*/
-    case ERROR_NOT_SAME_DEVICE:			return EXDEV;	/* 17	*/
-    case ERROR_NO_MORE_FILES:			return ENOENT;	/* 18	*/
-    case ERROR_WRITE_PROTECT:			return EACCES;	/* 19	*/
-    case ERROR_BAD_UNIT:			return EACCES;	/* 20	*/
-    case ERROR_NOT_READY:			return EACCES;	/* 21	*/
-    case ERROR_BAD_COMMAND:			return EACCES;	/* 22	*/
-    case ERROR_CRC:				return EACCES;	/* 23	*/
-    case ERROR_BAD_LENGTH:			return EACCES;	/* 24	*/
-    case ERROR_SEEK:				return EACCES;	/* 25	*/
-    case ERROR_NOT_DOS_DISK:			return EACCES;	/* 26	*/
-    case ERROR_SECTOR_NOT_FOUND:		return EACCES;	/* 27	*/
-    case ERROR_OUT_OF_PAPER:			return EACCES;	/* 28	*/
-    case ERROR_WRITE_FAULT:			return EACCES;	/* 29	*/
-    case ERROR_READ_FAULT:			return EACCES;	/* 30	*/
-    case ERROR_GEN_FAILURE:			return EACCES;	/* 31	*/
-    case ERROR_SHARING_VIOLATION:		return EACCES;	/* 32	*/
-    case ERROR_LOCK_VIOLATION:			return EACCES;	/* 33	*/
-    case ERROR_WRONG_DISK:			return EACCES;	/* 34	*/
-    case ERROR_SHARING_BUFFER_EXCEEDED:		return EACCES;	/* 36	*/
-    case ERROR_BAD_NETPATH:			return ENOENT;	/* 53	*/
-    case ERROR_NETWORK_ACCESS_DENIED:		return EACCES;	/* 65	*/
-    case ERROR_BAD_NET_NAME:			return ENOENT;	/* 67	*/
-    case ERROR_FILE_EXISTS:			return EEXIST;	/* 80	*/
-    case ERROR_CANNOT_MAKE:			return EACCES;	/* 82	*/
-    case ERROR_FAIL_I24:			return EACCES;	/* 83	*/
-    case ERROR_INVALID_PARAMETER:		return EINVAL;	/* 87	*/
-    case ERROR_NO_PROC_SLOTS:			return EAGAIN;	/* 89	*/
-    case ERROR_DRIVE_LOCKED:			return EACCES;	/* 108	*/
-    case ERROR_BROKEN_PIPE:			return EPIPE;	/* 109	*/
-    case ERROR_DISK_FULL:			return ENOSPC;	/* 112	*/
-    case ERROR_INVALID_TARGET_HANDLE:		return EBADF;	/* 114	*/
-    case ERROR_WAIT_NO_CHILDREN:		return ECHILD;	/* 128	*/
-    case ERROR_CHILD_NOT_COMPLETE:		return ECHILD;	/* 129	*/
-    case ERROR_DIRECT_ACCESS_HANDLE:		return EBADF;	/* 130	*/
-    case ERROR_NEGATIVE_SEEK:			return EINVAL;	/* 131	*/
-    case ERROR_SEEK_ON_DEVICE:			return EACCES;	/* 132	*/
-    case ERROR_DIR_NOT_EMPTY:			return ENOTEMPTY;/* 145	*/
-    case ERROR_NOT_LOCKED:			return EACCES;	/* 158	*/
-    case ERROR_BAD_PATHNAME:			return ENOENT;	/* 161	*/
-    case ERROR_MAX_THRDS_REACHED:		return EAGAIN;	/* 164	*/
-    case ERROR_LOCK_FAILED:			return EACCES;	/* 167	*/
-    case ERROR_ALREADY_EXISTS:			return EEXIST;	/* 183	*/
-    case ERROR_INVALID_STARTING_CODESEG:	return ENOEXEC;	/* 188	*/
-    case ERROR_INVALID_STACKSEG:		return ENOEXEC;	/* 189	*/
-    case ERROR_INVALID_MODULETYPE:		return ENOEXEC;	/* 190	*/
-    case ERROR_INVALID_EXE_SIGNATURE:		return ENOEXEC;	/* 191	*/
-    case ERROR_EXE_MARKED_INVALID:		return ENOEXEC;	/* 192	*/
-    case ERROR_BAD_EXE_FORMAT:			return ENOEXEC;	/* 193	*/
-    case ERROR_ITERATED_DATA_EXCEEDS_64k:	return ENOEXEC;	/* 194	*/
-    case ERROR_INVALID_MINALLOCSIZE:		return ENOEXEC;	/* 195	*/
-    case ERROR_DYNLINK_FROM_INVALID_RING:	return ENOEXEC;	/* 196	*/
-    case ERROR_IOPL_NOT_ENABLED:		return ENOEXEC;	/* 197	*/
-    case ERROR_INVALID_SEGDPL:			return ENOEXEC;	/* 198	*/
-    case ERROR_AUTODATASEG_EXCEEDS_64k:		return ENOEXEC;	/* 199	*/
-    case ERROR_RING2SEG_MUST_BE_MOVABLE:	return ENOEXEC;	/* 200	*/
-    case ERROR_RELOC_CHAIN_XEEDS_SEGLIM:	return ENOEXEC;	/* 201	*/
-    case ERROR_INFLOOP_IN_RELOC_CHAIN:		return ENOEXEC;	/* 202	*/
-    case ERROR_FILENAME_EXCED_RANGE:		return ENOENT;	/* 206	*/
-    case ERROR_NESTING_NOT_ALLOWED:		return EAGAIN;	/* 215	*/
-    case ERROR_NOT_ENOUGH_QUOTA:		return ENOMEM;	/* 1816	*/
-    default:					return EINVAL;
-    }
-}
-
-static ETHR_INLINE thr_data_ *
-tid2thr(ethr_tid tid)
-{
-    ethr_tid ix;
-    thr_data_ *td;
-    
-    if (tid < 0)
-	return NULL;
-    ix = THR_IX(tid);
-    if (ix >= ETHR_MAX_THREADS)
-	return NULL;
-    td = thr_table[ix];
-    if (!td)
-	return NULL;
-    if (td->thr_id != tid)
-	return NULL;
-    return td;
-}
-
-static ETHR_INLINE void
-new_tid(ethr_tid *new_tid, ethr_tid *new_serial, ethr_tid *new_ix)
-{
-    ethr_tid tmp_serial = last_serial;
-    ethr_tid tmp_ix = last_ix + 1;
-    ethr_tid start_ix = tmp_ix;
-
-
-    do {
-	if (tmp_ix >= ETHR_MAX_THREADS) {
-	    tmp_serial++;
-	    if ((tmp_serial << serial_shift) < 0)
-		tmp_serial = 0;
-	    tmp_ix = 0;
-	}
-	if (!thr_table[tmp_ix]) {
-	    *new_tid	= (tmp_serial << serial_shift) | tmp_ix;
-	    *new_serial	= tmp_serial;
-	    *new_ix	= tmp_ix;
-	    return;
-	}
-	tmp_ix++;
-    } while (tmp_ix != start_ix);
-
-    *new_tid	= INVALID_TID;
-    *new_serial	= INVALID_TID;
-    *new_ix	= INVALID_TID;
-
-}
-
-
-static void thr_exit_cleanup(thr_data_ *td, void *res)
-{
-
-    ASSERT(td == OWN_THR_DATA);
-
-    run_exit_handlers();
-
-    EnterCriticalSection(&thr_table_cs);
-    CloseHandle(td->wait_event.handle);
-    if (td->thr_handle == INVALID_HANDLE_VALUE) {
-	/* We are detached; cleanup thread table */
-	ASSERT(td->joiner == INVALID_TID);
-	ASSERT(td == thr_table[THR_IX(td->thr_id)]);
-	thr_table[THR_IX(td->thr_id)] = NULL;
-	if (td != &main_thr_data)
-	    (*freep)((void *) td);
-    }
-    else {
-	/* Save result and let joining thread cleanup */
-	td->result = res;
-    }
-    LeaveCriticalSection(&thr_table_cs);
-}
-
-static unsigned __stdcall thr_wrapper(LPVOID args)
-{
-    void *(*func)(void*) = ((thr_wrap_data_ *) args)->func;
-    void *arg = ((thr_wrap_data_ *) args)->arg;
-    thr_data_ *td = ((thr_wrap_data_ *) args)->td;
-
-    td->wait_event.handle = CreateEvent(NULL, FALSE, FALSE, NULL);
-    if (td->wait_event.handle == INVALID_HANDLE_VALUE
-	|| !TlsSetValue(tls_own_thr_data, (LPVOID) td)) {
-	((thr_wrap_data_ *) args)->res = get_errno();
-	if (td->wait_event.handle != INVALID_HANDLE_VALUE)
-	    CloseHandle(td->wait_event.handle);
-	SetEvent(((thr_wrap_data_ *) args)->ptd->wait_event.handle);
-	_endthreadex((unsigned) 0);
-	ASSERT(0);
-    }
-
-    td->wait_event.prev = NULL;
-    td->wait_event.next = NULL;
-    td->wait_event.in_queue = 0;
-
-    if (thread_create_child_func)
-	(*thread_create_child_func)(((thr_wrap_data_ *) args)->prep_func_res);
-
-    ASSERT(td == OWN_THR_DATA);
-
-    ((thr_wrap_data_ *) args)->res = 0;
-    SetEvent(((thr_wrap_data_ *) args)->ptd->wait_event.handle);
-
-    thr_exit_cleanup(td, (*func)(arg));
-    return 0;
-}
-
-int
-ethr_fake_static_mutex_init(ethr_mutex *mtx)
-{
-    EnterCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs);
-    /* Got here under race conditions; check again... */
-    if (!mtx->initialized) {
-	if (!InitializeCriticalSectionAndSpinCount(&mtx->cs,
-						   ETHR_MUTEX_SPIN_COUNT))
-	    return get_errno();
-	mtx->initialized = ETHR_MUTEX_INITIALIZED;
-    }
-    LeaveCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs);
-    return 0;
-}
-
-static int
-fake_static_cond_init(ethr_cond *cnd)
-{
-    EnterCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs);
-    /* Got here under race conditions; check again... */
-    if (!cnd->initialized) {
-	if (!InitializeCriticalSectionAndSpinCount(&cnd->cs,
-						   ETHR_COND_SPIN_COUNT))
-	    return get_errno();
-	cnd->queue = NULL;
-	cnd->queue_end = NULL;
-	cnd->initialized = ETHR_COND_INITIALIZED;
-    }
-    LeaveCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs);
-    return 0;
-}
-
-#ifdef __GNUC__
-#define LL_LITERAL(X) X##LL
-#else
-#define LL_LITERAL(X) X##i64
-#endif
-
-#define EPOCH_JULIAN_DIFF LL_LITERAL(11644473600)
-
-static ETHR_INLINE void
-get_curr_time(long *sec, long *nsec)
-{
-    SYSTEMTIME t;
-    FILETIME ft;
-    LONGLONG lft;
-
-    GetSystemTime(&t);
-    SystemTimeToFileTime(&t, &ft);
-    memcpy(&lft, &ft, sizeof(lft));
-    *nsec = ((long) (lft % LL_LITERAL(10000000)))*100;
-    *sec = (long) ((lft / LL_LITERAL(10000000)) - EPOCH_JULIAN_DIFF);
-}
-
-static cnd_wait_event_ *cwe_freelist;
-static CRITICAL_SECTION cwe_cs;
-
-static int
-alloc_cwe(cnd_wait_event_ **cwe_res)
-{
-    cnd_wait_event_ *cwe;
-    EnterCriticalSection(&cwe_cs);
-    cwe = cwe_freelist;
-    if (cwe) {
-	cwe_freelist = cwe->next;
-	LeaveCriticalSection(&cwe_cs);
-    }
-    else {
-	LeaveCriticalSection(&cwe_cs);
-	cwe = (*allocp)(sizeof(cnd_wait_event_));
-	if (!cwe)
-	    return ENOMEM;
-	cwe->handle = CreateEvent(NULL, FALSE, FALSE, NULL);
-	if (cwe->handle == INVALID_HANDLE_VALUE) {
-	    int res = get_errno();
-	    (*freep)(cwe);
-	    return res;
-	}
-    }
-    *cwe_res = cwe;
-    return 0;
-}
-
-static
-free_cwe(cnd_wait_event_ *cwe)
-{
-    EnterCriticalSection(&cwe_cs);
-    cwe->next = cwe_freelist;
-    cwe_freelist = cwe;
-    LeaveCriticalSection(&cwe_cs);
-}
-
-static ETHR_INLINE int
-condwait(ethr_cond *cnd,
-	 ethr_mutex *mtx,
-	 int with_timeout,
-	 ethr_timeval *timeout)
-{
-    int res;
-    thr_data_ *td;
-    cnd_wait_event_ *cwe;
-    DWORD code;
-    long time; /* time until timeout in milli seconds */
-
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-
-    if (!mtx
-	|| mtx->initialized != ETHR_MUTEX_INITIALIZED
-	|| !cnd
-	|| (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)
-	|| (with_timeout && !timeout)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    td = OWN_THR_DATA;
-    if (td)
-	cwe = &td->wait_event;
-    else { /* A non-ethread thread */
-	res = alloc_cwe(&cwe);
-	if (res != 0)
-	    return res;
-    }
-
-    if (!cnd->initialized)
-	fake_static_cond_init(cnd);
-    EnterCriticalSection(&cnd->cs);
-
-    ASSERT(!cwe->in_queue);
-    if (cnd->queue_end) {
-	ASSERT(cnd->queue);
-	cwe->prev = cnd->queue_end;
-	cwe->next = NULL;
-	cnd->queue_end->next = cwe;
-	cnd->queue_end = cwe;
-    }
-    else {
-	ASSERT(!cnd->queue);
-	cwe->prev = NULL;
-	cwe->next = NULL;
-	cnd->queue = cwe;
-	cnd->queue_end = cwe;
-    }
-    cwe->in_queue = 1;
-
-    LeaveCriticalSection(&cnd->cs);
- 
-    LeaveCriticalSection(&mtx->cs);
-
-    if (!with_timeout)
-	time = INFINITE;
-    else {
-	long sec, nsec;
-	ASSERT(timeout);
-	get_curr_time(&sec, &nsec);
-	time = (timeout->tv_sec - sec)*1000;
-	time += (timeout->tv_nsec - nsec + 500)/1000000;
-	if (time < 0)
-	    time = 0;
-    }
-
-    /* wait for event to signal */
-    code = WaitForSingleObject(cwe->handle, time);
-
-    EnterCriticalSection(&mtx->cs);
-
-    if (code == WAIT_OBJECT_0) {
-	/* We were woken by a signal or a broadcast ... */
-	res = 0;
-
-	/* ... no need to remove event from wait queue since this was
-	   taken care of by the signal or broadcast */
-#ifdef DEBUG
-	EnterCriticalSection(&cnd->cs);
-	ASSERT(!cwe->in_queue);
-	LeaveCriticalSection(&cnd->cs);
-#endif
-
-    }
-    else {
-	/* We timed out... */
-	res = ETIMEDOUT;
-
-	/* ... probably have to remove event from wait queue ... */
-	EnterCriticalSection(&cnd->cs);
-
-	if (cwe->in_queue) { /* ... but we must check that we are in queue
-				since a signal or broadcast after timeout
-				may have removed us from the queue */
-	    if (cwe->prev) {
-		cwe->prev->next = cwe->next;
-	    }
-	    else {
-		ASSERT(cnd->queue == cwe);
-		cnd->queue = cwe->next;
-	    }
-
-	    if (cwe->next) {
-		cwe->next->prev = cwe->prev;
-	    }
-	    else {
-		ASSERT(cnd->queue_end == cwe);
-		cnd->queue_end = cwe->prev;
-	    }
-	    cwe->in_queue = 0;
-	}
-
-	LeaveCriticalSection(&cnd->cs);
-
-    }
-
-    if (!td)
-	free_cwe(cwe);
-
-    return res;
-
-}
-
-
-/*
- * ----------------------------------------------------------------------------
- * Exported functions
- * ----------------------------------------------------------------------------
- */
-
-int
-ethr_init(ethr_init_data *id)
-{
-#ifdef _WIN32_WINNT
-    DWORD major = (_WIN32_WINNT >> 8) & 0xff;
-    DWORD minor = _WIN32_WINNT & 0xff;
-    OSVERSIONINFO os_version;
-#endif
-    int err = 0;
-    thr_data_ *td = &main_thr_data;
-    unsigned long i;
-
-    if (!ethr_not_inited)
-	return EINVAL;
-
-#ifdef _WIN32_WINNT
-    os_version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
-    GetVersionEx(&os_version);
-    if (os_version.dwPlatformId != VER_PLATFORM_WIN32_NT
-	|| os_version.dwMajorVersion < major
-	|| (os_version.dwMajorVersion == major
-	    && os_version.dwMinorVersion < minor))
-	return ENOTSUP;
-#endif
-
-    ASSERT(ETHR_MAX_THREADS > 0);
-    for (i = ETHR_MAX_THREADS - 1, serial_shift = 0;
-	 i;
-	 serial_shift++, i >>= 1);
-    thr_ix_mask = ~(~((ethr_tid) 0) << serial_shift);
-
-    tls_own_thr_data = TlsAlloc();
-    if (tls_own_thr_data == TLS_OUT_OF_INDEXES)
-	goto error;
-
-    last_serial = 0;
-    last_ix = 0;
-
-    td->thr_id = 0;
-    td->thr_handle = GetCurrentThread();
-    td->joiner = INVALID_TID;
-    td->result = NULL;
-    td->wait_event.handle = CreateEvent(NULL, FALSE, FALSE, NULL);
-    if (td->wait_event.handle == INVALID_HANDLE_VALUE)
-	goto error;
-    td->wait_event.prev = NULL;
-    td->wait_event.next = NULL;
-    td->wait_event.in_queue = 0;
-    thr_table[0] = td;
-
-    if (!TlsSetValue(tls_own_thr_data, (LPVOID) td))
-	goto error;
-
-    ASSERT(td == OWN_THR_DATA);
-
-
-    cwe_freelist = NULL;
-    if (!InitializeCriticalSectionAndSpinCount(&cwe_cs,
-					       ETHR_MUTEX_SPIN_COUNT))
-	goto error;
-
-    for (i = 1; i < ETHR_MAX_THREADS; i++)
-	thr_table[i] = NULL;
-
-    if (!InitializeCriticalSectionAndSpinCount(&thr_table_cs,
-					       ETHR_MUTEX_SPIN_COUNT))
-	goto error;
-    if (!InitializeCriticalSectionAndSpinCount(&fake_static_init_cs,
-					       ETHR_MUTEX_SPIN_COUNT))
-	goto error;
-    ethr_not_inited = 0;
-
-    err = init_common(id);
-    if (err)
-	goto error;
-
-    return 0;
-
- error:
-    ethr_not_inited = 1;
-    if (err == 0)
-	err = get_errno();
-    ASSERT(err != 0);
-    if (td->thr_handle != INVALID_HANDLE_VALUE)
-	CloseHandle(td->thr_handle);
-    if (td->wait_event.handle != INVALID_HANDLE_VALUE)
-	CloseHandle(td->wait_event.handle);
-    return err;
-}
-
-/*
- * Thread functions.
- */
-
-int
-ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg,
-		ethr_thr_opts *opts)
-{
-    int err = 0;
-    thr_wrap_data_ twd;
-    thr_data_ *my_td, *child_td = NULL;
-    ethr_tid child_tid, child_serial, child_ix;
-    DWORD code;
-    unsigned ID;
-    unsigned stack_size = 0; /* 0 = system default */
-    int use_stack_size = (opts && opts->suggested_stack_size >= 0
-			  ? opts->suggested_stack_size
-			  : -1 /* Use system default */);
-
-#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE
-    if (use_stack_size < 0)
-	use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE;
-#endif
-
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!tid || !func) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    my_td = OWN_THR_DATA;
-    if (!my_td) {
-	/* Only ethreads are allowed to call this function */
-	ASSERT(0);
-	return EACCES;
-    }
-
-    if (use_stack_size >= 0) {
-	size_t suggested_stack_size = (size_t) use_stack_size;
-#ifdef DEBUG
-	suggested_stack_size /= 2; /* Make sure we got margin */
-#endif
-	if (suggested_stack_size < min_stack_size)
-	    stack_size = (unsigned) ETHR_KW2B(min_stack_size);
-	else if (suggested_stack_size > max_stack_size)
-	    stack_size = (unsigned) ETHR_KW2B(max_stack_size);
-	else
-	    stack_size =
-		(unsigned) ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size));
-    }
-
-    EnterCriticalSection(&thr_table_cs);
-
-    /* Call prepare func if it exist */
-    if (thread_create_prepare_func)
-	twd.prep_func_res = (*thread_create_prepare_func)();
-    else
-	twd.prep_func_res = NULL;
-
-    /* Find a new thread id to use */
-    new_tid(&child_tid, &child_serial, &child_ix);
-    if (child_tid == INVALID_TID) {
-	err = EAGAIN;
-	goto error;
-    }
-
-    ASSERT(child_ix == THR_IX(child_tid));
-
-    *tid = child_tid;
-
-    ASSERT(!thr_table[child_ix]);
-
-    /* Alloc thread data */
-    thr_table[child_ix] = child_td = (thr_data_ *) (*allocp)(sizeof(thr_data_));
-    if (!child_td) {
-	err = ENOMEM;
-	goto error;
-    }
-
-    /* Init thread data */
-
-    child_td->thr_id		= child_tid;
-    child_td->thr_handle	= INVALID_HANDLE_VALUE;
-    child_td->joiner		= INVALID_TID;
-    child_td->result		= NULL;
-    /* 'child_td->wait_event' is initialized by child thread */
-
-
-    /* Init thread wrapper data */
-
-    twd.func			= func;
-    twd.arg			= arg;
-    twd.ptd			= my_td;
-    twd.td			= child_td;
-    twd.res			= 0;
-
-    ASSERT(!my_td->wait_event.in_queue);
-
-    /* spawn the thr_wrapper function */
-    child_td->thr_handle = (HANDLE) _beginthreadex(NULL,
-						   stack_size,
-						   thr_wrapper,
-						   (LPVOID) &twd,
-						   0,
-						   &ID);
-    if (child_td->thr_handle == (HANDLE) 0) {
-	child_td->thr_handle = INVALID_HANDLE_VALUE;
-	goto error;
-    }
-
-    ASSERT(child_td->thr_handle != INVALID_HANDLE_VALUE);
-
-    /* Wait for child to finish initialization */
-    code = WaitForSingleObject(my_td->wait_event.handle, INFINITE);
-    if (twd.res || code != WAIT_OBJECT_0) {
-	err = twd.res;
-	goto error;
-    }
-
-    if (opts && opts->detached) {
-	CloseHandle(child_td->thr_handle);
-	child_td->thr_handle = INVALID_HANDLE_VALUE;
-    }
-
-    last_serial	= child_serial;
-    last_ix	= child_ix;
-
-    ASSERT(thr_table[child_ix] == child_td);
-
-    if (thread_create_parent_func)
-	(*thread_create_parent_func)(twd.prep_func_res);
-
-    LeaveCriticalSection(&thr_table_cs);
-
-    return 0;
-
- error:
-
-    if (err == 0)
-	err = get_errno();
-    ASSERT(err != 0);
-
-    if (thread_create_parent_func)
-	(*thread_create_parent_func)(twd.prep_func_res);
-
-    if (child_ix != INVALID_TID) {
-
-	if (child_td) {
-	    ASSERT(thr_table[child_ix] == child_td);
-
-	    if (child_td->thr_handle != INVALID_HANDLE_VALUE) {
-		WaitForSingleObject(child_td->thr_handle, INFINITE);
-		CloseHandle(child_td->thr_handle);
-	    }
-
-	    (*freep)((void *) child_td);
-	    thr_table[child_ix] = NULL;
-	}
-    }
-
-    *tid = INVALID_TID;
-
-    LeaveCriticalSection(&thr_table_cs);
-    return err;
-}
-
-int ethr_thr_join(ethr_tid tid, void **res)
-{
-    int err = 0;
-    DWORD code;
-    thr_data_ *td;
-    thr_data_ *my_td;
-
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-
-    my_td = OWN_THR_DATA;
-
-    if (!my_td) {
-	/* Only ethreads are allowed to call this function */
-	ASSERT(0);
-	return EACCES;
-    }
-
-    EnterCriticalSection(&thr_table_cs);
-
-    td = tid2thr(tid);
-    if (!td)
-	err = ESRCH;
-    else if (td->thr_handle == INVALID_HANDLE_VALUE /* i.e. detached */
-	     || td->joiner != INVALID_TID) /* i.e. someone else is joining */
-	err = EINVAL;
-    else if (my_td == td)
-	err = EDEADLK;
-    else
-	td->joiner = my_td->thr_id;
-
-    LeaveCriticalSection(&thr_table_cs);
-
-    if (err)
-	goto error;
-
-    /* Wait for thread to terminate */
-    code = WaitForSingleObject(td->thr_handle, INFINITE);
-    if (code != WAIT_OBJECT_0)
-	goto error;
-
-    EnterCriticalSection(&thr_table_cs);
-
-    ASSERT(td == tid2thr(tid));
-    ASSERT(td->thr_handle != INVALID_HANDLE_VALUE);
-    ASSERT(td->joiner == my_td->thr_id);
-
-    if (res)
-	*res = td->result;
-
-    CloseHandle(td->thr_handle);
-    ASSERT(td == thr_table[THR_IX(td->thr_id)]);
-    thr_table[THR_IX(td->thr_id)] = NULL;
-    if (td != &main_thr_data)
-	(*freep)((void *) td);
-
-    LeaveCriticalSection(&thr_table_cs);
-
-    return 0;
-
- error:
-    if (err == 0)
-	err = get_errno();
-    ASSERT(err != 0);
-    return err;
-}
-
-
-int
-ethr_thr_detach(ethr_tid tid)
-{
-    int res;
-    DWORD code;
-    thr_data_ *td;
-
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-
-    if (!OWN_THR_DATA) {
-	/* Only ethreads are allowed to call this function */
-	ASSERT(0);
-	return EACCES;
-    }
-
-    EnterCriticalSection(&thr_table_cs);
-
-    td = tid2thr(tid);
-    if (!td)
-	res = ESRCH;
-    if (td->thr_handle == INVALID_HANDLE_VALUE /* i.e. detached */
-	|| td->joiner != INVALID_TID) /* i.e. someone is joining */
-	res = EINVAL;
-    else {
-	res = 0;
-	CloseHandle(td->thr_handle);
-	td->thr_handle = INVALID_HANDLE_VALUE;
-    }
-
-    LeaveCriticalSection(&thr_table_cs);
-
-    return res;
-}
-
-
-void
-ethr_thr_exit(void *res)
-{
-    thr_data_ *td;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return;
-    }
-#endif
-    td = OWN_THR_DATA;
-    if (!td) {
-	/* Only ethreads are allowed to call this function */
-	ASSERT(0);
-	return;
-    }
-    thr_exit_cleanup(td, res);
-    _endthreadex((unsigned) 0);
-}
-
-ethr_tid
-ethr_self(void)
-{
-    thr_data_ *td;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return INVALID_TID;
-    }
-#endif
-    /* It is okay for non-ethreads (i.e. native win32 threads) to call
-       ethr_self(). They will however be returned the INVALID_TID. */
-    td = OWN_THR_DATA;
-    if (!td)
-	return INVALID_TID;
-    return td->thr_id;
-}
-
-int
-ethr_equal_tids(ethr_tid tid1, ethr_tid tid2)
-{
-    /* INVALID_TID does not equal any tid, not even the INVALID_TID */
-    return tid1 == tid2 && tid1 != INVALID_TID;
-}
-
-/*
- * Mutex functions.
- */
-
-int
-ethr_mutex_init(ethr_mutex *mtx)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!InitializeCriticalSectionAndSpinCount(&mtx->cs, ETHR_MUTEX_SPIN_COUNT))
-	return get_errno();
-    mtx->initialized = ETHR_MUTEX_INITIALIZED;
-#if ETHR_XCHK
-    mtx->is_rec_mtx = 0;
-#endif
-    return 0;
-}
-
-int
-ethr_rec_mutex_init(ethr_mutex *mtx)
-{
-    int res;
-    res = ethr_mutex_init(mtx);
-#if ETHR_XCHK
-    mtx->is_rec_mtx = 1;
-#endif
-    return res;
-}
-
-int
-ethr_mutex_destroy(ethr_mutex *mtx)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    DeleteCriticalSection(&mtx->cs);
-    mtx->initialized = 0;
-    return 0;
-}
-
-int ethr_mutex_set_forksafe(ethr_mutex *mtx)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return 0; /* No fork() */
-}
-
-int ethr_mutex_unset_forksafe(ethr_mutex *mtx)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return 0; /* No fork() */
-}
-
-int
-ethr_mutex_trylock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx
-	|| (mtx->initialized && mtx->initialized != ETHR_MUTEX_INITIALIZED)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!mtx->initialized) {
-	int res = ethr_fake_static_mutex_init(mtx);
-	if (res != 0)
-	    return res;
-    }
-    return ethr_mutex_trylock__(mtx);
-}
-
-int
-ethr_mutex_lock(ethr_mutex *mtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx
-	|| (mtx->initialized && mtx->initialized != ETHR_MUTEX_INITIALIZED)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_lock__(mtx);
-}
-
-int
-ethr_mutex_unlock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    int res;
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_unlock__(mtx);
-}
-
-/*
- * Condition variable functions.
- */
-
-int
-ethr_cond_init(ethr_cond *cnd)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!InitializeCriticalSectionAndSpinCount(&cnd->cs, ETHR_COND_SPIN_COUNT))
-	return get_errno();
-    cnd->queue = NULL;
-    cnd->queue_end = NULL;
-    cnd->initialized = ETHR_COND_INITIALIZED;
-    return 0;
-}
-
-int
-ethr_cond_destroy(ethr_cond *cnd)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)
-	|| cnd->queue) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    DeleteCriticalSection(&cnd->cs);
-    cnd->initialized = 0;
-    return 0;
-}
-
-int
-ethr_cond_signal(ethr_cond *cnd)
-{
-    cnd_wait_event_ *cwe;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!cnd->initialized) {
-	int res = fake_static_cond_init(cnd);
-	if (res != 0)
-	    return res;
-    }
-    EnterCriticalSection(&cnd->cs);
-    cwe = cnd->queue;
-    if (cwe) {
-	ASSERT(cwe->in_queue);
-	SetEvent(cnd->queue->handle);
-	if (cwe->next)
-	    cwe->next->prev = NULL;
-	else {
-	    ASSERT(cnd->queue_end == cnd->queue);
-	    cnd->queue_end = NULL;
-	}
-	cnd->queue = cwe->next;
-	cwe->in_queue = 0;
-    }
-    LeaveCriticalSection(&cnd->cs);
-    return 0;
-}
-
-int
-ethr_cond_broadcast(ethr_cond *cnd)
-{
-    cnd_wait_event_ *cwe;
-
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!cnd->initialized) {
-	int res = fake_static_cond_init(cnd);
-	if (res != 0)
-	    return res;
-    }
-    EnterCriticalSection(&cnd->cs);
-    for (cwe = cnd->queue; cwe; cwe = cwe->next) {
-	ASSERT(cwe->in_queue);
-	SetEvent(cwe->handle);
-	cwe->in_queue = 0;
-    }
-    cnd->queue = NULL;
-    cnd->queue_end = NULL;
-    LeaveCriticalSection(&cnd->cs);
-    return 0;
-
-}
-
-int
-ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
-{
-    return condwait(cnd, mtx, 0, NULL);
-}
-
-int
-ethr_cond_timedwait(ethr_cond *cnd, ethr_mutex *mtx, ethr_timeval *timeout)
-{
-    return condwait(cnd, mtx, 1, timeout);
-}
-
-int
-ethr_time_now(ethr_timeval *time)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!time) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    get_curr_time(&time->tv_sec, &time->tv_nsec);
-    return 0;
-}
-
-/*
- * Thread specific data
- */
-
-int
-ethr_tsd_key_create(ethr_tsd_key *keyp)
-{
-    DWORD key;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!keyp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    key = TlsAlloc();
-    if (key == TLS_OUT_OF_INDEXES)
-	return get_errno();
-    *keyp = (ethr_tsd_key) key;
-    return 0;
-}
-
-int
-ethr_tsd_key_delete(ethr_tsd_key key)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    if (!TlsFree((DWORD) key))
-	return get_errno();
-    return 0;
-}
-
-int
-ethr_tsd_set(ethr_tsd_key key, void *value)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    if (!TlsSetValue((DWORD) key, (LPVOID) value))
-	return get_errno();
-    return 0;
-}
-
-void *
-ethr_tsd_get(ethr_tsd_key key)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return NULL;
-    }
-#endif
-    return (void *) TlsGetValue((DWORD) key);
-}
-
-/* Misc */
-
-#ifndef ETHR_HAVE_OPTIMIZED_LOCKS
-
-int
-ethr_do_spinlock_init(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (InitializeCriticalSectionAndSpinCount(&lock->cs, INT_MAX))
-	return 0;
-    else
-	return get_errno();
-}
-
-int
-ethr_do_rwlock_init(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    lock->counter = 0;
-    if (InitializeCriticalSectionAndSpinCount(&lock->cs, INT_MAX))
-	return 0;
-    else
-	return get_errno();    
-}
-
-#endif /* #ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS */
-
-#else
-#error "Missing thread implementation"
-#endif
-
-/* Atomics */
-
-int
-ethr_atomic_init(ethr_atomic_t *var, long i)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_init__(var, i);
-}
-
-int
-ethr_atomic_set(ethr_atomic_t *var, long i)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_set__(var, i);
-}
-
-int
-ethr_atomic_read(ethr_atomic_t *var, long *i)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var || !i) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_read__(var, i);
-}
-
-
-int
-ethr_atomic_addtest(ethr_atomic_t *var, long incr, long *testp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var || !testp) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_addtest__(var, incr, testp);
-}   
-    
-int
-ethr_atomic_inctest(ethr_atomic_t *incp, long *testp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!incp || !testp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_inctest__(incp, testp);
-}
-
-int
-ethr_atomic_dectest(ethr_atomic_t *decp, long *testp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!decp || !testp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_dectest__(decp, testp);
-}
-
-int
-ethr_atomic_add(ethr_atomic_t *var, long incr)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_add__(var, incr);
-}   
-    
-int 
-ethr_atomic_inc(ethr_atomic_t *incp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!incp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_inc__(incp);
-}
-
-int
-ethr_atomic_dec(ethr_atomic_t *decp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!decp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_dec__(decp);
-}
-
-int
-ethr_atomic_and_old(ethr_atomic_t *var, long mask, long *old)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!var || !old) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_and_old__(var, mask, old);
-}
-
-int
-ethr_atomic_or_old(ethr_atomic_t *var, long mask, long *old)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!var || !old) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_or_old__(var, mask, old);
-}
-
-int
-ethr_atomic_xchg(ethr_atomic_t *var, long new, long *old)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }  
-    if (!var || !old) {
-	ASSERT(0);
-	return EINVAL;
-    }  
-#endif
-    return ethr_atomic_xchg__(var, new, old);
-}   
-
-int
-ethr_atomic_cmpxchg(ethr_atomic_t *var, long new, long expected, long *old)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }  
-    if (!var || !old) {
-	ASSERT(0);
-	return EINVAL;
-    }  
-#endif
-    return ethr_atomic_cmpxchg__(var, new, expected, old);
-}
-
-/* Spinlocks and rwspinlocks */
-
-int
-ethr_spinlock_init(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_spinlock_init__(lock);
-}
-
-int
-ethr_spinlock_destroy(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_spinlock_destroy__(lock);
-}
-
-
-int
-ethr_spin_unlock(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_spin_unlock__(lock);
-}
-
-int
-ethr_spin_lock(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_spin_lock__(lock);
-}
-
-int
-ethr_rwlock_init(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwlock_init__(lock);
-}
-
-int
-ethr_rwlock_destroy(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwlock_destroy__(lock);
-}
-
-int
-ethr_read_unlock(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_read_unlock__(lock);
-}
-
-int
-ethr_read_lock(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_read_lock__(lock);
-}
-
-int
-ethr_write_unlock(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_write_unlock__(lock);
-}
-
-int
-ethr_write_lock(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_write_lock__(lock);
-}
-
-
-int
-ethr_gate_init(ethr_gate *gp)
-{
-    int res;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_init(&gp->mtx);
-    if (res != 0)
-	return res;
-    res = ethr_cond_init(&gp->cnd);
-    if (res != 0) {
-	ethr_mutex_destroy(&gp->mtx);
-	return res;
-    }
-    gp->open = 0;
-    return 0;
-}
-
-int
-ethr_gate_destroy(ethr_gate *gp)
-{
-    int res, dres;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_destroy(&gp->mtx);
-    dres = ethr_cond_destroy(&gp->cnd);
-    if (res == 0)
-	res = dres;
-    gp->open = 0;
-    return res;
-}
-
-int
-ethr_gate_close(ethr_gate *gp)
-{
-    int res;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&gp->mtx);
-    if (res != 0)
-	return res;
-    gp->open = 0;
-    res = ethr_mutex_unlock__(&gp->mtx);
-    return res;
-}
-
-int
-ethr_gate_let_through(ethr_gate *gp, unsigned no)
-{
-    int res, ures;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&gp->mtx);
-    if (res != 0)
-	return res;
-    gp->open += no;
-    res = (gp->open == 1
-	   ? ethr_cond_signal(&gp->cnd)
-	   : ethr_cond_broadcast(&gp->cnd));
-    ures = ethr_mutex_unlock__(&gp->mtx);
-    if (res != 0)
-	res = ures;
-    return res;
-}
-
-int
-ethr_gate_swait(ethr_gate *gp, int spincount)
-{
-    int res, ures, n;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    n = spincount;
-    res = ethr_mutex_lock__(&gp->mtx);
-    if (res != 0)
-	return res;
-    while (n >= 0 && !gp->open) {
-	res = ethr_mutex_unlock__(&gp->mtx);
-	if (res != 0)
-	    return res;
-	res = ethr_mutex_lock__(&gp->mtx);
-	if (res != 0)
-	    return res;
-	n--;
-    }
-    while (!gp->open) {
-	res = ethr_cond_wait(&gp->cnd, &gp->mtx);
-	if (res != 0 && res != EINTR)
-	    goto done;
-    }
-    gp->open--;
- done:
-    ures = ethr_mutex_unlock__(&gp->mtx);
-    if (res == 0)
-	res = ures;
-    return res;
-}
-
-
-int
-ethr_gate_wait(ethr_gate *gp)
-{
-    return ethr_gate_swait(gp, 0);
-}
-
-
-/* rwmutex fallback */
-#ifdef ETHR_USE_RWMTX_FALLBACK
-
-int
-ethr_rwmutex_init(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_init(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    ethr_cond_init(&rwmtx->rcnd);
-    if (res != 0)
-	goto error_cleanup1;
-    res = ethr_cond_init(&rwmtx->wcnd);
-    if (res != 0)
-	goto error_cleanup2;
-    rwmtx->readers = 0;
-    rwmtx->waiting_readers = 0;
-    rwmtx->waiting_writers = 0;
-#if ETHR_XCHK
-    rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED;
-#endif
-    return 0;
- error_cleanup2:
-    ethr_cond_destroy(&rwmtx->rcnd);
- error_cleanup1:
-    ethr_mutex_destroy(&rwmtx->mtx);
-    return res;
-}
-
-int
-ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
-{
-    int res, pres;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    rwmtx->initialized = 0;
-#endif
-    res = ethr_mutex_destroy(&rwmtx->mtx);
-    pres = ethr_cond_destroy(&rwmtx->rcnd);
-    if (res == 0)
-	res = pres;
-    pres = ethr_cond_destroy(&rwmtx->wcnd);
-    if (res == 0)
-	res = pres;
-    return res;
-}
-
-int
-ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_trylock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    if (!rwmtx->waiting_writers) {
-	res = ethr_mutex_unlock__(&rwmtx->mtx);
-	if (res == 0)
-	    return EBUSY;
-	return res;
-    }
-    rwmtx->readers++;
-    return ethr_mutex_unlock__(&rwmtx->mtx);
-}
-
-int
-ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    while (rwmtx->waiting_writers) {
-	rwmtx->waiting_readers++;
-	res = ethr_cond_wait(&rwmtx->rcnd, &rwmtx->mtx);
-	rwmtx->waiting_readers--;
-	if (res != 0 && res != EINTR) {
-	    (void) ethr_mutex_unlock__(&rwmtx->mtx);
-	    return res;
-	}
-    }
-    rwmtx->readers++;
-    return ethr_mutex_unlock__(&rwmtx->mtx);
-}
-
-int
-ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
-{
-    int res, ures;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    rwmtx->readers--;
-    if (!rwmtx->readers && rwmtx->waiting_writers)
-	res = ethr_cond_signal(&rwmtx->wcnd);
-    ures = ethr_mutex_unlock__(&rwmtx->mtx);
-    if (res == 0)
-	res = ures;
-    return res;
-}
-
-int
-ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_trylock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    if (!rwmtx->readers && !rwmtx->waiting_writers)
-	return 0;
-    else {
-	res = ethr_mutex_unlock__(&rwmtx->mtx);
-	if (res == 0)
-	    return EBUSY;
-	return res;
-    }
-}
-
-int
-ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    if (!rwmtx->readers && !rwmtx->waiting_writers) 
-	return 0;
-    
-    while (rwmtx->readers) {
-	rwmtx->waiting_writers++;
-	res = ethr_cond_wait(&rwmtx->wcnd, &rwmtx->mtx);
-	rwmtx->waiting_writers--;
-	if (res != 0 && res != EINTR) {
-	    (void) ethr_rwmutex_rwunlock(rwmtx);
-	    return res;
-	}
-    }
-    return 0;
-}
-
-int
-ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
-{
-    int res, ures;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = 0;
-    if (rwmtx->waiting_writers)
-	res = ethr_cond_signal(&rwmtx->wcnd);
-    else if (rwmtx->waiting_readers)
-	res = ethr_cond_broadcast(&rwmtx->rcnd);
-    ures = ethr_mutex_unlock__(&rwmtx->mtx);
-    if (res == 0)
-	res = ures;
-    return res;
-}
-
-#endif /* #ifdef ETHR_USE_RWMTX_FALLBACK */
-
-void
-ethr_compiler_barrier(void)
-{
-
-}
-
-#ifdef DEBUG
-
-#include <stdio.h>
-int ethr_assert_failed(char *f, int l, char *a)
-{
-    fprintf(stderr, "%s:%d: Assertion failed: %s\n", f, l, a);
-    abort();
-    return 0;
-}
-
-#endif
-
-
diff --git a/erts/lib_src/pthread/ethr_event.c b/erts/lib_src/pthread/ethr_event.c
new file mode 100644
index 0000000000..9434d60d0a
--- /dev/null
+++ b/erts/lib_src/pthread/ethr_event.c
@@ -0,0 +1,225 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Author: Rickard Green
+ */
+
+#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHR_EVENT_IMPL__
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "ethread.h"
+
+#if defined(ETHR_LINUX_FUTEX_IMPL__)
+/* --- Linux futex implementation of ethread events ------------------------- */
+
+#include <sched.h>
+#include <errno.h>
+
+#define ETHR_YIELD_AFTER_BUSY_LOOPS 50
+
+int
+ethr_event_init(ethr_event *e)
+{
+    ethr_atomic32_init(&e->futex, ETHR_EVENT_OFF__);
+    return 0;
+}
+
+int
+ethr_event_destroy(ethr_event *e)
+{
+    return 0;
+}
+
+static ETHR_INLINE int
+wait__(ethr_event *e, int spincount)
+{
+    unsigned sc = spincount;
+    int res;
+    ethr_sint32_t val;
+    int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+
+    if (spincount < 0)
+	ETHR_FATAL_ERROR__(EINVAL);
+
+    while (1) {
+	while (1) {
+	    val = ethr_atomic32_read(&e->futex);
+	    if (val == ETHR_EVENT_ON__)
+		return 0;
+	    if (sc == 0)
+		break;
+	    sc--;
+	    ETHR_SPIN_BODY;
+	    if (--until_yield == 0) {
+		until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		res = ETHR_YIELD();
+		if (res != 0)
+		    ETHR_FATAL_ERROR__(res);
+	    }
+	}
+
+	if (val != ETHR_EVENT_OFF_WAITER__) {
+	    val = ethr_atomic32_cmpxchg(&e->futex,
+					ETHR_EVENT_OFF_WAITER__,
+					ETHR_EVENT_OFF__);
+
+	    if (val == ETHR_EVENT_ON__)
+		return 0;
+	    ETHR_ASSERT(val == ETHR_EVENT_OFF__);
+	}
+
+	res = ETHR_FUTEX__(&e->futex,
+			   ETHR_FUTEX_WAIT__,
+			   ETHR_EVENT_OFF_WAITER__);
+	if (res == EINTR)
+	    break;
+	if (res != 0 && res != EWOULDBLOCK)
+	    ETHR_FATAL_ERROR__(res);
+    }
+
+    return res;
+}
+
+#elif defined(ETHR_PTHREADS)
+/* --- Posix mutex/cond implementation of events ---------------------------- */
+
+int
+ethr_event_init(ethr_event *e)
+{
+    int res;
+    ethr_atomic32_init(&e->state, ETHR_EVENT_OFF__);
+    res = pthread_mutex_init(&e->mtx, NULL);
+    if (res != 0)
+	return res;
+    res = pthread_cond_init(&e->cnd, NULL);
+    if (res != 0) {
+	pthread_mutex_destroy(&e->mtx);
+	return res;
+    }
+    return 0;
+}
+
+int
+ethr_event_destroy(ethr_event *e)
+{
+    int res;
+    res = pthread_mutex_destroy(&e->mtx);
+    if (res != 0)
+	return res;
+    res = pthread_cond_destroy(&e->cnd);
+    if (res != 0)
+	return res;
+    return 0;
+}
+
+static ETHR_INLINE int
+wait__(ethr_event *e, int spincount)
+{
+    int sc = spincount;
+    ethr_sint32_t val;
+    int res, ulres;
+    int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+
+    if (spincount < 0)
+	ETHR_FATAL_ERROR__(EINVAL);
+
+    while (1) {
+	val = ethr_atomic32_read(&e->state);
+	if (val == ETHR_EVENT_ON__)
+	    return 0;
+	if (sc == 0)
+	    break;
+	sc--;
+	ETHR_SPIN_BODY;
+	if (--until_yield == 0) {
+	    until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+	    res = ETHR_YIELD();
+	    if (res != 0)
+		ETHR_FATAL_ERROR__(res);
+	}
+    }
+
+    if (val != ETHR_EVENT_OFF_WAITER__) {
+	val = ethr_atomic32_cmpxchg(&e->state,
+				    ETHR_EVENT_OFF_WAITER__,
+				    ETHR_EVENT_OFF__);
+	if (val == ETHR_EVENT_ON__)
+	    return 0;
+	ETHR_ASSERT(val == ETHR_EVENT_OFF__);
+    }
+
+    ETHR_ASSERT(val == ETHR_EVENT_OFF_WAITER__
+		|| val == ETHR_EVENT_OFF__);
+
+    res = pthread_mutex_lock(&e->mtx);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+
+    while (1) {
+
+	val = ethr_atomic32_read(&e->state);
+	if (val == ETHR_EVENT_ON__)
+	    break;
+
+	res = pthread_cond_wait(&e->cnd, &e->mtx);
+	if (res == EINTR)
+	    break;
+	if (res != 0)
+	    ETHR_FATAL_ERROR__(res);
+    }
+
+    ulres = pthread_mutex_unlock(&e->mtx);
+    if (ulres != 0)
+	ETHR_FATAL_ERROR__(ulres);
+
+    return res; /* 0 || EINTR */
+}
+
+#else
+#error No ethread event implementation
+#endif
+
+void
+ethr_event_reset(ethr_event *e)
+{
+    ethr_event_reset__(e);
+}
+
+void
+ethr_event_set(ethr_event *e)
+{
+    ethr_event_set__(e);
+}
+
+int
+ethr_event_wait(ethr_event *e)
+{
+    return wait__(e, 0);
+}
+
+int
+ethr_event_swait(ethr_event *e, int spincount)
+{
+    return wait__(e, spincount);
+}
diff --git a/erts/lib_src/pthread/ethr_x86_sse2_asm.c b/erts/lib_src/pthread/ethr_x86_sse2_asm.c
new file mode 100644
index 0000000000..6cbe73cf16
--- /dev/null
+++ b/erts/lib_src/pthread/ethr_x86_sse2_asm.c
@@ -0,0 +1,31 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: sse2 asm:s
+ * Author: Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+/* ETHR_X86_SSE2_ASM_C__ will trigger asm:s to compile to be included */
+#define ETHR_X86_SSE2_ASM_C__
+#include "ethread.h"
diff --git a/erts/lib_src/pthread/ethread.c b/erts/lib_src/pthread/ethread.c
new file mode 100644
index 0000000000..fb7d135418
--- /dev/null
+++ b/erts/lib_src/pthread/ethread.c
@@ -0,0 +1,570 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Pthread implementation of the ethread library
+ * Author: Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#define ETHR_CHILD_WAIT_SPIN_COUNT 4000
+
+#include <stdio.h>
+#ifdef ETHR_TIME_WITH_SYS_TIME
+#  include <time.h>
+#  include <sys/time.h>
+#else
+#  ifdef ETHR_HAVE_SYS_TIME_H
+#    include <sys/time.h>
+#  else
+#    include <time.h>
+#  endif
+#endif
+#include <sys/types.h>
+#include <unistd.h>
+#include <signal.h>
+
+#include <limits.h>
+
+#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHREAD_IMPL__
+
+#include "ethread.h"
+#include "ethr_internal.h"
+
+#ifndef ETHR_HAVE_ETHREAD_DEFINES
+#error Missing configure defines
+#endif
+
+pthread_key_t ethr_ts_event_key__;
+static int child_wait_spin_count;
+
+/*
+ * --------------------------------------------------------------------------
+ * Static functions
+ * --------------------------------------------------------------------------
+ */
+
+static void thr_exit_cleanup(void)
+{
+    ethr_run_exit_handlers__();
+}
+
+
+/* Argument passed to thr_wrapper() */
+typedef struct {
+    ethr_atomic32_t result;
+    ethr_ts_event *tse;
+    void *(*thr_func)(void *);
+    void *arg;
+    void *prep_func_res;
+} ethr_thr_wrap_data__;
+
+static void *thr_wrapper(void *vtwd)
+{
+    ethr_sint32_t result;
+    void *res;
+    ethr_thr_wrap_data__ *twd = (ethr_thr_wrap_data__ *) vtwd;
+    void *(*thr_func)(void *) = twd->thr_func;
+    void *arg = twd->arg;
+    ethr_ts_event *tsep = NULL;
+
+    result = (ethr_sint32_t) ethr_make_ts_event__(&tsep);
+
+    if (result == 0) {
+	tsep->iflgs |= ETHR_TS_EV_ETHREAD;
+	if (ethr_thr_child_func__)
+	    ethr_thr_child_func__(twd->prep_func_res);
+    }
+
+    tsep = twd->tse; /* We aren't allowed to follow twd after
+			result has been set! */
+
+    ethr_atomic32_set(&twd->result, result);
+
+    ethr_event_set(&tsep->event);
+
+    res = result == 0 ? (*thr_func)(arg) : NULL;
+
+    thr_exit_cleanup();
+    return res;
+}
+
+/* internal exports */
+
+int ethr_set_tse__(ethr_ts_event *tsep)
+{
+    return pthread_setspecific(ethr_ts_event_key__, (void *) tsep);
+}
+
+ethr_ts_event *ethr_get_tse__(void)
+{
+    return pthread_getspecific(ethr_ts_event_key__);
+}
+
+#if defined(ETHR_PPC_RUNTIME_CONF__)
+
+#include <sys/wait.h>
+
+static void
+handle_lwsync_sigill(int signum)
+{
+    _exit(1);
+}
+
+static int
+ppc_init__(void)
+{
+    int pid;
+
+    /* If anything what so ever failes we assume no lwsync for safety */
+    ethr_runtime__.conf.have_lwsync = 0;
+
+    /*
+     * We perform the lwsync test (which might cause an illegal
+     * instruction signal) in a separate process in order to be
+     * completely certain that we do not mess up our own state.
+     */
+    pid = fork();
+    if (pid == 0) {
+	struct sigaction act, oact;
+
+	sigemptyset(&act.sa_mask);
+	act.sa_flags = SA_RESETHAND;
+	act.sa_handler = handle_lwsync_sigill;
+	if (sigaction(SIGILL, &act, &oact) != 0)
+	    _exit(2);
+
+	__asm__ __volatile__ ("lwsync\n\t" : : : "memory");
+
+	_exit(0);
+    }
+
+    if (pid != -1) {
+	while (1) {
+	    int status, res;
+	    res = waitpid(pid, &status, 0);
+	    if (res == pid) {
+		if (WIFEXITED(status) && WEXITSTATUS(status) == 0)
+		    ethr_runtime__.conf.have_lwsync = 1;
+		break;
+	    }
+	}
+    }
+    return 0;
+}
+
+#endif
+
+#if defined(ETHR_X86_RUNTIME_CONF__)
+
+void
+ethr_x86_cpuid__(int *eax, int *ebx, int *ecx, int *edx)
+{
+#if ETHR_SIZEOF_PTR == 4
+    int have_cpuid;
+    /*
+     * If it is possible to toggle eflags bit 21,
+     * we have the cpuid instruction.
+     */
+    __asm__ ("pushf\n\t"
+             "popl %%eax\n\t"
+             "movl %%eax, %%ecx\n\t"
+             "xorl $0x200000, %%eax\n\t"
+             "pushl %%eax\n\t"
+             "popf\n\t"
+             "pushf\n\t"
+             "popl %%eax\n\t"
+             "movl $0x0, %0\n\t"
+             "xorl %%ecx, %%eax\n\t"
+             "jz no_cpuid\n\t"
+	     "movl $0x1, %0\n\t"
+             "no_cpuid:\n\t"
+             : "=r"(have_cpuid)
+             :
+             : "%eax", "%ecx", "cc");
+    if (!have_cpuid) {
+	*eax = *ebx = *ecx = *edx = 0;
+	return;
+    }
+#endif
+#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__
+    /*
+     * When position independet code is used in 32-bit mode, the B register
+     * is used for storage of global offset table address, and we may not
+     * use it as input or output in an asm. We need to save and restore the
+     * B register explicitly (for some reason gcc doesn't provide this
+     * service to us).
+     */
+    __asm__ ("pushl %%ebx\n\t"
+	     "cpuid\n\t"
+	     "movl %%ebx, %1\n\t"
+	     "popl %%ebx\n\t"
+	     : "=a"(*eax), "=r"(*ebx), "=c"(*ecx), "=d"(*edx)
+	     : "0"(*eax)
+	     : "cc");
+#else
+    __asm__ ("cpuid\n\t"
+	     : "=a"(*eax), "=b"(*ebx), "=c"(*ecx), "=d"(*edx)
+	     : "0"(*eax)
+	     : "cc");
+#endif
+}
+
+#endif /* ETHR_X86_RUNTIME_CONF__ */
+
+/*
+ * --------------------------------------------------------------------------
+ * Exported functions
+ * --------------------------------------------------------------------------
+ */
+
+int
+ethr_init(ethr_init_data *id)
+{
+    int res;
+
+    if (!ethr_not_inited__)
+	return EINVAL;
+
+    ethr_not_inited__ = 0;
+
+#if defined(ETHR_PPC_RUNTIME_CONF__)
+    res = ppc_init__();
+    if (res != 0)
+	goto error;
+#endif
+
+    res = ethr_init_common__(id);
+    if (res != 0)
+	goto error;
+
+    child_wait_spin_count = ETHR_CHILD_WAIT_SPIN_COUNT;
+    if (erts_get_cpu_configured(ethr_cpu_info__) == 1)
+	child_wait_spin_count = 0;
+
+    res = pthread_key_create(&ethr_ts_event_key__, ethr_ts_event_destructor__);
+    if (res != 0)
+	goto error;
+
+    return 0;
+ error:
+    ethr_not_inited__ = 1;
+    return res;
+
+}
+
+int
+ethr_late_init(ethr_late_init_data *id)
+{
+    int res = ethr_late_init_common__(id);
+    if (res != 0)
+	return res;
+    ethr_not_completely_inited__ = 0;
+    return res;
+}
+
+int
+ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg,
+		ethr_thr_opts *opts)
+{
+    ethr_thr_wrap_data__ twd;
+    pthread_attr_t attr;
+    int res, dres;
+    int use_stack_size = (opts && opts->suggested_stack_size >= 0
+			  ? opts->suggested_stack_size
+			  : -1 /* Use system default */);
+
+#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE
+    if (use_stack_size < 0)
+	use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE;
+#endif
+
+#if ETHR_XCHK
+    if (ethr_not_completely_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!tid || !func) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+
+    ethr_atomic32_init(&twd.result, (ethr_sint32_t) -1);
+    twd.tse = ethr_get_ts_event();
+    twd.thr_func = func;
+    twd.arg = arg;
+
+    res = pthread_attr_init(&attr);
+    if (res != 0)
+	return res;
+
+    /* Error cleanup needed after this point */
+
+    /* Schedule child thread in system scope (if possible) ... */
+    res = pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM);
+    if (res != 0 && res != ENOTSUP)
+	goto error;
+
+    if (use_stack_size >= 0) {
+	size_t suggested_stack_size = (size_t) use_stack_size;
+	size_t stack_size;
+#ifdef ETHR_DEBUG
+	suggested_stack_size /= 2; /* Make sure we got margin */
+#endif
+#ifdef ETHR_STACK_GUARD_SIZE
+	/* The guard is at least on some platforms included in the stack size
+	   passed when creating threads */
+	suggested_stack_size += ETHR_B2KW(ETHR_STACK_GUARD_SIZE);
+#endif
+	if (suggested_stack_size < ethr_min_stack_size__)
+	    stack_size = ETHR_KW2B(ethr_min_stack_size__);
+	else if (suggested_stack_size > ethr_max_stack_size__)
+	    stack_size = ETHR_KW2B(ethr_max_stack_size__);
+	else
+	    stack_size = ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size));
+	(void) pthread_attr_setstacksize(&attr, stack_size);
+    }
+
+#ifdef ETHR_STACK_GUARD_SIZE
+    (void) pthread_attr_setguardsize(&attr, ETHR_STACK_GUARD_SIZE);
+#endif
+
+    /* Detached or joinable... */
+    res = pthread_attr_setdetachstate(&attr,
+				      (opts && opts->detached
+				       ? PTHREAD_CREATE_DETACHED
+				       : PTHREAD_CREATE_JOINABLE));
+    if (res != 0)
+	goto error;
+
+    /* Call prepare func if it exist */
+    if (ethr_thr_prepare_func__)
+	twd.prep_func_res = ethr_thr_prepare_func__();
+    else
+	twd.prep_func_res = NULL;
+
+    res = pthread_create((pthread_t *) tid, &attr, thr_wrapper, (void*) &twd);
+
+    if (res == 0) {
+	int spin_count = child_wait_spin_count;
+
+	/* Wait for child to initialize... */
+	while (1) {
+	    ethr_sint32_t result;
+	    ethr_event_reset(&twd.tse->event);
+
+	    result = ethr_atomic32_read(&twd.result);
+	    if (result == 0)
+		break;
+
+	    if (result > 0) {
+		res = (int) result;
+		goto error;
+	    }
+
+	    res = ethr_event_swait(&twd.tse->event, spin_count);
+	    if (res != 0 && res != EINTR)
+		goto error;
+	    spin_count = 0;
+	}
+    }
+
+    /* Cleanup... */
+
+ error:
+    dres = pthread_attr_destroy(&attr);
+    if (res == 0)
+	res = dres;
+    if (ethr_thr_parent_func__)
+	ethr_thr_parent_func__(twd.prep_func_res);
+    return res;
+}
+
+int
+ethr_thr_join(ethr_tid tid, void **res)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+    return pthread_join((pthread_t) tid, res);
+}
+
+int
+ethr_thr_detach(ethr_tid tid)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+    return pthread_detach((pthread_t) tid);
+}
+
+void
+ethr_thr_exit(void *res)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return;
+    }
+#endif
+    thr_exit_cleanup();
+    pthread_exit(res);
+}
+
+ethr_tid
+ethr_self(void)
+{
+    return (ethr_tid) pthread_self();
+}
+
+int
+ethr_equal_tids(ethr_tid tid1, ethr_tid tid2)
+{
+    return pthread_equal((pthread_t) tid1, (pthread_t) tid2);
+}
+
+
+/*
+ * Thread specific events
+ */
+
+ethr_ts_event *
+ethr_get_ts_event(void)
+{
+    return ethr_get_ts_event__();
+}
+
+void
+ethr_leave_ts_event(ethr_ts_event *tsep)
+{
+    ethr_leave_ts_event__(tsep);
+}
+
+/*
+ * Thread specific data
+ */
+
+int
+ethr_tsd_key_create(ethr_tsd_key *keyp)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!keyp) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    return pthread_key_create((pthread_key_t *) keyp, NULL);
+}
+
+int
+ethr_tsd_key_delete(ethr_tsd_key key)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+    return pthread_key_delete((pthread_key_t) key);
+}
+
+int
+ethr_tsd_set(ethr_tsd_key key, void *value)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+    return pthread_setspecific((pthread_key_t) key, value);
+}
+
+void *
+ethr_tsd_get(ethr_tsd_key key)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return NULL;
+    }
+#endif
+    return pthread_getspecific((pthread_key_t) key);
+}
+
+/*
+ * Signal functions
+ */
+
+#if ETHR_HAVE_ETHR_SIG_FUNCS
+
+int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!set && !oset) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+  return pthread_sigmask(how, set, oset);
+}
+
+int ethr_sigwait(const sigset_t *set, int *sig)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!set || !sig) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    if (sigwait(set, sig) < 0)
+	return errno;
+    return 0;
+}
+
+#endif /* #if ETHR_HAVE_ETHR_SIG_FUNCS */
+
+ETHR_IMPL_NORETURN__
+ethr_abort__(void)
+{
+    abort();
+}
diff --git a/erts/lib_src/utils/make_atomics_api b/erts/lib_src/utils/make_atomics_api
new file mode 100755
index 0000000000..74736c5a2d
--- /dev/null
+++ b/erts/lib_src/utils/make_atomics_api
@@ -0,0 +1,2286 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2011-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-mode(compile).
+
+%%%-------------------------------------------------------------------
+%%% @author Rickard Green <[email protected]>
+%%% @copyright (C) 2011, Rickard Green
+%%% @doc
+%%%   Generation of the ethread atomic API
+%%% @end
+%%% Created : 17 Jan 2011 by Rickard Green <[email protected]>
+%%%-------------------------------------------------------------------
+
+-define(H_FILE, "erts/include/internal/ethr_atomics.h").
+-define(C_FILE, "erts/lib_src/common/ethr_atomics.c").
+
+%% These order constraints are important:
+%% - 'cmpxchg' needs to appear before 'read'
+%% - 'xchg' needs to apper before 'set'
+%% - 'set' needs to apper before 'init'
+%% - 'add_read' needs to apper before 'add', 'inc_read', and 'dec_read'
+%% - 'inc_read' needs to apper before and 'inc'
+%% - 'dec_read' needs to apper before and 'dec'
+-define(ATOMIC_OPS, [cmpxchg, xchg, set, init, add_read,
+		     read, inc_read, dec_read, add, inc,
+		     dec, read_band, read_bor]).
+
+-define(DW_ATOMIC_OPS, [cmpxchg, set, read, init]).
+-define(DW_FUNC_MACRO, "ETHR_DW_ATOMIC_FUNC__").
+-define(DW_RTCHK_MACRO, "ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__").
+
+%% Barrier versions we implement
+-define(BARRIERS, [none, ddrb, rb, wb, acqb, relb, mb]).
+-define(NON_NATIVE_BARRIERS, [ddrb]).
+-define(NATIVE_BARRIERS, (?BARRIERS -- ?NON_NATIVE_BARRIERS)).
+
+-define(ATOMIC_SIZES, ["dword", "word", "32"]).
+
+-define(HAVE_NATIVE_ATOMIC, "ETHR_HAVE_ETHR_NATIVE_ATOMIC").
+
+-define(SU_DW_SINT_FIELD, "dw_sint").
+-define(DW_SINT_FIELD, "sint").
+
+%% Fallback
+-define(ETHR_ATMC_FLLBK_ADDR_BITS, "10").
+-define(ETHR_ATMC_FLLBK_ADDR_SHIFT, "6").
+
+-record(atomic_context, {dw,
+			 amc_fallback,
+			 ret_type,
+			 ret_var,
+			 arg1,
+			 arg2,
+			 arg3,
+			 have_native_atomic_ops,
+			 atomic,
+			 atomic_t,
+			 addr_aint_t,
+			 aint_t,
+			 naint_t,
+			 'NATMC',
+			 'ATMC',
+			 unusual_val}).
+
+atomic_context("dword") ->
+    #atomic_context{dw = true,
+		    amc_fallback = true,
+		    ret_type = "int",
+		    ret_var = "res",
+		    arg1 = "var",
+		    arg2 = "val",
+		    arg3 = "old_val",
+		    have_native_atomic_ops = "ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS",
+		    atomic = "ethr_dw_atomic",
+		    atomic_t = "ethr_dw_atomic_t",
+		    addr_aint_t = "ethr_sint_t",
+		    aint_t = "ethr_dw_sint_t",
+		    naint_t = "ETHR_SU_DW_NAINT_T__",
+		    'NATMC' = "DW_NATMC",
+		    'ATMC' = "DW_ATMC",
+		    unusual_val = "ETHR_UNUSUAL_SINT_VAL__"};    
+atomic_context(Size) ->
+    {SizeSuffix, HaveSize, AMC} = case Size of
+				      "word" -> {"", "WORD_SZ", true};
+				      _ -> {Size, Size++"BIT", false}
+				  end,
+    AintT = ["ethr_sint", SizeSuffix, "_t"],
+    #atomic_context{dw = false,
+		    amc_fallback = AMC,
+		    ret_type = AintT,
+		    ret_var = "res",
+		    arg1 = "var",
+		    arg2 = "val",
+		    arg3 = "old_val",
+		    have_native_atomic_ops = ["ETHR_HAVE_", HaveSize, "_NATIVE_ATOMIC_OPS"],
+		    atomic = ["ethr_atomic", SizeSuffix],
+		    atomic_t = ["ethr_atomic", SizeSuffix, "_t"],
+		    addr_aint_t = AintT,
+		    aint_t = AintT,
+		    naint_t = ["ETHR_NAINT", SizeSuffix, "_T__"],
+		    'NATMC' = ["NATMC", SizeSuffix],
+		    'ATMC' = ["ATMC", SizeSuffix],
+		    unusual_val = ["ETHR_UNUSUAL_SINT", SizeSuffix, "_VAL__"]}.
+
+-record(op_context, {ret, var, val1, val2}).
+
+-define(POTENTIAL_NBITS, ["64", "32"]).
+
+is_return_op(#atomic_context{dw = false}, add) -> false;
+is_return_op(#atomic_context{dw = false}, inc) -> false;
+is_return_op(#atomic_context{dw = false}, dec) -> false;
+is_return_op(#atomic_context{dw = true}, read) -> false;
+is_return_op(_AC, init) -> false;
+is_return_op(_AC, set) -> false;
+is_return_op(_AC, _OP) -> true.
+
+native(add_read) -> add_return;
+native(inc_read) -> inc_return;
+native(dec_read) -> dec_return;
+native(read_band) -> and_retold;
+native(read_bor) -> or_retold;
+native(Op) -> Op.
+
+op(Op, #op_context{var = Var, val1 = Val1}) when Op == init; Op == set ->
+    [Var, " = ", Val1];
+op(read, #op_context{ret = Ret, var = Var}) ->
+    [Ret, " = ", Var];
+op(add_read, OpC) ->
+    [op(add, OpC), "; ", op(read, OpC)];
+op(add, #op_context{var = Var, val1 = Val1}) ->
+    [Var, " += ", Val1];
+op(inc, #op_context{var = Var}) ->
+    ["++(", Var, ")"];
+op(dec, #op_context{var = Var}) ->
+    ["--(", Var, ")"];
+op(inc_read, #op_context{ret = Ret, var = Var}) ->
+    [Ret, " = ++(", Var, ")"];
+op(dec_read, #op_context{ret = Ret, var = Var}) ->
+    [Ret, " = --(", Var, ")"];
+op(read_band, #op_context{var = Var, val1 = Val1} = OpC) ->
+    [op(read, OpC), "; ", Var, " &= ", Val1];
+op(read_bor, #op_context{var = Var, val1 = Val1} = OpC) ->
+    [op(read, OpC), "; ", Var, " |= ", Val1];
+op(xchg, OpC) ->
+    [op(read, OpC), "; ", op(set, OpC)];
+op(cmpxchg, #op_context{ret = Ret, var = Var, val1 = Val1, val2 = Val2}) ->
+    [Ret, " = (", Var, " == ", Val2, " ? (", Var, " = ", Val1, ", ", Val2, ") : ", Var, ")"].
+
+dw_op(Op, #op_context{var = Var, val1 = Val1}) when Op == init; Op == set ->
+    [Var, "[0] = ", Val1, "[0]; ", Var, "[1] = ", Val1, "[1]"];
+dw_op(read, #op_context{var = Var, val1 = Val1}) ->
+    [Val1, "[0] = ", Var, "[0]; ", Val1, "[1] = ", Var, "[1]"];
+dw_op(cmpxchg, #op_context{ret = Ret, var = Var, val1 = Val1, val2 = Val2}) ->
+    ["
+	{
+	    ", Ret, " = (", Var, "[0] == ", Val2, "[0] && ", Var, "[1] == ", Val2, "[1]);
+	    if (", Ret, ") {
+		", Var, "[0] = ", Val1, "[0];
+		", Var, "[1] = ", Val1, "[1];
+	    }
+	    else {
+		", Val2, "[0] = ", Var, "[0];
+		", Val2, "[1] = ", Var, "[1];
+	    }
+	}"].
+
+op_head_tail(init) -> {undef, undef};
+op_head_tail(set) -> {store, store};
+op_head_tail(read) -> {load, load};
+op_head_tail(_) -> {load, undef}.
+
+op_barrier_ext(none) -> "";
+op_barrier_ext(Barrier) -> [$_, a2l(Barrier)].
+
+op_call(addr, _DW, Ret, Func, Arg1, _Arg2, _Arg3, _TypeCast) ->
+    [Ret, " ", Func, "(", Arg1, ");"];
+op_call(Op, false, Ret, Func, Arg1, _Arg2, _Arg3, _TypeCast) when Op == read;
+								  Op == inc_read;
+								  Op == inc_return;
+								  Op == dec_read;
+								  Op == dec_return ->
+    [Ret, " ", Func, "(", Arg1, ");"];
+op_call(Op, false, _Ret, Func, Arg1, _Arg2, _Arg3, _TypeCast) when Op == inc;
+								   Op == dec ->
+    [Func, "(", Arg1, ");"];
+op_call(Op, false, Ret, Func, Arg1, Arg2, _Arg3, TypeCast) when Op == add_return;
+								Op == add_read;
+								Op == read_band;
+								Op == and_retold;
+								Op == read_bor;
+								Op == or_retold;
+								Op == xchg ->
+    [Ret, " ", Func, "(", Arg1, ",", TypeCast, " ", Arg2, ");"];
+op_call(cmpxchg, _DW, Ret, Func, Arg1, Arg2, Arg3, TypeCast) ->
+    [Ret, " ", Func, "(", Arg1, ",", TypeCast, " ", Arg2, ",", TypeCast, " ", Arg3, ");"];
+op_call(_Op, _DW, _Ret, Func, Arg1, Arg2, _Arg3, TypeCast) ->
+    [Func, "(", Arg1, ",", TypeCast, " ", Arg2, ");"]. % set, init, add (!= dw), read (== dw)
+
+native_op_call(#atomic_context{dw = DW,
+			       ret_var = RetVar,
+			       arg1 = Arg1,
+			       arg2 = Arg2,
+			       arg3 = Arg3,
+			       aint_t = AintT,
+			       'NATMC' = NATMC,
+			       naint_t = NAintT},
+	       Op, B, TypeCasts) ->
+    op_call(Op,
+	    DW,
+	    [RetVar, " =",
+	     case TypeCasts of
+		 true -> [" (", AintT, ")"];
+		 false -> ""
+	     end],
+	    ["ETHR_", NATMC, "_FUNC__(", opstr(native(Op)), op_barrier_ext(B), ")"],
+	    Arg1,
+	    Arg2,
+	    Arg3,
+	    case TypeCasts of
+		true -> [" (", NAintT, ")"];
+		false -> ""
+	    end).
+
+simple_fallback(#atomic_context{arg1 = Arg1,
+				arg2 = Arg2,
+				'ATMC' = ATMC},
+		init, B) -> %% Also double word
+    ["    ETHR_", ATMC, "_FUNC__(set", op_barrier_ext(B),")(", Arg1, ", ", Arg2, ");\n"];
+simple_fallback(#atomic_context{dw = false,
+				arg1 = Arg1,
+				arg2 = Arg2,
+				'ATMC' = ATMC},
+		set, B) ->
+    ["    (void) ETHR_", ATMC, "_FUNC__(xchg", op_barrier_ext(B),")(", Arg1, ", ", Arg2, ");\n"];
+simple_fallback(#atomic_context{dw = false,
+				arg1 = Arg1,
+				arg2 = Arg2,
+				'ATMC' = ATMC},
+		add, B) ->
+    ["    (void) ETHR_", ATMC, "_FUNC__(add_read", op_barrier_ext(B), ")(", Arg1, ", ", Arg2, ");\n"];
+simple_fallback(#atomic_context{dw = false,
+				ret_var = RetVar,
+				arg1 = Arg1,
+				aint_t = AintT,
+				'ATMC' = ATMC},
+		inc_read, B) ->
+    ["    ", RetVar, " = ETHR_", ATMC, "_FUNC__(add_read", op_barrier_ext(B), ")(", Arg1, ", (", AintT,") 1);\n"];
+simple_fallback(#atomic_context{dw = false,
+				ret_var = RetVar,
+				arg1 = Arg1,
+				aint_t = AintT,
+				'ATMC' = ATMC},
+		dec_read, B) ->
+    ["    ", RetVar, " = ETHR_", ATMC, "_FUNC__(add_read", op_barrier_ext(B), ")(", Arg1, ", (", AintT,") -1);\n"];
+simple_fallback(#atomic_context{dw = false,
+				arg1 = Arg1,
+				'ATMC' = ATMC},
+		inc, B) ->
+    ["    (void) ETHR_", ATMC, "_FUNC__(inc_read", op_barrier_ext(B), ")(", Arg1, ");\n"];
+simple_fallback(#atomic_context{dw = false,
+				arg1 = Arg1,
+				'ATMC' = ATMC},
+		dec, B) ->
+    ["    (void) ETHR_", ATMC, "_FUNC__(dec_read", op_barrier_ext(B), ")(", Arg1, ");\n"];
+simple_fallback(#atomic_context{dw = false,
+				unusual_val = UnusualVal,
+				ret_var = RetVar,
+				arg1 = Arg1,
+				aint_t = AintT,
+				'ATMC' = ATMC},
+		read, B) ->
+    ["    ", RetVar, " = ETHR_", ATMC, "_FUNC__(cmpxchg", op_barrier_ext(B), ")(", Arg1, ", (", AintT, ") ", UnusualVal, ", (", AintT,") ", UnusualVal, ");\n"];
+simple_fallback(#atomic_context{dw = true,
+				unusual_val = UnusualVal,
+				arg1 = Arg1,
+				arg2 = Arg2,
+				aint_t = AintT,
+				'ATMC' = ATMC},
+		read, B) ->
+    ["    ", AintT, " tmp;
+    tmp.", ?DW_SINT_FIELD, "[0] = ", UnusualVal, ";
+    tmp.", ?DW_SINT_FIELD, "[1] = ", UnusualVal, ";
+    ", Arg2, "->", ?DW_SINT_FIELD, "[0] = ", UnusualVal, ";
+    ", Arg2, "->", ?DW_SINT_FIELD, "[1] = ", UnusualVal, ";
+    (void) ETHR_", ATMC, "_FUNC__(cmpxchg", op_barrier_ext(B), ")(", Arg1, ", &tmp, ", Arg2, ");
+"
+    ];
+simple_fallback(_AC, _Op, _B) ->
+    [].
+
+func_header(AC, prototype, MacroName, Op, B) ->
+    [func_header(AC, implementation, MacroName, Op, B), ";"];
+func_header(#atomic_context{'ATMC' = ATMC} = AC, inline_implementation, _MacroName, Op, B) ->
+    do_func_header(AC, Op, "static ETHR_INLINE ",
+		   ["ETHR_", ATMC, "_FUNC__(", opstr(Op), op_barrier_ext(B), ")"]);
+func_header(#atomic_context{atomic = Atomic} = AC, implementation, false, Op, B) ->
+    do_func_header(AC, Op, "", [Atomic, "_", opstr(Op), op_barrier_ext(B)]);
+func_header(AC, implementation, MacroName, Op, B) ->
+    do_func_header(AC, Op, "", [MacroName, "(", opstr(Op), op_barrier_ext(B), ")"]).
+
+
+do_func_header(#atomic_context{atomic_t = AtomicT,
+			       addr_aint_t = AddrAintT,
+			       arg1 = Arg1},
+	       addr, Inline, Func) ->
+    [Inline, AddrAintT, " *", Func, "(", AtomicT, " *", Arg1, ")"];
+do_func_header(#atomic_context{dw = false,
+			       atomic_t = AtomicT,
+			       aint_t = AintT,
+			       arg1 = Arg1,
+			       arg2 = Arg2},
+	       Op, Inline, Func) when Op == init;
+				      Op == set;
+				      Op == add ->
+    [Inline, "void ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " ", Arg2, ")"];
+do_func_header(#atomic_context{dw = false,
+			       atomic_t = AtomicT,
+			       arg1 = Arg1},
+	       Op, Inline, Func) when Op == inc;
+				      Op == dec ->
+    [Inline, "void ", Func, "(", AtomicT, " *", Arg1, ")"];
+do_func_header(#atomic_context{dw = false,
+			       atomic_t = AtomicT,
+			       aint_t = AintT,
+			       arg1 = Arg1},
+	       Op, Inline, Func) when Op == read;
+				      Op == inc_read;
+				      Op == dec_read ->
+    [Inline, AintT, " ", Func, "(", AtomicT, " *", Arg1, ")"];
+do_func_header(#atomic_context{dw = false,
+			       atomic_t = AtomicT,
+			       aint_t = AintT,
+			       arg1 = Arg1,
+			       arg2 = Arg2},
+	       Op, Inline, Func) when Op == add_read;
+				      Op == read_band;
+				      Op == read_bor;
+				      Op == xchg ->
+    [Inline, AintT, " ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " ", Arg2, ")"];
+do_func_header(#atomic_context{dw = false,
+			       atomic_t = AtomicT,
+			       aint_t = AintT,
+			       arg1 = Arg1,
+			       arg2 = Arg2,
+			       arg3 = Arg3},
+	       cmpxchg, Inline, Func) ->
+    [Inline, AintT, " ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " ", Arg2, ", ", AintT, " ", Arg3, ")"];
+do_func_header(#atomic_context{dw = true,
+			       atomic_t = AtomicT,
+			       aint_t = AintT,
+			       arg1 = Arg1,
+			       arg2 = Arg2},
+	       Op, Inline, Func) when Op == init;
+				      Op == set;
+				      Op == read ->
+    [Inline, "void ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " *", Arg2, ")"];
+do_func_header(#atomic_context{dw = true,
+			       atomic_t = AtomicT,
+			       aint_t = AintT,
+			       arg1 = Arg1,
+			       arg2 = Arg2,
+			       arg3 = Arg3},
+	       cmpxchg, Inline, Func) ->
+    [Inline, "int ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " *", Arg2, ", ", AintT, " *", Arg3, ")"].
+
+xbarriers(_Op, none, _NB) ->
+    {"", ""};
+
+xbarriers(_Op, acqb, NB) when NB == acqb; NB == mb ->
+    {"", ""};
+xbarriers(Op, acqb, NB) ->
+    case {op_head_tail(Op), NB} of
+	{{_, load}, rb} -> {"", "ETHR_MEMBAR(ETHR_LoadStore);"};
+	{{_, load}, _} -> {"", "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);"};
+	{{_, store}, _} -> {"", "ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);"};
+	{_, rb} -> {"", "ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);"};
+	_ -> {"", "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);"}
+    end;
+
+xbarriers(_Op, relb, NB) when NB == relb; NB == mb ->
+    {"", ""};
+xbarriers(Op, relb, NB) ->
+    case {op_head_tail(Op), NB} of
+	{{store, _}, wb} -> {"ETHR_MEMBAR(ETHR_LoadStore);", ""};
+	{{store, _}, _} -> {"ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);", ""};
+	{{load, _}, _} -> {"ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);", ""};
+	{_, wb} -> {"ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);", ""};
+	_ -> {"ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);", ""}
+    end;
+
+xbarriers(_Op, wb, NB) when NB == wb; NB == mb ->
+    {"", ""};
+xbarriers(_Op, wb, _NB) ->
+    {"ETHR_MEMBAR(ETHR_StoreStore);", ""};
+
+xbarriers(_Op, rb, NB) when NB == rb; NB == mb ->
+    {"", ""};
+xbarriers(_Op, rb, _NB) ->
+    {"", "ETHR_MEMBAR(ETHR_LoadLoad);"};
+
+xbarriers(_Op, mb, mb) ->
+    {"", ""};
+xbarriers(Op, mb, NB) ->
+    MB = "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);",
+    {Head, Tail} = op_head_tail(Op),
+    PreOp = case {Head, NB} of
+		{_, relb} -> "";
+		{store, wb} -> "ETHR_MEMBAR(ETHR_LoadStore);";
+		{store, _} ->  "ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);";
+		{load, _} -> "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);";
+		{_, wb} -> "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);";
+		_ -> MB
+	    end,
+    PostOp = case {Tail, NB} of
+		 {_, acqb} -> "";
+		 {load, rb} -> "ETHR_MEMBAR(ETHR_LoadStore);";
+		 {load, _} ->  "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);";
+		 {store, _} ->  "ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);";
+		 {_, rb} -> "ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);";
+		 _ -> MB
+	     end,
+    {PreOp, PostOp}.
+
+try_barrier_order_first(none) ->
+    [none, rb, wb, acqb, relb];
+try_barrier_order_first(acqb) ->
+    [acqb, rb, none, mb];
+try_barrier_order_first(relb) ->
+    [relb, wb, none, mb];
+try_barrier_order_first(rb) ->
+    [rb, none, mb];
+try_barrier_order_first(wb) ->
+    [wb, none, mb];
+try_barrier_order_first(mb) ->
+    [mb, relb, acqb, wb, rb, none].
+
+try_barrier_order(B) ->
+    First = try_barrier_order_first(B),
+    First ++ (?NATIVE_BARRIERS -- First).
+
+native_barrier_op(#atomic_context{'NATMC' = NATMC} = AC, If, ExtraDecl, Op, B, NB, TypeCasts) ->
+    NOpStr = opstr(native(Op)),
+    CapNOpStr = to_upper(NOpStr),
+    NBExt = op_barrier_ext(NB),
+    CapNBExt = to_upper(NBExt),
+    {PreB, PostB} = xbarriers(Op, B, NB),
+    [If, " defined(ETHR_HAVE_", NATMC, "_", CapNOpStr, CapNBExt, ")\n",
+     ExtraDecl,
+     case PreB of
+	 "" -> "";
+	 _ -> ["    ", PreB, "\n"]
+     end,
+     "    ", native_op_call(AC, Op, NB, TypeCasts), "\n",
+     case PostB of
+	 "" -> "";
+	 _ -> ["    ", PostB, "\n"]
+     end].
+
+dw_native_barrier_op(#atomic_context{arg1 = Arg1, arg2 = Arg2, arg3 = Arg3} = AC, If, ExtraDecl, Op, B, NB) ->
+    native_barrier_op(AC#atomic_context{arg1 = ["&", Arg1, "->native"],
+					arg2 = [Arg2, "->", ?DW_SINT_FIELD],
+					arg3 = [Arg3, "->", ?DW_SINT_FIELD]},
+		      If, ExtraDecl, Op, B, NB, false).
+
+su_dw_native_barrier_op(#atomic_context{dw = true,
+					naint_t = NAintT,
+					ret_var = RetVar,
+					arg1 = Arg1,
+					arg2 = Arg2,
+					arg3 = Arg3,
+					'NATMC' = NATMC} = AC, If, cmpxchg, B, NB) ->
+    SU = ["->", ?SU_DW_SINT_FIELD],
+    TmpVar = "act",
+    SUArg1 = ["&", Arg1, "->native"],
+    SUArg2 = [Arg2, SU],
+    SUArg3 = [Arg3, SU],
+    ExtraDecl = ["    ", NAintT, " ", TmpVar, ";\n"],
+    [native_barrier_op(AC#atomic_context{dw = false,
+					 ret_var = TmpVar,
+					 arg1 = SUArg1,
+					 arg2 = SUArg2,
+					 arg3 = SUArg3,
+					 'NATMC' = ["SU_", NATMC]},
+		       If, ExtraDecl, cmpxchg, B, NB, false),
+     "    ", RetVar, " = (", TmpVar, " == ", SUArg3, ");
+    ", SUArg3, " = ", TmpVar, ";
+"
+     ];
+su_dw_native_barrier_op(#atomic_context{dw = true,
+					arg1 = Arg1,
+					arg2 = Arg2,
+					 'NATMC' = NATMC} = AC, If, Op, B, NB) ->
+    SUArg1 = ["&", Arg1, "->native"],
+    SUArg2 = [Arg2, "->", ?SU_DW_SINT_FIELD],
+    native_barrier_op(AC#atomic_context{dw = false,
+					ret_var = SUArg2,
+					arg1 = SUArg1,
+					arg2 = SUArg2,
+					arg3 = not_used,
+					'NATMC' = ["SU_", NATMC]}, If, "", Op, B, NB, false).
+
+cmpxchg_fallback_define(#atomic_context{dw = false, aint_t = AintT} = AC) ->
+    do_cmpxchg_fallback_define(AC, true, AintT);
+cmpxchg_fallback_define(#atomic_context{dw = true,
+					'NATMC' = NATMC,
+					naint_t = NAintT} = AC) ->
+    ["\n\n#if defined(ETHR_HAVE_NATIVE_DW_ATOMIC)\n",
+     do_cmpxchg_fallback_define(AC, false, not_used),
+     "\n\n#elif defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)\n",
+     do_cmpxchg_fallback_define(AC#atomic_context{'NATMC' = ["SU_", NATMC],
+						  naint_t = NAintT},
+				true,
+				NAintT),
+     "
+
+#else
+#  error \"?!?\"
+#endif
+"].
+     
+do_cmpxchg_fallback_define(#atomic_context{'NATMC' = NATMC,
+					   aint_t = AintT,
+					   naint_t = NAintT},
+			  SU, SUType) ->
+
+    ReadFunc = fun (IF) ->
+		       fun (B) ->
+			       BExt = op_barrier_ext(B),
+			       CapBExt = to_upper(BExt),
+			       [IF, " defined(ETHR_HAVE_", NATMC, "_READ", CapBExt, ")",
+				case SU of
+				    true -> ["
+#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR) \\
+  ETHR_", NATMC, "_FUNC__(read", BExt, ")(VAR)
+"
+					    ];
+				    false -> ["
+#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, VAL) \\
+  ETHR_", NATMC, "_FUNC__(read", BExt, ")(VAR, VAL)
+#elif defined(ETHR_HAVE_SU_", NATMC, "_READ", CapBExt, ")
+#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, VAL) \\
+  VAL.", ?SU_DW_SINT_FIELD, " = ETHR_SU_", NATMC, "_FUNC__(read", BExt, ")(VAR)
+"
+					     ]
+				end]
+		       end
+	       end,
+    NotDefCMPXCHG = fun (B) ->
+		       CapBExt = to_upper(op_barrier_ext(B)),
+		       ["!defined(ETHR_HAVE_", NATMC, "_CMPXCHG", CapBExt, ")"]
+	       end,
+    NoneTryBarrierOrder = try_barrier_order(none),
+    %% First a sanity check
+    ["
+#if (", NotDefCMPXCHG(hd(?NATIVE_BARRIERS)) ,
+     lists:map(fun (B) ->
+		       [" \\
+     && ", NotDefCMPXCHG(B)]
+	       end,
+	       tl(?NATIVE_BARRIERS)), ")
+#  error \"No native cmpxchg() op available\"
+#endif
+
+
+/*
+ * Read op used together with cmpxchg() fallback when no native op present.
+ */
+",
+
+     %% Read op to use with cmpxchg fallback
+     (ReadFunc("#if"))(hd(NoneTryBarrierOrder)),
+     lists:map(ReadFunc("#elif"), tl(NoneTryBarrierOrder)),
+"#else
+/*
+ * We have no native read() op; guess zero and then use the
+ * the atomics actual value returned from cmpxchg().
+ */",
+     case SU of
+	 true -> ["
+#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR) \\
+  ((", NAintT, ") 0)"];
+	 false -> ["
+#define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, VAL) \\
+do { \\
+  VAL.", ?DW_SINT_FIELD, "[0] = (ethr_sint_t) 0; \\
+  VAL.", ?DW_SINT_FIELD, "[1] = (ethr_sint_t) 0; \\
+} while (0)"]
+     end, "
+#endif
+",
+
+     %% The fallback
+     "
+/*
+ * Native cmpxchg() fallback used when no native op present.
+ */
+#define ETHR_", NATMC, "_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \\
+do { \\",
+     case SU of
+	 true -> ["
+    ", SUType, " AVAL; \\
+    ", NAintT, " new__, act__, exp__; \\
+    act__ = ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR); \\
+    do { \\
+	exp__ = act__; \\
+	AVAL = (", SUType, ") act__; \\
+	{ OPS; } \\
+	new__ = (", NAintT, ") AVAL; \\
+	act__ = CMPXCHG(VAR, new__, exp__); \\
+    } while (__builtin_expect(act__ != exp__, 0)); \\"];
+	 false -> ["
+    int res__; \\
+    ", AintT, " AVAL, exp_act__; \\
+    ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, exp_act__); \\
+    do { \\
+	AVAL.", ?DW_SINT_FIELD, "[0] = exp_act__.", ?DW_SINT_FIELD, "[0]; \\
+	AVAL.", ?DW_SINT_FIELD, "[1] = exp_act__.", ?DW_SINT_FIELD, "[1]; \\
+	{ OPS; } \\
+	res__ = CMPXCHG(VAR, AVAL.", ?DW_SINT_FIELD, ", exp_act__.", ?DW_SINT_FIELD, "); \\
+    } while (__builtin_expect(res__ == 0, 0)); \\"]
+     end, "
+} while (0)
+"
+    ].
+
+cmpxchg_fallbacks(#atomic_context{}, _SUDW, cmpxchg, _B) ->
+    ""; %% No need for a fallback
+cmpxchg_fallbacks(#atomic_context{dw = DW,
+				  ret_var = RetVar,
+				  arg1 = Arg1,
+				  arg2 = Arg2,
+				  arg3 = Arg3,
+				  'NATMC' = NATMC},
+		  SUDW, Op, B) ->
+    Operation = case DW of
+		    false ->
+			op(Op, #op_context{ret = RetVar,
+					   var = "aval",
+					   val1 = Arg2,
+					   val2 = Arg3});
+		    true ->
+			case SUDW of
+			    true ->
+				op(Op, #op_context{ret = [Arg2, "->", ?SU_DW_SINT_FIELD],
+						   var = "aval",
+						   val1 = [Arg2, "->", ?SU_DW_SINT_FIELD]});
+			    false ->
+				dw_op(Op, #op_context{ret = RetVar,
+						      var = ["aval.", ?DW_SINT_FIELD],
+						      val1 = [Arg2, "->", ?DW_SINT_FIELD]})
+			end
+		end,
+    [lists:map(fun (NB) ->
+		       NativeVar = case DW of
+				       true -> ["&", Arg1, "->native"];
+				       false -> Arg1
+				   end,
+		       NBExt = op_barrier_ext(NB),
+		       CapNBExt = to_upper(NBExt),
+		       {PreB, PostB} = xbarriers(cmpxchg, B, NB),
+		       ["#elif defined(ETHR_HAVE_", NATMC, "_CMPXCHG", CapNBExt, ")\n",
+			case PreB of
+			    "" -> "";
+			    _ -> ["    ", PreB, "\n"]
+			end,
+			"    ETHR_", NATMC, "_CMPXCHG_FALLBACK__(ETHR_", NATMC, "_FUNC__(cmpxchg", NBExt, "), ", NativeVar, ", aval, ", Operation, ");\n",
+			case PostB of
+			    "" -> "";
+			    _ -> ["    ", PostB, "\n"]
+			end]
+	       end,
+	       try_barrier_order(B))].
+
+translate_have_defs(#atomic_context{dw = DW, 'NATMC' = NATMC}) ->
+    ["
+#if !defined(ETHR_", NATMC, "_BITS__)
+#  error \"Missing native atomic implementation\"",
+     lists:map(fun (NBits) ->
+		       {HaveInPrefix,
+			HaveOutPrefix,
+			HaveInPrefixExtra,
+			HaveOutPrefixExtra,
+			NativeTypeCheck} = case NBits of
+					  "dw" ->
+						   {"ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC",
+						    ["ETHR_HAVE_", NATMC],
+						    "ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC",
+						    ["ETHR_HAVE_SU_", NATMC],
+						    "\n#elif defined(ETHR_HAVE_NATIVE_DW_ATOMIC)  || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)"};
+					  _ ->
+						   {[?HAVE_NATIVE_ATOMIC, NBits],
+						    case DW of
+							true -> ["ETHR_HAVE_SU_", NATMC];
+							false -> ["ETHR_HAVE_", NATMC]
+						    end,
+						    false,
+						    ["ETHR_HAVE_", NATMC],
+						    ["\n#elif ETHR_", NATMC, "_BITS__ == ", NBits]}
+				      end,
+		       [NativeTypeCheck,
+			lists:map(fun (Op) ->
+					  NOpStr = opstr(native(Op)),
+					  CapNOpStr = to_upper(NOpStr),
+					  lists:map(fun (B) ->
+							    NBExt = op_barrier_ext(B),
+							    CapNBExt = to_upper(NBExt),
+							    HaveOutDef = [HaveOutPrefix, "_", CapNOpStr, CapNBExt],
+							    HaveOutDefExtra = [HaveOutPrefixExtra, "_", CapNOpStr, CapNBExt],
+							    [case DW of
+								 true ->
+								     ["\n#  undef ", HaveOutDefExtra];
+								 false ->
+								     ""
+							     end, "
+#  undef ", HaveOutDef,"
+#  ifdef ", HaveInPrefix, "_", CapNOpStr, CapNBExt, "
+#    define ", HaveOutDef, " 1
+#  endif",
+							     case HaveInPrefixExtra of
+								 false -> "";
+								 _ -> ["
+#  ifdef ", HaveInPrefixExtra, "_", CapNOpStr, CapNBExt, "
+#    define ", HaveOutDefExtra, " 1
+#  endif"
+								      ]
+							     end]
+						    end,
+						    ?NATIVE_BARRIERS)
+				  end,
+				  case DW of
+				      true -> ?DW_ATOMIC_OPS;
+				      false -> ?ATOMIC_OPS
+				  end)]
+	       end,
+	       case DW of
+		   true -> ["dw", "64"];
+		   false -> ?POTENTIAL_NBITS
+	       end),
+     "
+#else
+#  error \"Invalid native atomic size\"
+#endif
+"].
+
+
+
+make_prototypes(#atomic_context{dw = DW, 'ATMC' = ATMC} = AC) ->
+    MkProt = fun (MacroName) ->
+		     %% addr() is special
+		     [func_header(AC, prototype, MacroName, addr, none), "\n",
+		      lists:map(fun (Op) ->
+					lists:map(fun (B) ->
+							  [func_header(AC, prototype, MacroName, Op, B), "\n"]
+						  end,
+						  ?BARRIERS)
+				end,
+				case DW of
+				    true -> ?DW_ATOMIC_OPS;
+				    false -> ?ATOMIC_OPS
+				end)]
+	     end,
+    ["
+#ifdef ETHR_NEED_", ATMC, "_PROTOTYPES__
+",
+     MkProt(false),
+     case DW of
+	 true -> ["#if defined(", ?DW_RTCHK_MACRO, ")\n",
+		  MkProt(?DW_FUNC_MACRO),
+		 "#endif\n"];
+	 false -> ""
+     end,
+     "#endif /* ETHR_NEED_", ATMC, "_PROTOTYPES__ */\n"].
+
+rtchk_fallback_call(Return, #atomic_context{dw = DW,
+					    ret_var = RetVar,
+					    arg1 = Arg1,
+					    arg2 = Arg2,
+					    arg3 = Arg3},
+		    Op, B) ->
+    op_call(Op, DW, case Return of
+			true -> "return";
+			false -> [RetVar, " ="]
+		    end, [?DW_FUNC_MACRO, "(", opstr(Op), op_barrier_ext(B), ")"], Arg1, Arg2, Arg3, "").
+
+non_native_barrier(B) ->
+    lists:member(B, ?NON_NATIVE_BARRIERS).
+
+non_native_barrier_impl(AC, inline_implementation = Type, Op, B) ->
+    ["
+", func_header(AC, Type, false, Op, B), "
+{",
+     case B of
+	 ddrb ->
+	     ["
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ", func_call(AC, Type, Op, none, true), "
+#else
+    ", func_call(AC, Type, Op, rb, true), "
+#endif
+"
+	     ]
+     end,
+     "}
+"
+    ];
+non_native_barrier_impl(#atomic_context{have_native_atomic_ops = HaveNative} = AC,
+			implementation = Type,
+			Op,
+			B) ->
+    ["
+", func_header(AC, Type, false, Op, B), "
+{",
+     case B of
+	 ddrb ->
+	     ["
+#if defined(", HaveNative, ")
+    ", func_call(AC, Type, Op, B, true), "
+#elif defined(ETHR_ORDERED_READ_DEPEND)
+    ", func_call(AC, symbol_implementation, Op, none, true), "
+#else
+    ", func_call(AC, symbol_implementation, Op, rb, true), "
+#endif
+"
+	     ]
+     end,
+     "}
+"
+    ].
+
+func_call(#atomic_context{'ATMC' = ATMC} = AC, inline_implementation, Op, B, RetStatement) ->
+    func_call(AC, Op, ["ETHR_", ATMC, "_FUNC__(", opstr(Op), op_barrier_ext(B), ")"], RetStatement);
+func_call(#atomic_context{atomic = Atomic} = AC, implementation, Op, B, RetStatement) ->
+    func_call(AC, Op, [Atomic, "_", opstr(Op), op_barrier_ext(B), "__"], RetStatement);
+func_call(#atomic_context{atomic = Atomic} = AC, symbol_implementation, Op, B, RetStatement) ->
+    func_call(AC, Op, [Atomic, "_", opstr(Op), op_barrier_ext(B)], RetStatement).
+
+func_call(#atomic_context{dw = DW, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3} = AC, Op, Func, true) ->
+    op_call(Op, DW, case is_return_op(AC, Op) of
+			true -> "return";
+			false -> ""
+		    end, Func, Arg1, Arg2, Arg3, "");
+func_call(#atomic_context{dw = DW, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3, ret_var = RetVar} = AC, Op, Func, false) ->
+    op_call(Op, DW, case is_return_op(AC, Op) of
+			true -> [RetVar, " = "];
+			false -> ""
+		    end, Func, Arg1, Arg2, Arg3, "").
+
+make_implementations(#atomic_context{dw = DW,
+				     ret_type = RetType,
+				     ret_var = RetVar,
+				     arg1 = Arg1,
+				     addr_aint_t = AddrAintT,
+				     atomic = Atomic,
+				     have_native_atomic_ops = HaveNativeAtomicOps,
+				     'ATMC' = ATMC,
+				     'NATMC' = NATMC} = AC) ->
+    NativeVar = case DW of
+		    true -> ["(&", Arg1, "->native)"];
+		    false -> Arg1
+		end,
+    RtchkBegin = ["
+#if defined(", ?DW_RTCHK_MACRO, ")
+    if (", ?DW_RTCHK_MACRO, ") {
+#endif
+"],
+    RtchkEnd = fun (Return, Operation, Barrier) ->
+		       ["
+#if defined(", ?DW_RTCHK_MACRO, ")
+    } else { ", rtchk_fallback_call(Return, AC, Operation, Barrier), " }
+#endif\n"
+			]
+	       end,
+    ["
+#if (defined(", HaveNativeAtomicOps, ") \\
+     && (defined(ETHR_", ATMC, "_INLINE__) || defined(ETHR_ATOMIC_IMPL__)))
+",
+     translate_have_defs(AC),
+     cmpxchg_fallback_define(AC),
+     %% addr() is special
+     "
+
+
+/* --- addr() --- */
+
+", func_header(AC, inline_implementation, false, addr, none), "
+{", case DW of
+	true -> RtchkBegin;
+	false -> ""
+    end, "
+    return (", AddrAintT, " *) ETHR_", NATMC, "_ADDR_FUNC__(", NativeVar, ");
+",case DW of
+      true -> RtchkEnd(true, addr, none);
+      false -> ""
+  end, "
+}
+",
+     lists:map(fun (Op) ->
+		       OpStr = opstr(Op),
+		       ["
+
+/* --- ", OpStr, "() --- */
+
+",
+			lists:map(fun (B) ->
+					  case non_native_barrier(B) of
+					      true ->
+						  non_native_barrier_impl(AC, inline_implementation, Op, B);
+					      false ->
+						  TryBarriers = try_barrier_order(B),
+						  ["
+", func_header(AC, inline_implementation, false, Op, B), "
+{
+",
+						   case is_return_op(AC, Op) of
+						       true ->
+							   ["    ", RetType, " ", RetVar, ";\n"];
+						       _ -> ""
+						   end,
+						   case DW of
+						       true ->
+							   [RtchkBegin,
+							    "\n",
+							    su_dw_native_barrier_op(AC, "#if", Op, B, hd(TryBarriers)),
+							    lists:map(fun (NB) ->
+									      su_dw_native_barrier_op(AC, "#elif", Op, B, NB)
+								      end,
+								      tl(TryBarriers)),
+							    lists:map(fun (NB) ->
+									      dw_native_barrier_op(AC, "#elif", "", Op, B, NB)
+								      end,
+								      TryBarriers),
+							    case simple_fallback(AC, Op, B) of
+								"" ->
+								    %% No simple fallback available;
+								    %% use cmpxchg() fallbacks...
+								    [cmpxchg_fallbacks(AC#atomic_context{'NATMC' = ["SU_", NATMC]}, true, Op, B),
+								     cmpxchg_fallbacks(AC, false, Op, B),
+								     "#else
+#error \"Missing implementation of ", Atomic, "_", opstr(Op), op_barrier_ext(B), "()!\"
+#endif
+"
+								    ];
+								SimpleFallback ->
+								    ["#else\n", SimpleFallback, "#endif\n"]
+							    end,
+							    RtchkEnd(false, Op, B), "\n"];
+						       false ->
+							   [native_barrier_op(AC, "#if", "", Op, B, hd(TryBarriers), true),
+							    lists:map(fun (NB) ->
+									      native_barrier_op(AC, "#elif", "", Op, B, NB, true)
+								      end,
+								      tl(TryBarriers)),
+							    case simple_fallback(AC, Op, B) of
+								"" ->
+								    %% No simple fallback available;
+								    %% use cmpxchg() fallbacks...
+								    [cmpxchg_fallbacks(AC, false, Op, B),
+								     "#else
+#error \"Missing implementation of ", Atomic, "_", opstr(Op), op_barrier_ext(B), "()!\"
+#endif
+"
+								    ];
+								SimpleFallback ->
+								    ["#else\n", SimpleFallback, "#endif\n"]
+							    end]
+						   end,
+						   case is_return_op(AC, Op) of
+						       true ->
+							   ["    return ", RetVar, ";\n"];
+						       false ->
+							   ""
+						   end,
+						   "}\n"]
+					  end
+				  end,
+				  ?NATIVE_BARRIERS ++ ?NON_NATIVE_BARRIERS)] %% non-native needs to be after native...
+	       end,
+	       case DW of
+		   true -> ?DW_ATOMIC_OPS;
+		   false -> ?ATOMIC_OPS
+	       end),
+     "
+#endif /* ETHR_", ATMC, "_INLINE__ */
+"
+    ].
+
+atomic_implementation_comment(AtomicSize) ->
+    CSz = case AtomicSize of
+	      "dword" -> "Double word size";
+	      "word" -> "Word size";
+	      _ -> AtomicSize ++ "-bit"
+	  end,
+    ["
+
+/* ---------- ", CSz, " atomic implementation ---------- */
+
+"
+    ].
+
+write_h_file(FileName) ->
+    {ok, FD} = file:open(FileName, [write, latin1]),
+    ok = file:write(FD, comments()),
+    ok = file:write(FD, "
+#ifndef ETHR_ATOMICS_H__
+#define ETHR_ATOMICS_H__
+"
+		   ),
+    ok = file:write(FD, h_top()),
+    ok = lists:foreach(fun (AtomicSize) ->
+			       AC = atomic_context(AtomicSize),
+			       ok = file:write(FD,
+					       [atomic_implementation_comment(AtomicSize),
+						make_prototypes(AC),
+						make_implementations(AC)])
+		       end,
+		       ?ATOMIC_SIZES),
+    ok = file:write(FD, "
+#endif /* ETHR_ATOMICS_H__ */
+"
+		   ),
+    ok = file:close(FD).
+
+
+make_native_impl_op(#atomic_context{dw = DW,
+				    atomic = Atomic,
+				    have_native_atomic_ops = HaveNativeAtomicOps,
+				    ret_var = RetVar,
+				    arg1 = Arg1,
+				    arg2 = Arg2,
+				    arg3 = Arg3}, Op, B) ->
+    ["#if defined(", HaveNativeAtomicOps, ")",
+     case DW of
+	 true -> [" && !defined(", ?DW_RTCHK_MACRO, ")"];
+	 false -> ""
+     end,
+     "\n",
+     "    ", op_call(Op, DW, [RetVar, " = "], [Atomic, "_", opstr(Op), op_barrier_ext(B), "__"], Arg1, Arg2, Arg3, ""),
+     "\n"].
+
+amc_op_dw_arg(#atomic_context{dw = false}) ->
+    "0";
+amc_op_dw_arg(#atomic_context{dw = true}) ->
+    "1".
+
+amc_op_arg_prefix(#atomic_context{dw = false}) ->
+    "&";
+amc_op_arg_prefix(#atomic_context{dw = true}) ->
+    "".
+
+amc_sint_arg(#atomic_context{dw = DW, arg2 = Arg}, arg2) ->
+    amc_sint_arg(DW, Arg);
+amc_sint_arg(#atomic_context{dw = DW, arg3 = Arg}, arg3) ->
+    amc_sint_arg(DW, Arg);
+amc_sint_arg(#atomic_context{dw = DW, ret_var = Arg}, ret_var) ->
+    amc_sint_arg(DW, Arg);
+amc_sint_arg(true, Arg) ->
+    [Arg, "->" ?DW_SINT_FIELD];
+amc_sint_arg(false, Arg) ->
+    ["&", Arg].
+
+amc_op_call(#atomic_context{arg1 = Arg1} = AC, init) ->
+    ["    amc_init(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ");\n"]; 
+amc_op_call(#atomic_context{arg1 = Arg1} = AC, set) ->
+    ["    amc_set(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ");\n"]; 
+amc_op_call(#atomic_context{dw = false, arg1 = Arg1} = AC, read) ->
+    ["    amc_read(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, ret_var), ");\n"]; 
+amc_op_call(#atomic_context{dw = true, arg1 = Arg1} = AC, read) ->
+    ["    amc_read(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ");\n"]; 
+amc_op_call(#atomic_context{dw = false, arg1 = Arg1, arg3 = Arg3, ret_var = RetVar} = AC, cmpxchg) ->
+    ["    ", RetVar, " = ", Arg3, ";
+    (void) amc_cmpxchg(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ", ", amc_sint_arg(AC, ret_var), ");\n"];
+amc_op_call(#atomic_context{dw = true, arg1 = Arg1, ret_var = RetVar} = AC, cmpxchg) ->
+    ["    ", RetVar, " = amc_cmpxchg(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ", ", amc_sint_arg(AC, arg3), ");\n"];
+amc_op_call(#atomic_context{dw = DW, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3, ret_var = RetVar}, Op) ->
+    OpCtxt = #op_context{ret = RetVar, var = [Arg1,"->sint"], val1 = Arg2, val2 = Arg3},
+    OpStr = case DW of
+		true -> dw_op(Op, OpCtxt);
+		false -> op(Op, OpCtxt)
+	    end,
+    ["    ETHR_AMC_MODIFICATION_OPS__(&", Arg1, "->amc, ", OpStr, ");\n"].
+
+make_amc_fallback_op(#atomic_context{amc_fallback = false}, _Op, _B) ->
+    "";
+make_amc_fallback_op(#atomic_context{amc_fallback = true} = AC, Op, B) ->
+    NB = case Op of
+	     read -> rb;
+	     _ -> none
+	 end,
+    {PreB, PostB} = xbarriers(Op, B, NB),
+    ["#elif defined(ETHR_AMC_FALLBACK__)\n",
+     case PreB of
+	 "" -> "";
+	 _ -> ["    ", PreB, "\n"]
+     end,
+     amc_op_call(AC, Op),
+     case PostB of
+	 "" -> "";
+	 _ -> ["    ", PostB, "\n"]
+     end].
+
+make_locked_fallback_op(#atomic_context{dw = DW,
+					ret_var = RetVar,
+					arg1 = Arg1,
+					arg2 = Arg2,
+					arg3 = Arg3}, Op, B) ->
+    OpStr = case DW of
+		true ->
+		    dw_op(Op, #op_context{ret = RetVar,
+					  var = [Arg1, "->" ?DW_SINT_FIELD],
+					  val1 = [Arg2, "->" ?DW_SINT_FIELD],
+					  val2 = [Arg3, "->" ?DW_SINT_FIELD]});
+		false ->
+		    op(Op, #op_context{ret = RetVar,
+				       var = ["*", Arg1],
+				       val1 = Arg2,
+				       val2 = Arg3})
+	    end,
+    {PreB, PostB} = xbarriers(Op, B, none),
+    ["#else\n",
+     case PreB of
+	 "" -> "";
+	 _ -> ["    ", PreB, "\n"]
+     end,
+     ["    ETHR_ATOMIC_OP_FALLBACK_IMPL__(", Arg1, ", ", OpStr, ");\n"],
+     case PostB of
+	 "" -> "";
+	 _ -> ["    ", PostB, "\n"]
+     end,
+     "#endif\n"].
+
+make_symbol_to_fallback_impl(#atomic_context{dw = true,
+					     atomic = Atomic,
+					     arg1 = Arg1,
+					     arg2 = Arg2,
+					     arg3 = Arg3} = AC,
+			     Op, B) ->
+    ["
+#ifdef ", ?DW_RTCHK_MACRO, "
+", func_header(AC, implementation, false, Op, B), "
+{",
+     case Op of
+	 init -> "";
+	 _ -> ["\n    ETHR_ASSERT(!ethr_not_inited__);"]
+     end, "
+    ETHR_ASSERT(", Arg1, ");
+    ", op_call(Op, true, "return", [Atomic, "_", opstr(Op), op_barrier_ext(B), "__"], Arg1, Arg2, Arg3, ""), "
+}
+#endif
+"
+    ];
+make_symbol_to_fallback_impl(_, _, _) ->
+    "".
+
+make_symbol_implementations(#atomic_context{dw = DW,
+					    amc_fallback = AMC,
+					    ret_type = RetType,
+					    addr_aint_t = AddrAintT,
+					    ret_var = RetVar,
+					    arg1 = Arg1} = AC) ->
+    FallbackVar = case DW of
+			true -> ["(&", Arg1, "->fallback)"];
+			false -> Arg1
+		  end,
+    ["
+",
+     case DW of
+	 true -> ["
+/*
+ * Double word atomics need runtime test.
+ */
+
+int ethr_have_native_dw_atomic(void)
+{
+    return ethr_have_native_dw_atomic__();
+}
+     "];
+	 false -> ""
+     end, "
+
+/* --- addr() --- */
+
+", func_header(AC, implementation,
+	       case DW of
+		   true -> ?DW_FUNC_MACRO;
+		   false -> false
+	       end, addr, none), "
+{
+    ", AddrAintT, " *", RetVar, ";
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(", Arg1, ");
+", make_native_impl_op(AC, addr, none),
+     case AMC of
+	 true -> ["#elif defined(ETHR_AMC_FALLBACK__)
+    ", RetVar ," = (", AddrAintT, " *) (", FallbackVar, ")->sint;"];
+	 false -> ""
+     end, "
+#else
+    ", RetVar, " = (", AddrAintT, " *) ", FallbackVar, ";
+#endif
+    return ", RetVar, ";
+}
+",
+     make_symbol_to_fallback_impl(AC, addr, none),
+     lists:map(fun (Op) ->
+		       ["
+
+/* -- ", opstr(Op), "() -- */
+
+",
+			lists:map(fun (B) ->
+					Macro = case DW of
+						   true -> ?DW_FUNC_MACRO;
+						   false -> false
+						end,
+					case non_native_barrier(B) of
+					    true ->
+						non_native_barrier_impl(AC, implementation, Op, B);
+					    false ->
+						["\n",
+					   	func_header(AC, implementation, Macro,  Op, B),
+						   "\n{\n",
+						   case is_return_op(AC, Op) of
+						       true -> ["    ", RetType, " ", RetVar, ";\n"];
+						       false -> ""
+						   end,
+						   case Op of
+						       init -> "";
+						       _ -> ["    ETHR_ASSERT(!ethr_not_inited__);\n"]
+						   end,
+						   ["    ETHR_ASSERT(", Arg1, ");\n"],
+						   make_native_impl_op(AC, Op, B),
+						   make_amc_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B), 
+						   make_locked_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B),
+						   case is_return_op(AC, Op) of
+						       true -> ["    return ", RetVar, ";"
+							       ];
+						       false ->
+							   ""
+						   end,
+						   "\n}\n",
+						   make_symbol_to_fallback_impl(AC, Op, B)]
+					  end
+				  end,
+				  ?BARRIERS)]
+	       end,
+	       case DW of
+		   true -> ?DW_ATOMIC_OPS;
+		   false -> ?ATOMIC_OPS
+	       end)].
+
+make_info_functions() ->
+    ["
+
+
+/* --------- Info functions --------- */
+
+#if defined(", ?DW_RTCHK_MACRO, ")
+char *zero_ops[] = {NULL};
+#endif
+",
+     [lists:map(fun (NBits) ->
+			{DW, Bits} = case NBits of
+					 "su_dw" -> {"su_dw_", ""};
+					 "dw" -> {"dw_", ""};
+					 _ -> {"", NBits}
+				     end,
+			["
+
+static char *native_", DW, "atomic", Bits, "_ops[] = {",
+			 lists:map(fun (Op) ->
+					   NOpStr = opstr(native(Op)),
+					   CapNOpStr = to_upper(NOpStr),
+					   lists:map(fun (B) ->
+							     HaveNative = case NBits of
+									      "dw" ->
+										  "ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC";
+									      "su_dw" ->
+										  "ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC";
+									      _ ->
+										  [?HAVE_NATIVE_ATOMIC, NBits]
+									  end,
+							     NBExt = op_barrier_ext(B),
+							     CapNBExt = to_upper(NBExt),
+							     ["
+#ifdef ", HaveNative, "_", CapNOpStr, CapNBExt, "
+    \"", NOpStr, NBExt, "\",
+#endif"
+							     ]
+						     end,
+						     ?NATIVE_BARRIERS)
+				   end,
+				   case NBits of
+				       "dw" -> ?DW_ATOMIC_OPS;
+				       "su_dw" -> ?DW_ATOMIC_OPS;
+				       _ -> ?ATOMIC_OPS
+				   end), "
+    NULL
+};
+
+char **
+ethr_native_", DW, "atomic", Bits, "_ops(void)
+{
+",
+     case DW of
+	 "" -> "";
+	 _ -> ["
+#if defined(", ?DW_RTCHK_MACRO, ")
+    if (!", ?DW_RTCHK_MACRO, ")
+	return &zero_ops[0];
+#endif"
+	     ]
+   end, "
+    return &native_", DW, "atomic", Bits, "_ops[0];
+}
+"
+    ]
+	       end, ["su_dw", "dw" | ?POTENTIAL_NBITS])]].
+
+write_c_file(FileName) ->
+    {ok, FD} = file:open(FileName, [write, latin1]),
+    ok = file:write(FD, comments()),
+    ok = file:write(FD, c_top()),
+    lists:foreach(fun (AtomicSize) ->
+			  ok = file:write(FD,
+					  [atomic_implementation_comment(AtomicSize),
+					   make_symbol_implementations(atomic_context(AtomicSize))])
+		  end,
+		  ?ATOMIC_SIZES),
+    ok = file:write(FD, make_info_functions()).
+			      
+
+main([]) ->
+    case os:getenv("ERL_TOP") of
+	false ->
+	    io:format("$ERL_TOP not set!~n", []),
+	    halt(1);
+	ErlTop ->
+	    HFile = filename:join(ErlTop, ?H_FILE),
+	    WHFile = fun () ->
+			     write_h_file(HFile)
+		     end,
+	    CFile = filename:join(ErlTop, ?C_FILE),
+	    WCFile = fun () ->
+			     write_c_file(CFile)
+		     end,
+	    case erlang:system_info(schedulers_online) of
+		1 ->
+		    WHFile(),
+		    WCFile();
+		_ ->
+		    {HPid, HMon} = spawn_monitor(WHFile),
+		    {CPid, CMon} = spawn_monitor(WCFile),
+		    receive
+			{'DOWN', HMon, process, HPid, HReason} ->
+			    normal = HReason
+		    end,
+		    receive
+			{'DOWN', CMon, process, CPid, CReason} ->
+			    normal = CReason
+		    end
+	    end,
+	    io:format("Wrote: ~s~n", [HFile]),
+	    io:format("Wrote: ~s~n", [CFile]),
+	    init:stop()
+    end.
+
+a2l(A) ->
+    atom_to_list(A).
+
+opstr(A) ->
+    a2l(A).
+
+to_upper([]) ->
+    [];
+to_upper([C|Cs]) when is_list(C) ->
+    [to_upper(C)|to_upper(Cs)];
+to_upper([C|Cs]) when is_integer(C), 97 =< C, C =< 122 ->
+    [C-32|to_upper(Cs)];
+to_upper([C|Cs]) ->
+    [C|to_upper(Cs)].
+
+
+comments() ->
+    Years = case erlang:date() of
+		{2011, _, _} -> "2011";
+		{Y, _, _} -> "2011-"++integer_to_list(Y)
+	    end,
+    ["/*
+ * --------------- DO NOT EDIT THIS FILE! ---------------
+ * This file was automatically generated by the
+ * \$ERL_TOP/erts/lib_src/utils/make_atomics_api script.
+ * If you need to make changes, edit the script and
+ * regenerate this file.
+ * --------------- DO NOT EDIT THIS FILE! ---------------
+ */
+
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB ", Years, ". All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the \"License\"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an \"AS IS\"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: The ethread atomics API
+ * Author: Rickard Green
+ */
+
+/*
+ * This file maps native atomic implementations to ethread
+ * API atomics. If no native atomic implementation
+ * is available, a less efficient fallback is used instead.
+ * The API consists of 32-bit size, word size (pointer size),
+ * and double word size atomics.
+ *
+ * The following atomic operations are implemented for
+ * 32-bit size, and word size atomics:
+",
+     lists:map(fun (Op) ->
+		       [" * - ", opstr(Op), "\n"]
+	       end,
+	       ?ATOMIC_OPS),
+     " *
+ * The following atomic operations are implemented for
+ * double word size atomics:
+",
+     lists:map(fun (Op) ->
+		       [" * - ", opstr(Op), "\n"]
+	       end,
+	       ?DW_ATOMIC_OPS),
+     " *
+ * Appart from a function implementing the atomic operation
+ * with unspecified memory barrier semantics, there are
+ * functions implementing each operation with the following
+ * implied memory barrier semantics:",
+     lists:map(fun (none) ->
+		       "";
+		   (mb) ->
+		       ["
+ * - mb   - Full memory barrier. Orders both loads, and
+ *          stores before, and after the atomic operation.
+ *          No load or store is allowed to be reordered
+ *          over the atomic operation."];
+		   (acqb) ->
+		       ["
+ * - acqb - Acquire barrier. Orders both loads, and stores
+ *          appearing *after* the atomic operation. These
+ *          are not allowed to be reordered over the
+ *          atomic operation."];
+		   (relb) ->
+		       ["
+ * - relb - Release barrier. Orders both loads, and
+ *          stores appearing *before* the atomic
+ *          operation. These are not allowed to be
+ *          reordered over the atomic operation."];
+		   (rb) ->
+		       ["
+ * - rb   - Read barrier. Orders *only* loads. These are
+ *          not allowed to be reordered over the barrier.
+ *          Load in atomic operation is ordered *before*
+ *          the barrier. "];
+		   (ddrb) ->
+		       ["
+ * - ddrb - Data dependency read barrier. Orders *only*
+ *          loads according to data dependency across the
+ *          barrier. Load in atomic operation is ordered
+ *          before the barrier."];
+		   (wb) ->
+		       ["
+ * - wb   - Write barrier. Orders *only* stores. These are
+ *          not allowed to be reordered over the barrier.
+ *          Store in atomic operation is ordered *after*
+ *          the barrier."];
+		   (B) ->
+		       [" * - ", a2l(B), "\n"]
+	       end,
+	       lists:reverse(?BARRIERS)),
+     "
+ *
+ * We implement all of these operation/barrier
+ * combinations, regardless of whether they are useful
+ * or not (some of them are useless).
+ *
+ * Double word size atomic functions are on the followning
+ * form:
+ *   ethr_dw_atomic_<OP>[_<BARRIER>]
+ *
+ * Word size atomic functions are on the followning
+ * form:
+ *   ethr_atomic_<OP>[_<BARRIER>]
+ *
+ * 32-bit size atomic functions are on the followning
+ * form:
+ *   ethr_atomic32_<OP>[_<BARRIER>]
+ *
+ * Apart from the operation/barrier functions
+ * described above also 'addr' functions are implemented
+ * which return the actual memory address used of the
+ * atomic variable. The 'addr' functions have no barrier
+ * versions.
+ *
+ * The native atomic implementation does not need to
+ * implement all operation/barrier combinations.
+ * Functions that have no native implementation will be
+ * constructed from existing native functionality. These
+ * functions will perform the wanted operation and will
+ * produce sufficient memory barriers, but may
+ * in some cases be less efficient than pure native
+ * versions.
+ *
+ * When we create ethread API operation/barrier functions by
+ * adding barriers before and after native operations it is
+ * assumed that:
+ * - A native read operation begins, and ends with a load.
+ * - A native set operation begins, and ends with a store.
+ * - An init operation begins with either a load, or a store,
+ *   and ends with either a load, or a store.
+ * - All other operations begins with a load, and ends with
+ *   either a load, or a store.
+ *
+ * This is the minimum functionality that a native
+ * implementation needs to provide:
+ *
+ * - Functions that need to be implemented:
+ *
+ *   - ethr_native_[dw_|su_dw_]atomic[BITS]_addr
+ *   - ethr_native_[dw_|su_dw_]atomic[BITS]_cmpxchg[_<BARRIER>]
+ *     (at least one cmpxchg of optional barrier)
+ *
+ * - Macros that needs to be defined:
+ *
+ *   A macro informing about the presence of the native
+ *   implementation:
+ *
+ *   - ETHR_HAVE_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]
+ *
+ *   A macro naming (a string constant) the implementation:
+ *
+ *   - ETHR_NATIVE_[DW_]ATOMIC[BITS]_IMPL
+ *
+ *   Each implemented native atomic function has to
+ *   be accompanied by a defined macro on the following
+ *   form informing about its presence:
+ *
+ *   - ETHR_HAVE_ETHR_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]_<OP>[_<BARRIER>]
+ *
+ *   A (sparc-v9 style) membar macro:
+ *
+ *   - ETHR_MEMBAR(B)
+ *
+ *     Which takes a combination of the following macros
+ *     or:ed (using |) together:
+ *
+ *     - ETHR_LoadLoad
+ *     - ETHR_LoadStore
+ *     - ETHR_StoreLoad
+ *     - ETHR_StoreStore
+ *
+ */
+"
+    ].
+
+h_top() ->
+    ["
+#undef ETHR_AMC_FALLBACK__
+#undef ETHR_AMC_NO_ATMCS__
+#undef ETHR_AMC_ATMC_T__
+#undef ETHR_AMC_ATMC_FUNC__
+
+/* -- 32-bit atomics -- */
+
+#undef ETHR_NAINT32_T__
+#undef ETHR_NATMC32_FUNC__
+#undef ETHR_NATMC32_ADDR_FUNC__
+#undef ETHR_NATMC32_BITS__
+#if defined(ETHR_HAVE_NATIVE_ATOMIC32)
+#    define ETHR_NEED_NATMC32_ADDR
+#  define ETHR_NATMC32_ADDR_FUNC__ ethr_native_atomic32_addr
+typedef ethr_native_atomic32_t ethr_atomic32_t;
+#  define ETHR_NAINT32_T__ ethr_sint32_t
+#  define ETHR_NATMC32_FUNC__(X) ethr_native_atomic32_ ## X
+#  define ETHR_NATMC32_BITS__ 32
+#elif defined(ETHR_HAVE_NATIVE_ATOMIC64)
+#  define ETHR_NEED_NATMC64_ADDR
+#ifdef ETHR_BIGENDIAN
+#  define ETHR_NATMC32_ADDR_FUNC__(VAR) \\
+  (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1)
+#else
+#  define ETHR_NATMC32_ADDR_FUNC__(VAR) \\
+  ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR)))
+#endif
+typedef ethr_native_atomic64_t ethr_atomic32_t;
+#  define ETHR_NAINT32_T__ ethr_sint64_t
+#  define ETHR_NATMC32_FUNC__(X) ethr_native_atomic64_ ## X
+#  define ETHR_NATMC32_BITS__ 64
+#else
+/*
+ * No native atomics usable for 32-bits atomics :(
+ * Use fallback...
+ */
+typedef ethr_sint32_t ethr_atomic32_t;
+#endif
+
+#undef ETHR_ATMC32_INLINE__
+#ifdef ETHR_NATMC32_BITS__
+#  ifdef ETHR_TRY_INLINE_FUNCS
+#    define ETHR_ATMC32_INLINE__
+#  endif
+#  define ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS
+#endif
+
+#if !defined(ETHR_ATMC32_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
+#  define ETHR_NEED_ATMC32_PROTOTYPES__
+#endif
+
+#ifndef ETHR_INLINE_ATMC32_FUNC_NAME_
+#  define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X
+#endif
+
+#undef ETHR_ATMC32_FUNC__
+#define ETHR_ATMC32_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
+
+
+/* -- Word size atomics -- */
+
+#undef ETHR_NEED_NATMC32_ADDR
+#undef ETHR_NEED_NATMC64_ADDR
+
+#undef ETHR_NAINT_T__
+#undef ETHR_NATMC_FUNC__
+#undef ETHR_NATMC_ADDR_FUNC__
+#undef ETHR_NATMC_BITS__
+#if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_NATIVE_ATOMIC64)
+#  ifndef ETHR_NEED_NATMC64_ADDR
+#    define ETHR_NEED_NATMC64_ADDR
+#  endif
+#  define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic64_addr
+typedef ethr_native_atomic64_t ethr_atomic_t;
+#  define ETHR_NAINT_T__ ethr_sint64_t
+#  define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#  define ETHR_NATMC_BITS__ 64
+#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC32)
+#  ifndef ETHR_NEED_NATMC64_ADDR
+#    define ETHR_NEED_NATMC32_ADDR
+#  endif
+#  define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic32_addr
+typedef ethr_native_atomic32_t ethr_atomic_t;
+#  define ETHR_NAINT_T__ ethr_sint32_t
+#  define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+#  define ETHR_NATMC_BITS__ 32
+#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64)
+#  ifndef ETHR_NEED_NATMC64_ADDR
+#    define ETHR_NEED_NATMC64_ADDR
+#  endif
+#ifdef ETHR_BIGENDIAN
+#  define ETHR_NATMC_ADDR_FUNC__(VAR) \\
+  (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1)
+#else
+#  define ETHR_NATMC_ADDR_FUNC__(VAR) \\
+  ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR)))
+#endif
+typedef ethr_native_atomic64_t ethr_atomic_t;
+#  define ETHR_NATMC_T__ ethr_native_atomic64_t
+#  define ETHR_NAINT_T__ ethr_sint64_t
+#  define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#  define ETHR_NATMC_BITS__ 64
+#else
+/*
+ * No native atomics usable for pointer size atomics :(
+ * Use fallback...
+ */
+
+#  if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+#    define ETHR_AMC_FALLBACK__
+#    define ETHR_AMC_NO_ATMCS__ 2
+#    define ETHR_AMC_SINT_T__ ethr_sint32_t
+#    define ETHR_AMC_ATMC_T__ ethr_atomic32_t
+#    define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
+typedef struct {
+    ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__];
+} ethr_amc_t;
+typedef struct {
+    ethr_amc_t amc;
+    ethr_sint_t sint;
+} ethr_atomic_t;
+#  else /* locked fallback */
+typedef ethr_sint_t ethr_atomic_t;
+#  endif
+#endif
+
+#undef ETHR_ATMC_INLINE__
+#ifdef ETHR_NATMC_BITS__
+#  ifdef ETHR_TRY_INLINE_FUNCS
+#    define ETHR_ATMC_INLINE__
+#  endif
+#  define ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS
+#endif
+
+#if !defined(ETHR_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
+#  define ETHR_NEED_ATMC_PROTOTYPES__
+#endif
+
+#ifndef ETHR_INLINE_ATMC_FUNC_NAME_
+#  define ETHR_INLINE_ATMC_FUNC_NAME_(X) X
+#endif
+
+#undef ETHR_ATMC_FUNC__
+#define ETHR_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X)
+
+/* -- Double word atomics -- */
+
+#undef ETHR_SU_DW_NAINT_T__
+#undef ETHR_SU_DW_NATMC_FUNC__
+#undef ETHR_SU_DW_NATMC_ADDR_FUNC__
+#undef ETHR_DW_NATMC_FUNC__
+#undef ETHR_DW_NATMC_ADDR_FUNC__
+#undef ETHR_DW_NATMC_BITS__
+#if defined(ETHR_HAVE_NATIVE_DW_ATOMIC) || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)
+#  define ETHR_NEED_DW_NATMC_ADDR
+#  define ETHR_DW_NATMC_ADDR_FUNC__ ethr_native_dw_atomic_addr
+#  define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_dw_atomic_t
+#  define ETHR_DW_NATMC_FUNC__(X) ethr_native_dw_atomic_ ## X
+#  define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_su_dw_atomic_ ## X
+#  if ETHR_SIZEOF_PTR == 8
+#    define ETHR_DW_NATMC_BITS__ 128
+#  elif ETHR_SIZEOF_PTR == 4
+#    define ETHR_DW_NATMC_BITS__ 64
+#  else
+#    error \"Word size not supported\"
+#  endif
+#  ifdef ETHR_NATIVE_SU_DW_SINT_T
+#    define ETHR_SU_DW_NAINT_T__ ETHR_NATIVE_SU_DW_SINT_T
+#  endif
+#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64)
+#  define ETHR_HAVE_NATIVE_SU_DW_ATOMIC
+#  ifndef ETHR_NEED_NATMC64_ADDR
+#    define ETHR_NEED_NATMC64_ADDR
+#  endif
+#  define ETHR_DW_NATMC_ADDR_FUNC__(VAR) \\
+  ((ethr_dw_sint_t *) ethr_native_atomic64_addr((VAR)))
+#  define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_atomic64_t
+#  define ETHR_SU_DW_NAINT_T__ ethr_sint64_t
+#  define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#  define ETHR_DW_NATMC_BITS__ 64
+#endif
+
+#if defined(", ?DW_RTCHK_MACRO, ")
+#define ", ?DW_FUNC_MACRO, "(X) ethr_dw_atomic_ ## X ## _fallback__
+#else
+#define ", ?DW_FUNC_MACRO, "(X) ethr_dw_atomic_ ## X
+#endif
+
+#if !defined(ETHR_DW_NATMC_BITS__) || defined(", ?DW_RTCHK_MACRO, ")
+#  define ETHR_NEED_DW_FALLBACK__
+#endif
+
+#if defined(ETHR_NEED_DW_FALLBACK__)
+/*
+ * No native atomics usable for double word atomics :(
+ * Use fallback...
+ */
+
+#  ifndef ETHR_AMC_FALLBACK__
+#    if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
+#      define ETHR_AMC_FALLBACK__
+#      define ETHR_AMC_NO_ATMCS__ 1
+#      define ETHR_AMC_SINT_T__ ethr_sint_t
+#      define ETHR_AMC_ATMC_T__ ethr_atomic_t
+#      define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X)
+#    elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
+#      define ETHR_AMC_FALLBACK__
+#      define ETHR_AMC_NO_ATMCS__ 2
+#      define ETHR_AMC_SINT_T__ ethr_sint32_t
+#      define ETHR_AMC_ATMC_T__ ethr_atomic32_t
+#      define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
+#    endif
+#    ifdef ETHR_AMC_FALLBACK__
+typedef struct {
+    ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__];
+} ethr_amc_t;
+#    endif
+#  endif
+
+typedef struct {
+#ifdef ETHR_AMC_FALLBACK__
+    ethr_amc_t amc;
+#endif
+    ethr_sint_t sint[2];
+} ethr_dw_atomic_fallback_t;
+
+#endif
+
+typedef union {
+#ifdef ETHR_NATIVE_DW_ATOMIC_T__
+    ETHR_NATIVE_DW_ATOMIC_T__ native;
+#endif
+#ifdef ETHR_NEED_DW_FALLBACK__
+    ethr_dw_atomic_fallback_t fallback;
+#endif
+    ethr_sint_t sint[2];
+} ethr_dw_atomic_t;
+
+typedef union {
+#ifdef ETHR_SU_DW_NAINT_T__
+    ETHR_SU_DW_NAINT_T__ ", ?SU_DW_SINT_FIELD, ";
+#endif
+    ethr_sint_t ", ?DW_SINT_FIELD, "[2];
+} ethr_dw_sint_t;
+
+#ifdef ETHR_BIGENDIAN
+#  define ETHR_DW_SINT_LOW_WORD 1
+#  define ETHR_DW_SINT_HIGH_WORD 0
+#else
+#  define ETHR_DW_SINT_LOW_WORD 0
+#  define ETHR_DW_SINT_HIGH_WORD 1
+#endif
+
+#undef ETHR_DW_ATMC_INLINE__
+#ifdef ETHR_DW_NATMC_BITS__
+#  ifdef ETHR_TRY_INLINE_FUNCS
+#    define ETHR_ATMC32_INLINE__
+#  endif
+#  define ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS
+#endif
+
+#if !defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
+#  define ETHR_NEED_DW_ATMC_PROTOTYPES__
+#endif
+
+#ifndef ETHR_INLINE_DW_ATMC_FUNC_NAME_
+#  define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X
+#endif
+
+#undef ETHR_DW_ATMC_FUNC__
+#define ETHR_DW_ATMC_FUNC__(X) ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_dw_atomic_ ## X)
+
+#if defined(ETHR_NEED_DW_ATMC_PROTOTYPES__)
+int ethr_have_native_dw_atomic(void);
+#endif
+#if defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
+static ETHR_INLINE int
+ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_have_native_dw_atomic)(void)
+{
+#if defined(", ?DW_RTCHK_MACRO, ")
+    return ", ?DW_RTCHK_MACRO, ";
+#elif defined(ETHR_DW_NATMC_BITS__)
+    return 1;
+#else
+    return 0;
+#endif
+}
+#endif
+
+/* -- Misc -- */
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+/*
+ * Unusual values are used by read() fallbacks implemented via cmpxchg().
+ * We want to use an unusual value in hope that it is more efficient
+ * not to match the value in memory.
+ *
+ * - Negative integer values are probably more unusual.
+ * - Very large absolute integer values are probably more unusual.
+ * - Odd pointers are probably more unusual (only char pointers can be odd).
+ */
+#  define ETHR_UNUSUAL_SINT32_VAL__ ((ethr_sint32_t) 0x81818181)
+#  if ETHR_SIZEOF_PTR == 4
+#    define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) ETHR_UNUSUAL_SINT32_VAL__)
+#  elif ETHR_SIZEOF_PTR == 8
+#    define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) 0x8181818181818181L)
+#  else
+#    error \"Word size not supported\"
+#  endif
+#  if defined(ETHR_NEED_DW_NATMC_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR)
+#    error \"No ethr_native_dw_atomic_addr() available\"
+#  endif
+#  if defined(ETHR_NEED_NATMC32_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR)
+#    error \"No ethr_native_atomic32_addr() available\"
+#  endif
+#  if defined(ETHR_NEED_NATMC64_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR)
+#    error \"No ethr_native_atomic64_addr() available\"
+#  endif
+#endif
+
+#if defined(__GNUC__)
+#  ifndef ETHR_COMPILER_BARRIER
+#    define ETHR_COMPILER_BARRIER __asm__ __volatile__(\"\" : : : \"memory\")
+#  endif
+#elif defined(ETHR_WIN32_THREADS)
+#  ifndef ETHR_COMPILER_BARRIER
+#    include <intrin.h>
+#    pragma intrinsic(_ReadWriteBarrier)
+#    define ETHR_COMPILER_BARRIER _ReadWriteBarrier()
+#  endif
+#endif
+
+void ethr_compiler_barrier_fallback(void);
+#ifndef ETHR_COMPILER_BARRIER
+#  define ETHR_COMPILER_BARRIER ethr_compiler_barrier_fallback()
+#endif
+
+int ethr_init_atomics(void);
+
+/* info */
+char **ethr_native_atomic32_ops(void);
+char **ethr_native_atomic64_ops(void);
+char **ethr_native_dw_atomic_ops(void);
+char **ethr_native_su_dw_atomic_ops(void);
+
+#if !defined(ETHR_DW_NATMC_BITS__) && !defined(ETHR_NATMC_BITS__) && !defined(ETHR_NATMC32_BITS__)
+/*
+ * ETHR_*MEMORY_BARRIER orders between locked and atomic accesses only,
+ * i.e. when no native atomic implementation exist and only our lock
+ * based atomic fallback is used, a noop is sufficient.
+ */
+#  undef ETHR_MEMORY_BARRIER
+#  undef ETHR_WRITE_MEMORY_BARRIER
+#  undef ETHR_READ_MEMORY_BARRIER
+#  undef ETHR_READ_DEPEND_MEMORY_BARRIER
+#  undef ETHR_MEMBAR
+#  define ETHR_MEMBAR(B) do { } while (0)
+#endif
+
+#ifndef ETHR_MEMBAR
+#  error \"No ETHR_MEMBAR defined\"
+#endif
+
+#define ETHR_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore)
+#define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMBAR(ETHR_StoreStore)
+#define ETHR_READ_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad)
+#ifdef ETHR_READ_DEPEND_MEMORY_BARRIER
+#  undef ETHR_ORDERED_READ_DEPEND
+#else
+#  define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER
+#  define ETHR_ORDERED_READ_DEPEND
+#endif
+"].
+
+c_top() ->
+    ["
+
+#ifdef HAVE_CONFIG_H
+#include \"config.h\"
+#endif
+
+#define ETHR_TRY_INLINE_FUNCS
+#define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X ## __
+#define ETHR_INLINE_ATMC_FUNC_NAME_(X) X ## __
+#define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X ## __
+#define ETHR_ATOMIC_IMPL__
+
+#include \"ethread.h\"
+#include \"ethr_internal.h\"
+
+#if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \\
+     || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS))
+/*
+ * Spinlock based fallback for atomics used in absence of a native
+ * implementation.
+ */
+
+#define ETHR_ATMC_FLLBK_ADDR_BITS ", ?ETHR_ATMC_FLLBK_ADDR_BITS, "
+#define ETHR_ATMC_FLLBK_ADDR_SHIFT ", ?ETHR_ATMC_FLLBK_ADDR_SHIFT, "
+
+typedef struct {
+    union {
+	ethr_spinlock_t lck;
+	char buf[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_spinlock_t))];
+    } u;
+} ethr_atomic_protection_t;
+
+extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS];
+
+#define ETHR_ATOMIC_PTR2LCK__(PTR) \\
+(&ethr_atomic_protection__[((((ethr_uint_t) (PTR)) >> ETHR_ATMC_FLLBK_ADDR_SHIFT) \\
+			   & ((1 << ETHR_ATMC_FLLBK_ADDR_BITS) - 1))].u.lck)
+
+
+#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS)			\\
+do {									\\
+    ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP));		\\
+    ethr_spin_lock(slp__);						\\
+    { EXPS; }								\\
+    ethr_spin_unlock(slp__);						\\
+} while (0)
+
+ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS];
+
+#endif
+
+", make_amc_fallback(), "
+
+int
+ethr_init_atomics(void)
+{
+#if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \\
+     || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS))
+    int i;
+    for (i = 0; i < (1 << ETHR_ATMC_FLLBK_ADDR_BITS); i++) {
+	int res = ethr_spinlock_init(&ethr_atomic_protection__[i].u.lck);
+	if (res != 0)
+	    return res;
+    }
+#endif
+    return 0;
+}
+"].
+
+make_amc_fallback() ->
+    ["
+#if defined(ETHR_AMC_FALLBACK__)
+
+/*
+ * Fallback for large sized (word and/or double word size) atomics using
+ * an \"Atomic Modification Counter\" based on smaller sized native atomics.
+ *
+ * We use a 63-bit modification counter and a one bit exclusive flag.
+ * If 32-bit native atomics are used, we need two 32-bit native atomics.
+ * The exclusive flag is the least significant bit, or if multiple atomics
+ * are used, the least significant bit of the least significant atomic.
+ *
+ * When using the AMC fallback the following is true:
+ * - Reads of the same atomic variable can be done in parallel.
+ * - Uncontended reads doesn't cause any cache line invalidations,
+ *   since no modifications are done.
+ * - Assuming that the AMC atomic(s) and the integer(s) containing the
+ *   value of the implemented atomic resides in the same cache line,
+ *   modifications will only cause invalidations of one cache line.
+ *
+ * When using the spinlock based fallback none of the above is true,
+ * however, the spinlock based fallback consumes less memory.
+ */
+
+#  if ETHR_AMC_NO_ATMCS__ != 1  && ETHR_AMC_NO_ATMCS__ != 2
+#    error \"Not supported\"
+#  endif
+#  define ETHR_AMC_MAX_TRY_READ__ 10
+#  ifdef ETHR_DEBUG
+#    define ETHR_DBG_CHK_EXCL_STATE(ASP, S) \\
+do { \\
+    ETHR_AMC_SINT_T__ act = ETHR_AMC_ATMC_FUNC__(read)(&(ASP)->atomic[0]); \\
+    ETHR_ASSERT(act == (S) + 1); \\
+    ETHR_ASSERT(act & 1); \\
+} while (0)
+#  else
+#    define ETHR_DBG_CHK_EXCL_STATE(ASP, S)
+#  endif
+
+static ETHR_INLINE void
+amc_init(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val)
+{
+    avar[0] = val[0];
+    if (dw)
+	avar[1] = val[1];
+#if ETHR_AMC_NO_ATMCS__ == 2
+    ETHR_AMC_ATMC_FUNC__(init)(&amc->atomic[1], 0);
+#endif
+    ETHR_AMC_ATMC_FUNC__(init_wb)(&amc->atomic[0], 0);
+}
+
+static ETHR_INLINE ETHR_AMC_SINT_T__
+amc_set_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ prev_state0)
+{
+    ETHR_AMC_SINT_T__ state0 = prev_state0;
+    /* Set exclusive flag. */
+    while (1) {
+	ETHR_AMC_SINT_T__ act_state0, new_state0;
+	while (state0 & 1) { /* Wait until exclusive bit has been cleared */
+	    ETHR_SPIN_BODY;
+	    state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
+	}
+	/* Try to set exclusive bit */
+	new_state0 = state0 + 1;
+	act_state0 = ETHR_AMC_ATMC_FUNC__(cmpxchg_acqb)(&amc->atomic[0],
+							new_state0,
+							state0);
+	if (state0 == act_state0)
+	    return state0; /* old state0 */
+	state0 = act_state0;
+    }
+}
+
+static ETHR_INLINE void
+amc_inc_mc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0)
+{
+    ETHR_AMC_SINT_T__ state0 = old_state0;
+
+    /* Increment modification counter and reset exclusive flag. */
+
+    ETHR_DBG_CHK_EXCL_STATE(amc, state0);
+
+    state0 += 2;
+
+    ETHR_ASSERT((state0 & 1) == 0);
+
+#if ETHR_AMC_NO_ATMCS__ == 2
+    if (state0 == 0) {
+	/*
+	 * state0 wrapped, so we need to increment state1. There is no need
+	 * for atomic inc op, since this is always done while having exclusive
+	 * flag.
+	 */
+	ETHR_AMC_SINT_T__ state1 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1]);
+	state1++;
+	ETHR_AMC_ATMC_FUNC__(set)(&amc->atomic[1], state1);
+    }
+#endif
+    ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], state0);
+}
+
+static ETHR_INLINE void
+amc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0)
+{
+    ETHR_DBG_CHK_EXCL_STATE(amc, old_state0);
+    /*
+     * Reset exclusive flag, but leave modification counter unchanged,
+     * i.e., restore state to what it was before setting exclusive
+     * flag.
+     */
+    ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], old_state0);
+}
+
+static ETHR_INLINE void
+amc_set(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val)
+{
+    ETHR_AMC_SINT_T__ state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
+
+    state0 = amc_set_excl(amc, state0);
+
+    avar[0] = val[0];
+    if (dw)
+	avar[1] = val[1];
+
+    amc_inc_mc_unset_excl(amc, state0);
+}
+
+static ETHR_INLINE int
+amc_try_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar,
+	     ethr_sint_t *val, ETHR_AMC_SINT_T__ *state0p)
+{
+    /* *state0p should contain last read value if aborting */
+    ETHR_AMC_SINT_T__ old_state0;
+#if ETHR_AMC_NO_ATMCS__ == 2
+    ETHR_AMC_SINT_T__ state1;
+    int abrt;
+#endif
+
+    *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]);
+    if ((*state0p) & 1) 
+	return 0; /* exclusive flag set; abort */
+#if ETHR_AMC_NO_ATMCS__ == 2
+    state1 = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[1]);
+#else
+    ETHR_COMPILER_BARRIER;
+#endif
+
+    val[0] = avar[0];
+    if (dw)
+	val[1] = avar[1];
+
+    ETHR_READ_MEMORY_BARRIER;
+
+    /*
+     * Abort if state has changed (i.e, either the exclusive
+     * flag is set, or modification counter changed).
+     */
+    old_state0 = *state0p;
+#if ETHR_AMC_NO_ATMCS__ == 2
+    *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]);
+    abrt = (old_state0 != *state0p);
+    abrt |= (state1 != ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1]));
+    return abrt == 0;
+#else
+    *state0p = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
+    return old_state0 == *state0p;
+#endif
+}
+
+static ETHR_INLINE void
+amc_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val)
+{
+    ETHR_AMC_SINT_T__ state0;
+    int i;
+
+#if ETHR_AMC_MAX_TRY_READ__ == 0
+    state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
+#else
+    for (i = 0; i < ETHR_AMC_MAX_TRY_READ__; i++) {
+	if (amc_try_read(amc, dw, avar, val, &state0))
+	    return; /* read success */
+	ETHR_SPIN_BODY;
+    }
+#endif
+
+    state0 = amc_set_excl(amc, state0);
+
+    val[0] = avar[0];
+    if (dw)
+	val[1] = avar[1];
+
+    amc_unset_excl(amc, state0);
+}
+
+static ETHR_INLINE int
+amc_cmpxchg(ethr_amc_t *amc, int dw, ethr_sint_t *avar,
+	    ethr_sint_t *new, ethr_sint_t *xchg)
+{
+    ethr_sint_t val[2];
+    ETHR_AMC_SINT_T__ state0;
+
+    if (amc_try_read(amc, dw, avar, val, &state0)) {
+	if (val[0] != xchg[0] || (dw && val[1] != xchg[1])) {
+	    xchg[0] = val[0];
+	    if (dw)
+		xchg[1] = val[1];
+	    return 0; /* failed */
+	}
+	/* Operation will succeed if not interrupted */
+    }
+
+    state0 = amc_set_excl(amc, state0);
+
+    if (xchg[0] != avar[0] || (dw && xchg[1] != avar[1])) {
+	xchg[0] = avar[0];
+	if (dw)
+	    xchg[1] = avar[1];
+
+	ETHR_DBG_CHK_EXCL_STATE(amc, state0);
+
+	amc_unset_excl(amc, state0);
+	return 0; /* failed */
+    }
+
+    avar[0] = new[0];
+    if (dw)
+	avar[1] = new[1];
+
+    amc_inc_mc_unset_excl(amc, state0);
+    return 1;
+}
+
+
+#define ETHR_AMC_MODIFICATION_OPS__(AMC, OPS)			\\
+do {								\\
+    ETHR_AMC_SINT_T__ state0__;					\\
+    state0__ = ETHR_AMC_ATMC_FUNC__(read)(&(AMC)->atomic[0]);	\\
+    state0__ = amc_set_excl((AMC), state0__);			\\
+    { OPS; }							\\
+    amc_inc_mc_unset_excl((AMC), state0__);			\\
+} while (0)
+
+#endif /* amc fallback */
+"].
+
diff --git a/erts/lib_src/win/ethr_event.c b/erts/lib_src/win/ethr_event.c
new file mode 100644
index 0000000000..bc2f635c26
--- /dev/null
+++ b/erts/lib_src/win/ethr_event.c
@@ -0,0 +1,115 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2009-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Author: Rickard Green
+ */
+
+#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHR_EVENT_IMPL__
+
+#include "ethread.h"
+
+/* --- Windows implementation of thread events ------------------------------ */
+
+int
+ethr_event_init(ethr_event *e)
+{
+    ethr_atomic32_init(&e->state, ETHR_EVENT_OFF__);
+    e->handle = CreateEvent(NULL, FALSE, FALSE, NULL);
+    if (e->handle == INVALID_HANDLE_VALUE)
+	return ethr_win_get_errno__();
+    return 0;
+}
+
+int
+ethr_event_destroy(ethr_event *e)
+{
+    BOOL res = CloseHandle(e->handle);
+    return res == 0 ? ethr_win_get_errno__() : 0;
+}
+
+void
+ethr_event_set(ethr_event *e)
+{
+    ethr_event_set__(e);
+}
+
+void
+ethr_event_reset(ethr_event *e)
+{
+    ethr_event_reset__(e);
+}
+
+static ETHR_INLINE int
+wait(ethr_event *e, int spincount)
+{
+    DWORD code;
+    int sc, res, until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+
+    if (spincount < 0)
+	ETHR_FATAL_ERROR__(EINVAL);
+
+    sc = spincount;
+
+    while (1) {
+	ethr_sint32_t state;
+	while (1) {
+	    state = ethr_atomic32_read(&e->state);
+	    if (state == ETHR_EVENT_ON__)
+		return 0;
+	    if (sc == 0)
+		break;
+	    sc--;
+	    ETHR_SPIN_BODY;
+	    if (--until_yield == 0) {
+		until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		res = ETHR_YIELD();
+		if (res != 0)
+		    ETHR_FATAL_ERROR__(res);
+	    }
+	}
+
+	if (state != ETHR_EVENT_OFF_WAITER__) {
+	    state = ethr_atomic32_cmpxchg(&e->state,
+					  ETHR_EVENT_OFF_WAITER__,
+					  ETHR_EVENT_OFF__);
+	    if (state == ETHR_EVENT_ON__)
+		return 0;
+	    ETHR_ASSERT(state == ETHR_EVENT_OFF__);
+	}
+
+	code = WaitForSingleObject(e->handle, INFINITE);
+	if (code != WAIT_OBJECT_0)
+	    ETHR_FATAL_ERROR__(ethr_win_get_errno__());
+    }
+
+}
+
+int
+ethr_event_wait(ethr_event *e)
+{
+    return wait(e, 0);
+}
+
+int
+ethr_event_swait(ethr_event *e, int spincount)
+{
+    return wait(e, spincount);
+}
diff --git a/erts/lib_src/win/ethread.c b/erts/lib_src/win/ethread.c
new file mode 100644
index 0000000000..3abda6de4c
--- /dev/null
+++ b/erts/lib_src/win/ethread.c
@@ -0,0 +1,606 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Windows native threads implementation of the ethread library
+ * Author: Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#define ETHR_CHILD_WAIT_SPIN_COUNT 4000
+
+#undef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include <process.h>
+#include <winerror.h>
+#include <stdio.h>
+#include <limits.h>
+#include <intrin.h>
+
+#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHREAD_IMPL__
+
+#include "ethread.h"
+#include "ethr_internal.h"
+
+#ifndef ETHR_HAVE_ETHREAD_DEFINES
+#error Missing configure defines
+#endif
+
+/* Argument passed to thr_wrapper() */
+typedef struct {
+    ethr_tid *tid;
+    ethr_atomic32_t result;
+    ethr_ts_event *tse;
+    void *(*thr_func)(void *);
+    void *arg;
+    void *prep_func_res;
+} ethr_thr_wrap_data__;
+
+#define ETHR_INVALID_TID_ID -1
+
+struct ethr_join_data_ {
+    HANDLE handle;
+    void *res;
+};
+
+static ethr_atomic_t thread_id_counter;
+static DWORD own_tid_key;
+static ethr_tid main_thr_tid;
+static int child_wait_spin_count;
+
+DWORD ethr_ts_event_key__;
+
+#define ETHR_GET_OWN_TID__	((ethr_tid *) TlsGetValue(own_tid_key))
+
+/*
+ * --------------------------------------------------------------------------
+ * Static functions
+ * --------------------------------------------------------------------------
+ */
+
+static void thr_exit_cleanup(ethr_tid *tid, void *res)
+{
+
+    ETHR_ASSERT(tid == ETHR_GET_OWN_TID__);
+
+    if (tid->jdata)
+	tid->jdata->res = res;
+
+    ethr_run_exit_handlers__();
+    ethr_ts_event_destructor__((void *) ethr_get_tse__());
+}
+
+static unsigned __stdcall thr_wrapper(LPVOID vtwd)
+{
+    ethr_tid my_tid;
+    ethr_sint32_t result;
+    void *res;
+    ethr_thr_wrap_data__ *twd = (ethr_thr_wrap_data__ *) vtwd;
+    void *(*thr_func)(void *) = twd->thr_func;
+    void *arg = twd->arg;
+    ethr_ts_event *tsep = NULL;
+
+    result = (ethr_sint32_t) ethr_make_ts_event__(&tsep);
+
+    if (result == 0) {
+	tsep->iflgs |= ETHR_TS_EV_ETHREAD;
+	my_tid = *twd->tid;
+	if (!TlsSetValue(own_tid_key, (LPVOID) &my_tid)) {
+	    result = (ethr_sint32_t) ethr_win_get_errno__();
+	    ethr_free_ts_event__(tsep);
+	}
+	else {
+	    if (ethr_thr_child_func__)
+		ethr_thr_child_func__(twd->prep_func_res);
+	}
+    }
+
+    tsep = twd->tse; /* We aren't allowed to follow twd after
+			result has been set! */
+
+    ethr_atomic32_set(&twd->result, result);
+
+    ethr_event_set(&tsep->event);
+
+    res = result == 0 ? (*thr_func)(arg) : NULL;
+
+    thr_exit_cleanup(&my_tid, res);
+    return 0;
+}
+
+/* internal exports */
+
+int
+ethr_win_get_errno__(void)
+{
+    return erts_get_last_win_errno();
+}
+
+int ethr_set_tse__(ethr_ts_event *tsep)
+{
+    return (TlsSetValue(ethr_ts_event_key__, (LPVOID) tsep)
+	    ? 0
+	    : ethr_win_get_errno__());
+}
+
+ethr_ts_event *ethr_get_tse__(void)
+{
+    return (ethr_ts_event *) TlsGetValue(ethr_ts_event_key__);
+}
+
+ETHR_IMPL_NORETURN__
+ethr_abort__(void)
+{
+#if 1
+    DebugBreak();
+#else
+    abort();
+#endif
+}
+
+#if defined(ETHR_X86_RUNTIME_CONF__)
+
+#pragma intrinsic(__cpuid)
+
+void
+ethr_x86_cpuid__(int *eax, int *ebx, int *ecx, int *edx)
+{
+    int CPUInfo[4];
+
+    __cpuid(CPUInfo, *eax);
+
+    *eax = CPUInfo[0];
+    *ebx = CPUInfo[1];
+    *ecx = CPUInfo[2];
+    *edx = CPUInfo[3];
+}
+
+#endif /* ETHR_X86_RUNTIME_CONF__ */
+
+/*
+ * ----------------------------------------------------------------------------
+ * Exported functions
+ * ----------------------------------------------------------------------------
+ */
+
+int
+ethr_init(ethr_init_data *id)
+{
+#ifdef _WIN32_WINNT
+    DWORD major = (_WIN32_WINNT >> 8) & 0xff;
+    DWORD minor = _WIN32_WINNT & 0xff;
+    OSVERSIONINFO os_version;
+#endif
+    int err = 0;
+    unsigned long i;
+
+    if (!ethr_not_inited__)
+	return EINVAL;
+
+#ifdef _WIN32_WINNT
+    os_version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
+    GetVersionEx(&os_version);
+    if (os_version.dwPlatformId != VER_PLATFORM_WIN32_NT
+	|| os_version.dwMajorVersion < major
+	|| (os_version.dwMajorVersion == major
+	    && os_version.dwMinorVersion < minor))
+	return ENOTSUP;
+#endif
+    err = ethr_init_common__(id);
+    if (err)
+	goto error;
+
+    own_tid_key = TlsAlloc();
+    if (own_tid_key == TLS_OUT_OF_INDEXES)
+	goto error;
+
+    ethr_atomic_init(&thread_id_counter, 0);
+
+    main_thr_tid.id = 0;
+    main_thr_tid.jdata = NULL;
+
+    if (!TlsSetValue(own_tid_key, (LPVOID) &main_thr_tid))
+	goto error;
+
+    ETHR_ASSERT(&main_thr_tid == ETHR_GET_OWN_TID__);
+
+    ethr_ts_event_key__ = TlsAlloc();
+    if (ethr_ts_event_key__ == TLS_OUT_OF_INDEXES)
+	goto error;
+
+    child_wait_spin_count = ETHR_CHILD_WAIT_SPIN_COUNT;
+    if (erts_get_cpu_configured(ethr_cpu_info__) == 1)
+	child_wait_spin_count = 0;
+
+    ethr_not_inited__ = 0;
+
+    return 0;
+
+ error:
+    ethr_not_inited__ = 1;
+    if (err == 0)
+	err = ethr_win_get_errno__();
+    ETHR_ASSERT(err != 0);
+    return err;
+}
+
+int
+ethr_late_init(ethr_late_init_data *id)
+{
+    int res = ethr_late_init_common__(id);
+    if (res != 0)
+	return res;
+    ethr_not_completely_inited__ = 0;
+    return res;
+}
+
+
+/*
+ * Thread functions.
+ */
+
+int
+ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg,
+		ethr_thr_opts *opts)
+{
+    HANDLE handle = INVALID_HANDLE_VALUE;
+    int err = 0;
+    ethr_thr_wrap_data__ twd;
+    DWORD code;
+    unsigned ID;
+    unsigned stack_size = 0; /* 0 = system default */
+    int use_stack_size = (opts && opts->suggested_stack_size >= 0
+			  ? opts->suggested_stack_size
+			  : -1 /* Use system default */);
+
+#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE
+    if (use_stack_size < 0)
+	use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE;
+#endif
+
+#if ETHR_XCHK
+    if (ethr_not_completely_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!tid || !func) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+
+    do {
+	tid->id = ethr_atomic_inc_read(&thread_id_counter);
+    } while (tid->id == ETHR_INVALID_TID_ID);
+
+    if (opts && opts->detached)
+	tid->jdata = NULL;
+    else {
+	tid->jdata = ethr_mem__.std.alloc(sizeof(struct ethr_join_data_));
+	if (!tid->jdata)
+	    return ENOMEM;
+	tid->jdata->handle = INVALID_HANDLE_VALUE;
+	tid->jdata->res = NULL;
+    }
+
+    if (use_stack_size >= 0) {
+	size_t suggested_stack_size = (size_t) use_stack_size;
+#ifdef ETHR_DEBUG
+	suggested_stack_size /= 2; /* Make sure we got margin */
+#endif
+	if (suggested_stack_size < ethr_min_stack_size__)
+	    stack_size = (unsigned) ETHR_KW2B(ethr_min_stack_size__);
+	else if (suggested_stack_size > ethr_max_stack_size__)
+	    stack_size = (unsigned) ETHR_KW2B(ethr_max_stack_size__);
+	else
+	    stack_size = (unsigned)
+	      ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size));
+    }
+
+    ethr_atomic32_init(&twd.result, -1);
+
+    twd.tid = tid;
+    twd.thr_func = func;
+    twd.arg = arg;
+    twd.tse = ethr_get_ts_event();
+
+    /* Call prepare func if it exist */
+    if (ethr_thr_prepare_func__)
+	twd.prep_func_res = ethr_thr_prepare_func__();
+    else
+	twd.prep_func_res = NULL;
+
+    /* spawn the thr_wrapper function */
+    handle = (HANDLE) _beginthreadex(NULL, stack_size, thr_wrapper,
+				     (LPVOID) &twd, 0, &ID);
+    if (handle == (HANDLE) 0) {
+	handle = INVALID_HANDLE_VALUE;
+	goto error;
+    }
+    else {
+	int spin_count = child_wait_spin_count;
+
+	ETHR_ASSERT(handle != INVALID_HANDLE_VALUE);
+
+	if (!tid->jdata)
+	    CloseHandle(handle);
+	else
+	    tid->jdata->handle = handle;
+
+	/* Wait for child to initialize... */
+	while (1) {
+	    ethr_sint32_t result;
+	    int err;
+	    ethr_event_reset(&twd.tse->event);
+
+	    result = ethr_atomic32_read(&twd.result);
+	    if (result == 0)
+		break;
+
+	    if (result > 0) {
+		err = (int) result;
+		goto error;
+	    }
+
+	    err = ethr_event_swait(&twd.tse->event, spin_count);
+	    if (err && err != EINTR)
+		goto error;
+	    spin_count = 0;
+	}
+    }
+
+    if (ethr_thr_parent_func__)
+	ethr_thr_parent_func__(twd.prep_func_res);
+
+    if (twd.tse)
+	ethr_leave_ts_event(twd.tse);
+
+    return 0;
+
+ error:
+
+    if (err == 0)
+	err = ethr_win_get_errno__();
+    ETHR_ASSERT(err != 0);
+
+    if (ethr_thr_parent_func__)
+	ethr_thr_parent_func__(twd.prep_func_res);
+
+    if (handle != INVALID_HANDLE_VALUE) {
+	WaitForSingleObject(handle, INFINITE);
+	CloseHandle(handle);
+    }
+
+    if (tid->jdata) {
+	ethr_mem__.std.free(tid->jdata);
+	tid->jdata = NULL;
+    }
+
+    tid->id = ETHR_INVALID_TID_ID;
+
+    if (twd.tse)
+	ethr_leave_ts_event(twd.tse);
+
+    return err;
+}
+
+int ethr_thr_join(ethr_tid tid, void **res)
+{
+    DWORD code;
+
+#if ETHR_XCHK 
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+
+    if (tid.id == ETHR_INVALID_TID_ID || !tid.jdata)
+	return EINVAL;
+
+    /* Wait for thread to terminate */
+    code = WaitForSingleObject(tid.jdata->handle, INFINITE);
+    if (code != WAIT_OBJECT_0)
+	return ethr_win_get_errno__();
+
+    CloseHandle(tid.jdata->handle);
+    tid.jdata->handle = INVALID_HANDLE_VALUE;
+
+    if (res)
+	*res = tid.jdata->res;
+
+    /*
+     * User better not try to join or detach again; or
+     * bad things will happen... (users responsibility)
+     */
+
+    ethr_mem__.std.free(tid.jdata);
+
+    return 0;
+}
+
+
+int
+ethr_thr_detach(ethr_tid tid)
+{
+#if ETHR_XCHK 
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+
+    if (tid.id == ETHR_INVALID_TID_ID || !tid.jdata)
+	return EINVAL;
+
+    CloseHandle(tid.jdata->handle);
+    tid.jdata->handle = INVALID_HANDLE_VALUE;
+
+    /*
+     * User better not try to join or detach again; or
+     * bad things will happen... (users responsibility)
+     */
+
+    ethr_mem__.std.free(tid.jdata);
+
+    return 0;
+}
+
+
+void
+ethr_thr_exit(void *res)
+{
+    ethr_tid *tid;
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return;
+    }
+#endif
+    tid = ETHR_GET_OWN_TID__;
+    if (!tid) {
+	ETHR_ASSERT(0);
+	_endthreadex((unsigned) 0);
+    }
+    thr_exit_cleanup(tid, res);
+    _endthreadex((unsigned) 0);
+}
+
+ethr_tid
+ethr_self(void)
+{
+    ethr_tid *tid;
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ethr_tid dummy_tid = {ETHR_INVALID_TID_ID, NULL};
+	ETHR_ASSERT(0);
+	return dummy_tid;
+    }
+#endif
+    /* It is okay for non-ethreads (i.e. native win32 threads) to call
+       ethr_self(). They will however be returned an invalid tid. */
+    tid = ETHR_GET_OWN_TID__;
+    if (!tid) {
+	ethr_tid dummy_tid = {ETHR_INVALID_TID_ID, NULL};
+	return dummy_tid;
+    }
+    return *tid;
+}
+
+int
+ethr_equal_tids(ethr_tid tid1, ethr_tid tid2)
+{
+    /* An invalid tid does not equal any tid, not even an invalid tid */
+    return tid1.id == tid2.id && tid1.id != ETHR_INVALID_TID_ID;
+}
+
+/*
+ * Thread specific data
+ */
+
+int
+ethr_tsd_key_create(ethr_tsd_key *keyp)
+{
+    DWORD key;
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!keyp) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    key = TlsAlloc();
+    if (key == TLS_OUT_OF_INDEXES)
+	return ethr_win_get_errno__();
+    *keyp = (ethr_tsd_key) key;
+    return 0;
+}
+
+int
+ethr_tsd_key_delete(ethr_tsd_key key)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+    if (!TlsFree((DWORD) key))
+	return ethr_win_get_errno__();
+    return 0;
+}
+
+int
+ethr_tsd_set(ethr_tsd_key key, void *value)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+    if (!TlsSetValue((DWORD) key, (LPVOID) value))
+	return ethr_win_get_errno__();
+    return 0;
+}
+
+void *
+ethr_tsd_get(ethr_tsd_key key)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return NULL;
+    }
+#endif
+    return (void *) TlsGetValue((DWORD) key);
+}
+
+
+/*
+ * Thread specific events
+ */
+
+ethr_ts_event *
+ethr_get_ts_event(void)
+{
+    return ethr_get_ts_event__();
+}
+
+void
+ethr_leave_ts_event(ethr_ts_event *tsep)
+{
+    ethr_leave_ts_event__(tsep);
+}
+
+ethr_ts_event *
+ethr_create_ts_event__(void)
+{
+    ethr_ts_event *tsep;
+    ethr_make_ts_event__(&tsep);
+    return tsep;
+}
diff --git a/erts/ntbuild.erl b/erts/ntbuild.erl
deleted file mode 100644
index e48be58c17..0000000000
--- a/erts/ntbuild.erl
+++ /dev/null
@@ -1,332 +0,0 @@
-%%
-%% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
-%% 
-%% The contents of this file are subject to the Erlang Public License,
-%% Version 1.1, (the "License"); you may not use this file except in
-%% compliance with the License. You should have received a copy of the
-%% Erlang Public License along with this software. If not, it can be
-%% retrieved online at http://www.erlang.org/.
-%% 
-%% Software distributed under the License is distributed on an "AS IS"
-%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
-%% the License for the specific language governing rights and limitations
-%% under the License.
-%% 
-%% %CopyrightEnd%
-%%
-%% To be used from makefiles on the unix side executing things on the NT-side
--module(ntbuild).
-
--export([nmake/1, omake/1, waitnode/1, restart/1,
-	 setdir/1, run_tests/1, run_command/1]).
--export([serv_nmake/2, serv_omake/2, serv_restart/0, serv_run_tests/2,
-	 serv_run_command/1]).
-
-waitnode([NtNode]) ->
-    % First, wait for node to disappear.
-    case wait_disappear(NtNode, 0) of
-	ok ->
-	    case wait_appear(NtNode, 0) of
-		ok ->
-		    halt(0);
-		fail ->
-		    halt(1)
-	    end;
-	fail ->
-	    halt(1)
-    end.
-
-% Wait for nt node to appear within 5 minutes.
-wait_appear(_NtNode, 300) ->
-    fail;
-wait_appear(NtNode, N) ->
-    receive after 1000 -> ok end,
-    case nt_node_alive(NtNode, quiet) of
-	no ->
-	    wait_appear(NtNode, N+1);
-	yes ->
-	    ok
-    end.
-    
-
-
-% Waits for nt node to disappear within 3 minutes.
-wait_disappear(NtNode, 300) ->
-    fail;
-wait_disappear(NtNode, N) ->
-    receive after 1000 -> ok end,
-    case nt_node_alive(NtNode, quiet) of
-	yes ->
-	    wait_disappear(NtNode, N+1);
-	no ->
-	    ok
-    end.
-			    
-restart([NtNode]) ->
-    case nt_node_alive(NtNode) of
-	yes ->
-	    case rpc:call(NtNode, ntbuild, serv_restart, []) of
-		ok ->
-		    io:format("halt(0)~n"),
-		    halt();
-		Error ->
-		    io:format("halt(1)~n"),
-		    halt(1)
-	    end;
-	no ->
-	    halt(1)
-    end.
-
-
-setdir([NtNode, Dir0]) ->    
-    Dir = atom_to_list(Dir0),
-    case nt_node_alive(NtNode) of
-	yes ->
-	    case rpc:call(NtNode, file, set_cwd, [Dir]) of
-		ok ->
-		    io:format("halt(0)~n"),
-		    halt();
-		Error ->
-		    io:format("halt(1) (Error: ~p) (~p not found) ~n", [Error, Dir]),
-		    halt(1)
-	    end;
-	no ->
-	    halt(1)
-    end.
-
-run_tests([NtNode, Vsn0, Logdir]) ->
-    Vsn = atom_to_list(Vsn0),
-    case nt_node_alive(NtNode) of
-	yes ->
-	    case rpc:call(NtNode, ntbuild, serv_run_tests, [Vsn, Logdir]) of
-		ok ->
-		    io:format("halt(0)~n"),
-		    halt();
-		Error ->
-		    io:format("RPC To Windows Node Failed: ~p~n", [Error]),
-		    io:format("halt(1)~n"),
-		    halt(1)
-	    end;
-	no ->
-	    halt(1)
-    end.
-
-run_command([NtNode, Cmd]) ->
-    case nt_node_alive(NtNode) of
-	yes ->
-	    case rpc:call(NtNode, ntbuild, serv_run_command, [Cmd]) of
-		ok ->
-		    io:format("halt(0)~n"),
-		    halt();
-		Error ->
-		    io:format("RPC To Windows Node Failed: ~p~n", [Error]),
-		    io:format("halt(1)~n"),
-		    halt(1)
-	    end;
-	no ->
-	    halt(1)
-    end.
-
-nmake([NtNode, Path, Options]) ->
-%    io:format("nmake2(~w,~w)~n",[Path, Options]),
-    Dir=atom_to_list(Path),
-    Opt=atom_to_list(Options),
-    case nt_node_alive(NtNode) of
-	yes ->
-	    case rpc:call(NtNode, ntbuild, serv_nmake, [Dir, Opt]) of
-		ok ->
-		    io:format("halt(0)~n"),
-		    halt();
-		Error ->
-		    io:format("Error: ~n", [Error]),
-		    halt(1)
-	    end;
-	no ->
-	    halt(1)
-    end.
-
-omake([NtNode, Path, Options]) ->
-    Dir=atom_to_list(Path),
-    Opt=atom_to_list(Options),
-    case nt_node_alive(NtNode) of
-	yes ->
-	    case rpc:call(NtNode, ntbuild, serv_omake, [Dir, Opt]) of
-		ok ->
-		    io:format("halt(0)~n"),
-		    halt();
-		Error ->
-		    io:format("RPC To Windows Node Failed: ~p~n", [Error]),
-		    io:format("~p        ~p~n", [Dir, Opt]),
-		    io:format("halt(1)~n"),
-		    halt(1)
-	    end;
-	no ->
-	    halt(1)
-    end.
-
-
-
-
-
-nt_node_alive(NtNode) ->
-    case net:ping(NtNode) of
-	pong ->
-	    yes;
-	pang ->
-	    io:format("The NT node (~p) is not up. ~n",[NtNode]),
-	    no
-    end.
-
-nt_node_alive(NtNode, quiet) ->
-    case net:ping(NtNode) of
-	pong ->
-	    yes;
-	pang ->
-	    no
-    end.
-
-
-
-%%%
-%%% The 'serv_' functions. Theese are the routines run on the WinNT node.
-%%%
-
-%%-----------------------
-%% serv_run_tests()
-%% Runs the tests.
-serv_run_tests(Vsn, Logdir) ->
-    {ok, Cwd}=file:get_cwd(),
-    io:format("serv_run_tests ~p ~p ~n", [Vsn, Logdir]),
-    Cmd0= "set central_log_dir=" ++ Logdir,
-    Erl = "C:/progra~1/erl"++Vsn++"/bin/erl",
-    Cmd1 = Erl++" -sname a -setcookie a -noshell -noinput -s ts install -s ts run -s ts save -s erlang halt",
-%%    Dir = "C:/temp/test_suite/test_server",
-    Cmd= Cmd0 ++ "/r/n" ++ Cmd1,
-    Dir = "C:/temp/test_server/p7a/test_server",
-    file:set_cwd(Dir),
-    Res=run_make_bat(Dir, Cmd),
-    file:set_cwd(Cwd),
-    Res.
-
-%%-----------------------
-%% serv_run_command()
-%% Runs a command.
-serv_run_command(Cmd) ->
-    {ok, Cwd}=file:get_cwd(),
-    Res=run_make_bat("", Cmd),
-    file:set_cwd(Cwd),
-    Res.
-
-%%-----------------------
-%% serv_restart()
-%% Reboots the NT machine.
-serv_restart() ->
-    Exe="\\erts\\install_nt\\reboot.exe",
-    open_port({spawn, Exe}, [stream, eof, in]),
-    ok.
-
-
-%%-----------------------
-%% serv_nmake(Path, Options) 
-%% Runs `nmake' in the given directory.
-%% Result: ok | error
-serv_nmake(Path, Options) ->
-    {ok, Cwd}=file:get_cwd(),
-    Command="nmake -e -f Makefile.win32 " ++ Options ++	" 2>&1",
-    Res=run_make_bat(Path, Command),
-    file:set_cwd(Cwd),
-    Res.
-
-%%-----------------------
-%% serv_omake(Path, Options) 
-%% Runs `omake' in the given directory.
-%% Result: ok | error
-serv_omake(Path, Options) ->
-    {ok, Cwd}=file:get_cwd(),
-    Command="omake -W -E -EN -f Makefile.win32 " ++ Options ++ " 2>&1",
-    Res=run_make_bat(Path, Command),
-    file:set_cwd(Cwd),
-    Res.
-
-
-read_output(Port, SoFar) ->
-%    io:format("(read_output)~n"),
-    case get_data_from_port(Port) of
-	eof ->
-	    io:format("*** eof ***~n"),
-	    io:format("Never reached a real message"),
-	    halt(1);
-	{ok, Data} ->
-	    case print_line([SoFar|Data]) of
-		{ok, Rest} ->
-		    read_output(Port, Rest);
-		{done, Res} ->
-		    Res
-	    end
-    end.
-
-print_line(Data) ->
-    print_line(Data, []).
-
-print_line([], Acc) ->
-    {ok, lists:reverse(Acc)};
-print_line([$*,$o,$k,$*|Rest], _Acc) ->
-    io:format("*ok*~n"),
-    {done, ok};
-print_line([$*,$e,$r,$r,$o,$r|Rest], _Acc) ->
-    io:format("*error*~n"),
-    {done, error};
-print_line([$\r,$\n|Rest], Acc) ->
-    io:format("~s~n", [lists:reverse(Acc)]),
-    print_line(Rest, []);
-print_line([Chr|Rest], Acc) ->
-    print_line(Rest, [Chr|Acc]).
-    
-get_data_from_port(Port) ->
-    receive
-	{Port, {data, Bytes}} ->
-	    {ok, Bytes};
-	{Port, eof} ->
-	    unlink(Port),
-	    exit(Port, die),
-	    eof;
-	Other ->
-	    io:format("Strange message received: ~p~n", [Other]),
-	    get_data_from_port(Port)
-    end.
-
-
-run_make_bat(Dir, Make) ->
-    {Name, Exe, Script}=create_make_script(Dir, Make),
-    io:format("Exe:~p  Cwd:~p Script:~p ~n",[Exe, Dir, Script]),
-    case file:write_file(Name, Script) of
-	ok ->
-	    case catch open_port({spawn, Exe}, [stderr_to_stdout, stream, hide,
-						eof, in]) of
-		Port when port(Port) ->
-		    read_output(Port, []);
-		Other ->
-		    io:format("Error, open_port failed: ~p~n", [Other]),
-		    {open_port, Other, Exe}
-	    end;
-	Error ->
-	    {write_file, Error, Name}
-    end.
-
-create_make_script(Dir, Make) when atom(Make) ->
-    create_make_script(Dir, atom_to_list(Make));
-create_make_script(Dir, Make) ->
-    {"run_make_bs.bat",
-     "run_make_bs 2>&1",
-     ["@echo off\r\n",
-      "@cd ", Dir, "\r\n",
-      Make++"\r\n",
-      "if errorlevel 1 echo *run_make_bs error*\r\n",
-      "if not errorlevel 1 echo *ok*\r\n"]}.
-
-
-
-
-
diff --git a/erts/preloaded/ebin/erl_prim_loader.beam b/erts/preloaded/ebin/erl_prim_loader.beam
index 304ac41dce..80fc79f396 100644
--- a/erts/preloaded/ebin/erl_prim_loader.beam
+++ b/erts/preloaded/ebin/erl_prim_loader.beam
diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam
index 21e5525b2f..b78747bc84 100644
--- a/erts/preloaded/ebin/erlang.beam
+++ b/erts/preloaded/ebin/erlang.beam
diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam
index 8ccc553c13..cc170b86b2 100644
--- a/erts/preloaded/ebin/init.beam
+++ b/erts/preloaded/ebin/init.beam
diff --git a/erts/preloaded/ebin/otp_ring0.beam b/erts/preloaded/ebin/otp_ring0.beam
index 8ae73ea9a7..45a409738c 100644
--- a/erts/preloaded/ebin/otp_ring0.beam
+++ b/erts/preloaded/ebin/otp_ring0.beam
diff --git a/erts/preloaded/ebin/prim_file.beam b/erts/preloaded/ebin/prim_file.beam
index 3acda843fd..6400cda2b5 100644
--- a/erts/preloaded/ebin/prim_file.beam
+++ b/erts/preloaded/ebin/prim_file.beam
diff --git a/erts/preloaded/ebin/prim_inet.beam b/erts/preloaded/ebin/prim_inet.beam
index 9457b6d360..b2f3ab6c5b 100644
--- a/erts/preloaded/ebin/prim_inet.beam
+++ b/erts/preloaded/ebin/prim_inet.beam
diff --git a/erts/preloaded/ebin/prim_zip.beam b/erts/preloaded/ebin/prim_zip.beam
index 6837cb4661..411fc8d524 100644
--- a/erts/preloaded/ebin/prim_zip.beam
+++ b/erts/preloaded/ebin/prim_zip.beam
diff --git a/erts/preloaded/ebin/zlib.beam b/erts/preloaded/ebin/zlib.beam
index 4e4f85d312..7a1f896d36 100644
--- a/erts/preloaded/ebin/zlib.beam
+++ b/erts/preloaded/ebin/zlib.beam
diff --git a/erts/preloaded/src/Makefile b/erts/preloaded/src/Makefile
index 785ad531f3..d5f5ba3c37 100644
--- a/erts/preloaded/src/Makefile
+++ b/erts/preloaded/src/Makefile
@@ -1,7 +1,7 @@
 # 
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 2008-2009. All Rights Reserved.
+# Copyright Ericsson AB 2008-2010. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -20,8 +20,7 @@
 # be used when the preloaded modules actually are to be updated (i.e. the 
 # beam files are to be recompiled, which is normally not done). 
 # The beam files are placed in the current directory and should be copied 
-# to the ../ebin directory by using the commit target (only works in 
-# clearcase). 
+# to the ../ebin directory by using the copy target.
 
 include $(ERL_TOP)/make/target.mk
 include $(ERL_TOP)/make/$(TARGET)/otp.mk
@@ -62,34 +61,17 @@ debug opt: $(TARGET_FILES)
 clean:
 	rm -f $(TARGET_FILES)
 
-prepare:
-	cleartool co -nc $(STATIC_EBIN)/*
-	cleartool co -nc $(STATIC_EBIN)
-
 copy:
-	for x in *.beam; do\
-	  if test '!' -f $(STATIC_EBIN)/$$x; then\
-	    cleartool mkelem -nc $$x;\
-	  fi;\
-	done
 	cp *.beam $(STATIC_EBIN)
 
-commit:
-	cleartool ci -ident -nc $(STATIC_EBIN)/*.beam
-	cleartool ci -ident -nc $(STATIC_EBIN)
-
-cancel:
-	-cleartool unco -rm $(STATIC_EBIN)
-	-cleartool unco -rm $(STATIC_EBIN)/*.beam
-
 
 include $(ERL_TOP)/make/otp_release_targets.mk
 
 release_spec: 
-	$(INSTALL_DIR) $(RELSYSDIR)/src
-	$(INSTALL_DATA) $(ERL_FILES) $(RELSYSDIR)/src
-	$(INSTALL_DIR) $(RELSYSDIR)/ebin
-	$(INSTALL_DATA) $(STATIC_TARGET_FILES) $(RELSYSDIR)/ebin
+	$(INSTALL_DIR) "$(RELSYSDIR)/src"
+	$(INSTALL_DATA) $(ERL_FILES) "$(RELSYSDIR)/src"
+	$(INSTALL_DIR) "$(RELSYSDIR)/ebin"
+	$(INSTALL_DATA) $(STATIC_TARGET_FILES) "$(RELSYSDIR)/ebin"
 
 release_docs_spec:
 
diff --git a/erts/preloaded/src/erl_prim_loader.erl b/erts/preloaded/src/erl_prim_loader.erl
index a13292d5ab..5ee3d03fef 100644
--- a/erts/preloaded/src/erl_prim_loader.erl
+++ b/erts/preloaded/src/erl_prim_loader.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1996-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2012. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -49,7 +49,7 @@
          prim_read_file_info/2, prim_get_cwd/2]).
 
 %% Used by escript and code
--export([set_primary_archive/3, release_archives/0]).
+-export([set_primary_archive/4, release_archives/0]).
 
 -include_lib("kernel/include/file.hrl").
 
@@ -102,8 +102,14 @@ debug(#prim_state{debug = Deb}, Term) ->
 %%% Interface Functions. 
 %%% --------------------------------------------------------
 
--spec start(term(), atom() | string(), host() | [host()]) ->
-	    {'ok', pid()} | {'error', term()}.
+-spec start(Id, Loader, Hosts) ->
+	    {'ok', Pid} | {'error', What} when
+      Id :: term(),
+      Loader :: atom() | string(),
+      Hosts :: Host | [Host],
+      Host :: host(),
+      Pid :: pid(),
+      What :: term().
 start(Id, Pgm, Hosts) when is_atom(Hosts) ->
     start(Id, Pgm, [Hosts]);
 start(Id, Pgm0, Hosts) ->
@@ -123,19 +129,7 @@ start(Id, Pgm0, Hosts) ->
             {error,Reason}
     end.
 
-start_it("ose_inet"=Cmd, Id, Pid, Hosts) ->
-    %% Setup reserved port for ose_inet driver (only OSE)
-    case catch erlang:open_port({spawn,Cmd}, [binary]) of
-        {'EXIT',Why} ->
-            ?dbg(ose_inet_port_open_fail, Why),
-            Why;
-        OseInetPort ->
-            ?dbg(ose_inet_port, OseInetPort),
-            OseInetPort
-    end,
-    start_it("inet", Id, Pid, Hosts);
-
-%% Hosts must be a list on form ['1.2.3.4' ...]
+%% Hosts must be a list of form ['1.2.3.4' ...]
 start_it("inet", Id, Pid, Hosts) ->
     process_flag(trap_exit, true),
     ?dbg(inet, {Id,Pid,Hosts}),
@@ -174,15 +168,20 @@ init_ack(Pid) ->
     Pid ! {self(),ok},
     ok.
 
--spec set_path([string()]) -> 'ok'.
+-spec set_path(Path) -> 'ok' when
+      Path :: [Dir :: string()].
 set_path(Paths) when is_list(Paths) ->
     request({set_path,Paths}).
 
--spec get_path() -> {'ok', [string()]}.
+-spec get_path() -> {'ok', Path} when
+      Path :: [Dir :: string()].
 get_path() ->
     request({get_path,[]}).
 
--spec get_file(atom() | string()) -> {'ok', binary(), string()} | 'error'.
+-spec get_file(Filename) -> {'ok', Bin, FullName} | 'error' when
+      Filename :: atom() | string(),
+      Bin :: binary(),
+      FullName :: string().
 get_file(File) when is_atom(File) ->
     get_file(atom_to_list(File));
 get_file(File) ->
@@ -202,13 +201,15 @@ get_files(ModFiles, Fun) ->
             ok
     end.
 
--spec list_dir(string()) -> {'ok', [string()]} | 'error'.
+-spec list_dir(Dir) -> {'ok', Filenames} | 'error' when
+      Dir :: string(),
+      Filenames :: [Filename :: string()].
 list_dir(Dir) ->
     check_file_result(list_dir, Dir, request({list_dir,Dir})).
 
-%% -> {ok,Info} | error
--spec read_file_info(string()) -> {'ok', tuple()} | 'error'.
-
+-spec read_file_info(Filename) -> {'ok', FileInfo} | 'error' when
+      Filename :: string(),
+      FileInfo :: file:file_info().
 read_file_info(File) ->
     check_file_result(read_file_info, File, request({read_file_info,File})).
 
@@ -221,15 +222,16 @@ get_cwd(Drive) ->
     check_file_result(get_cwd, Drive, request({get_cwd,[Drive]})).
 
 -spec set_primary_archive(File :: string() | 'undefined', 
-                          ArchiveBin :: binary() | 'undefined',
-			  FileInfo :: #file_info{} | 'undefined')
+			  ArchiveBin :: binary() | 'undefined',
+			  FileInfo :: #file_info{} | 'undefined',
+			  ParserFun :: fun())
 			 -> {ok, [string()]} | {error,_}.
 
-set_primary_archive(undefined, undefined, undefined) ->
-    request({set_primary_archive, undefined, undefined, undefined});
-set_primary_archive(File, ArchiveBin, FileInfo)
+set_primary_archive(undefined, undefined, undefined, ParserFun) ->
+    request({set_primary_archive, undefined, undefined, undefined, ParserFun});
+set_primary_archive(File, ArchiveBin, FileInfo, ParserFun)
   when is_list(File), is_binary(ArchiveBin), is_record(FileInfo, file_info) ->
-    request({set_primary_archive, File, ArchiveBin, FileInfo}).
+    request({set_primary_archive, File, ArchiveBin, FileInfo, ParserFun}).
 
 -spec release_archives() -> 'ok' | {'error', _}.
 
@@ -317,8 +319,11 @@ loop(State, Parent, Paths) ->
                     {get_cwd,[_]=Args} ->
                         {Res,State1} = handle_get_cwd(State, Args),
                         {Res,State1,Paths};
-                    {set_primary_archive,File,Bin,FileInfo} ->
-                        {Res,State1} = handle_set_primary_archive(State, File, Bin, FileInfo),
+                    {set_primary_archive,File,ArchiveBin,FileInfo,ParserFun} ->
+                        {Res,State1} =
+			    handle_set_primary_archive(State, File,
+						       ArchiveBin, FileInfo,
+						       ParserFun),
                         {Res,State1,Paths};
                     release_archives ->
                         {Res,State1} = handle_release_archives(State),
@@ -358,8 +363,8 @@ handle_get_file(State = #state{loader = efile}, Paths, File) ->
 handle_get_file(State = #state{loader = inet}, Paths, File) ->
     ?SAFE2(inet_get_file_from_port(State, File, Paths), State).
 
-handle_set_primary_archive(State= #state{loader = efile}, File, Bin, FileInfo) ->
-    ?SAFE2(efile_set_primary_archive(State, File, Bin, FileInfo), State).
+handle_set_primary_archive(State= #state{loader = efile}, File, ArchiveBin, FileInfo, ParserFun) ->
+    ?SAFE2(efile_set_primary_archive(State, File, ArchiveBin, FileInfo, ParserFun), State).
 
 handle_release_archives(State= #state{loader = efile}) ->
     ?SAFE2(efile_release_archives(State), State).
@@ -395,7 +400,7 @@ handle_timeout(State = #state{loader = inet}, Parent) ->
     inet_timeout_handler(State, Parent).
 
 %%% --------------------------------------------------------
-%%% Functions which handles efile as prim_loader (default).
+%%% Functions which handle efile as prim_loader (default).
 %%% --------------------------------------------------------
 
 %%% Reading many files in parallel is an optimization. 
@@ -405,7 +410,7 @@ handle_timeout(State = #state{loader = inet}, Parent) ->
 efile_multi_get_file_from_port(State, ModFiles, Paths, Fun) ->
     Ref = make_ref(),
     %% More than 200 processes is no gain.
-    Max = min(200, erlang:system_info(thread_pool_size)),
+    Max = erlang:min(200, erlang:system_info(thread_pool_size)),
     efile_multi_get_file_from_port2(ModFiles, 0, Max, State, Paths, Fun, Ref, ok).
 
 efile_multi_get_file_from_port2([MF | MFs], Out, Max, State, Paths, Fun, Ref, Ret) when Out < Max ->
@@ -469,7 +474,7 @@ efile_get_file_from_port2(#state{prim_state = PS} = State, File) ->
     end.
 
 efile_get_file_from_port3(State, File, [P | Paths]) ->
-    case efile_get_file_from_port2(State, concat([P,"/",File])) of
+    case efile_get_file_from_port2(State, join(P, File)) of
         {{error,Reason},State1} when Reason =/= emfile ->
             case Paths of
                 [] ->                           % return last error
@@ -483,8 +488,10 @@ efile_get_file_from_port3(State, File, [P | Paths]) ->
 efile_get_file_from_port3(State, _File, []) ->
     {{error,enoent},State}.
 
-efile_set_primary_archive(#state{prim_state = PS} = State, File, Bin, FileInfo) ->
-    {Res, PS2} = prim_set_primary_archive(PS, File, Bin, FileInfo),
+efile_set_primary_archive(#state{prim_state = PS} = State, File,
+			  ArchiveBin, FileInfo, ParserFun) ->
+    {Res, PS2} = prim_set_primary_archive(PS, File, ArchiveBin,
+					  FileInfo, ParserFun),
     {Res,State#state{prim_state = PS2}}.
 
 efile_release_archives(#state{prim_state = PS} = State) ->
@@ -522,7 +529,7 @@ efile_timeout_handler(#state{n_timeouts = N} = State, _Parent) ->
     end.
 
 %%% --------------------------------------------------------
-%%% Functions which handles inet prim_loader
+%%% Functions which handle inet prim_loader
 %%% --------------------------------------------------------
 
 %%
@@ -643,7 +650,7 @@ inet_get_file_from_port(State, File, Paths) ->
     end.
 
 inet_get_file_from_port1(File, [P | Paths], State) ->
-    File1 = concat([P,"/",File]),
+    File1 = join(P, File),
     case inet_send_and_rcv({get,File1}, File1, State) of
         {{error,Reason},State1} ->
             case Paths of
@@ -728,7 +735,7 @@ udp_options() ->
 %% INET version IPv4 addresses
 %%
 ll_tcp_connect(LocalPort, IP, RemotePort) ->
-    case ll_open_set_bind(tcp, ?INET_FAMILY, tcp_options(),
+    case ll_open_set_bind(tcp, ?INET_FAMILY, stream, tcp_options(),
                           ?INET_ADDRESS, LocalPort) of
         {ok,S} ->
             case prim_inet:connect(S, IP, RemotePort, tcp_timeout()) of
@@ -742,11 +749,11 @@ ll_tcp_connect(LocalPort, IP, RemotePort) ->
 %% Open and initialize an udp port for broadcast
 %%
 ll_udp_open(P) ->
-    ll_open_set_bind(udp, ?INET_FAMILY, udp_options(), ?INET_ADDRESS, P).
+    ll_open_set_bind(udp, ?INET_FAMILY, dgram, udp_options(), ?INET_ADDRESS, P).
 
 
-ll_open_set_bind(Protocol, Family, SOpts, IP, Port) ->
-    case prim_inet:open(Protocol, Family) of
+ll_open_set_bind(Protocol, Family, Type, SOpts, IP, Port) ->
+    case prim_inet:open(Protocol, Family, Type) of
         {ok, S} ->
             case prim_inet:setopts(S, SOpts) of
                 ok ->
@@ -790,7 +797,7 @@ prim_release_archives(PS) ->
 prim_do_release_archives(PS, [{ArchiveFile, DictVal} | KeyVals], Acc) ->
     Res = 
 	case DictVal of
-	    {primary, _PrimZip, _FI} ->
+	    {primary, _PrimZip, _FI, _ParserFun} ->
 		ok; % Keep primary archive
 	    {Cache, _FI} ->
 		debug(PS, {release, cache, ArchiveFile}),
@@ -808,7 +815,7 @@ prim_do_release_archives(PS, [], []) ->
 prim_do_release_archives(PS, [], Errors) ->
     {{error, Errors}, PS#prim_state{primary_archive = undefined}}.
 
-prim_set_primary_archive(PS, undefined, undefined, undefined) ->
+prim_set_primary_archive(PS, undefined, undefined, undefined, _ParserFun) ->
     debug(PS, {set_primary_archive, clean}),
     case PS#prim_state.primary_archive of
         undefined ->
@@ -816,48 +823,40 @@ prim_set_primary_archive(PS, undefined, undefined, undefined) ->
             debug(PS, {return, Res}),
             {Res, PS};
         ArchiveFile ->
-            {primary, PrimZip, _FI} = erase(ArchiveFile),
+            {primary, PrimZip, _FI, _ParserFun2} = erase(ArchiveFile),
             ok = prim_zip:close(PrimZip),
             PS2 = PS#prim_state{primary_archive = undefined},
             Res = {ok, []},
             debug(PS2, {return, Res}),
             {Res, PS2}
     end;
-prim_set_primary_archive(PS, ArchiveFile, ArchiveBin, #file_info{} = FileInfo)
-  when is_list(ArchiveFile), is_binary(ArchiveBin) ->
+
+prim_set_primary_archive(PS, ArchiveFile0, ArchiveBin,
+			 #file_info{} = FileInfo, ParserFun)
+  when is_list(ArchiveFile0), is_binary(ArchiveBin) ->
     %% Try the archive file
-    debug(PS, {set_primary_archive, ArchiveFile, byte_size(ArchiveBin)}),
+    debug(PS, {set_primary_archive, ArchiveFile0, byte_size(ArchiveBin)}),
+    ArchiveFile = real_path(absname(ArchiveFile0)),
     {Res3, PS3} =
         case PS#prim_state.primary_archive of
             undefined ->
-                Fun =
-                    fun({Funny, _GI, _GB}, A) ->
-                            case Funny of
-                                ["", "nibe", RevApp] -> % Reverse ebin
-                                    %% Collect ebin directories in archive
-                                    Ebin = reverse(RevApp) ++ "/ebin",
-				    {true, [Ebin | A]};
-                                _ ->
-                                    {true, A}
-                            end
-                    end,
-                Ebins0 = [ArchiveFile],
-                case open_archive({ArchiveFile, ArchiveBin}, FileInfo, Ebins0, Fun) of
-                    {ok, PrimZip, {RevEbins, FI, _}} ->
-                        Ebins = reverse(RevEbins),
+                case load_prim_archive(ArchiveFile, ArchiveBin, FileInfo) of
+                    {ok, PrimZip, FI, Ebins} ->
                         debug(PS, {set_primary_archive, Ebins}),
-                        put(ArchiveFile, {primary, PrimZip, FI}),
-                        {{ok, Ebins}, PS#prim_state{primary_archive = ArchiveFile}};
+                        put(ArchiveFile, {primary, PrimZip, FI, ParserFun}),
+                        {{ok, Ebins},
+                         PS#prim_state{primary_archive = ArchiveFile}};
                     Error ->
                         debug(PS, {set_primary_archive, Error}),
                         {Error, PS}
                 end;
             OldArchiveFile ->
                 debug(PS, {set_primary_archive, clean}),
-                {primary, PrimZip, _FI} = erase(OldArchiveFile),
+                {primary, PrimZip, _FI, _ParserFun} = erase(OldArchiveFile),
                 ok = prim_zip:close(PrimZip),
                 PS2 = PS#prim_state{primary_archive = undefined},
-                prim_set_primary_archive(PS2, ArchiveFile, ArchiveBin, FileInfo)
+                prim_set_primary_archive(PS2, ArchiveFile, ArchiveBin,
+					 FileInfo, ParserFun)
         end,
     debug(PS3, {return, Res3}),
     {Res3, PS3}.
@@ -872,11 +871,11 @@ prim_get_file(PS, File) ->
                 {Res, PS};
             {archive, ArchiveFile, FileInArchive} ->
                 debug(PS, {archive_get_file, ArchiveFile, FileInArchive}),
-                FunnyFile = funny_split(FileInArchive, $/),
+                FileComponents = path_split(FileInArchive),
                 Fun =
-                    fun({Funny, _GetInfo, GetBin}, Acc) ->
+                    fun({Components, _GetInfo, GetBin}, Acc) ->
                             if
-                                Funny =:= FunnyFile ->
+                                Components =:= FileComponents ->
                                     {false, {ok, GetBin()}};
                                 true ->
                                     {true, Acc}
@@ -899,11 +898,11 @@ prim_list_dir(PS, Dir) ->
                 {Res, PS};
             {archive, ArchiveFile, FileInArchive} ->
                 debug(PS, {archive_list_dir, ArchiveFile, FileInArchive}),
-                FunnyDir = funny_split(FileInArchive, $/),
+                DirComponents = path_split(FileInArchive),
                 Fun =
-                    fun({Funny, _GetInfo, _GetBin}, {Status, Names} = Acc) ->
-                            case Funny of
-                                [RevName | FD] when FD =:= FunnyDir ->
+                    fun({Components, _GetInfo, _GetBin}, {Status, Names} = Acc) ->
+                            case Components of
+                                [RevName | DC] when DC =:= DirComponents ->
                                     case RevName of
                                         "" ->
                                             %% The listed directory
@@ -913,16 +912,16 @@ prim_list_dir(PS, Dir) ->
                                             Name = reverse(RevName),
                                             {true, {Status, [Name | Names]}}
                                     end;
-                                ["", RevName | FD] when FD =:= FunnyDir ->
+                                ["", RevName | DC] when DC =:= DirComponents ->
                                     %% Directory
                                     Name = reverse(RevName),
                                     {true, {Status, [Name | Names]}};
-                                [RevName] when FunnyDir =:= [""] ->
-                                    %% Top file
+                                [RevName] when DirComponents =:= [""] ->
+                                    %% File in top directory
                                     Name = reverse(RevName),
                                     {true, {ok, [Name | Names]}};
-                                ["", RevName] when FunnyDir =:= [""] ->
-                                    %% Top file
+                                ["", RevName] when DirComponents =:= [""] ->
+                                    %% Directory in top directory
                                     Name = reverse(RevName),
                                     {true, {ok, [Name | Names]}};
                                 _ ->
@@ -961,15 +960,14 @@ prim_read_file_info(PS, File) ->
                 end;
             {archive, ArchiveFile, FileInArchive} ->
                 debug(PS, {archive_read_file_info, File}),
-                FunnyFile = funny_split(FileInArchive, $/),
+                FileComponents = path_split(FileInArchive),
                 Fun =
-                    fun({Funny, GetInfo, _GetBin}, Acc)  ->
-			    case Funny of
-				[H | T] when  H =:= "",
-					      T =:= FunnyFile ->
+                    fun({Components, GetInfo, _GetBin}, Acc)  ->
+			    case Components of
+				["" | F] when F =:= FileComponents ->
                                     %% Directory
                                     {false, {ok, GetInfo()}};
-                                F when F =:= FunnyFile ->
+                                F when F =:= FileComponents ->
                                     %% Plain file
                                     {false, {ok, GetInfo()}};
                                 _ ->
@@ -1010,7 +1008,7 @@ apply_archive(PS, Fun, Acc, Archive) ->
 		    %% put(Archive, {Error, FI}),
 		    {Error, PS}
 	    end;
-        {primary, PrimZip, FI} ->
+        {primary, PrimZip, FI, ParserFun} ->
 	    case prim_file:read_file_info(Archive) of
                 {ok, FI2} 
 		  when FI#file_info.mtime =:= FI2#file_info.mtime ->
@@ -1021,6 +1019,16 @@ apply_archive(PS, Fun, Acc, Archive) ->
 			    debug(PS, {primary, Error}),
 			    {Error, PS}
 		    end;
+		{ok, FI2} ->
+		    ok = clear_cache(Archive, {ok, PrimZip}),
+		    case load_prim_archive(Archive, FI2, ParserFun) of
+			{ok, PrimZip2, FI3, _Ebins} ->
+			    debug(PS, {cache, {update, Archive}}),
+			    put(Archive, {primary, PrimZip2, FI3, ParserFun});
+			Error2 ->
+			    debug(PS, {cache, {clear, Error2}})
+		    end,
+		    apply_archive(PS, Fun, Acc, Archive);
 		Error ->
 		    debug(PS, {cache, {clear, Error}}),
 		    clear_cache(Archive, {ok, PrimZip}),
@@ -1062,50 +1070,69 @@ open_archive(Archive, Acc, Fun) ->
 	    {error, Reason}
     end.
 
+%% Open the given archive and iterate through all files with an own
+%% wrapper fun in order to identify each file as a component list as
+%% returned from path_split/1.
+%%
+%% In the archive (zip) file, directory elements might or might not be
+%% present. To ensure consistency, a directory element is added if it
+%% does not already exist (ensure_virtual_dirs/6). NOTE that there will
+%% be no such directory element for the top directory of the archive.
 open_archive(Archive, FileInfo, Acc, Fun) ->
     FakeFI = FileInfo#file_info{type = directory},
     Wrapper =
-	fun({N, GI, GB}, {A, I, FunnyDirs}) -> % Full iteration at open
-		Funny = funny_split(N, $/),
-		FunnyDirs2 =
-		    case Funny of
-			["" | FunnyDir] ->
-			    [FunnyDir | FunnyDirs];
+	fun({N, GI, GB}, {A, I, Dirs}) ->
+		Components = path_split(N),
+		Dirs2 =
+		    case Components of
+			["" | Dir] ->
+			    %% This is a directory
+			    [Dir | Dirs];
 			_ ->
-			    FunnyDirs
+			    %% This is a regular file
+			    Dirs
 		    end,
-		{Includes, FunnyDirs3, A2} = 
-		    ensure_virtual_dirs(Funny, Fun, FakeFI, [{true, Funny}], FunnyDirs2, A),
-		{_Continue, A3} = Fun({Funny, GI, GB}, A2),
-		{true, Includes, {A3, I, FunnyDirs3}}
+		{Includes, Dirs3, A2} =
+		    ensure_virtual_dirs(Components, Fun, FakeFI,
+					[{true, Components}], Dirs2, A),
+		{_Continue, A3} = Fun({Components, GI, GB}, A2),
+		{true, Includes, {A3, I, Dirs3}}
 	end,
     prim_zip:open(Wrapper, {Acc, FakeFI, []}, Archive).
 
-ensure_virtual_dirs(Funny, Fun, FakeFI, Includes, FunnyDirs, Acc) ->
-    case Funny of
-	[_ | FunnyDir] ->
-	    case lists:member(FunnyDir, FunnyDirs) of % BIF
+ensure_virtual_dirs(Components, Fun, FakeFI, Includes, Dirs, Acc) ->
+    case Components of
+	[_] ->
+	    %% Don't add virtual dir for top directory
+	    {Includes, Dirs, Acc};
+	[_ | Dir] ->
+	    case lists:member(Dir, Dirs) of % BIF
 		false ->
+		    %% The directory does not yet exist - add it
 		    GetInfo = fun() -> FakeFI end,
 		    GetBin = fun() -> <<>> end,
-		    VirtualDir = ["" | FunnyDir],
+		    VirtualDir = ["" | Dir],
 		    Includes2 = [{true, VirtualDir, GetInfo, GetBin} | Includes],
-		    FunnyDirs2 = [FunnyDir | FunnyDirs],
-		    {I, F, Acc2} = ensure_virtual_dirs(FunnyDir, Fun, FakeFI, Includes2, FunnyDirs2, Acc),
+		    Dirs2 = [Dir | Dirs],
+
+		    %% Recursively ensure dir elements on all levels
+		    {I, F, Acc2} = ensure_virtual_dirs(Dir, Fun, FakeFI,
+						       Includes2, Dirs2, Acc),
+
 		    {_Continue, Acc3} = Fun({VirtualDir, GetInfo, GetBin}, Acc2),
 		    {I, F, Acc3};
 		true ->
-		    {reverse(Includes), FunnyDirs, Acc}
-	    end;
-	[] ->
-	    {reverse(Includes), FunnyDirs, Acc}
+		    %% The directory element does already exist
+		    %% Recursivly ensure dir elements on all levels
+		    ensure_virtual_dirs(Dir,Fun,FakeFI,Includes,Dirs,Acc)
+	    end
     end.
 
 foldl_archive(PrimZip, Acc, Fun) ->
     Wrapper =
-        fun({Funny, GI, GB}, A) ->
+        fun({Components, GI, GB}, A) ->
                 %% Allow partial iteration at foldl
-                {Continue, A2} = Fun({Funny, GI, GB}, A),
+                {Continue, A2} = Fun({Components, GI, GB}, A),
                 {Continue, true, A2}
         end,                        
     prim_zip:foldl(Wrapper, Acc, PrimZip).
@@ -1151,14 +1178,8 @@ send_all(U, [IP | AL], Cmd) ->
     send_all(U, AL, Cmd);
 send_all(_U, [], _) -> ok.
 
-%%concat([A|T]) when is_atom(A) ->              %Atom
-%%    atom_to_list(A) ++ concat(T);
-concat([C|T]) when C >= 0, C =< 255 ->
-    [C|concat(T)];
-concat([S|T]) ->                                %String
-    S ++ concat(T);
-concat([]) ->
-    [].
+join(P, F) ->
+    P ++ "/" ++ F.
 
 member(X, [X|_]) -> true;
 member(X, [_|Y]) -> member(X, Y);
@@ -1189,9 +1210,6 @@ keyins(X, I, [Y | T]) when X < element(I,Y) -> [X,Y|T];
 keyins(X, I, [Y | T]) -> [Y | keyins(X, I, T)];
 keyins(X, _I, []) -> [X].
 
-min(X, Y) when X < Y -> X;
-min(_X, Y) -> Y.
-
 to_strs([P|Paths]) when is_atom(P) ->
     [atom_to_list(P)|to_strs(Paths)];
 to_strs([P|Paths]) when is_list(P) ->
@@ -1210,17 +1228,32 @@ reverse([A, B]) ->
 reverse([A, B | L]) ->
     lists:reverse(L, [B, A]). % BIF
                         
-%% Returns all lists in reverse order
-funny_split(List, Sep) ->
-   funny_split(List, Sep, [], []).
-
-funny_split([Sep | Tail], Sep, Path, Paths) ->
-    funny_split(Tail, Sep, [], [Path | Paths]);
-funny_split([Head | Tail], Sep, Path, Paths) ->
-    funny_split(Tail, Sep, [Head | Path], Paths);
-funny_split([], _Sep, Path, Paths) ->
+%% Returns a reversed list of path components, each component itself a
+%% reversed list (string), e.g.
+%% /path/to/file -> ["elif","ot","htap",""]
+%% /path/to/dir/ -> ["","rid","ot","htap",""]
+%% Note the "" marking leading and trailing / (slash).
+path_split(List) ->
+   path_split(List, [], []).
+
+path_split([$/ | Tail], Path, Paths) ->
+    path_split(Tail, [], [Path | Paths]);
+path_split([Head | Tail], Path, Paths) ->
+    path_split(Tail, [Head | Path], Paths);
+path_split([], Path, Paths) ->
     [Path | Paths].
 
+%% The opposite of path_split/1
+path_join(Paths) ->
+    path_join(Paths,[]).
+
+path_join([""],Acc) ->
+    Acc;
+path_join([Path],Acc) ->
+    reverse(Path) ++ Acc;
+path_join([Path|Paths],Acc) ->
+    path_join(Paths,"/" ++ reverse(Path) ++ Acc).
+
 name_split(ArchiveFile, File0) ->
     File = absname(File0),
     do_name_split(ArchiveFile, File).
@@ -1243,26 +1276,22 @@ do_name_split(undefined, File) ->
 	    %% False match. Assume plain file
             {file, File}
     end;
-do_name_split(ArchiveFile0, File) ->
+do_name_split(ArchiveFile, File) ->
     %% Look first in primary archive
-    ArchiveFile = absname(ArchiveFile0),
-    case string_match(File, ArchiveFile, []) of
+    case string_match(real_path(File), ArchiveFile, []) of
         no_match ->
             %% Archive or plain file
             do_name_split(undefined, File);
         {match, _RevPrimArchiveFile, FileInArchive} ->
             %% Primary archive
-            case FileInArchive of
-                [$/ | FileInArchive2] ->
-                    {archive, ArchiveFile, FileInArchive2};
-                _ ->
-                    {archive, ArchiveFile, FileInArchive}
-            end
+	    {archive, ArchiveFile, FileInArchive}
     end.
 
 string_match([Char | File], [Char | Archive], RevTop) ->
     string_match(File, Archive, [Char | RevTop]);
-string_match(File, [], RevTop) ->
+string_match([] = File, [], RevTop) ->
+    {match, RevTop, File};
+string_match([$/ | File], [], RevTop) ->
     {match, RevTop, File};
 string_match(_File, _Archive, _RevTop) ->
     no_match.
@@ -1322,14 +1351,14 @@ absname(Name) ->
 	    case prim_file:get_cwd() of
 		{ok, Cwd} ->
 		    Cwd ++ "/" ++ Name2;
-		{error, _} -> 
+		{error, _} ->
 		    Name2
 	    end;
 	volumerelative ->
 	    case prim_file:get_cwd() of
 		{ok, Cwd} ->
 		    absname_vr(Name2, Cwd);
-		{error, _} -> 
+		{error, _} ->
 		    Name2
 	    end
     end.
@@ -1363,9 +1392,7 @@ pathtype(Name) when is_list(Name) ->
 		    absolute;
 		_ ->
 		    relative
-	    end;
-	{ose,_} ->
-	    unix_pathtype(Name)
+	    end
     end.
 
 unix_pathtype(Name) ->
@@ -1390,22 +1417,12 @@ win32_pathtype(Name) ->
 	    win32_pathtype([Char | List++Rest]);
 	[$/, $/|_] -> 
 	    absolute;
-	[$\\, $/|_] -> 
-	    absolute;
-	[$/, $\\|_] ->
-	    absolute;
-	[$\\, $\\|_] -> 
-	    absolute;
 	[$/|_] -> 
 	    volumerelative;
-	[$\\|_] -> 
-	    volumerelative;
 	[C1, C2, List | Rest] when is_list(List) ->
-	    pathtype([C1, C2|List ++ Rest]);
+	    win32_pathtype([C1, C2|List ++ Rest]);
 	[_Letter, $:, $/|_] -> 
 	    absolute;
-	[_Letter, $:, $\\|_] ->
-	    absolute;
 	[_Letter, $:|_] -> 
 	    volumerelative;
 	_ -> 
@@ -1418,8 +1435,6 @@ vxworks_first(Name) ->
 	    {not_device, [], []};
 	[$/ | T] ->
 	    vxworks_first2(device, T, [$/]);
-	[$\\ | T] ->
-	    vxworks_first2(device, T, [$/]);
 	[H | T] when is_list(H) ->
 	    vxworks_first(H ++ T);
 	[H | T] ->
@@ -1432,8 +1447,6 @@ vxworks_first2(Devicep, Name, FirstComp) ->
     {Devicep, [], FirstComp};
 	[$/ |T ] ->
 	    {Devicep, [$/ | T], FirstComp};
-	[$\\ | T] ->
-	    {Devicep, [$/ | T], FirstComp};
 	[$: | T]->
 	    {device, T, [$: | FirstComp]};
 	[H | T] when is_list(H) ->
@@ -1455,3 +1468,70 @@ normalize(Name, Acc) ->
 	[] ->
 	    reverse(Acc)
     end.
+
+%% Remove .. and . from the path, e.g.
+%% /path/./to/this/../file -> /path/to/file
+%% This includes resolving symlinks.
+%%
+%% This is done to ensure that paths are totally normalized before
+%% comparing to find out if a file is inside the primary archive or
+%% not.
+real_path(Name) ->
+    real_path(Name,reverse(path_split(Name)),[],[]).
+
+real_path(_Name,[],Acc,_Links) ->
+    path_join(Acc);
+real_path(Name,["."|Paths],Acc,Links) ->
+    real_path(Name,Paths,Acc,Links);
+real_path(Name,[".."|Paths],[""]=Acc,Links) ->
+    %% /.. -> / (can't get higher than root)
+    real_path(Name,Paths,Acc,Links);
+real_path(Name,[".."|Paths],[Prev|Acc],Links) when Prev=/=".." ->
+    real_path(Name,Paths,Acc,Links);
+real_path(Name,[Path|Paths],Acc,Links) ->
+    This = [Path|Acc],
+    ThisFile = path_join(This),
+    case lists:member(ThisFile,Links) of
+	true -> % circular!!
+	    Name;
+	false ->
+	    case prim_file:read_link(ThisFile) of
+		{ok,Link} ->
+		    case reverse(path_split(Link)) of
+			[""|_] = LinkPaths ->
+			    real_path(Name,LinkPaths++Paths,[],[ThisFile|Links]);
+			LinkPaths ->
+			    real_path(Name,LinkPaths++Paths,Acc,[ThisFile|Links])
+		    end;
+		_ ->
+		    real_path(Name,Paths,This,Links)
+	    end
+    end.
+
+load_prim_archive(ArchiveFile, ArchiveBin, #file_info{}=FileInfo) ->
+    Fun = fun({Components, _GI, _GB}, A) ->
+		  case Components of
+		      ["", "nibe", RevApp] -> % Reverse ebin
+			  %% Collect ebin directories in archive
+			  Ebin = lists:reverse(RevApp, "/ebin"),
+			  {true, [Ebin | A]};
+		      _ ->
+			  {true, A}
+		  end
+	  end,
+    Ebins0 = [ArchiveFile],
+    case open_archive({ArchiveFile, ArchiveBin}, FileInfo,
+		      Ebins0, Fun) of
+	{ok, PrimZip, {RevEbins, FI, _}} ->
+	    Ebins = reverse(RevEbins),
+	    {ok, PrimZip, FI, Ebins};
+	Error ->
+	    Error
+    end;
+load_prim_archive(ArchiveFile, FileInfo, ParserFun) ->
+    case ParserFun(ArchiveFile) of
+	{ok, ArchiveBin} ->
+	    load_prim_archive(ArchiveFile, ArchiveBin, FileInfo);
+	Error ->
+	    Error
+    end.
diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl
index 1edb5e72db..1cbce5b80b 100644
--- a/erts/preloaded/src/erlang.erl
+++ b/erts/preloaded/src/erlang.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1996-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2012. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -35,33 +35,53 @@
 -export([delay_trap/2]).
 -export([set_cookie/2, get_cookie/0]).
 -export([nodes/0]).
--export([concat_binary/1]).
+
 -export([list_to_integer/2,integer_to_list/2]).
 -export([flush_monitor_message/2]).
 -export([set_cpu_topology/1, format_cpu_topology/1]).
 -export([await_proc_exit/3]).
+-export([memory/0, memory/1]).
+-export([alloc_info/1, alloc_sizes/1]).
+
+-export([gather_sched_wall_time_result/1,
+	 await_sched_wall_time_modifications/2]).
 
 -deprecated([hash/2]).
--deprecated([concat_binary/1]).
 
--compile(nowarn_bif_clash).
+% Get rid of autoimports of spawn to avoid clashes with ourselves.
+-compile({no_auto_import,[spawn/1]}).
+-compile({no_auto_import,[spawn/4]}).
+-compile({no_auto_import,[spawn_link/1]}).
+-compile({no_auto_import,[spawn_link/4]}).
+-compile({no_auto_import,[spawn_opt/2]}).
+-compile({no_auto_import,[spawn_opt/4]}).
+-compile({no_auto_import,[spawn_opt/5]}).
 
-%%--------------------------------------------------------------------------
+-export_type([timestamp/0]).
 
--type date() :: {pos_integer(), pos_integer(), pos_integer()}.
--type time() :: {non_neg_integer(), non_neg_integer(), non_neg_integer()}.
--type date_time() :: {date(), time()}.
+-type timestamp() :: {MegaSecs :: non_neg_integer(),
+                      Secs :: non_neg_integer(),
+                      MicroSecs :: non_neg_integer()}.
 
 %%--------------------------------------------------------------------------
 
+-spec apply(Fun, Args) -> term() when
+      Fun :: function(),
+      Args :: [term()].
 apply(Fun, Args) ->
-    apply(Fun, Args).
+    erlang:apply(Fun, Args).
 
+-spec apply(Module, Function, Args) -> term() when
+      Module :: module(),
+      Function :: atom(),
+      Args :: [term()].
 apply(Mod, Name, Args) ->
-    apply(Mod, Name, Args).
+    erlang:apply(Mod, Name, Args).
 
 %% Spawns with a fun
 
+-spec spawn(Fun) -> pid() when
+      Fun :: function().
 spawn(F) when is_function(F) ->
     spawn(erlang, apply, [F, []]);
 spawn({M,F}=MF) when is_atom(M), is_atom(F) ->
@@ -69,6 +89,9 @@ spawn({M,F}=MF) when is_atom(M), is_atom(F) ->
 spawn(F) ->
     erlang:error(badarg, [F]).
 
+-spec spawn(Node, Fun) -> pid() when
+      Node :: node(),
+      Fun :: function().
 spawn(N, F) when N =:= node() ->
     spawn(F);
 spawn(N, F) when is_function(F) ->
@@ -78,6 +101,8 @@ spawn(N, {M,F}=MF) when is_atom(M), is_atom(F) ->
 spawn(N, F) ->
     erlang:error(badarg, [N, F]).
 
+-spec spawn_link(Fun) -> pid() when
+      Fun :: function().
 spawn_link(F) when is_function(F) ->
     spawn_link(erlang, apply, [F, []]);
 spawn_link({M,F}=MF) when is_atom(M), is_atom(F) ->
@@ -85,6 +110,9 @@ spawn_link({M,F}=MF) when is_atom(M), is_atom(F) ->
 spawn_link(F) ->
     erlang:error(badarg, [F]).
 
+-spec spawn_link(Node, Fun) -> pid() when
+      Node :: node(),
+      Fun :: function().
 spawn_link(N, F) when N =:= node() ->
     spawn_link(F);
 spawn_link(N, F) when is_function(F) ->
@@ -96,16 +124,30 @@ spawn_link(N, F) ->
 
 %% Spawn and atomically set up a monitor.
 
+-spec spawn_monitor(Fun) -> {pid(), reference()} when
+      Fun :: function().
 spawn_monitor(F) when is_function(F, 0) ->
     erlang:spawn_opt({erlang,apply,[F,[]],[monitor]});
 spawn_monitor(F) ->
     erlang:error(badarg, [F]).
 
+-spec spawn_monitor(Module, Function, Args) -> {pid(), reference()} when
+      Module :: module(),
+      Function :: atom(),
+      Args :: [term()].
 spawn_monitor(M, F, A) when is_atom(M), is_atom(F), is_list(A) ->
     erlang:spawn_opt({M,F,A,[monitor]});
 spawn_monitor(M, F, A) ->
     erlang:error(badarg, [M,F,A]).
 
+-spec spawn_opt(Fun, Options) -> pid() | {pid(), reference()} when
+      Fun :: function(),
+      Options :: [Option],
+      Option :: link | monitor | {priority, Level}
+              | {fullsweep_after, Number :: non_neg_integer()}
+              | {min_heap_size, Size :: non_neg_integer()}
+              | {min_bin_vheap_size, VSize :: non_neg_integer()},
+      Level :: low | normal | high.
 spawn_opt(F, O) when is_function(F) ->
     spawn_opt(erlang, apply, [F, []], O);
 spawn_opt({M,F}=MF, O) when is_atom(M), is_atom(F) ->
@@ -115,6 +157,15 @@ spawn_opt({M,F,A}, O) -> % For (undocumented) backward compatibility
 spawn_opt(F, O) ->
     erlang:error(badarg, [F, O]).
 
+-spec spawn_opt(Node, Fun, Options) -> pid() | {pid(), reference()} when
+      Node :: node(),
+      Fun :: function(),
+      Options :: [Option],
+      Option :: link | monitor | {priority, Level}
+              | {fullsweep_after, Number :: non_neg_integer()}
+              | {min_heap_size, Size :: non_neg_integer()}
+              | {min_bin_vheap_size, VSize :: non_neg_integer()},
+      Level :: low | normal | high.
 spawn_opt(N, F, O) when N =:= node() ->
     spawn_opt(F, O);
 spawn_opt(N, F, O) when is_function(F) ->
@@ -126,6 +177,11 @@ spawn_opt(N, F, O) ->
 
 %% Spawns with MFA
 
+-spec spawn(Node, Module, Function, Args) -> pid() when
+      Node :: node(),
+      Module :: module(),
+      Function :: atom(),
+      Args :: [term()].
 spawn(N,M,F,A) when N =:= node(), is_atom(M), is_atom(F), is_list(A) ->
     spawn(M,F,A);
 spawn(N,M,F,A) when is_atom(N), is_atom(M), is_atom(F) ->
@@ -151,6 +207,11 @@ spawn(N,M,F,A) when is_atom(N), is_atom(M), is_atom(F) ->
 spawn(N,M,F,A) ->
     erlang:error(badarg, [N, M, F, A]).
 
+-spec spawn_link(Node, Module, Function, Args) -> pid() when
+      Node :: node(),
+      Module :: module(),
+      Function :: atom(),
+      Args :: [term()].
 spawn_link(N,M,F,A) when N =:= node(), is_atom(M), is_atom(F), is_list(A) ->
     spawn_link(M,F,A);
 spawn_link(N,M,F,A) when is_atom(N), is_atom(M), is_atom(F) ->
@@ -176,6 +237,17 @@ spawn_link(N,M,F,A) when is_atom(N), is_atom(M), is_atom(F) ->
 spawn_link(N,M,F,A) ->
     erlang:error(badarg, [N, M, F, A]).
 
+-spec spawn_opt(Module, Function, Args, Options) ->
+                       pid() | {pid(), reference()} when
+      Module :: module(),
+      Function :: atom(),
+      Args :: [term()],
+      Options :: [Option],
+      Option :: link | monitor | {priority, Level}
+              | {fullsweep_after, Number :: non_neg_integer()}
+              | {min_heap_size, Size :: non_neg_integer()}
+              | {min_bin_vheap_size, VSize :: non_neg_integer()},
+      Level :: low | normal | high.
 spawn_opt(M, F, A, Opts) ->
     case catch erlang:spawn_opt({M,F,A,Opts}) of
 	{'EXIT',{Reason,_}} ->
@@ -184,6 +256,18 @@ spawn_opt(M, F, A, Opts) ->
 	    Res
     end.
 
+-spec spawn_opt(Node, Module, Function, Args, Options) ->
+                       pid() | {pid(), reference()} when
+      Node :: node(),
+      Module :: module(),
+      Function :: atom(),
+      Args :: [term()],
+      Options :: [Option],
+      Option :: link | monitor | {priority, Level}
+              | {fullsweep_after, Number :: non_neg_integer()}
+              | {min_heap_size, Size :: non_neg_integer()}
+              | {min_bin_vheap_size, VSize :: non_neg_integer()},
+      Level :: low | normal | high.
 spawn_opt(N, M, F, A, O) when N =:= node(),
 			      is_atom(M), is_atom(F), is_list(A),
 			      is_list(O) ->
@@ -253,18 +337,25 @@ crasher(Node,Mod,Fun,Args,Opts,Reason) ->
 			     [Mod,Fun,Args,Opts,Node]),
     exit(Reason).
 
--spec yield() -> 'true'.
+-spec erlang:yield() -> 'true'.
 yield() ->
     erlang:yield().
 
--spec nodes() -> [node()].
+-spec nodes() -> Nodes when
+      Nodes :: [node()].
 nodes() ->
     erlang:nodes(visible).
 
--spec disconnect_node(node()) -> boolean().
+-spec disconnect_node(Node) -> boolean() | ignored when
+      Node :: node().
 disconnect_node(Node) ->
     net_kernel:disconnect(Node).
 
+-spec erlang:fun_info(Fun) -> [{Item, Info}] when
+      Fun :: function(),
+      Item :: arity | env | index | name
+            | module | new_index | new_uniq | pid | type | uniq,
+      Info :: term().
 fun_info(Fun) when is_function(Fun) ->
     Keys = [type,env,arity,name,uniq,index,new_uniq,new_index,module,pid],
     fun_info_1(Keys, Fun, []).
@@ -276,24 +367,37 @@ fun_info_1([K|Ks], Fun, A) ->
     end;
 fun_info_1([], _, A) -> A.
 
--type dst() :: pid() | port() | atom() | {atom(), node()}.
+-type dst() :: pid()
+             | port()
+             | (RegName :: atom())
+             | {RegName :: atom(), Node :: node()}.
 
--spec send_nosuspend(dst(), term()) -> boolean().
+-spec erlang:send_nosuspend(Dest, Msg) -> boolean() when
+      Dest :: dst(),
+      Msg :: term().
 send_nosuspend(Pid, Msg) ->
     send_nosuspend(Pid, Msg, []).
 
--spec send_nosuspend(dst(), term(), ['noconnect' | 'nosuspend']) -> boolean().
+-spec erlang:send_nosuspend(Dest, Msg, Options) -> boolean() when
+      Dest :: dst(),
+      Msg :: term(),
+      Options :: [noconnect].
 send_nosuspend(Pid, Msg, Opts) ->
     case erlang:send(Pid, Msg, [nosuspend|Opts]) of
 	ok -> true;
 	_  -> false
     end.
 
--spec localtime_to_universaltime(date_time()) -> date_time().
+-spec erlang:localtime_to_universaltime({Date1, Time1}) -> {Date2, Time2} when
+      Date1 :: calendar:date(),
+      Date2 :: calendar:date(),
+      Time1 :: calendar:time(),
+      Time2 :: calendar:time().
 localtime_to_universaltime(Localtime) ->
     erlang:localtime_to_universaltime(Localtime, undefined).
 
--spec suspend_process(pid()) -> 'true'.
+-spec erlang:suspend_process(Suspendee) -> 'true' when
+      Suspendee :: pid().
 suspend_process(P) ->
     case catch erlang:suspend_process(P, []) of
 	{'EXIT', {Reason, _}} -> erlang:error(Reason, [P]);
@@ -419,28 +523,25 @@ delay_trap(Result, Timeout) -> receive after Timeout -> Result end.
 %% Messages to us use our cookie. IF we change our cookie, other nodes 
 %% have to reflect that, which we cannot forsee.
 %%
+-spec erlang:set_cookie(Node, Cookie) -> true when
+      Node :: node(),
+      Cookie :: atom().
 set_cookie(Node, C) when Node =/= nonode@nohost, is_atom(Node) ->
-    Res = case C of
-	      _ when is_atom(C) ->
-		  auth:set_cookie(Node, C);
-	      {CI,CO} when is_atom(CI), is_atom(CO) ->
-		  auth:set_cookie(Node, {CI, CO});
-	      _ ->
-		  error
-	  end,
-    case Res of
-	error -> exit(badarg);
-	Other -> Other
+    case is_atom(C) of
+	true ->
+	    auth:set_cookie(Node, C);
+	false ->
+	    error(badarg)
     end.
 
--spec get_cookie() -> atom().
+-spec erlang:get_cookie() -> Cookie | nocookie when
+      Cookie :: atom().
 get_cookie() ->
     auth:get_cookie().
 
-concat_binary(List) ->
-    list_to_binary(List).
-
--spec integer_to_list(integer(), 1..255) -> string().
+-spec integer_to_list(Integer, Base) -> string() when
+      Integer :: integer(),
+      Base :: 2..36.
 integer_to_list(I, 10) ->
     erlang:integer_to_list(I);
 integer_to_list(I, Base) 
@@ -468,6 +569,9 @@ integer_to_list(I0, Base, R0) ->
     end.
 
 
+-spec list_to_integer(String, Base) -> integer() when
+      String :: string(),
+      Base :: 2..36.
 list_to_integer(L, 10) ->
     erlang:list_to_integer(L);
 list_to_integer(L, Base)
@@ -688,10 +792,447 @@ await_proc_exit(Proc, Op, Data) ->
 	    end
     end.
 
--spec min(term(), term()) -> term().
+-spec min(Term1, Term2) -> Minimum when
+      Term1 :: term(),
+      Term2 :: term(),
+      Minimum :: term().
 min(A, B) when A > B -> B;
 min(A, _) -> A.
 
--spec max(term(), term()) -> term().
+-spec max(Term1, Term2) -> Maximum when
+      Term1 :: term(),
+      Term2 :: term(),
+      Maximum :: term().
 max(A, B) when A < B -> B;
 max(A, _) -> A.
+
+
+%%
+%% erlang:memory/[0,1]
+%%
+%% NOTE! When updating these functions, make sure to also update
+%%       erts_memory() in $ERL_TOP/erts/emulator/beam/erl_alloc.c
+%%
+
+-type memory_type() :: 'total' | 'processes' | 'processes_used' | 'system' | 'atom' | 'atom_used' | 'binary' | 'code' | 'ets' | 'low' | 'maximum'.
+
+-define(CARRIER_ALLOCS, [mseg_alloc, sbmbc_alloc, sbmbc_low_alloc]).
+-define(LOW_ALLOCS, [sbmbc_low_alloc, ll_low_alloc, std_low_alloc]).
+-define(ALL_NEEDED_ALLOCS, (erlang:system_info(alloc_util_allocators)
+			    -- ?CARRIER_ALLOCS)).
+
+-record(memory, {total = 0,
+		 processes = 0,
+		 processes_used = 0,
+		 system = 0,
+		 atom = 0,
+		 atom_used = 0,
+		 binary = 0,
+		 code = 0,
+		 ets = 0,
+		 low = 0,
+		 maximum = 0}).
+
+-spec memory() -> [{memory_type(), non_neg_integer()}].
+memory() ->
+    case aa_mem_data(au_mem_data(?ALL_NEEDED_ALLOCS)) of
+	notsup ->
+	    erlang:error(notsup);
+	Mem ->
+	    InstrTail = case Mem#memory.maximum of
+			    0 -> [];
+			    _ -> [{maximum, Mem#memory.maximum}]
+			end,
+	    Tail = case Mem#memory.low of
+		       0 -> InstrTail;
+		       _ -> [{low, Mem#memory.low} | InstrTail]
+		   end,
+	    [{total, Mem#memory.total},
+	     {processes, Mem#memory.processes},
+	     {processes_used, Mem#memory.processes_used},
+	     {system, Mem#memory.system},
+	     {atom, Mem#memory.atom},
+	     {atom_used, Mem#memory.atom_used},
+	     {binary, Mem#memory.binary},
+	     {code, Mem#memory.code},
+	     {ets, Mem#memory.ets} | Tail]
+    end.
+
+-spec memory(memory_type()|[memory_type()]) -> non_neg_integer() | [{memory_type(), non_neg_integer()}].
+memory(Type) when is_atom(Type) ->
+    {AA, ALCU, ChkSup, BadArgZero} = need_mem_info(Type),
+    case get_mem_data(ChkSup, ALCU, AA) of
+	notsup ->
+	    erlang:error(notsup, [Type]);
+	Mem ->
+	    Value = get_memval(Type, Mem),
+	    case {BadArgZero, Value} of
+		{true, 0} -> erlang:error(badarg, [Type]);
+		_ -> Value
+	    end
+    end;
+memory(Types) when is_list(Types) ->
+    {AA, ALCU, ChkSup, BadArgZeroList} = need_mem_info_list(Types),
+    case get_mem_data(ChkSup, ALCU, AA) of
+	notsup ->
+	    erlang:error(notsup, [Types]);
+	Mem ->
+	    case memory_result_list(Types, BadArgZeroList, Mem) of
+		badarg -> erlang:error(badarg, [Types]);
+		Result -> Result
+	    end
+    end.
+
+memory_result_list([], [], _Mem) ->
+    [];
+memory_result_list([T|Ts], [BAZ|BAZs], Mem) ->
+    case memory_result_list(Ts, BAZs, Mem) of
+	badarg -> badarg;
+	TVs ->
+	    V = get_memval(T, Mem),
+	    case {BAZ, V} of
+		{true, 0} -> badarg;
+		_ -> [{T, V}| TVs]
+	    end
+    end.
+
+get_mem_data(true, AlcUAllocs, NeedAllocatedAreas) ->
+    case memory_is_supported() of
+	false -> notsup;
+	true -> get_mem_data(false, AlcUAllocs, NeedAllocatedAreas)
+    end;
+get_mem_data(false, AlcUAllocs, NeedAllocatedAreas) ->
+    AlcUMem = case AlcUAllocs of
+		  [] -> #memory{};
+		  _ ->
+		      au_mem_data(AlcUAllocs)
+	      end,
+    case NeedAllocatedAreas of
+	true -> aa_mem_data(AlcUMem);
+	false -> AlcUMem
+    end.
+
+need_mem_info_list([]) ->
+    {false, [], false, []};
+need_mem_info_list([T|Ts]) ->
+    {MAA, MALCU, MChkSup, MBadArgZero} = need_mem_info_list(Ts),
+    {AA, ALCU, ChkSup, BadArgZero} = need_mem_info(T),
+    {case AA of
+	 true -> true;
+	 _ -> MAA
+     end,
+     ALCU ++ (MALCU -- ALCU),
+     case ChkSup of
+	 true -> true;
+	 _ -> MChkSup
+     end,
+     [BadArgZero|MBadArgZero]}.
+
+need_mem_info(Type) when Type == total;
+			 Type == system ->
+    {true, ?ALL_NEEDED_ALLOCS, false, false};
+need_mem_info(Type) when Type == processes;
+			 Type == processes_used ->
+    {true, [eheap_alloc, fix_alloc], true, false};
+need_mem_info(Type) when Type == atom;
+			 Type == atom_used;
+			 Type == code ->
+    {true, [], true, false};
+need_mem_info(binary) ->
+    {false, [binary_alloc], true, false};
+need_mem_info(ets) ->
+    {true, [ets_alloc], true, false};
+need_mem_info(low) ->
+    LowAllocs = ?LOW_ALLOCS -- ?CARRIER_ALLOCS,
+    {_, _, FeatureList, _} = erlang:system_info(allocator),
+    AlcUAllocs = case LowAllocs -- FeatureList of
+		     [] -> LowAllocs;
+		     _ -> []
+		 end,
+    {false, AlcUAllocs, true, true};
+need_mem_info(maximum) ->
+    {true, [], true, true};
+need_mem_info(_) ->
+    {false, [], false, true}.
+
+get_memval(total, #memory{total = V}) -> V;
+get_memval(processes, #memory{processes = V}) -> V;
+get_memval(processes_used, #memory{processes_used = V}) -> V;
+get_memval(system, #memory{system = V}) -> V;
+get_memval(atom, #memory{atom = V}) -> V;
+get_memval(atom_used, #memory{atom_used = V}) -> V;
+get_memval(binary, #memory{binary = V}) -> V;
+get_memval(code, #memory{code = V}) -> V;
+get_memval(ets, #memory{ets = V}) -> V;
+get_memval(low, #memory{low = V}) -> V;
+get_memval(maximum, #memory{maximum = V}) -> V;
+get_memval(_, #memory{}) -> 0.
+
+memory_is_supported() ->
+    {_, _, FeatureList, _} = erlang:system_info(allocator),
+    case ((erlang:system_info(alloc_util_allocators) 
+	   -- ?CARRIER_ALLOCS)
+	  -- FeatureList) of
+	[] -> true;
+	_ -> false
+    end.
+
+get_blocks_size([{blocks_size, Sz, _, _} | Rest], Acc) ->
+    get_blocks_size(Rest, Acc+Sz);
+get_blocks_size([{_, _, _, _} | Rest], Acc) ->
+    get_blocks_size(Rest, Acc);
+get_blocks_size([], Acc) ->
+    Acc.
+
+blocks_size([{Carriers, SizeList} | Rest], Acc) when Carriers == mbcs;
+						     Carriers == sbcs;
+						     Carriers == sbmbcs ->
+    blocks_size(Rest, get_blocks_size(SizeList, Acc));
+blocks_size([_ | Rest], Acc) ->
+    blocks_size(Rest, Acc);
+blocks_size([], Acc) ->
+    Acc.
+
+get_fix_proc([{ProcType, A1, U1}| Rest], {A0, U0}) when ProcType == proc;
+							ProcType == monitor_sh;
+							ProcType == nlink_sh;
+							ProcType == msg_ref ->
+    get_fix_proc(Rest, {A0+A1, U0+U1});
+get_fix_proc([_|Rest], Acc) ->
+    get_fix_proc(Rest, Acc);
+get_fix_proc([], Acc) ->
+    Acc.
+
+fix_proc([{fix_types, SizeList} | _Rest], Acc) ->
+    get_fix_proc(SizeList, Acc);
+fix_proc([_ | Rest], Acc) ->
+    fix_proc(Rest, Acc);
+fix_proc([], Acc) ->
+    Acc.
+
+is_low_alloc(_A, []) ->
+    false;
+is_low_alloc(A, [A|_As]) ->
+    true;
+is_low_alloc(A, [_A|As]) ->
+    is_low_alloc(A, As).
+
+is_low_alloc(A) ->
+    is_low_alloc(A, ?LOW_ALLOCS).
+
+au_mem_data(notsup, _) ->
+    notsup;
+au_mem_data(_, [{_, false} | _]) ->
+    notsup;
+au_mem_data(#memory{total = Tot,
+		    processes = Proc,
+		    processes_used = ProcU} = Mem,
+	    [{eheap_alloc, _, Data} | Rest]) ->
+    Sz = blocks_size(Data, 0),
+    au_mem_data(Mem#memory{total = Tot+Sz,
+			   processes = Proc+Sz,
+			   processes_used = ProcU+Sz},
+		Rest);
+au_mem_data(#memory{total = Tot,
+		    system = Sys,
+		    ets = Ets} = Mem,
+	    [{ets_alloc, _, Data} | Rest]) ->
+    Sz = blocks_size(Data, 0),
+    au_mem_data(Mem#memory{total = Tot+Sz,
+			   system = Sys+Sz,
+			   ets = Ets+Sz},
+		Rest);
+au_mem_data(#memory{total = Tot,
+		    system = Sys,
+		    binary = Bin} = Mem,
+	    [{binary_alloc, _, Data} | Rest]) ->
+    Sz = blocks_size(Data, 0),
+    au_mem_data(Mem#memory{total = Tot+Sz,
+			   system = Sys+Sz,
+			   binary = Bin+Sz},
+		Rest);
+au_mem_data(#memory{total = Tot,
+		    processes = Proc,
+		    processes_used = ProcU,
+		    system = Sys} = Mem,
+	    [{fix_alloc, _, Data} | Rest]) ->
+    {A, U} = fix_proc(Data, {0, 0}),
+    Sz = blocks_size(Data, 0),
+    au_mem_data(Mem#memory{total = Tot+Sz,
+			   processes = Proc+A,
+			   processes_used = ProcU+U,
+			   system = Sys+Sz-A},
+		Rest);
+au_mem_data(#memory{total = Tot,
+		    system = Sys,
+		    low = Low} = Mem,
+	    [{A, _, Data} | Rest]) ->
+    Sz = blocks_size(Data, 0),
+    au_mem_data(Mem#memory{total = Tot+Sz,
+			   system = Sys+Sz,
+			   low = case is_low_alloc(A) of
+				     true -> Low+Sz;
+				     false -> Low
+				 end},
+		Rest);
+au_mem_data(EMD, []) ->
+    EMD.
+
+au_mem_data(Allocs) ->
+    Ref = make_ref(),
+    erlang:system_info({allocator_sizes, Ref, Allocs}),
+    receive_emd(Ref).
+
+receive_emd(_Ref, EMD, 0) ->
+    EMD;
+receive_emd(Ref, EMD, N) ->
+    receive
+	{Ref, _, Data} ->
+	    receive_emd(Ref, au_mem_data(EMD, Data), N-1)
+    end.
+
+receive_emd(Ref) ->
+    receive_emd(Ref, #memory{}, erlang:system_info(schedulers)).
+
+aa_mem_data(#memory{} = Mem,
+	    [{maximum, Max} | Rest]) ->
+    aa_mem_data(Mem#memory{maximum = Max},
+		Rest);
+aa_mem_data(#memory{} = Mem,
+	    [{total, Tot} | Rest]) ->
+    aa_mem_data(Mem#memory{total = Tot,
+			   system = 0}, % system will be adjusted later
+		Rest);
+aa_mem_data(#memory{atom = Atom,
+		    atom_used = AtomU} = Mem,
+	    [{atom_space, Alloced, Used} | Rest]) ->
+    aa_mem_data(Mem#memory{atom = Atom+Alloced,
+			   atom_used = AtomU+Used},
+		Rest);
+aa_mem_data(#memory{atom = Atom,
+		    atom_used = AtomU} = Mem,
+	    [{atom_table, Sz} | Rest]) ->
+    aa_mem_data(Mem#memory{atom = Atom+Sz,
+			   atom_used = AtomU+Sz},
+		Rest);
+aa_mem_data(#memory{ets = Ets} = Mem,
+	    [{ets_misc, Sz} | Rest]) ->
+    aa_mem_data(Mem#memory{ets = Ets+Sz},
+		Rest);
+aa_mem_data(#memory{processes = Proc,
+		    processes_used = ProcU,
+		    system = Sys} = Mem,
+	    [{ProcData, Sz} | Rest]) when ProcData == bif_timer;
+					  ProcData == link_lh;
+					  ProcData == process_table ->
+    aa_mem_data(Mem#memory{processes = Proc+Sz,
+			   processes_used = ProcU+Sz,
+			   system = Sys-Sz},
+		Rest);
+aa_mem_data(#memory{code = Code} = Mem,
+	    [{CodeData, Sz} | Rest]) when CodeData == module_table;
+					  CodeData == export_table;
+					  CodeData == export_list;
+					  CodeData == fun_table;
+					  CodeData == module_refs;
+					  CodeData == loaded_code ->
+    aa_mem_data(Mem#memory{code = Code+Sz},
+		Rest);
+aa_mem_data(EMD, [{_, _} | Rest]) ->
+    aa_mem_data(EMD, Rest);
+aa_mem_data(#memory{total = Tot,
+		    processes = Proc,
+		    system = Sys} = Mem,
+	    []) when Sys =< 0 ->
+    %% Instrumented runtime system -> Sys = Tot - Proc
+    Mem#memory{system = Tot - Proc};
+aa_mem_data(EMD, []) ->
+    EMD.
+
+aa_mem_data(notsup) ->
+    notsup;
+aa_mem_data(EMD) ->
+    aa_mem_data(EMD, erlang:system_info(allocated_areas)).
+
+%%
+%% alloc_info/1 and alloc_sizes/1 are for internal use only (used by
+%% erlang:system_info({allocator|allocator_sizes, _})).
+%%
+
+alloc_info(Allocs) ->
+    get_alloc_info(allocator, Allocs).
+
+alloc_sizes(Allocs) ->
+    get_alloc_info(allocator_sizes, Allocs).
+
+get_alloc_info(Type, AAtom) when is_atom(AAtom) ->
+    [{AAtom, Result}] = get_alloc_info(Type, [AAtom]),
+    Result;
+get_alloc_info(Type, AList) when is_list(AList) ->
+    Ref = make_ref(),
+    erlang:system_info({Type, Ref, AList}),
+    receive_allocator(Ref,
+		      erlang:system_info(schedulers),
+		      mk_res_list(AList)).
+
+mk_res_list([]) ->
+    [];
+mk_res_list([Alloc | Rest]) ->
+    [{Alloc, []} | mk_res_list(Rest)].
+
+insert_instance(I, N, []) ->
+    [{instance, N, I}];
+insert_instance(I, N, [{instance, M, _}|_] = Rest) when N < M ->
+    [{instance, N, I} | Rest];
+insert_instance(I, N, [Prev|Rest]) ->
+    [Prev | insert_instance(I, N, Rest)].
+
+insert_info([], Ys) ->
+    Ys;
+insert_info([{A, false}|Xs], [{A, _IList}|Ys]) ->
+    insert_info(Xs, [{A, false}|Ys]);
+insert_info([{A, N, I}|Xs], [{A, IList}|Ys]) ->
+    insert_info(Xs, [{A, insert_instance(I, N, IList)}|Ys]);
+insert_info([{A1, _}|_] = Xs, [{A2, _} = Y | Ys]) when A1 /= A2 ->
+    [Y | insert_info(Xs, Ys)];
+insert_info([{A1, _, _}|_] = Xs, [{A2, _} = Y | Ys]) when A1 /= A2 ->
+    [Y | insert_info(Xs, Ys)].
+
+receive_allocator(_Ref, 0, Acc) ->
+    Acc;
+receive_allocator(Ref, N, Acc) ->
+    receive
+	{Ref, _, InfoList} ->
+	    receive_allocator(Ref, N-1, insert_info(InfoList, Acc))
+    end.
+
+-spec await_sched_wall_time_modifications(Ref, Result) -> boolean() when
+      Ref :: reference(),
+      Result :: boolean().
+
+await_sched_wall_time_modifications(Ref, Result) ->
+    sched_wall_time(Ref, erlang:system_info(schedulers)),
+    Result.
+
+-spec gather_sched_wall_time_result(Ref) -> [{pos_integer(),
+					      non_neg_integer(),
+					      non_neg_integer()}] when
+      Ref :: reference().
+
+gather_sched_wall_time_result(Ref) when is_reference(Ref) ->
+    sched_wall_time(Ref, erlang:system_info(schedulers), []).
+
+sched_wall_time(_Ref, 0) ->
+    ok;
+sched_wall_time(Ref, N) ->
+    receive Ref -> sched_wall_time(Ref, N-1) end.
+
+sched_wall_time(_Ref, 0, Acc) ->
+    Acc;
+sched_wall_time(Ref, N, undefined) ->
+    receive {Ref, _} -> sched_wall_time(Ref, N-1, undefined) end;
+sched_wall_time(Ref, N, Acc) ->
+    receive
+	{Ref, undefined} -> sched_wall_time(Ref, N-1, undefined);
+	{Ref, SWT} -> sched_wall_time(Ref, N-1, [SWT|Acc])
+    end.
diff --git a/erts/preloaded/src/init.erl b/erts/preloaded/src/init.erl
index 3b98b9cddc..c9c434dea0 100644
--- a/erts/preloaded/src/init.erl
+++ b/erts/preloaded/src/init.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1996-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -28,10 +28,10 @@
 %%                       : $Var in the boot script is expanded to
 %%                         Value.
 %%        -loader LoaderMethod
-%%                       : efile, inet, ose_inet
+%%                       : efile, inet
 %%                         (Optional - default efile)
 %%        -hosts [Node]  : List of hosts from which we can boot.
-%%                         (Mandatory if -loader inet or ose_inet)
+%%                         (Mandatory if -loader inet)
 %%        -mode embedded : Load all modules at startup, no automatic loading
 %%        -mode interactive : Auto load modules (default system behaviour).
 %%        -path          : Override path in bootfile.
@@ -51,6 +51,9 @@
 	 get_status/0,boot/1,get_arguments/0,get_plain_arguments/0,
 	 get_argument/1,script_id/0]).
 
+%% for the on_load functionality; not for general use
+-export([run_on_load_handlers/0]).
+
 %% internal exports
 -export([fetch_loaded/0,ensure_loaded/1,make_permanent/2,
 	 notify_when_started/1,wait_until_started/0, 
@@ -69,25 +72,31 @@
 		script_id = [],
 		loaded = [],
 		subscribed = []}).
+-type state() :: #state{}.
 
 -define(ON_LOAD_HANDLER, init__boot__on_load_handler).
 
 debug(false, _) -> ok;
 debug(_, T)     -> erlang:display(T).
 
--spec get_arguments() -> [{atom(), [string()]}].
+-spec get_arguments() -> Flags when
+      Flags :: [{Flag :: atom(), Values :: [string()]}].
 get_arguments() ->
     request(get_arguments).
 
--spec get_plain_arguments() -> [string()].
+-spec get_plain_arguments() -> [Arg] when
+      Arg :: string().
 get_plain_arguments() ->
     bs2ss(request(get_plain_arguments)).
 
--spec get_argument(atom()) -> 'error' | {'ok', [[string()]]}.
+-spec get_argument(Flag) -> {'ok', Arg} | 'error' when
+      Flag :: atom(),
+      Arg :: [Values :: [string()]].
 get_argument(Arg) ->
     request({get_argument, Arg}).
 
--spec script_id() -> term().
+-spec script_id() -> Id when
+      Id :: term().
 script_id() ->
     request(script_id).
 
@@ -101,7 +110,9 @@ bs2ss(L0) when is_list(L0) ->
 bs2ss(L) ->
     L.
 
--spec get_status() -> {internal_status(), term()}.
+-spec get_status() -> {InternalStatus, ProvidedStatus} when
+      InternalStatus :: internal_status(),
+      ProvidedStatus :: term().
 get_status() ->
     request(get_status).
 
@@ -143,13 +154,15 @@ restart() -> init ! {stop,restart}, ok.
 -spec reboot() -> 'ok'.
 reboot() -> init ! {stop,reboot}, ok.
 
--spec stop() -> no_return().
+-spec stop() -> 'ok'.
 stop() -> init ! {stop,stop}, ok.
 
--spec stop(non_neg_integer() | string()) -> no_return().
+-spec stop(Status) -> 'ok' when
+      Status :: non_neg_integer() | string().
 stop(Status) -> init ! {stop,{stop,Status}}, ok.
 
--spec boot([binary()]) -> no_return().
+-spec boot(BootArgs) -> no_return() when
+      BootArgs :: [binary()].
 boot(BootArgs) ->
     register(init, self()),
     process_flag(trap_exit, true),
@@ -275,7 +288,7 @@ crash(String, List) ->
     halt(halt_string(String, List)).
 
 %% Status is {InternalStatus,ProvidedStatus}
--spec boot_loop(pid(), #state{}) -> no_return().
+-spec boot_loop(pid(), state()) -> no_return().
 boot_loop(BootPid, State) ->
     receive
 	{BootPid,loaded,ModLoaded} ->
@@ -308,24 +321,6 @@ boot_loop(BootPid, State) ->
 	{stop,Reason} ->
 	    stop(Reason,State);
 	{From,fetch_loaded} ->   %% Fetch and reset initially loaded modules.
-	    case whereis(?ON_LOAD_HANDLER) of
-		undefined ->
-		    %% There is no on_load handler process,
-		    %% probably because init:restart/0 has been
-		    %% called and it is not the first time we
-		    %% pass through here.
-		    ok;
-		Pid when is_pid(Pid) ->
-		    Pid ! run_on_load,
-		    receive
-			{'EXIT',Pid,on_load_done} ->
-			    ok;
-			{'EXIT',Pid,Res} ->
-			    %% Failure to run an on_load handler.
-			    %% This is fatal during start-up.
-			    exit(Res)
-		    end
-	    end,
 	    From ! {init,State#state.loaded},
 	    garb_boot_loop(BootPid,State#state{loaded = []});
 	{From,{ensure_loaded,Module}} ->
@@ -350,7 +345,7 @@ notify(Pids) ->
     lists:foreach(fun(Pid) -> Pid ! {init,started} end, Pids).
 
 %% Garbage collect all info about initially loaded modules.
-%% This information is temporary stored until the code_server
+%% This information is temporarily stored until the code_server
 %% is started.
 %% We force the garbage collection as the init process holds
 %% this information during the initialisation of the system and
@@ -736,6 +731,7 @@ do_boot(Init,Flags,Start) ->
     BootList = get_boot(BootFile,Root),
     LoadMode = b2a(get_flag('-mode',Flags,false)),
     Deb = b2a(get_flag('-init_debug',Flags,false)),
+    catch ?ON_LOAD_HANDLER ! {init_debug_flag,Deb},
     BootVars = get_flag_args('-boot_var',Flags),
     ParallelLoad = 
 	(Pgm =:= "efile") and (erlang:system_info(thread_pool_size) > 0),
@@ -1037,7 +1033,7 @@ start_it({eval,Bin}) ->
 	      TsR -> reverse([{dot,1} | TsR])
 	  end,
     {ok,Expr} = erl_parse:parse_exprs(Ts1),
-    erl_eval:exprs(Expr, []),
+    erl_eval:exprs(Expr, erl_eval:new_bindings()),
     ok;
 start_it([_|_]=MFA) ->
     Ref = make_ref(),
@@ -1335,23 +1331,44 @@ archive_extension() ->
 %%% Support for handling of on_load functions.
 %%%
 
+run_on_load_handlers() ->
+    Ref = monitor(process, ?ON_LOAD_HANDLER),
+    catch ?ON_LOAD_HANDLER ! run_on_load,
+    receive
+	{'DOWN',Ref,process,_,noproc} ->
+	    %% There is no on_load handler process,
+	    %% probably because init:restart/0 has been
+	    %% called and it is not the first time we
+	    %% pass through here.
+	    ok;
+	{'DOWN',Ref,process,_,on_load_done} ->
+	    ok;
+	{'DOWN',Ref,process,_,Res} ->
+	    %% Failure to run an on_load handler.
+	    %% This is fatal during start-up.
+	    exit(Res)
+    end.
+
 start_on_load_handler_process() ->
     register(?ON_LOAD_HANDLER,
-	     spawn_link(fun on_load_handler_init/0)).
+	     spawn(fun on_load_handler_init/0)).
 
 on_load_handler_init() ->
-    on_load_loop([]).
+    on_load_loop([], false).
 
-on_load_loop(Mods) ->
+on_load_loop(Mods, Debug0) ->
     receive
+	{init_debug_flag,Debug} ->
+	    on_load_loop(Mods, Debug);
 	{loaded,Mod} ->
-	    on_load_loop([Mod|Mods]);
+	    on_load_loop([Mod|Mods], Debug0);
 	run_on_load ->
-	    run_on_load_handlers(Mods),
+	    run_on_load_handlers(Mods, Debug0),
 	    exit(on_load_done)
     end.
 
-run_on_load_handlers([M|Ms]) ->
+run_on_load_handlers([M|Ms], Debug) ->
+    debug(Debug, {running_on_load_handler,M}),
     Fun = fun() ->
 		  Res = erlang:call_on_load_function(M),
 		  exit(Res)
@@ -1363,9 +1380,12 @@ run_on_load_handlers([M|Ms]) ->
 	    erlang:finish_after_on_load(M, Keep),
 	    case Keep of
 		false ->
-		    exit({on_load_function_failed,M});
+		    Error = {on_load_function_failed,M},
+		    debug(Debug, Error),
+		    exit(Error);
 		true ->
-		    run_on_load_handlers(Ms)
+		    debug(Debug, {on_load_handler_returned_ok,M}),
+		    run_on_load_handlers(Ms, Debug)
 	    end
     end;
-run_on_load_handlers([]) -> ok.
+run_on_load_handlers([], _) -> ok.
diff --git a/erts/preloaded/src/prim_file.erl b/erts/preloaded/src/prim_file.erl
index 43e6f6cd88..0fc56c8c8e 100644
--- a/erts/preloaded/src/prim_file.erl
+++ b/erts/preloaded/src/prim_file.erl
@@ -1,19 +1,19 @@
 %%
 %% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 2000-2009. All Rights Reserved.
-%% 
+%%
+%% Copyright Ericsson AB 2000-2012. All Rights Reserved.
+%%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
 %% compliance with the License. You should have received a copy of the
 %% Erlang Public License along with this software. If not, it can be
 %% retrieved online at http://www.erlang.org/.
-%% 
+%%
 %% Software distributed under the License is distributed on an "AS IS"
 %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 %% the License for the specific language governing rights and limitations
 %% under the License.
-%% 
+%%
 %% %CopyrightEnd%
 %%
 -module(prim_file).
@@ -25,8 +25,9 @@
 %%% Interface towards a single file's contents. Uses ?FD_DRV.
 
 %% Generic file contents operations
--export([open/2, close/1, sync/1, position/2, truncate/1,
-	 write/2, pwrite/2, pwrite/3, read/2, read_line/1, pread/2, pread/3, copy/3]).
+-export([open/2, close/1, datasync/1, sync/1, advise/4, position/2, truncate/1,
+	 write/2, pwrite/2, pwrite/3, read/2, read_line/1, pread/2, pread/3,
+	 copy/3, sendfile/10]).
 
 %% Specialized file operations
 -export([open/1, open/3]).
@@ -44,13 +45,13 @@
 	 rename/2, rename/3, 
 	 make_dir/1, make_dir/2,
 	 del_dir/1, del_dir/2,
-	 read_file_info/1, read_file_info/2,
+	 read_file_info/1, read_file_info/2, read_file_info/3,
 	 altname/1, altname/2,
-	 write_file_info/2, write_file_info/3,
+	 write_file_info/2, write_file_info/3, write_file_info/4,
 	 make_link/2, make_link/3,
 	 make_symlink/2, make_symlink/3,
 	 read_link/1, read_link/2,
-	 read_link_info/1, read_link_info/2,
+	 read_link_info/1, read_link_info/2, read_link_info/3,
 	 list_dir/1, list_dir/2]).
 %% How to start and stop the ?DRV port.
 -export([start/0, stop/1]).
@@ -63,8 +64,8 @@
 
 -include("file.hrl").
 
--define(DRV,    efile).
--define(FD_DRV, efile).
+-define(DRV,    "efile").
+-define(FD_DRV, "efile").
 
 -define(LARGEFILESIZE, (1 bsl 63)).
 
@@ -96,6 +97,9 @@
 -define(FILE_IPREAD,           27).
 -define(FILE_ALTNAME,          28).
 -define(FILE_READ_LINE,        29).
+-define(FILE_FDATASYNC,        30).
+-define(FILE_ADVISE,           31).
+-define(FILE_SENDFILE,         32).
 
 %% Driver responses
 -define(FILE_RESP_OK,          0).
@@ -107,6 +111,9 @@
 -define(FILE_RESP_LDATA,       6).
 -define(FILE_RESP_N2DATA,      7).
 -define(FILE_RESP_EOF,         8).
+-define(FILE_RESP_FNAME,       9).
+-define(FILE_RESP_ALL_DATA,   10).
+-define(FILE_RESP_LFNAME,     11).
 
 %% Open modes for the driver's open function.
 -define(EFILE_MODE_READ,       1).
@@ -114,9 +121,10 @@
 -define(EFILE_MODE_READ_WRITE, 3).  
 -define(EFILE_MODE_APPEND,     4).
 -define(EFILE_COMPRESSED,      8).
+-define(EFILE_MODE_EXCL,       16).
 
 %% Use this mask to get just the mode bits to be passed to the driver.
--define(EFILE_MODE_MASK, 15).
+-define(EFILE_MODE_MASK, 31).
 
 %% Seek modes for the driver's seek function.
 -define(EFILE_SEEK_SET, 0).
@@ -130,6 +138,13 @@
 %% IPREAD variants
 -define(IPREAD_S32BU_P32BU, 0).
 
+%% POSIX file advises
+-define(POSIX_FADV_NORMAL,     0).
+-define(POSIX_FADV_RANDOM,     1).
+-define(POSIX_FADV_SEQUENTIAL, 2).
+-define(POSIX_FADV_WILLNEED,   3).
+-define(POSIX_FADV_DONTNEED,   4).
+-define(POSIX_FADV_NOREUSE,    5).
 
 
 %%%-----------------------------------------------------------------
@@ -143,7 +158,7 @@
 %% Opens a file using the driver port Port. Returns {error, Reason}
 %% | {ok, FileDescriptor}
 open(Port, File, ModeList) when is_port(Port), 
-                                is_list(File), 
+                                (is_list(File) orelse is_binary(File)), 
                                 is_list(ModeList) ->
     case open_mode(ModeList) of
 	{Mode, _Portopts, _Setopts} ->
@@ -155,10 +170,11 @@ open(_,_,_) ->
     {error, badarg}.
 
 %% Opens a file. Returns {error, Reason} | {ok, FileDescriptor}.
-open(File, ModeList) when is_list(File), is_list(ModeList) ->
+open(File, ModeList) when (is_list(File) orelse is_binary(File)), 
+			  is_list(ModeList) ->
     case open_mode(ModeList) of
 	{Mode, Portopts, Setopts} ->
-	    open_int({?FD_DRV, Portopts}, File, Mode, Setopts);
+	    open_int({?FD_DRV, Portopts},File, Mode, Setopts);
 	Reason ->
 	    {error, Reason}
     end;
@@ -186,7 +202,7 @@ open_int({Driver, Portopts}, File, Mode, Setopts) ->
     end;
 open_int(Port, File, Mode, Setopts) ->
     M = Mode band ?EFILE_MODE_MASK,
-    case drv_command(Port, [<<?FILE_OPEN, M:32>>, File, 0]) of
+    case drv_command(Port, [<<?FILE_OPEN, M:32>>, pathname(File)]) of
 	{ok, Number} ->
 	    open_int_setopts(Port, Number, Setopts);
 	Error ->
@@ -220,11 +236,39 @@ close(#file_descriptor{module = ?MODULE, data = {Port, _}}) ->
 close(Port) when is_port(Port) ->
     drv_close(Port).
 
+-define(ADVISE(Offs, Len, Adv),
+	<<?FILE_ADVISE, Offs:64/signed, Len:64/signed,
+	  Adv:32/signed>>).
 
+%% Returns {error, Reason} | ok.
+advise(#file_descriptor{module = ?MODULE, data = {Port, _}},
+       Offset, Length, Advise) ->
+    case Advise of
+	normal ->
+	    Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_NORMAL),
+	    drv_command(Port, Cmd);
+	random ->
+	    Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_RANDOM),
+	    drv_command(Port, Cmd);
+	sequential ->
+	    Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_SEQUENTIAL),
+	    drv_command(Port, Cmd);
+	will_need ->
+	    Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_WILLNEED),
+	    drv_command(Port, Cmd);
+	dont_need ->
+	    Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_DONTNEED),
+	    drv_command(Port, Cmd);
+	no_reuse ->
+	    Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_NOREUSE),
+	    drv_command(Port, Cmd);
+	_ ->
+	    {error, einval}
+    end.
 
 %% Returns {error, Reason} | ok.
 write(#file_descriptor{module = ?MODULE, data = {Port, _}}, Bytes) ->
-    case drv_command(Port, [?FILE_WRITE,Bytes]) of
+    case drv_command_nt(Port, [?FILE_WRITE,erlang:dt_prepend_vm_tag_data(Bytes)],undefined) of
 	{ok, _Size} ->
 	    ok;
 	Error ->
@@ -239,8 +283,8 @@ pwrite(#file_descriptor{module = ?MODULE, data = {Port, _}}, L)
 pwrite_int(_, [], 0, [], []) ->
     ok;
 pwrite_int(Port, [], N, Spec, Data) ->
-    Header = list_to_binary([<<?FILE_PWRITEV, N:32>> | reverse(Spec)]),
-    case drv_command_raw(Port, [Header | reverse(Data)]) of
+    Header = list_to_binary([?FILE_PWRITEV, erlang:dt_prepend_vm_tag_data(<<N:32>>) | reverse(Spec)]),
+    case drv_command_nt(Port, [Header | reverse(Data)], undefined) of
 	{ok, _Size} ->
 	    ok;
 	Error ->
@@ -292,6 +336,9 @@ pwrite(#file_descriptor{module = ?MODULE}, _, _) ->
     {error, badarg}.
 
 
+%% Returns {error, Reason} | ok.
+datasync(#file_descriptor{module = ?MODULE, data = {Port, _}}) ->
+    drv_command(Port, [?FILE_FDATASYNC]).
 
 %% Returns {error, Reason} | ok.
 sync(#file_descriptor{module = ?MODULE, data = {Port, _}}) ->
@@ -330,7 +377,7 @@ read(#file_descriptor{module = ?MODULE, data = {Port, _}}, Size)
 		    {ok, Data};
 		{error, enomem} ->
 		    %% Garbage collecting here might help if
-		    %% the current processes has some old binaries left.
+		    %% the current processes have some old binaries left.
 		    erlang:garbage_collect(),
 		    case drv_command(Port, <<?FILE_READ, Size:64>>) of
 			{ok, {0, _Data}} when Size =/= 0 ->
@@ -355,7 +402,7 @@ pread(#file_descriptor{module = ?MODULE, data = {Port, _}}, L)
 pread_int(_, [], 0, []) ->
     {ok, []};
 pread_int(Port, [], N, Spec) ->
-    drv_command(Port, [<<?FILE_PREADV, 0:32, N:32>> | reverse(Spec)]);
+    drv_command_nt(Port, [?FILE_PREADV, erlang:dt_prepend_vm_tag_data(<<0:32, N:32>>) | reverse(Spec)],undefined);
 pread_int(Port, [{Offs, Size} | T], N, Spec)
   when is_integer(Offs), is_integer(Size), 0 =< Size ->
     if
@@ -376,9 +423,9 @@ pread(#file_descriptor{module = ?MODULE, data = {Port, _}}, Offs, Size)
     if
 	-(?LARGEFILESIZE) =< Offs, Offs < ?LARGEFILESIZE,
 	Size < ?LARGEFILESIZE ->
-	    case drv_command(Port, 
-			     <<?FILE_PREADV, 0:32, 1:32,
-			      Offs:64/signed, Size:64>>) of
+	    case drv_command_nt(Port, 
+				[?FILE_PREADV, erlang:dt_prepend_vm_tag_data(<<0:32, 1:32,
+									     Offs:64/signed, Size:64>>)], undefined) of
 		{ok, [eof]} ->
 		    eof;
 		{ok, [Data]} ->
@@ -448,7 +495,7 @@ ipread_s32bu_p32bu(#file_descriptor{module = ?MODULE, data = {_, _}},
 
 
 %% Returns {ok, Contents} | {error, Reason}
-read_file(File) ->
+read_file(File) when (is_list(File) orelse is_binary(File)) ->
     case drv_open(?FD_DRV, [binary]) of
 	{ok, Port} ->
 	    Result = read_file(Port, File),
@@ -456,11 +503,14 @@ read_file(File) ->
 	    Result;
 	{error, _} = Error ->
 	    Error
-    end.
+    end;
+read_file(_) ->
+    {error, badarg}.
 
 %% Takes a Port opened with open/1.
-read_file(Port, File) when is_port(Port) ->
-    Cmd = [?FILE_READ_FILE | File],
+read_file(Port, File) when is_port(Port),
+			   (is_list(File) orelse is_binary(File)) ->
+    Cmd = [?FILE_READ_FILE | pathname(File)],
     case drv_command(Port, Cmd) of
 	{error, enomem} ->
 	    %% It could possibly help to do a 
@@ -471,12 +521,14 @@ read_file(Port, File) when is_port(Port) ->
 	    drv_command(Port, Cmd);
 	Result ->
 	    Result
-    end.
+    end;
+read_file(_,_) ->
+    {error, badarg}.
 
     
 
 %% Returns {error, Reason} | ok.
-write_file(File, Bin) ->
+write_file(File, Bin) when (is_list(File) orelse is_binary(File)) ->
     case open(File, [binary, write]) of
 	{ok, Handle} ->
 	    Result = write(Handle, Bin),
@@ -484,9 +536,35 @@ write_file(File, Bin) ->
 	    Result;
 	Error ->
 	    Error
-    end.
+    end;
+write_file(_, _) -> 
+    {error, badarg}.
 
 
+%% Returns {error, Reason} | {ok, BytesCopied}
+%sendfile(_,_,_,_,_,_,_,_,_,_) ->
+%    {error, enotsup};
+sendfile(#file_descriptor{module = ?MODULE, data = {Port, _}},
+	 Dest, Offset, Bytes, _ChunkSize, Headers, Trailers,
+	 _Nodiskio, _MNowait, _Sync) ->
+    case erlang:port_get_data(Dest) of
+	Data when Data == inet_tcp; Data == inet6_tcp ->
+	    ok = inet:lock_socket(Dest,true),
+	    {ok, DestFD} = prim_inet:getfd(Dest),
+	    try drv_command(Port, [<<?FILE_SENDFILE, DestFD:32,
+				     0:8,
+				     Offset:64/unsigned,
+				     Bytes:64/unsigned,
+				     (iolist_size(Headers)):32/unsigned,
+				     (iolist_size(Trailers)):32/unsigned>>,
+				   Headers,Trailers])
+	    after
+		ok = inet:lock_socket(Dest,false)
+	    end;
+	_Else ->
+	    {error,badarg}
+    end.
+
 
 %%%-----------------------------------------------------------------
 %%% Functions operating on files without handle to the file. ?DRV.
@@ -498,7 +576,7 @@ write_file(File, Bin) ->
 %% Returns {ok, Port}, the Port should be used as first argument in all
 %% the following functions. Returns {error, Reason} upon failure.
 start() ->
-    try erlang:open_port({spawn, atom_to_list(?DRV)}, []) of
+    try erlang:open_port({spawn, ?DRV}, [binary]) of
 	Port ->
 	    {ok, Port}
     catch
@@ -555,7 +633,7 @@ get_cwd(_, _) ->
     {error, badarg}.
 
 get_cwd_int(Drive) ->
-    get_cwd_int({?DRV, []}, Drive).
+    get_cwd_int({?DRV, [binary]}, Drive).
 
 get_cwd_int(Port, Drive) ->
     drv_command(Port, <<?FILE_PWD, Drive>>).
@@ -565,7 +643,7 @@ get_cwd_int(Port, Drive) ->
 %% set_cwd/{1,2}
 
 set_cwd(Dir) ->
-    set_cwd_int({?DRV, []}, Dir).
+    set_cwd_int({?DRV, [binary]}, Dir).
 
 set_cwd(Port, Dir) when is_port(Port) ->
     set_cwd_int(Port, Dir).
@@ -591,180 +669,235 @@ set_cwd_int(Port, Dir0) ->
 	 end),
     %% Dir is now either a string or an EXIT tuple.
     %% An EXIT tuple will fail in the following catch.
-    drv_command(Port, [?FILE_CHDIR, Dir, 0]).
+    drv_command(Port, [?FILE_CHDIR, pathname(Dir)]).
 
 
 
 %% delete/{1,2}
 
 delete(File) ->
-    delete_int({?DRV, []}, File).
+    delete_int({?DRV, [binary]}, File).
 
 delete(Port, File) when is_port(Port) ->
     delete_int(Port, File).
 
 delete_int(Port, File) ->
-    drv_command(Port, [?FILE_DELETE, File, 0]).
+    drv_command(Port, [?FILE_DELETE, pathname(File)]).
 
 
 
 %% rename/{2,3}
 
 rename(From, To) ->
-    rename_int({?DRV, []}, From, To).
+    rename_int({?DRV, [binary]}, From, To).
 
 rename(Port, From, To) when is_port(Port) ->
     rename_int(Port, From, To).
 
 rename_int(Port, From, To) ->
-    drv_command(Port, [?FILE_RENAME, From, 0, To, 0]).
+    drv_command(Port, [?FILE_RENAME, pathname(From), pathname(To)]).
 
 
 
 %% make_dir/{1,2}
 
 make_dir(Dir) ->
-    make_dir_int({?DRV, []}, Dir).
+    make_dir_int({?DRV, [binary]}, Dir).
 
 make_dir(Port, Dir) when is_port(Port) ->
     make_dir_int(Port, Dir).
 
 make_dir_int(Port, Dir) ->
-    drv_command(Port, [?FILE_MKDIR, Dir, 0]).
+    drv_command(Port, [?FILE_MKDIR, pathname(Dir)]).
 
 
 
 %% del_dir/{1,2}
 
 del_dir(Dir) ->
-    del_dir_int({?DRV, []}, Dir).
+    del_dir_int({?DRV, [binary]}, Dir).
 
 del_dir(Port, Dir) when is_port(Port) ->
     del_dir_int(Port, Dir).
 
 del_dir_int(Port, Dir) ->
-    drv_command(Port, [?FILE_RMDIR, Dir, 0]).
+    drv_command(Port, [?FILE_RMDIR, pathname(Dir)]).
 
 
 
-%% read_file_info/{1,2}
+%% read_file_info/{1,2,3}
 
 read_file_info(File) ->
-    read_file_info_int({?DRV, []}, File).
+    read_file_info_int({?DRV, [binary]}, File, local).
 
 read_file_info(Port, File) when is_port(Port) ->
-    read_file_info_int(Port, File).
+    read_file_info_int(Port, File, local);
+read_file_info(File, Opts) ->
+    read_file_info_int({?DRV, [binary]}, File, plgv(time, Opts, local)).
+
+read_file_info(Port, File, Opts) when is_port(Port) ->
+    read_file_info_int(Port, File, plgv(time, Opts, local)).
+
+read_file_info_int(Port, File, TimeType) ->
+    try
+	case drv_command(Port, [?FILE_FSTAT, pathname(File)]) of
+	    {ok, FI} -> {ok, FI#file_info{
+			ctime = from_seconds(FI#file_info.ctime, TimeType),
+			mtime = from_seconds(FI#file_info.mtime, TimeType),
+			atime = from_seconds(FI#file_info.atime, TimeType)
+		    }};
+	    Error    -> Error
+	end
+    catch
+	error:_ -> {error, badarg}
+    end.
 
-read_file_info_int(Port, File) ->
-    drv_command(Port, [?FILE_FSTAT, File, 0]).
 
 %% altname/{1,2}
 
 altname(File) ->
-    altname_int({?DRV, []}, File).
+    altname_int({?DRV, [binary]}, File).
 
 altname(Port, File) when is_port(Port) ->
     altname_int(Port, File).
 
 altname_int(Port, File) ->
-    drv_command(Port, [?FILE_ALTNAME, File, 0]).
-
+    drv_command(Port, [?FILE_ALTNAME, pathname(File)]).
 
-%% write_file_info/{2,3}
+%% write_file_info/{2,3,4}
 
 write_file_info(File, Info) ->
-    write_file_info_int({?DRV, []}, File, Info).
+    write_file_info_int({?DRV, [binary]}, File, Info, local).
 
 write_file_info(Port, File, Info) when is_port(Port) ->
-    write_file_info_int(Port, File, Info).
+    write_file_info_int(Port, File, Info, local);
+write_file_info(File, Info, Opts) ->
+    write_file_info_int({?DRV, [binary]}, File, Info, plgv(time, Opts, local)).
 
-write_file_info_int(Port, 
-		    File, 
+write_file_info(Port, File, Info, Opts) when is_port(Port) ->
+    write_file_info_int(Port, File, Info, plgv(time, Opts, local)).
+
+write_file_info_int(Port, File, 
 		    #file_info{mode=Mode, 
 			       uid=Uid, 
 			       gid=Gid,
 			       atime=Atime0, 
 			       mtime=Mtime0, 
-			       ctime=Ctime}) ->
-    {Atime, Mtime} =
-	case {Atime0, Mtime0} of
-	    {undefined, Mtime0} -> {erlang:localtime(), Mtime0};
-	    {Atime0, undefined} -> {Atime0, Atime0};
-	    Complete -> Complete
-	end,
-    drv_command(Port, [?FILE_WRITE_INFO, 
-			int_to_bytes(Mode), 
-			int_to_bytes(Uid), 
-			int_to_bytes(Gid),
-			date_to_bytes(Atime), 
-			date_to_bytes(Mtime), 
-			date_to_bytes(Ctime),
-			File, 0]).
+			       ctime=Ctime0},
+		       TimeType) ->
+
+    % Atime and/or Mtime might be undefined
+    %  - use localtime() for atime, if atime is undefined
+    %  - use atime as mtime if mtime is undefined
+    %  - use mtime as ctime if ctime is undefined
+
+    try
+	Atime = file_info_validate_atime(Atime0, TimeType),
+	Mtime = file_info_validate_mtime(Mtime0, Atime),
+	Ctime = file_info_validate_ctime(Ctime0, Mtime),
+
+	drv_command(Port, [?FILE_WRITE_INFO, 
+		int_to_int32bytes(Mode), 
+		int_to_int32bytes(Uid), 
+		int_to_int32bytes(Gid),
+		int_to_int64bytes(to_seconds(Atime, TimeType)), 
+		int_to_int64bytes(to_seconds(Mtime, TimeType)), 
+		int_to_int64bytes(to_seconds(Ctime, TimeType)),
+		pathname(File)])
+    catch
+	error:_ -> {error, badarg}
+    end.
+
 
+file_info_validate_atime(Atime, _) when Atime =/= undefined -> Atime;
+file_info_validate_atime(undefined, local)     -> erlang:localtime();
+file_info_validate_atime(undefined, universal) -> erlang:universaltime();
+file_info_validate_atime(undefined, posix)     -> erlang:universaltime_to_posixtime(erlang:universaltime()).
 
+file_info_validate_mtime(undefined, Atime) -> Atime;
+file_info_validate_mtime(Mtime, _)         -> Mtime.
+
+file_info_validate_ctime(undefined, Mtime) -> Mtime;
+file_info_validate_ctime(Ctime, _)         -> Ctime.
 
 %% make_link/{2,3}
 
 make_link(Old, New) ->
-    make_link_int({?DRV, []}, Old, New).
+    make_link_int({?DRV, [binary]}, Old, New).
 
 make_link(Port, Old, New) when is_port(Port) ->
     make_link_int(Port, Old, New).
 
 make_link_int(Port, Old, New) ->
-    drv_command(Port, [?FILE_LINK, Old, 0, New, 0]).
+    drv_command(Port, [?FILE_LINK, pathname(Old), pathname(New)]).
 
 
 
 %% make_symlink/{2,3}
 
 make_symlink(Old, New) ->
-    make_symlink_int({?DRV, []}, Old, New).
+    make_symlink_int({?DRV, [binary]}, Old, New).
 
 make_symlink(Port, Old, New) when is_port(Port) ->
     make_symlink_int(Port, Old, New).
 
 make_symlink_int(Port, Old, New) ->
-    drv_command(Port, [?FILE_SYMLINK, Old, 0, New, 0]).
+    drv_command(Port, [?FILE_SYMLINK, pathname(Old), pathname(New)]).
 
 
 
 %% read_link/{2,3}
 
 read_link(Link) ->
-    read_link_int({?DRV, []}, Link).
+    read_link_int({?DRV, [binary]}, Link).
 
 read_link(Port, Link) when is_port(Port) ->
     read_link_int(Port, Link).
 
 read_link_int(Port, Link) ->
-    drv_command(Port, [?FILE_READLINK, Link, 0]).
+    drv_command(Port, [?FILE_READLINK, pathname(Link)]).
 
 
 
 %% read_link_info/{2,3}
 
 read_link_info(Link) ->
-    read_link_info_int({?DRV, []}, Link).
+    read_link_info_int({?DRV, [binary]}, Link, local).
 
 read_link_info(Port, Link) when is_port(Port) ->
-    read_link_info_int(Port, Link).
+    read_link_info_int(Port, Link, local);
+
+read_link_info(Link, Opts) ->
+    read_link_info_int({?DRV, [binary]}, Link, plgv(time, Opts, local)).
 
-read_link_info_int(Port, Link) ->
-    drv_command(Port, [?FILE_LSTAT, Link, 0]).
+read_link_info(Port, Link, Opts) when is_port(Port) ->
+    read_link_info_int(Port, Link, plgv(time, Opts, local)).
 
 
+read_link_info_int(Port, Link, TimeType) ->
+    try
+	case drv_command(Port, [?FILE_LSTAT, pathname(Link)]) of
+	    {ok, FI} -> {ok, FI#file_info{
+			ctime = from_seconds(FI#file_info.ctime, TimeType),
+			mtime = from_seconds(FI#file_info.mtime, TimeType),
+			atime = from_seconds(FI#file_info.atime, TimeType)
+		    }};
+	    Error    -> Error
+	end
+    catch
+	error:_ -> {error, badarg}
+    end.
 
 %% list_dir/{1,2}
 
 list_dir(Dir) ->
-    list_dir_int({?DRV, []}, Dir).
+    list_dir_int({?DRV, [binary]}, Dir).
 
 list_dir(Port, Dir) when is_port(Port) ->
     list_dir_int(Port, Dir).
 
 list_dir_int(Port, Dir) ->
-    drv_command(Port, [?FILE_READDIR, Dir, 0], []).
+    drv_command(Port, [?FILE_READDIR, pathname(Dir)], []).
 
 
 
@@ -774,7 +907,7 @@ list_dir_int(Port, Dir) ->
 
 
 %% Opens a driver port and converts any problems into {error, emfile}.
-%% Returns {ok, Port} when succesful.
+%% Returns {ok, Port} when successful.
 
 drv_open(Driver, Portopts) ->
     try erlang:open_port({spawn, Driver}, Portopts) of
@@ -782,7 +915,7 @@ drv_open(Driver, Portopts) ->
 	    {ok, Port}
     catch
 	error:Reason ->
-	    {error,Reason}
+	    {error, Reason}
     end.
 
 
@@ -790,12 +923,17 @@ drv_open(Driver, Portopts) ->
 %% Closes a port in a safe way. Returns ok.
 
 drv_close(Port) ->
-    try erlang:port_close(Port) catch error:_ -> ok end,
-    receive %% Ugly workaround in case the caller==owner traps exits
-	{'EXIT', Port, _Reason} -> 
-	    ok
-    after 0 -> 
-	    ok
+    Save = erlang:dt_spread_tag(false),
+    try
+	try erlang:port_close(Port) catch error:_ -> ok end,
+	receive %% Ugly workaround in case the caller==owner traps exits
+	    {'EXIT', Port, _Reason} -> 
+		ok
+	after 0 -> 
+		ok
+	end
+    after
+	erlang:dt_restore_tag(Save)
     end.
 
 
@@ -805,25 +943,23 @@ drv_close(Port) ->
 %% then closed after the result has been received.
 %% Returns {ok, Result} or {error, Reason}.
 
-drv_command_raw(Port, Command) ->
-    drv_command(Port, Command, false, undefined).
-
 drv_command(Port, Command) ->
     drv_command(Port, Command, undefined).
 
 drv_command(Port, Command, R) when is_binary(Command) ->
     drv_command(Port, Command, true, R);
 drv_command(Port, Command, R) ->
-    try erlang:iolist_to_binary(Command) of
-	Bin ->
-	    drv_command(Port, Bin, true, R)
+    try erlang:iolist_size(Command) of
+	_ ->
+	    drv_command(Port, Command, true, R)
     catch
 	error:Reason ->
 	    {error, Reason}
     end.
 
 drv_command(Port, Command, Validated, R) when is_port(Port) ->
-    try erlang:port_command(Port, Command) of
+    Save = erlang:dt_spread_tag(false),
+    try erlang:port_command(Port, erlang:dt_append_vm_tag_data(Command)) of
 	true ->
 	    drv_get_response(Port, R)
     catch
@@ -842,6 +978,8 @@ drv_command(Port, Command, Validated, R) when is_port(Port) ->
 	    end;
 	error:Reason ->
 	    {error, Reason}
+    after
+	erlang:dt_restore_tag(Save)
     end;
 drv_command({Driver, Portopts}, Command, Validated, R) ->
     case drv_open(Driver, Portopts) of
@@ -852,6 +990,25 @@ drv_command({Driver, Portopts}, Command, Validated, R) ->
 	Error ->
 	    Error
     end.
+drv_command_nt(Port, Command, R) when is_port(Port) ->
+    Save = erlang:dt_spread_tag(false),
+    try erlang:port_command(Port, Command) of
+	true ->
+	    drv_get_response(Port, R)
+    catch
+	error:badarg ->
+	    try erlang:iolist_size(Command) of
+		_ -> % Valid
+		    {error, einval}
+	    catch
+		error:_ ->
+		    {error, badarg}
+	    end;
+	error:Reason ->
+	    {error, Reason}
+    after
+	erlang:dt_restore_tag(Save)
+    end.
 
 
     
@@ -864,6 +1021,8 @@ drv_get_response(Port, R) when is_list(R) ->
 	    {ok, R};
 	{ok, Name} ->
 	    drv_get_response(Port, [Name|R]);
+	{append, Names} ->
+	    drv_get_response(Port, append(Names, R));
 	Error ->
 	    Error
     end;
@@ -888,9 +1047,11 @@ drv_get_response(Port) ->
 %%%-----------------------------------------------------------------
 %%% Utility functions.
 
+append([I | Is], R) when is_list(R) -> append(Is, [I | R]);
+append([], R) -> R.
 
 
-%% Converts a list of mode atoms into an mode word for the driver.
+%% Converts a list of mode atoms into a mode word for the driver.
 %% Returns {Mode, Portopts, Setopts} where Portopts is a list of 
 %% options for erlang:open_port/2 and Setopts is a list of 
 %% setopt commands to send to the port, or error Reason upon failure.
@@ -918,6 +1079,8 @@ open_mode([compressed|Rest], Mode, Portopts, Setopts) ->
 open_mode([append|Rest], Mode, Portopts, Setopts) ->
     open_mode(Rest, Mode bor ?EFILE_MODE_APPEND bor ?EFILE_MODE_WRITE, 
 	      Portopts, Setopts);
+open_mode([exclusive|Rest], Mode, Portopts, Setopts) ->
+    open_mode(Rest, Mode bor ?EFILE_MODE_EXCL, Portopts, Setopts);
 open_mode([delayed_write|Rest], Mode, Portopts, Setopts) ->
     open_mode([{delayed_write, 64*1024, 2000}|Rest], Mode,
 	      Portopts, Setopts);
@@ -983,18 +1146,16 @@ lseek_position(_) ->
 
 translate_response(?FILE_RESP_OK, []) ->
     ok;
-translate_response(?FILE_RESP_OK, Data) ->
-    {ok, Data};
 translate_response(?FILE_RESP_ERROR, List) when is_list(List) ->
     {error, list_to_atom(List)};
 translate_response(?FILE_RESP_NUMBER, List) ->
     {N, []} = get_uint64(List),
     {ok, N};
 translate_response(?FILE_RESP_DATA, List) ->
-    {N, Data} = get_uint64(List),
-    {ok, {N, Data}};
+    {_N, _Data} = ND = get_uint64(List),
+    {ok, ND};
 translate_response(?FILE_RESP_INFO, List) when is_list(List) ->
-    {ok, transform_info_ints(get_uint32s(List))};
+    {ok, transform_info(List)};
 translate_response(?FILE_RESP_NUMERR, L0) ->
     {N, L1} = get_uint64(L0),
     {error, {N, list_to_atom(L1)}};
@@ -1017,7 +1178,7 @@ translate_response(?FILE_RESP_N2DATA,
     {ok, {Size, Offset, D}};
 translate_response(?FILE_RESP_N2DATA = X, L0) when is_list(L0) ->
     {Offset, L1}    = get_uint64(L0),
-    {ReadSize, L2} = get_uint64(L1),
+    {ReadSize, L2}  = get_uint64(L1),
     {Size, L3}      = get_uint64(L2),
     case {ReadSize, L3} of
 	{0, []} ->
@@ -1031,30 +1192,54 @@ translate_response(?FILE_RESP_N2DATA = X, L0) when is_list(L0) ->
     end;
 translate_response(?FILE_RESP_EOF, []) ->
     eof;
+translate_response(?FILE_RESP_FNAME, []) ->
+    ok;
+translate_response(?FILE_RESP_FNAME, Data) when is_binary(Data) ->
+    {ok, prim_file:internal_native2name(Data)};
+translate_response(?FILE_RESP_FNAME, Data) ->
+    {ok, Data};
+translate_response(?FILE_RESP_LFNAME, []) ->
+    ok;
+translate_response(?FILE_RESP_LFNAME, Data) when is_binary(Data) ->
+    {append, transform_lfname(Data)};
+translate_response(?FILE_RESP_LFNAME, Data) ->
+    {append, transform_lfname(Data)};
+translate_response(?FILE_RESP_ALL_DATA, Data) ->
+    {ok, Data};
 translate_response(X, Data) ->
     {error, {bad_response_from_port, [X | Data]}}.
 
-transform_info_ints(Ints) ->
-    [HighSize, LowSize, Type|Tail0] = Ints,
-    Size = HighSize * 16#100000000 + LowSize,
-    [Ay, Am, Ad, Ah, Ami, As|Tail1]  = Tail0,
-    [My, Mm, Md, Mh, Mmi, Ms|Tail2] = Tail1,
-    [Cy, Cm, Cd, Ch, Cmi, Cs|Tail3] = Tail2,
-    [Mode, Links, Major, Minor, Inode, Uid, Gid, Access] = Tail3,
+transform_info([
+    Hsize1, Hsize2, Hsize3, Hsize4, 
+    Lsize1, Lsize2, Lsize3, Lsize4,
+    Type1,  Type2,  Type3,  Type4,
+    Atime1, Atime2, Atime3, Atime4, Atime5, Atime6, Atime7, Atime8,
+    Mtime1, Mtime2, Mtime3, Mtime4, Mtime5, Mtime6, Mtime7, Mtime8,
+    Ctime1, Ctime2, Ctime3, Ctime4, Ctime5, Ctime6, Ctime7, Ctime8,
+    Mode1,  Mode2,  Mode3,  Mode4,
+    Links1, Links2, Links3, Links4,
+    Major1, Major2, Major3, Major4,
+    Minor1, Minor2, Minor3, Minor4,
+    Inode1, Inode2, Inode3, Inode4,
+    Uid1,   Uid2,   Uid3,   Uid4,
+    Gid1,   Gid2,   Gid3,   Gid4,
+    Access1,Access2,Access3,Access4]) ->
     #file_info {
-		size = Size,
-		type = file_type(Type),
-		access = file_access(Access),
-		atime = {{Ay, Am, Ad}, {Ah, Ami, As}},
-		mtime = {{My, Mm, Md}, {Mh, Mmi, Ms}},
-		ctime = {{Cy, Cm, Cd}, {Ch, Cmi, Cs}},
-		mode = Mode,
-		links = Links,
-		major_device = Major,
-		minor_device = Minor,
-		inode = Inode,
-		uid = Uid,
-		gid = Gid}.
+		size   = uint32(Hsize1,Hsize2,Hsize3,Hsize4)*16#100000000 + uint32(Lsize1,Lsize2,Lsize3,Lsize4),
+		type   = file_type(uint32(Type1,Type2,Type3,Type4)),
+		access = file_access(uint32(Access1,Access2,Access3,Access4)),
+		atime  = sint64(Atime1, Atime2, Atime3, Atime4, Atime5, Atime6, Atime7, Atime8),
+		mtime  = sint64(Mtime1, Mtime2, Mtime3, Mtime4, Mtime5, Mtime6, Mtime7, Mtime8),
+		ctime  = sint64(Ctime1, Ctime2, Ctime3, Ctime4, Ctime5, Ctime6, Ctime7, Ctime8),
+		mode   = uint32(Mode1,Mode2,Mode3,Mode4),
+		links  = uint32(Links1,Links2,Links3,Links4),
+		major_device = uint32(Major1,Major2,Major3,Major4),
+		minor_device = uint32(Minor1,Minor2,Minor3,Minor4),
+		inode = uint32(Inode1,Inode2,Inode3,Inode4),
+		uid   = uint32(Uid1,Uid2,Uid3,Uid4),
+		gid   = uint32(Gid1,Gid2,Gid3,Gid4)
+	    }.
+    
     
 file_type(1) -> device;
 file_type(2) -> directory;
@@ -1067,24 +1252,22 @@ file_access(1) -> write;
 file_access(2) -> read;
 file_access(3) -> read_write.
 
-int_to_bytes(Int) when is_integer(Int) ->
+int_to_int32bytes(Int) when is_integer(Int) ->
     <<Int:32>>;
-int_to_bytes(undefined) ->
+int_to_int32bytes(undefined) ->
     <<-1:32>>.
 
-date_to_bytes(undefined) ->
-    <<-1:32, -1:32, -1:32, -1:32, -1:32, -1:32>>;
-date_to_bytes({{Y, Mon, D}, {H, Min, S}}) ->
-    <<Y:32, Mon:32, D:32, H:32, Min:32, S:32>>.
+int_to_int64bytes(Int) when is_integer(Int) ->
+    <<Int:64/signed>>.
 
-% uint64([[X1, X2, X3, X4] = Y1 | [X5, X6, X7, X8] = Y2]) ->
-%     (uint32(Y1) bsl 32) bor uint32(Y2).
 
-% uint64(X1, X2, X3, X4, X5, X6, X7, X8) ->
-%     (uint32(X1, X2, X3, X4) bsl 32) bor uint32(X5, X6, X7, X8).
+sint64(I1,I2,I3,I4,I5,I6,I7,I8) when I1 > 127 ->
+    ((I1 bsl 56) bor (I2 bsl 48) bor (I3 bsl 40) bor (I4 bsl 32) bor
+	(I5 bsl 24) bor (I6 bsl 16) bor (I7 bsl  8) bor I8) - (1 bsl 64);
+sint64(I1,I2,I3,I4,I5,I6,I7,I8) ->
+    ((I1 bsl 56) bor (I2 bsl 48) bor (I3 bsl 40) bor (I4 bsl 32) bor
+	(I5 bsl 24) bor (I6 bsl 16) bor (I7 bsl  8) bor I8).
 
-% uint32([X1,X2,X3,X4]) ->
-%     (X1 bsl 24) bor (X2 bsl 16) bor (X3 bsl 8) bor X4.
 
 uint32(X1,X2,X3,X4) ->
     (X1 bsl 24) bor (X2 bsl 16) bor (X3 bsl 8) bor X4.
@@ -1097,11 +1280,6 @@ get_uint64(L0) ->
 get_uint32([X1,X2,X3,X4|List]) ->
     {(((((X1 bsl 8) bor X2) bsl 8) bor X3) bsl 8) bor X4, List}.
 
-get_uint32s([X1,X2,X3,X4|Tail]) ->
-    [uint32(X1,X2,X3,X4) | get_uint32s(Tail)];
-get_uint32s([]) -> [].
-
-
 
 %% Binary mode
 transform_ldata(<<0:32, 0:32>>) ->
@@ -1141,6 +1319,14 @@ transform_ldata(0, List, [Size | Sizes], R) ->
     {Front, Rear} = lists_split(List, Size),
     transform_ldata(0, Rear, Sizes, [Front | R]).
 
+transform_lfname(<<>>) -> [];
+transform_lfname(<<L:16, Name:L/binary, Names/binary>>) -> 
+    [ prim_file:internal_native2name(Name) | transform_lfname(Names)];
+transform_lfname([]) -> [];
+transform_lfname([L1,L2|Names]) ->
+    L = (L1 bsl 8) bor L2,
+    {Name, Rest} = lists_split(Names, L),
+    [Name | transform_lfname(Rest)].
 
 
 lists_split(List, 0) when is_list(List) ->
@@ -1166,3 +1352,34 @@ lists_split([Hd | Tl], N, Rev) ->
 
 reverse(X) -> lists:reverse(X, []).
 reverse(L, T) -> lists:reverse(L, T).
+
+% Will add zero termination too
+% The 'EXIT' tuple from a bad argument will eventually generate an error
+% in list_to_binary, which is caught and generates the {error,badarg} return
+pathname(File) ->
+    (catch prim_file:internal_name2native(File)).
+
+
+%% proplist:get_value/3
+plgv(K, [{K, V}|_], _) -> V;
+plgv(K, [_|KVs], D)    -> plgv(K, KVs, D);
+plgv(_, [], D)         -> D.
+
+%%
+%% We don't actually want this here
+%% We want to use posix time in all prim but erl_prim_loader makes that tricky
+%% It is probably needed to redo the whole erl_prim_loader
+
+from_seconds(Seconds, posix) when is_integer(Seconds) ->
+    Seconds;
+from_seconds(Seconds, universal) when is_integer(Seconds) ->
+    erlang:posixtime_to_universaltime(Seconds);
+from_seconds(Seconds, local) when is_integer(Seconds) ->
+    erlang:universaltime_to_localtime(erlang:posixtime_to_universaltime(Seconds)).
+
+to_seconds(Seconds, posix) when is_integer(Seconds) ->
+    Seconds;
+to_seconds({_,_} = Datetime, universal) ->
+    erlang:universaltime_to_posixtime(Datetime);
+to_seconds({_,_} = Datetime, local) ->
+    erlang:universaltime_to_posixtime(erlang:localtime_to_universaltime(Datetime)).
diff --git a/erts/preloaded/src/prim_inet.erl b/erts/preloaded/src/prim_inet.erl
index 0feb591efb..846ae97ed2 100644
--- a/erts/preloaded/src/prim_inet.erl
+++ b/erts/preloaded/src/prim_inet.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2000-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2000-2012. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -25,8 +25,8 @@
 
 %% Primitive inet_drv interface
 
--export([open/1, open/2, fdopen/2, fdopen/3, close/1]).
--export([bind/3, listen/1, listen/2]). 
+-export([open/3, fdopen/4, close/1]).
+-export([bind/3, listen/1, listen/2, peeloff/2]).
 -export([connect/3, connect/4, async_connect/4]).
 -export([accept/1, accept/2, async_accept/2]).
 -export([shutdown/2]).
@@ -36,8 +36,9 @@
 -export([recvfrom/2, recvfrom/3]).
 -export([setopt/3, setopts/2, getopt/2, getopts/2, is_sockopt_val/2]).
 -export([chgopt/3, chgopts/2]).
--export([getstat/2, getfd/1, getindex/1, getstatus/1, gettype/1, 
-	 getiflist/1, ifget/3, ifset/3,
+-export([getstat/2, getfd/1, ignorefd/2,
+	 getindex/1, getstatus/1, gettype/1,
+	 getifaddrs/1, getiflist/1, ifget/3, ifset/3,
 	 gethostname/1]).
 -export([getservbyname/3, getservbyport/3]).
 -export([peername/1, setpeername/2]).
@@ -56,58 +57,48 @@
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%
-%% OPEN(tcp | udp | sctp, inet | inet6)  ->
+%% OPEN(tcp | udp | sctp, inet | inet6, stream | dgram | seqpacket)  ->
 %%       {ok, insock()} |
 %%       {error, Reason}
 %%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-open(Protocol)          -> open1(Protocol, ?INET_AF_INET).
+open(Protocol, Family, Type) ->
+    open(Protocol, Family, Type, ?INET_REQ_OPEN, []).
 
-open(Protocol,   inet)  -> open1(Protocol, ?INET_AF_INET);
-open(Protocol,  inet6)  -> open1(Protocol, ?INET_AF_INET6);
-open(_, _)              -> {error, einval}.
+fdopen(Protocol, Family, Type, Fd) when is_integer(Fd) ->
+    open(Protocol, Family, Type, ?INET_REQ_FDOPEN, ?int32(Fd)).
 
-fdopen(Protocol, Fd)        -> fdopen1(Protocol, ?INET_AF_INET, Fd).
-
-fdopen(Protocol, Fd, inet)  -> fdopen1(Protocol, ?INET_AF_INET, Fd);
-fdopen(Protocol, Fd, inet6) -> fdopen1(Protocol, ?INET_AF_INET6, Fd);
-fdopen(_, _, _)             -> {error, einval}.
-
-open1(Protocol, Family) ->
-    case open0(Protocol) of
-	{ok, S} ->
-	    case ctl_cmd(S, ?INET_REQ_OPEN, [Family]) of
-		{ok, _} ->
-		    {ok,S};
-		Error ->
-		    close(S), Error
-	    end;
-	Error -> Error
+open(Protocol, Family, Type, Req, Data) ->
+    Drv = protocol2drv(Protocol),
+    AF = enc_family(Family),
+    T = enc_type(Type),
+    try erlang:open_port({spawn_driver,Drv}, [binary]) of
+	S ->
+	    case ctl_cmd(S, Req, [AF,T,Data]) of
+		{ok,_} -> {ok,S};
+		{error,_}=Error ->
+		    close(S),
+		    Error
+	    end
+    catch
+	%% The only (?) way to get here is to try to open
+	%% the sctp driver when it does not exist (badarg)
+	error:badarg       -> {error, eprotonosupport};
+	%% system_limit if out of port slots
+	error:system_limit -> {error, system_limit}
     end.
 
-fdopen1(Protocol, Family, Fd) when is_integer(Fd) ->
-    case open0(Protocol) of
-	{ok, S} ->
-	    case ctl_cmd(S,?INET_REQ_FDOPEN,[Family,?int32(Fd)]) of
-		{ok, _} -> {ok,S};
-		Error -> close(S), Error
-	    end;
-	Error -> Error
-    end.
+enc_family(inet) -> ?INET_AF_INET;
+enc_family(inet6) -> ?INET_AF_INET6.
 
-open0(Protocol) ->
-    try	erlang:open_port({spawn_driver,protocol2drv(Protocol)}, [binary]) of
-	Port -> {ok,Port}
-    catch
-	error:Reason -> {error,Reason}
-    end.
+enc_type(stream) -> ?INET_TYPE_STREAM;
+enc_type(dgram) -> ?INET_TYPE_DGRAM;
+enc_type(seqpacket) -> ?INET_TYPE_SEQPACKET.
 
 protocol2drv(tcp)  -> "tcp_inet";
 protocol2drv(udp)  -> "udp_inet";
-protocol2drv(sctp) -> "sctp_inet";
-protocol2drv(_) ->
-    erlang:error(eprotonosupport).
+protocol2drv(sctp) -> "sctp_inet".
 
 drv2protocol("tcp_inet")  -> tcp;
 drv2protocol("udp_inet")  -> udp;
@@ -139,7 +130,7 @@ shutdown_1(S, How) ->
 shutdown_2(S, How) ->
     case ctl_cmd(S, ?TCP_REQ_SHUTDOWN, [How]) of
 	{ok, []} -> ok;
-	Error -> Error
+	{error,_}=Error -> Error
     end.
 
 shutdown_pend_loop(S, N0) ->
@@ -195,7 +186,7 @@ close_pend_loop(S, N) ->
 bind(S,IP,Port) when is_port(S), is_integer(Port), Port >= 0, Port =< 65535 ->
     case ctl_cmd(S,?INET_REQ_BIND,[?int16(Port),ip_to_bytes(IP)]) of
 	{ok, [P1,P0]} -> {ok, ?u16(P1, P0)};
-	Error -> Error
+	{error,_}=Error -> Error
     end;
 
 %% Multi-homed "bind": sctp_bindx(). The Op is 'add' or 'remove'.
@@ -216,12 +207,13 @@ bindx(S, AddFlag, Addrs) ->
 	sctp ->
 	    %% Really multi-homed "bindx". Stringified args:
 	    %% [AddFlag, (Port, IP)+]:
-	    Args = ?int8(AddFlag) ++
-		lists:concat([?int16(Port)++ip_to_bytes(IP) ||
-				 {IP, Port} <- Addrs]),
+	    Args =
+		[?int8(AddFlag)|
+		 [[?int16(Port)|ip_to_bytes(IP)] ||
+		     {IP, Port} <- Addrs]],
 	    case ctl_cmd(S, ?SCTP_REQ_BINDX, Args) of
 		{ok,_} -> {ok, S};
-		Error  -> Error
+		{error,_}=Error  -> Error
 	    end;
 	_ -> {error, einval}
     end.
@@ -264,7 +256,7 @@ async_connect(S, IP, Port, Time) ->
     case ctl_cmd(S, ?INET_REQ_CONNECT,
 		 [enc_time(Time),?int16(Port),ip_to_bytes(IP)]) of
 	{ok, [R1,R0]} -> {ok, S, ?u16(R1,R0)};
-	Error -> Error
+	{error,_}=Error -> Error
     end.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -317,9 +309,9 @@ accept_opts(L, S) ->
     end.
 
 async_accept(L, Time) ->
-    case ctl_cmd(L,?TCP_REQ_ACCEPT, [enc_time(Time)]) of
+    case ctl_cmd(L,?INET_REQ_ACCEPT, [enc_time(Time)]) of
 	{ok, [R1,R0]} -> {ok, ?u16(R1,R0)};
-	Error -> Error
+	{error,_}=Error -> Error
     end.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -333,16 +325,30 @@ async_accept(L, Time) ->
 %% listening) is also accepted:
 
 listen(S) -> listen(S, ?LISTEN_BACKLOG).
-    
+
+listen(S, true) -> listen(S, ?LISTEN_BACKLOG);
+listen(S, false) -> listen(S, 0);
 listen(S, BackLog) when is_port(S), is_integer(BackLog) ->
-    case ctl_cmd(S, ?TCP_REQ_LISTEN, [?int16(BackLog)]) of
+    case ctl_cmd(S, ?INET_REQ_LISTEN, [?int16(BackLog)]) of
 	{ok, _} -> ok;
-	Error   -> Error
-    end;
-listen(S, Flag)   when is_port(S), is_boolean(Flag) ->
-    case ctl_cmd(S, ?SCTP_REQ_LISTEN, enc_value(set, bool8, Flag)) of
-	{ok,_} -> ok;
-	Error -> Error
+	{error,_}=Error   -> Error
+    end.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%
+%% PEELOFF(insock(), AssocId) -> {ok,outsock()} | {error, Reason}
+%%
+%% SCTP: Peel off one association into a type stream socket
+%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+peeloff(S, AssocId) ->
+    case ctl_cmd(S, ?SCTP_REQ_PEELOFF, [?int32(AssocId)]) of
+	inet_reply ->
+	    receive
+		{inet_reply,S,Res} -> Res
+	    end;
+	{error,_}=Error -> Error
     end.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -394,12 +400,12 @@ sendto(S, IP, Port, Data) when is_port(S), Port >= 0, Port =< 65535 ->
 	true -> 
 	    receive
 		{inet_reply,S,Reply} ->
-		    ?DBG_FORMAT("prim_inet:send() -> ~p~n", [Reply]),
+		    ?DBG_FORMAT("prim_inet:sendto() -> ~p~n", [Reply]),
 		     Reply
 	    end
     catch
 	error:_ ->
-	    ?DBG_FORMAT("prim_inet:send() -> {error,einval}~n", []),
+	    ?DBG_FORMAT("prim_inet:sendto() -> {error,einval}~n", []),
 	     {error,einval}
     end.
 
@@ -454,7 +460,7 @@ recv0(S, Length, Time) when is_port(S), is_integer(Length), Length >= 0 ->
 async_recv(S, Length, Time) ->
     case ctl_cmd(S, ?TCP_REQ_RECV, [enc_time(Time), ?int32(Length)]) of
 	{ok,[R1,R0]} -> {ok, ?u16(R1,R0)};
-	Error -> Error
+	{error,_}=Error -> Error
     end.	    
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -500,7 +506,7 @@ recvfrom0(S, Length, Time)
 		{inet_async, S, Ref, Error={error, _}} ->
 		    Error
 	    end;
-	Error ->
+	{error,_}=Error ->
 	    Error % Front-end error
     end;
 recvfrom0(_, _, _) -> {error,einval}.
@@ -516,18 +522,18 @@ peername(S) when is_port(S) ->
 	{ok, [F, P1,P0 | Addr]} ->
 	    {IP, _} = get_ip(F, Addr),
 	    {ok, { IP, ?u16(P1, P0) }};
-	Error -> Error
+	{error,_}=Error -> Error
     end.
 
 setpeername(S, {IP,Port}) when is_port(S) ->
     case ctl_cmd(S, ?INET_REQ_SETPEER, [?int16(Port),ip_to_bytes(IP)]) of
 	{ok,[]} -> ok;
-	Error -> Error
+	{error,_}=Error -> Error
     end;
 setpeername(S, undefined) when is_port(S) ->
     case ctl_cmd(S, ?INET_REQ_SETPEER, []) of
 	{ok,[]} -> ok;
-	Error -> Error
+	{error,_}=Error -> Error
     end.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -541,18 +547,18 @@ sockname(S) when is_port(S) ->
 	{ok, [F, P1, P0 | Addr]} ->
 	    {IP, _} = get_ip(F, Addr),
 	    {ok, { IP, ?u16(P1, P0) }};
-	Error -> Error
+	{error,_}=Error -> Error
     end.
 
 setsockname(S, {IP,Port}) when is_port(S) ->
     case ctl_cmd(S, ?INET_REQ_SETNAME, [?int16(Port),ip_to_bytes(IP)]) of
 	{ok,[]} -> ok;
-	Error -> Error
+	{error,_}=Error -> Error
     end;
 setsockname(S, undefined) when is_port(S) ->
     case ctl_cmd(S, ?INET_REQ_SETNAME, []) of
 	{ok,[]} -> ok;
-	Error -> Error
+	{error,_}=Error -> Error
     end.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -572,7 +578,7 @@ setopts(S, Opts) when is_port(S) ->
 	{ok, Buf} ->
 	    case ctl_cmd(S, ?INET_REQ_SETOPTS, Buf) of
 		{ok, _} -> ok;
-		Error -> Error
+		{error,_}=Error -> Error
 	    end;
 	Error  -> Error
     end.	    
@@ -598,12 +604,12 @@ getopts(S, Opts) when is_port(S), is_list(Opts) ->
 		{ok,Rep} ->
 		    %% Non-SCTP: "Rep" contains the encoded option vals:
 		    decode_opt_val(Rep);
-		{error,sctp_reply} ->
+		inet_reply ->
 		    %% SCTP: Need to receive the full value:
 		    receive
 			{inet_reply,S,Res} -> Res
 		    end;
-		Error -> Error
+		{error,_}=Error -> Error
 	    end;
 	Error -> Error
     end.
@@ -623,7 +629,7 @@ chgopt(S, Opt, Value) when is_port(S) ->
     chgopts(S, [{Opt,Value}]).
 
 chgopts(S, Opts) when is_port(S), is_list(Opts) ->
-    case inet:getopts(S, need_template(Opts)) of
+    case getopts(S, need_template(Opts)) of
 	{ok,Templates} ->
 	    try merge_options(Opts, Templates) of
 		NewOpts ->
@@ -636,7 +642,94 @@ chgopts(S, Opts) when is_port(S), is_list(Opts) ->
     
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%
-%% IFLIST(insock()) -> {ok,IfNameList} | {error, Reason}
+%% getifaddrs(insock()) -> {ok,IfAddrsList} | {error, Reason}
+%%
+%%   IfAddrsList = [{Name,[Opts]}]
+%%   Name = string()
+%%   Opts = {flags,[Flag]} | {addr,Addr} | {netmask,Addr} | {broadaddr,Addr}
+%%        | {dstaddr,Addr} | {hwaddr,HwAddr} | {mtu,integer()}
+%%   Flag = up | broadcast | loopback | running | multicast
+%%   Addr = ipv4addr() | ipv6addr()
+%%   HwAddr = ethernet_addr()
+%%
+%% get interface name and addresses list
+%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+getifaddrs(S) when is_port(S) ->
+    case ctl_cmd(S, ?INET_REQ_GETIFADDRS, []) of
+	{ok, Data} ->
+	    {ok, comp_ifaddrs(build_ifaddrs(Data), ktree_empty())};
+	{error,enotsup} ->
+	    case getiflist(S) of
+		{ok, IFs} ->
+		    {ok, getifaddrs_ifget(S, IFs)};
+		Err1 -> Err1
+	    end;
+	Err2 -> Err2
+    end.
+
+%% Restructure interface properties per interface and remove duplicates
+
+comp_ifaddrs([{If,Opts}|IfOpts], T) ->
+    case ktree_is_defined(If, T) of
+	true ->
+	    OptSet = comp_ifaddrs_add(ktree_get(If, T), Opts),
+	    comp_ifaddrs(IfOpts, ktree_update(If, OptSet, T));
+	false ->
+	    OptSet = comp_ifaddrs_add(ktree_empty(), Opts),
+	    comp_ifaddrs(IfOpts, ktree_insert(If, OptSet, T))
+     end;
+comp_ifaddrs([], T) ->
+    [{If,ktree_keys(ktree_get(If, T))} || If <- ktree_keys(T)].
+
+comp_ifaddrs_add(OptSet, [Opt|Opts]) ->
+    case ktree_is_defined(Opt, OptSet) of
+	true
+	  when element(1, Opt) =:= flags;
+	       element(1, Opt) =:= hwaddr ->
+	    comp_ifaddrs_add(OptSet, Opts);
+	_ ->
+	    comp_ifaddrs_add(ktree_insert(Opt, undefined, OptSet), Opts)
+    end;
+comp_ifaddrs_add(OptSet, []) -> OptSet.
+
+%% Legacy emulation of getifaddrs
+
+getifaddrs_ifget(_, []) -> [];
+getifaddrs_ifget(S, [IF|IFs]) ->
+    case ifget(S, IF, [flags]) of
+	{ok,[{flags,Flags}]=FlagsVals} ->
+	    BroadOpts =
+		case member(broadcast, Flags) of
+		    true ->
+			[broadaddr,hwaddr];
+		    false ->
+			[hwaddr]
+		end,
+	    P2POpts =
+		case member(pointtopoint, Flags) of
+		    true ->
+			[dstaddr|BroadOpts];
+		    false ->
+			BroadOpts
+		end,
+	    getifaddrs_ifget(S, IFs, IF, FlagsVals, [addr,netmask|P2POpts]);
+	_ ->
+	    getifaddrs_ifget(S, IFs, IF, [], [addr,netmask,hwaddr])
+    end.
+
+getifaddrs_ifget(S, IFs, IF, FlagsVals, Opts) ->
+    OptVals =
+	case ifget(S, IF, Opts) of
+	    {ok,OVs} -> OVs;
+	    _ -> []
+	end,
+    [{IF,FlagsVals++OptVals}|getifaddrs_ifget(S, IFs)].
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%
+%% getiflist(insock()) -> {ok,IfNameList} | {error, Reason}
 %%
 %% get interface name list
 %%
@@ -645,7 +738,7 @@ chgopts(S, Opts) when is_port(S), is_list(Opts) ->
 getiflist(S) when is_port(S) ->
     case ctl_cmd(S, ?INET_REQ_GETIFLIST, []) of
 	{ok, Data} -> {ok, build_iflist(Data)};
-	Error -> Error
+	{error,_}=Error -> Error
     end.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -663,7 +756,7 @@ ifget(S, Name, Opts) ->
 		{ok, Buf2} ->
 		    case ctl_cmd(S, ?INET_REQ_IFGET, [Buf1,Buf2]) of
 			{ok, Data} -> decode_ifopts(Data,[]);
-			Error -> Error
+			{error,_}=Error -> Error
 		    end;
 		Error -> Error
 	    end;
@@ -685,7 +778,7 @@ ifset(S, Name, Opts) ->
 		{ok, Buf2} ->
 		    case ctl_cmd(S, ?INET_REQ_IFSET, [Buf1,Buf2]) of
 			{ok, _} -> ok;
-			Error -> Error
+			{error,_}=Error -> Error
 		    end;
 		Error -> Error
 	    end;
@@ -713,7 +806,7 @@ subscribe(S, Sub) when is_port(S), is_list(Sub) ->
 	{ok, Bytes} ->
 	    case ctl_cmd(S, ?INET_REQ_SUBSCRIBE, Bytes) of
 		{ok, Data} -> decode_subs(Data);
-		Error -> Error
+		{error,_}=Error -> Error
 	    end;
 	Error -> Error
     end.
@@ -731,7 +824,7 @@ getstat(S, Stats) when is_port(S), is_list(Stats) ->
 	{ok, Bytes} ->
 	    case ctl_cmd(S, ?INET_REQ_GETSTAT, Bytes) of
 		{ok, Data} -> decode_stats(Data);
-		Error -> Error
+		{error,_}=Error -> Error
 	    end;
 	Error -> Error
     end.
@@ -747,11 +840,26 @@ getstat(S, Stats) when is_port(S), is_list(Stats) ->
 getfd(S) when is_port(S) ->
     case ctl_cmd(S, ?INET_REQ_GETFD, []) of
 	{ok, [S3,S2,S1,S0]} -> {ok, ?u32(S3,S2,S1,S0)};
-	Error -> Error
+	{error,_}=Error -> Error
     end.        
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%
+%% IGNOREFD(insock(),boolean()) -> {ok,integer()} | {error, Reason}
+%%
+%% steal internal file descriptor
+%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+ignorefd(S,Bool) when is_port(S) ->
+    Val = if Bool -> 1; true -> 0 end,
+    case ctl_cmd(S, ?INET_REQ_IGNOREFD, [Val]) of
+	{ok, _} -> ok;
+	Error -> Error
+    end.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%
 %% GETIX(insock()) -> {ok,integer()} | {error, Reason}
 %%
 %% get internal socket index
@@ -785,7 +893,7 @@ gettype(S) when is_port(S) ->
 			_		     -> undefined
 		   end,
 	    {ok, {Family, Type}};
-	Error -> Error
+	{error,_}=Error -> Error
     end.
 
 getprotocol(S) when is_port(S) ->
@@ -813,7 +921,7 @@ getstatus(S) when is_port(S) ->
     case ctl_cmd(S, ?INET_REQ_GETSTATUS, []) of
 	{ok, [S3,S2,S1,S0]} ->	
 	    {ok, dec_status(?u32(S3,S2,S1,S0))};
-	Error -> Error
+	{error,_}=Error -> Error
     end.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -855,7 +963,7 @@ getservbyname1(S,Name,Proto) ->
 	    case ctl_cmd(S, ?INET_REQ_GETSERVBYNAME, [L1,Name,L2,Proto]) of
 		{ok, [P1,P0]} ->
 		    {ok, ?u16(P1,P0)};
-		Error -> 
+		{error,_}=Error ->
 		    Error
 	    end
     end.
@@ -883,7 +991,7 @@ getservbyport1(S,Port,Proto) ->
        true ->
 	    case ctl_cmd(S, ?INET_REQ_GETSERVBYPORT, [?int16(Port),L,Proto]) of
 		{ok, Name} -> {ok, Name};
-		Error -> Error
+		{error,_}=Error -> Error
 	    end
     end.
 
@@ -897,7 +1005,7 @@ getservbyport1(S,Port,Proto) ->
 unrecv(S, Data) ->
     case ctl_cmd(S, ?TCP_REQ_UNRECV, Data) of
 	{ok, _} -> ok;
-	Error  -> Error
+	{error,_}=Error  -> Error
     end.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -1206,7 +1314,7 @@ type_opt_1(sctp_default_send_param) ->
 	timetolive        = [uint32,0],
 	tsn               = [],
 	cumtsn            = [],
-	assoc_id          = [sctp_assoc_id,0]}}];
+	assoc_id          = [[sctp_assoc_id,0]]}}];
 %% for SCTP_OPT_EVENTS
 type_opt_1(sctp_events) ->
     [{record,#sctp_event_subscribe{
@@ -1325,6 +1433,19 @@ type_value_2({enum,List}, Enum) ->
 	{value,_} 				    -> true;
 	false                                       -> false
     end;
+type_value_2(sockaddr, Addr) ->
+    case Addr of
+	any                                         -> true;
+	loopback                                    -> true;
+	{A,B,C,D} when ?ip(A,B,C,D)                 -> true;
+	{A,B,C,D,E,F,G,H} when ?ip6(A,B,C,D,E,F,G,H) -> true;
+	_                                           -> false
+    end;
+type_value_2(linkaddr, Addr) when is_list(Addr) ->
+    case len(Addr, 32768) of
+	undefined                                   -> false;
+	_                                           -> true
+    end;
 type_value_2({bitenumlist,List}, EnumList) -> 
     case enum_vals(EnumList, List) of
 	Ls when is_list(Ls)                         -> true;
@@ -1413,14 +1534,21 @@ enc_value_2(addr, {any,Port}) ->
     [?INET_AF_ANY|?int16(Port)];
 enc_value_2(addr, {loopback,Port}) ->
     [?INET_AF_LOOPBACK|?int16(Port)];
-enc_value_2(addr, {IP,Port}) ->
-    case tuple_size(IP) of
-	4 ->
-	    [?INET_AF_INET,?int16(Port)|ip4_to_bytes(IP)];
-	8 ->
-	    [?INET_AF_INET6,?int16(Port)|ip6_to_bytes(IP)]
-    end;
-enc_value_2(ether, [X1,X2,X3,X4,X5,X6]) -> [X1,X2,X3,X4,X5,X6];
+enc_value_2(addr, {IP,Port}) when tuple_size(IP) =:= 4 ->
+    [?INET_AF_INET,?int16(Port)|ip4_to_bytes(IP)];
+enc_value_2(addr, {IP,Port}) when tuple_size(IP) =:= 8 ->
+    [?INET_AF_INET6,?int16(Port)|ip6_to_bytes(IP)];
+enc_value_2(ether, [_,_,_,_,_,_]=Xs) -> Xs;
+enc_value_2(sockaddr, any) ->
+    [?INET_AF_ANY];
+enc_value_2(sockaddr, loopback) ->
+    [?INET_AF_LOOPBACK];
+enc_value_2(sockaddr, IP) when tuple_size(IP) =:= 4 ->
+    [?INET_AF_INET|ip4_to_bytes(IP)];
+enc_value_2(sockaddr, IP) when tuple_size(IP) =:= 8 ->
+    [?INET_AF_INET6|ip6_to_bytes(IP)];
+enc_value_2(linkaddr, Linkaddr) ->
+    [?int16(length(Linkaddr)),Linkaddr];
 enc_value_2(sctp_assoc_id, Val) -> ?int32(Val);
 %% enc_value_2(sctp_assoc_id, Bin) -> [byte_size(Bin),Bin];
 enc_value_2({enum,List}, Enum) ->
@@ -1464,7 +1592,11 @@ dec_value(time, [X3,X2,X1,X0|T]) ->
 	Val -> {Val, T}
     end;
 dec_value(ip, [A,B,C,D|T])             -> {{A,B,C,D}, T};
-dec_value(ether,[X1,X2,X3,X4,X5,X6|T]) -> {[X1,X2,X3,X4,X5,X6],T};
+%% dec_value(ether, [X1,X2,X3,X4,X5,X6|T]) -> {[X1,X2,X3,X4,X5,X6],T};
+dec_value(sockaddr, [X|T]) ->
+    get_ip(X, T);
+dec_value(linkaddr, [X1,X0|T]) ->
+    split(?i16(X1,X0), T);
 dec_value({enum,List}, [X3,X2,X1,X0|T]) ->
     Val = ?i32(X3,X2,X1,X0),
     case enum_name(Val, List) of
@@ -1480,7 +1612,7 @@ dec_value({bitenumlist,List}, [X3,X2,X1,X0|T]) ->
 %%     {enum_names(Val, List), T};
 dec_value(binary,[L0,L1,L2,L3|List]) ->
     Len = ?i32(L0,L1,L2,L3),
-    {X,T}=lists:split(Len,List),
+    {X,T}=split(Len,List),
     {list_to_binary(X),T};
 dec_value(Types, List) when is_tuple(Types) ->
     {L,T} = dec_value_tuple(Types, List, 1, []),
@@ -1495,7 +1627,7 @@ dec_value_tuple(Types, List, N, Acc)
     {Term,Tail} = dec_value(element(N, Types), List),
     dec_value_tuple(Types, Tail, N+1, [Term|Acc]);
 dec_value_tuple(_, List, _, Acc) ->
-    {lists:reverse(Acc),List}.
+    {rev(Acc),List}.
 
 borlist([V|Vs], Value) ->
     borlist(Vs, V bor Value);
@@ -1702,11 +1834,11 @@ merge_fields(_, _, _) -> [].
 %%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-type_ifopt(addr)      -> ip;
-type_ifopt(broadaddr) -> ip;
-type_ifopt(dstaddr)   -> ip;
+type_ifopt(addr)      -> sockaddr;
+type_ifopt(broadaddr) -> sockaddr;
+type_ifopt(dstaddr)   -> sockaddr;
 type_ifopt(mtu)       -> int;
-type_ifopt(netmask)   -> ip;
+type_ifopt(netmask)   -> sockaddr;
 type_ifopt(flags)     ->
     {bitenumlist,
      [{up, ?INET_IFF_UP},
@@ -1718,7 +1850,7 @@ type_ifopt(flags)     ->
       {no_pointtopoint, ?INET_IFF_NPOINTTOPOINT},
       {running, ?INET_IFF_RUNNING},
       {multicast, ?INET_IFF_MULTICAST}]};
-type_ifopt(hwaddr)    -> ether;
+type_ifopt(hwaddr)    -> linkaddr;
 type_ifopt(Opt) when is_atom(Opt) -> undefined.
 
 enc_ifopt(addr)      -> ?INET_IFOPT_ADDR;
@@ -1903,6 +2035,30 @@ encode_ifname(Name) ->
     if N > 255 -> {error, einval};
        true -> {ok,[N | Name]}
     end.
+
+build_ifaddrs(Cs) ->
+    build_ifaddrs(Cs, []).
+%%
+build_ifaddrs([], []) ->
+    [];
+build_ifaddrs([0|Cs], Acc) ->
+    Name = utf8_to_characters(rev(Acc)),
+    {Opts,Rest} = build_ifaddrs_opts(Cs, []),
+    [{Name,Opts}|build_ifaddrs(Rest)];
+build_ifaddrs([C|Cs], Acc) ->
+    build_ifaddrs(Cs, [C|Acc]).
+
+build_ifaddrs_opts([0|Cs], Acc) ->
+    {rev(Acc),Cs};
+build_ifaddrs_opts([C|Cs]=CCs, Acc) ->
+    case dec_ifopt(C) of
+	undefined ->
+	    erlang:error(badarg, [CCs,Acc]);
+	Opt ->
+	    Type = type_ifopt(Opt),
+	    {Val,Rest} = dec_value(Type, Cs),
+	    build_ifaddrs_opts(Rest, [{Opt,Val}|Acc])
+    end.
     
 build_iflist(Cs) ->
     build_iflist(Cs, [], []).
@@ -1927,6 +2083,80 @@ rev(L) -> rev(L,[]).
 rev([C|L],Acc) -> rev(L,[C|Acc]);
 rev([],Acc) -> Acc.
 
+split(N, L) -> split(N, L, []).
+split(0, L, R) when is_list(L) -> {rev(R),L};
+split(N, [H|T], R) when is_integer(N), N > 0 -> split(N-1, T, [H|R]).
+
+len(L, N) -> len(L, N, 0).
+len([], N, C) when is_integer(N), N >= 0 -> C;
+len(L, 0, _) when is_list(L) -> undefined;
+len([_|L], N, C) when is_integer(N), N >= 0 -> len(L, N-1, C+1).
+
+member(X, [X|_]) -> true;
+member(X, [_|Xs]) -> member(X, Xs);
+member(_, []) -> false.
+
+
+
+%% Lookup tree that keeps key insert order
+
+ktree_empty() -> {[],tree()}.
+ktree_is_defined(Key, {_,T}) -> tree(T, Key, is_defined).
+ktree_get(Key, {_,T}) -> tree(T, Key, get).
+ktree_insert(Key, V, {Keys,T}) -> {[Key|Keys],tree(T, Key, {insert,V})}.
+ktree_update(Key, V, {Keys,T}) -> {Keys,tree(T, Key, {update,V})}.
+ktree_keys({Keys,_}) -> rev(Keys).
+
+%% Simple lookup tree. Hash the key to get statistical balance.
+%% Key is matched equal, not compared equal.
+
+tree() -> nil.
+tree(T, Key, Op) -> tree(T, Key, Op, erlang:phash2(Key)).
+
+tree(nil, _, is_defined, _) -> false;
+tree(nil, K, {insert,V}, _) -> {K,V,nil,nil};
+tree({K,_,_,_}, K, is_defined, _) -> true;
+tree({K,V,_,_}, K, get, _) -> V;
+tree({K,_,L,R}, K, {update,V}, _) -> {K,V,L,R};
+tree({K0,V0,L,R}, K, Op, H) ->
+    H0 = erlang:phash2(K0),
+    if  H0 < H;  H0 =:= H, K0 < K ->
+	    if  is_tuple(Op) ->
+		    {K0,V0,tree(L, K, Op, H),R};
+		true ->
+		    tree(L, K, Op, H)
+	    end;
+	true ->
+	    if  is_tuple(Op) ->
+		    {K0,V0,L,tree(R, K, Op, H)};
+		true ->
+		    tree(R, K, Op, H)
+	    end
+    end.
+
+
+
+utf8_to_characters([]) -> [];
+utf8_to_characters([B|Bs]=Arg) when (B band 16#FF) =:= B ->
+    if  16#F8 =< B ->
+	    erlang:error(badarg, [Arg]);
+	16#F0 =< B ->
+	    utf8_to_characters(Bs, B band 16#07, 3);
+	16#E0 =< B ->
+	    utf8_to_characters(Bs, B band 16#0F, 2);
+	16#C0 =< B ->
+	    utf8_to_characters(Bs, B band 16#1F, 1);
+	16#80 =< B ->
+	    erlang:error(badarg, [Arg]);
+	true ->
+	    [B|utf8_to_characters(Bs)]
+    end.
+%%
+utf8_to_characters(Bs, U, 0) ->
+    [U|utf8_to_characters(Bs)];
+utf8_to_characters([B|Bs], U, N) when ((B band 16#3F) bor 16#80) =:= B ->
+    utf8_to_characters(Bs, (U bsl 6) bor (B band 16#3F), N-1).
+
 ip_to_bytes(IP) when tuple_size(IP) =:= 4 -> ip4_to_bytes(IP);
 ip_to_bytes(IP) when tuple_size(IP) =:= 8 -> ip6_to_bytes(IP).
 
@@ -1953,7 +2183,7 @@ ctl_cmd(Port, Cmd, Args) ->
     Result =
 	try erlang:port_control(Port, Cmd, Args) of
 	    [?INET_REP_OK|Reply]  -> {ok,Reply};
-	    [?INET_REP_SCTP]  -> {error,sctp_reply};
+	    [?INET_REP]  -> inet_reply;
 	    [?INET_REP_ERROR|Err] -> {error,list_to_atom(Err)}
 	catch
 	    error:_               -> {error,einval}
diff --git a/erts/preloaded/src/prim_zip.erl b/erts/preloaded/src/prim_zip.erl
index 3f5a5b9721..d29f17ae56 100644
--- a/erts/preloaded/src/prim_zip.erl
+++ b/erts/preloaded/src/prim_zip.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2008-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -21,7 +21,7 @@
 
 -module(prim_zip).
 
-%% unzipping piecemal
+%% unzipping piecemeal
 -export([
 	 open/1,
 	 open/3,
@@ -65,34 +65,55 @@ filter_fun() ->
 open(F) ->
     open(filter_fun(), undefined, F).
 
-open(FilterFun, FilterAcc, F) ->
-    case ?CATCH do_open(FilterFun, FilterAcc, F) of
-	{ok, PrimZip, Acc} ->
-	    {ok, PrimZip, Acc};
-	Error ->
-	    {error, Error}
-    end.
+open(FilterFun, FilterAcc, F) when is_function(FilterFun, 2) ->
+    try
+	do_open(FilterFun, FilterAcc, F)
+    catch
+	throw:{filter_fun_throw, Reason} ->
+	    throw(Reason);
+	throw:InternalReason ->
+	    {error, InternalReason};
+	Class:Reason ->
+	    erlang:error(erlang:raise(Class, Reason, erlang:get_stacktrace()))
+    end;
+open(_, _, _) ->
+    {error, einval}.
 
 do_open(FilterFun, FilterAcc, F) ->
     Input = get_zip_input(F),
     In0 = Input({open, F, [read, binary, raw]}, []),
     Z = zlib:open(),
     PrimZip = #primzip{files = [], zlib = Z, in = In0, input = Input},
-    {PrimZip2, FilterAcc2} = get_central_dir(PrimZip, FilterFun, FilterAcc),
-    {ok, PrimZip2, FilterAcc2}.
+    try
+	{PrimZip2, FilterAcc2} = get_central_dir(PrimZip, FilterFun, FilterAcc),
+	{ok, PrimZip2, FilterAcc2}
+    catch
+	Class:Reason ->
+	    close(PrimZip),
+	    erlang:error(erlang:raise(Class, Reason, erlang:get_stacktrace()))
+    end.
 
 %% iterate over all files in a zip archive
-foldl(FilterFun, FilterAcc, #primzip{files = Files} = PrimZip) ->
-    case ?CATCH do_foldl(FilterFun, FilterAcc, Files, [], PrimZip, PrimZip) of
-	{ok, FilterAcc2, PrimZip2} -> {ok, PrimZip2, FilterAcc2};
-	Error                      -> {error, Error}
+foldl(FilterFun, FilterAcc, #primzip{files = Files} = PrimZip)
+  when is_function(FilterFun, 2) ->
+    try
+	{ok, FilterAcc2, PrimZip2} =
+	    do_foldl(FilterFun, FilterAcc, Files, [], PrimZip, PrimZip),
+	{ok, PrimZip2, FilterAcc2}
+    catch
+	throw:{filter_fun_throw, Reason} ->
+	    throw(Reason);
+	throw:InternalReason ->
+	    {error, InternalReason};
+	Class:Reason ->
+	    erlang:error(erlang:raise(Class, Reason, erlang:get_stacktrace()))
     end;
 foldl(_, _, _) ->
     {error, einval}.
 
 do_foldl(FilterFun, FilterAcc, [PF | Tail], Acc0, PrimZip, PrimZipOrig) ->
     #primzip_file{name = F, get_info = GetInfo, get_bin = GetBin} = PF,
-    case FilterFun({F, GetInfo, GetBin}, FilterAcc) of
+    try FilterFun({F, GetInfo, GetBin}, FilterAcc) of
 	{Continue, Include, FilterAcc2} ->
 	    Acc1 = include_acc(Include, PF, Acc0),
 	    case Continue of
@@ -103,6 +124,9 @@ do_foldl(FilterFun, FilterAcc, [PF | Tail], Acc0, PrimZip, PrimZipOrig) ->
 	    end;
 	FilterRes ->
 	    throw({illegal_filter, FilterRes})
+    catch
+	throw:Reason ->
+	    throw({filter_fun_throw, Reason})
     end;
 do_foldl(_FilterFun, FilterAcc, [], Acc, PrimZip, _PrimZipOrig) ->
     {ok, FilterAcc, PrimZip#primzip{files = reverse(Acc)}}.
@@ -121,12 +145,14 @@ include_acc(Include, PF, Acc) ->
 	List when is_list(List) ->
 	    %% Add new entries
 	    Fun = fun(I, A) -> include_acc(I, PF, A) end,
-	    lists_foldl(Fun, Acc, List)
+	    lists_foldl(Fun, Acc, List);
+	Bad ->
+	    throw({illegal_filter, Bad})
     end.
 
 lists_foldl(F, Accu, [Hd|Tail]) ->
     lists_foldl(F, F(Hd, Accu), Tail);
-lists_foldl(F, Accu, []) when is_function(F, 2) -> 
+lists_foldl(F, Accu, []) when is_function(F, 2) ->
     Accu.
 
 %% close a zip archive
@@ -139,7 +165,9 @@ close(_) ->
 get_zip_input({F, B}) when is_binary(B), is_list(F) ->
     fun binary_io/2;
 get_zip_input(F) when is_list(F) ->
-    fun prim_file_io/2.
+    fun prim_file_io/2;
+get_zip_input(_) ->
+    throw(einval).
 
 %% get a file from the archive
 get_z_file(F, Offset, ChunkSize, #primzip{zlib = Z, in = In0, input = Input}) ->
@@ -218,15 +246,15 @@ get_cd_loop(N, BCD, Acc0, PrimZip, FileName, Offset, CFH, EndOffset, FilterFun,
     GetInfo = fun() -> cd_file_header_to_file_info(FileName, CFH, <<>>) end,
     GetBin = fun() -> get_z_file(FileName, Offset, Size, PrimZip) end,
     PF = #primzip_file{name = FileName, get_info = GetInfo, get_bin = GetBin},
-    case FilterFun({FileName, GetInfo, GetBin}, FilterAcc) of
+    try FilterFun({FileName, GetInfo, GetBin}, FilterAcc) of
 	{Continue, Include, FilterAcc2} ->
 	    Acc1 =
 		case Include of
-		    false -> 
+		    false ->
 			Acc0;
 		    true ->
 			[PF | Acc0];
-		    {true, Nick} -> 
+		    {true, Nick} ->
 			[PF#primzip_file{name = Nick} | Acc0];
 		    {true, Nick, GI, GB} ->
 			PF2 = #primzip_file{name = Nick, get_info = GI, get_bin = GB},
@@ -247,13 +275,16 @@ get_cd_loop(N, BCD, Acc0, PrimZip, FileName, Offset, CFH, EndOffset, FilterFun,
 	    end;
 	FilterRes ->
 	    throw({illegal_filter, FilterRes})
+    catch
+	throw:Reason ->
+	    throw({filter_fun_throw, Reason})
     end.
 
 get_file_header(BCD) ->
     BCFH =
 	case BCD of
 	    <<?CENTRAL_FILE_MAGIC:32/little,
-	     B:(?CENTRAL_FILE_HEADER_SZ-4)/binary, 
+	     B:(?CENTRAL_FILE_HEADER_SZ-4)/binary,
 	     _/binary>> ->
 		B;
 	    _ ->
@@ -266,11 +297,11 @@ get_file_header(BCD) ->
     ToGet = FileNameLen + ExtraLen + CommentLen,
     {B2, BCDRest} =
 	case BCD of
-	    <<_:?CENTRAL_FILE_HEADER_SZ/binary, 
+	    <<_:?CENTRAL_FILE_HEADER_SZ/binary,
 	     G:ToGet/binary,
-	     Rest/binary>> -> 
+	     Rest/binary>> ->
 		{G, Rest};
-	    _ -> 
+	    _ ->
 		throw(bad_central_directory)
 	end,
     FileName = get_filename_from_b2(B2, FileNameLen, ExtraLen, CommentLen),
@@ -319,9 +350,9 @@ prim_file_io({file_info, F}, _) ->
     end;
 prim_file_io({open, FN, Opts}, _) ->
     case ?CATCH prim_file:open(FN, Opts++[binary]) of
-	{ok, H} -> 
+	{ok, H} ->
 	    H;
-	{error, E} -> 
+	{error, E} ->
 	    throw(E)
     end;
 prim_file_io({read, N}, H) ->
@@ -401,7 +432,7 @@ binary_io({file_info, B}, _) ->
 	    is_binary(B) -> {regular, byte_size(B)};
 	    B =:= directory -> {directory, 0}
 	end,
-    Now = calendar:local_time(),
+    Now = erlang:localtime(),
     #file_info{size = Size, type = Type, access = read_write,
 	       atime = Now, mtime = Now, ctime = Now,
 	       mode = 0, links = 1, major_device = 0,
@@ -476,7 +507,7 @@ cd_file_header_to_file_info(FileName,
 %%     FI;     % not yet supported, and not widely used
 add_extra_info(FI, _) ->
     FI.
-%% 
+%%
 %% unix_extra_field_and_var_from_bin(<<TSize:16/little,
 %% 				   ATime:32/little,
 %% 				   MTime:32/little,
@@ -500,7 +531,7 @@ dos_date_time_to_datetime(DosDate, DosTime) ->
     <<Hour:5, Min:6, Sec:5>> = <<DosTime:16>>,
     <<YearFrom1980:7, Month:4, Day:5>> = <<DosDate:16>>,
     {{YearFrom1980+1980, Month, Day},
-     {Hour, Min, Sec}}. 
+     {Hour, Min, Sec}}.
 
 cd_file_header_from_bin(<<VersionMadeBy:16/little,
 			 VersionNeeded:16/little,
@@ -622,7 +653,7 @@ reverse(X) ->
 
 reverse([H|T], Y) ->
     reverse(T, [H|Y]);
-reverse([], X) -> 
+reverse([], X) ->
     X.
 
 last([E|Es]) -> last(E, Es).
diff --git a/erts/preloaded/src/zlib.erl b/erts/preloaded/src/zlib.erl
index 51d6cd0a0b..1faae1c1f4 100644
--- a/erts/preloaded/src/zlib.erl
+++ b/erts/preloaded/src/zlib.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2003-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2003-2012. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -124,7 +124,6 @@
 -type zwindowbits() :: -15..-9 | 9..47.
 -type zmemlevel()   :: 1..9.
 -type zstrategy()   :: 'default' | 'filtered' | 'huffman_only'.
--type zflush()      :: 'none' | 'sync' | 'full' | 'finish'.
 
 %%------------------------------------------------------------------------
 
@@ -134,7 +133,8 @@ open() ->
     open_port({spawn, "zlib_drv"}, [binary]).
 
 %% close and release z_stream
--spec close(zstream()) -> 'ok'.
+-spec close(Z) -> 'ok' when
+      Z :: zstream().
 close(Z) ->
     try
 	true = port_close(Z),
@@ -145,16 +145,25 @@ close(Z) ->
     catch _:_ -> erlang:error(badarg)
     end.
 
--spec deflateInit(zstream()) -> 'ok'.
+-spec deflateInit(Z) -> 'ok' when
+      Z :: zstream().
 deflateInit(Z) ->
     call(Z, ?DEFLATE_INIT, <<?Z_DEFAULT_COMPRESSION:32>>).
 
--spec deflateInit(zstream(), zlevel()) -> 'ok'.
+-spec deflateInit(Z, Level) -> 'ok' when
+      Z :: zstream(),
+      Level :: zlevel().
 deflateInit(Z, Level) ->
     call(Z, ?DEFLATE_INIT, <<(arg_level(Level)):32>>).
 
--spec deflateInit(zstream(), zlevel(), zmethod(),
-		  zwindowbits(), zmemlevel(), zstrategy()) -> 'ok'.
+-spec deflateInit(Z, Level, Method,
+		  WindowBits, MemLevel, Strategy) -> 'ok' when
+      Z :: zstream(),
+      Level :: zlevel(),
+      Method :: zmethod(),
+      WindowBits :: zwindowbits(),
+      MemLevel :: zmemlevel(),
+      Strategy :: zstrategy().
 deflateInit(Z, Level, Method, WindowBits, MemLevel, Strategy) ->
     call(Z, ?DEFLATE_INIT2, <<(arg_level(Level)):32, 
 			     (arg_method(Method)):32,
@@ -162,24 +171,38 @@ deflateInit(Z, Level, Method, WindowBits, MemLevel, Strategy) ->
 			     (arg_mem(MemLevel)):32,
 			     (arg_strategy(Strategy)):32>>).
 
--spec deflateSetDictionary(zstream(), binary()) -> integer().
+-spec deflateSetDictionary(Z, Dictionary) -> Adler32 when
+      Z :: zstream(),
+      Dictionary :: iodata(),
+      Adler32 :: integer().
 deflateSetDictionary(Z, Dictionary) ->
     call(Z, ?DEFLATE_SETDICT, Dictionary).
 
--spec deflateReset(zstream()) -> 'ok'.
+-spec deflateReset(Z) -> 'ok' when
+      Z :: zstream().
 deflateReset(Z) ->
     call(Z, ?DEFLATE_RESET, []).
 
--spec deflateParams(zstream(), zlevel(), zstrategy()) -> 'ok'.
+-spec deflateParams(Z, Level, Strategy) -> ok when
+      Z :: zstream(),
+      Level :: zlevel(),
+      Strategy :: zstrategy().
 deflateParams(Z, Level, Strategy) ->
     call(Z, ?DEFLATE_PARAMS, <<(arg_level(Level)):32, 
 			      (arg_strategy(Strategy)):32>>).
 
--spec deflate(zstream(), iodata()) -> iolist().
+-spec deflate(Z, Data) -> Compressed when
+      Z :: zstream(),
+      Data :: iodata(),
+      Compressed :: iolist().
 deflate(Z, Data) ->
     deflate(Z, Data, none).
 
--spec deflate(zstream(), iodata(), zflush()) -> iolist().
+-spec deflate(Z, Data, Flush) -> Compressed when
+      Z :: zstream(),
+      Data :: iodata(),
+      Flush :: none | sync | full | finish,
+      Compressed :: iolist().
 deflate(Z, Data, Flush) ->
     try port_command(Z, Data) of
 	true ->
@@ -191,19 +214,25 @@ deflate(Z, Data, Flush) ->
 	    erlang:error(badarg) 
     end.
 
--spec deflateEnd(zstream()) -> 'ok'.
+-spec deflateEnd(Z) -> 'ok' when
+      Z :: zstream().
 deflateEnd(Z) ->
     call(Z, ?DEFLATE_END, []).    
 
--spec inflateInit(zstream()) -> 'ok'.
+-spec inflateInit(Z) -> 'ok' when
+      Z :: zstream().
 inflateInit(Z) ->
     call(Z, ?INFLATE_INIT, []).
 
--spec inflateInit(zstream(), zwindowbits()) -> 'ok'.
+-spec inflateInit(Z, WindowBits) -> 'ok' when
+      Z :: zstream(),
+      WindowBits :: zwindowbits().
 inflateInit(Z, WindowBits) -> 
     call(Z, ?INFLATE_INIT2, <<(arg_bitsz(WindowBits)):32>>).
 
--spec inflateSetDictionary(zstream(), binary()) -> 'ok'.
+-spec inflateSetDictionary(Z, Dictionary) -> 'ok' when
+      Z :: zstream(),
+      Dictionary :: iodata().
 inflateSetDictionary(Z, Dictionary) -> 
     call(Z, ?INFLATE_SETDICT, Dictionary).
 
@@ -211,11 +240,15 @@ inflateSetDictionary(Z, Dictionary) ->
 inflateSync(Z) -> 
     call(Z, ?INFLATE_SYNC, []).
 
--spec inflateReset(zstream()) -> 'ok'.
+-spec inflateReset(Z) -> 'ok' when
+      Z :: zstream().
 inflateReset(Z) -> 
     call(Z, ?INFLATE_RESET, []).
 
--spec inflate(zstream(), iodata()) -> iolist().
+-spec inflate(Z, Data) -> Decompressed when
+      Z :: zstream(),
+      Data :: iodata(),
+      Decompressed :: iolist().
 inflate(Z, Data) ->
     try port_command(Z, Data) of
 	true -> 
@@ -227,50 +260,79 @@ inflate(Z, Data) ->
 	    erlang:error(badarg) 
     end.
 
--spec inflateEnd(zstream()) -> 'ok'.
+-spec inflateEnd(Z) -> 'ok' when
+      Z :: zstream().
 inflateEnd(Z) ->
     call(Z, ?INFLATE_END, []).
 
--spec setBufSize(zstream(), non_neg_integer()) -> 'ok'.
+-spec setBufSize(Z, Size) -> 'ok' when
+      Z :: zstream(),
+      Size :: non_neg_integer().
 setBufSize(Z, Size) ->
     call(Z, ?SET_BUFSZ, <<Size:32>>).
 
--spec getBufSize(zstream()) -> non_neg_integer().
+-spec getBufSize(Z) -> Size when
+      Z :: zstream(),
+      Size :: non_neg_integer().
 getBufSize(Z) ->
     call(Z, ?GET_BUFSZ, []).
 
--spec crc32(zstream()) -> integer().
+-spec crc32(Z) -> CRC when
+      Z :: zstream(),
+      CRC :: integer().
 crc32(Z) ->
     call(Z, ?CRC32_0, []).
 
--spec crc32(zstream(), binary()) -> integer().
-crc32(Z, Binary) ->
-    call(Z, ?CRC32_1, Binary).
-
--spec crc32(zstream(), integer(), binary()) -> integer().
-crc32(Z, CRC, Binary) when is_binary(Binary), is_integer(CRC) ->
-    call(Z, ?CRC32_2, <<CRC:32, Binary/binary>>);
-crc32(_Z, _CRC, _Binary)  ->
-    erlang:error(badarg).
-
--spec adler32(zstream(), binary()) -> integer().
-adler32(Z, Binary) ->
-    call(Z, ?ADLER32_1, Binary).
-
--spec adler32(zstream(), integer(), binary()) -> integer().
-adler32(Z, Adler, Binary) when is_binary(Binary), is_integer(Adler) ->
-    call(Z, ?ADLER32_2, <<Adler:32, Binary/binary>>);
-adler32(_Z, _Adler, _Binary)  ->
+-spec crc32(Z, Data) -> CRC when
+      Z :: zstream(),
+      Data :: iodata(),
+      CRC :: integer().
+crc32(Z, Data) ->
+    call(Z, ?CRC32_1, Data).
+
+-spec crc32(Z, PrevCRC, Data) -> CRC when
+      Z :: zstream(),
+      PrevCRC :: integer(),
+      Data :: iodata(),
+      CRC :: integer().
+crc32(Z, CRC, Data) ->
+    call(Z, ?CRC32_2, [<<CRC:32>>, Data]).
+
+-spec adler32(Z, Data) -> CheckSum when
+      Z :: zstream(),
+      Data :: iodata(),
+      CheckSum :: integer().
+adler32(Z, Data) ->
+    call(Z, ?ADLER32_1, Data).
+
+-spec adler32(Z, PrevAdler, Data) -> CheckSum when
+      Z :: zstream(),
+      PrevAdler :: integer(),
+      Data :: iodata(),
+      CheckSum :: integer().
+adler32(Z, Adler, Data) when is_integer(Adler) ->
+    call(Z, ?ADLER32_2, [<<Adler:32>>, Data]);
+adler32(_Z, _Adler, _Data)  ->
     erlang:error(badarg).
 
--spec crc32_combine(zstream(), integer(), integer(), integer()) -> integer().
+-spec crc32_combine(Z, CRC1, CRC2, Size2) -> CRC when
+      Z :: zstream(),
+      CRC :: integer(),
+      CRC1 :: integer(),
+      CRC2 :: integer(),
+      Size2 :: integer().
 crc32_combine(Z, CRC1, CRC2, Len2) 
   when is_integer(CRC1), is_integer(CRC2), is_integer(Len2) ->
     call(Z, ?CRC32_COMBINE, <<CRC1:32, CRC2:32, Len2:32>>);
 crc32_combine(_Z, _CRC1, _CRC2, _Len2) ->
     erlang:error(badarg).
 
--spec adler32_combine(zstream(), integer(), integer(), integer()) -> integer().
+-spec adler32_combine(Z, Adler1, Adler2, Size2) -> Adler when
+      Z :: zstream(),
+      Adler :: integer(),
+      Adler1 :: integer(),
+      Adler2 :: integer(),
+      Size2 :: integer().
 adler32_combine(Z, Adler1, Adler2, Len2) 
   when is_integer(Adler1), is_integer(Adler2), is_integer(Len2) ->
     call(Z, ?ADLER32_COMBINE, <<Adler1:32, Adler2:32, Len2:32>>);
@@ -282,64 +344,107 @@ getQSize(Z) ->
     call(Z, ?GET_QSIZE, []).
 
 %% compress/uncompress zlib with header
--spec compress(binary()) -> binary().
-compress(Binary) ->
-    Z = open(),
-    deflateInit(Z, default),
-    Bs = deflate(Z, Binary,finish),
-    deflateEnd(Z),
-    close(Z),
-    list_to_binary(Bs).
-
--spec uncompress(binary()) -> binary().
-uncompress(Binary) when byte_size(Binary) >= 8 ->
+-spec compress(Data) -> Compressed when
+      Data :: iodata(),
+      Compressed :: binary().
+compress(Data) ->
     Z = open(),
-    inflateInit(Z),
-    Bs = inflate(Z, Binary),
-    inflateEnd(Z),
-    close(Z),
-    list_to_binary(Bs);
-uncompress(Binary) when is_binary(Binary) -> erlang:error(data_error);
-uncompress(_) -> erlang:error(badarg).
+    Bs = try
+	     deflateInit(Z, default),
+	     B = deflate(Z, Data, finish),
+	     deflateEnd(Z),
+	     B
+	 after
+	     close(Z)
+	 end,
+    iolist_to_binary(Bs).
+
+-spec uncompress(Data) -> Decompressed when
+      Data  :: iodata(),
+      Decompressed :: binary().
+uncompress(Data) ->
+    try iolist_size(Data) of
+        Size ->
+            if
+                Size >= 8 ->
+                    Z = open(),
+		    Bs = try
+			     inflateInit(Z),
+			     B = inflate(Z, Data),
+			     inflateEnd(Z),
+			     B
+			 after
+			     close(Z)
+			 end,
+                    iolist_to_binary(Bs);
+                true ->
+                    erlang:error(data_error)
+            end
+    catch
+        _:_ ->
+            erlang:error(badarg)
+    end.
 
 %% unzip/zip zlib without header (zip members)
--spec zip(binary()) -> binary().
-zip(Binary) ->
+-spec zip(Data) -> Compressed when
+      Data :: iodata(),
+      Compressed :: binary().
+zip(Data) ->
     Z = open(),
-    deflateInit(Z, default, deflated, -?MAX_WBITS, 8, default),
-    Bs = deflate(Z, Binary, finish),
-    deflateEnd(Z),
-    close(Z),
-    list_to_binary(Bs).
-
--spec unzip(binary()) -> binary().
-unzip(Binary) ->
+    Bs = try
+	     deflateInit(Z, default, deflated, -?MAX_WBITS, 8, default),
+	     B = deflate(Z, Data, finish),
+	     deflateEnd(Z),
+	     B
+	 after
+	     close(Z)
+	 end,
+    iolist_to_binary(Bs).
+
+-spec unzip(Data) -> Decompressed when
+      Data :: iodata(),
+      Decompressed :: binary().
+unzip(Data) ->
     Z = open(),
-    inflateInit(Z, -?MAX_WBITS),
-    Bs = inflate(Z, Binary),
-    inflateEnd(Z),
-    close(Z),
-    list_to_binary(Bs).
+    Bs = try
+	     inflateInit(Z, -?MAX_WBITS),
+	     B = inflate(Z, Data),
+	     inflateEnd(Z),
+	     B
+	 after
+	     close(Z)
+	 end,
+    iolist_to_binary(Bs).
     
--spec gzip(iodata()) -> binary().
-gzip(Data) when is_binary(Data); is_list(Data) ->
+-spec gzip(Data) -> Compressed when
+      Data :: iodata(),
+      Compressed :: binary().
+gzip(Data) ->
     Z = open(),
-    deflateInit(Z, default, deflated, 16+?MAX_WBITS, 8, default),
-    Bs = deflate(Z, Data, finish),
-    deflateEnd(Z),
-    close(Z),
-    iolist_to_binary(Bs);
-gzip(_) -> erlang:error(badarg).
-
--spec gunzip(iodata()) -> binary().
-gunzip(Data) when is_binary(Data); is_list(Data) ->
+    Bs = try
+	     deflateInit(Z, default, deflated, 16+?MAX_WBITS, 8, default),
+	     B = deflate(Z, Data, finish),
+	     deflateEnd(Z),
+	     B
+	 after
+	     close(Z)
+	 end,
+    iolist_to_binary(Bs).
+
+-spec gunzip(Data) -> Decompressed when
+      Data :: iodata(),
+      Decompressed :: binary().
+gunzip(Data) ->
     Z = open(),
-    inflateInit(Z, 16+?MAX_WBITS),
-    Bs = inflate(Z, Data),
-    inflateEnd(Z),
-    close(Z),
-    iolist_to_binary(Bs);
-gunzip(_) -> erlang:error(badarg).
+    Bs = try
+	     inflateInit(Z, 16+?MAX_WBITS),
+	     B = inflate(Z, Data),
+	     inflateEnd(Z),
+	     B
+	 after
+	     close(Z)
+	 end,
+    iolist_to_binary(Bs).
 
 -spec collect(zstream()) -> iolist().
 collect(Z) -> 
diff --git a/erts/start_scripts/Makefile b/erts/start_scripts/Makefile
index 4df7568484..55584bb057 100644
--- a/erts/start_scripts/Makefile
+++ b/erts/start_scripts/Makefile
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1997-2010. All Rights Reserved.
+# Copyright Ericsson AB 1997-2011. All Rights Reserved.
 #
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -74,7 +74,7 @@ $(SS_ROOT)/start_clean.script \
 $(SS_ROOT)/start_clean.boot:	$(SS_ROOT)/start_clean.rel
 	$(INSTALL_DIR) $(SS_TMP)
 	( cd $(SS_TMP) && \
-	$(ERLC) $(SASL_FLAGS) $(SCRIPT_PATH) -o $(SS_ROOT) $< )
+	$(ERLC) $(SASL_FLAGS) $(SCRIPT_PATH) +no_warn_sasl -o $(SS_ROOT) $< )
 
 $(SS_ROOT)/start_sasl.script \
 $(SS_ROOT)/start_sasl.boot:	$(SS_ROOT)/start_sasl.rel
@@ -128,7 +128,7 @@ $(SS_ROOT)/start_all_example.rel:	$(SS_ROOT)/start_all_example.rel.src \
 $(ERL_TOP)/bin/start.script:
 	$(INSTALL_DIR) $(SS_TMP)
 	( cd $(SS_TMP) && \
-	  $(ERLC) $(SCRIPT_PATH) +otp_build -o $@ $(SS_ROOT)/start_clean.rel )
+	  $(ERLC) $(SCRIPT_PATH) +no_warn_sasl +otp_build -o $@ $(SS_ROOT)/start_clean.rel )
 
 $(ERL_TOP)/bin/start_sasl.script:
 	$(INSTALL_DIR) $(SS_TMP)
@@ -138,7 +138,7 @@ $(ERL_TOP)/bin/start_sasl.script:
 $(ERL_TOP)/bin/start_clean.script:
 	$(INSTALL_DIR) $(SS_TMP)
 	( cd $(SS_TMP) && \
-	  $(ERLC) $(SCRIPT_PATH) +otp_build -o $@ $(SS_ROOT)/start_clean.rel )
+	  $(ERLC) $(SCRIPT_PATH) +no_warn_sasl +otp_build -o $@ $(SS_ROOT)/start_clean.rel )
 
 ## Special target used from system/build/Makefile for source code release bootstrap.
 bootstrap_scripts: $(SS_ROOT)/start_clean.rel
@@ -163,14 +163,14 @@ docs:
 include $(ERL_TOP)/make/otp_release_targets.mk
 
 release_spec: script
-	$(INSTALL_DIR) $(RELEASE_PATH)/releases/$(SYSTEM_VSN)
+	$(INSTALL_DIR) "$(RELEASE_PATH)/releases/$(SYSTEM_VSN)"
 ifneq ($(findstring win32,$(TARGET)),win32)
-	$(INSTALL_DATA) RELEASES.src $(RELEASE_PATH)/releases
+	$(INSTALL_DATA) RELEASES.src "$(RELEASE_PATH)/releases"
 endif
 	$(INSTALL_DATA) $(INSTALL_SCRIPTS) $(REL_SCRIPTS) \
-	   $(RELEASE_PATH)/releases/$(SYSTEM_VSN)
-	$(INSTALL_DATA) start_clean.script $(RELEASE_PATH)/releases/$(SYSTEM_VSN)/start.script
-	$(INSTALL_DATA) start_clean.boot $(RELEASE_PATH)/releases/$(SYSTEM_VSN)/start.boot
+	   "$(RELEASE_PATH)/releases/$(SYSTEM_VSN)"
+	$(INSTALL_DATA) start_clean.script "$(RELEASE_PATH)/releases/$(SYSTEM_VSN)/start.script"
+	$(INSTALL_DATA) start_clean.boot "$(RELEASE_PATH)/releases/$(SYSTEM_VSN)/start.boot"
 
 
 release_docs_spec:
diff --git a/erts/test/Makefile b/erts/test/Makefile
index 796403e182..79be5497b0 100644
--- a/erts/test/Makefile
+++ b/erts/test/Makefile
@@ -28,6 +28,7 @@ EBIN = .
 # ----------------------------------------------------
 
 MODULES= \
+	autoimport_SUITE \
 	erlc_SUITE \
 	install_SUITE \
 	nt_SUITE \
@@ -38,12 +39,14 @@ MODULES= \
 	erlexec_SUITE \
 	z_SUITE
 
+ERL_XML = $(ERL_TOP)/erts/doc/src/erlang.xml
+ERL_XML_TARGET = autoimport_SUITE_data/erlang.xml
 
 ERL_FILES= $(MODULES:%=%.erl)
 
 TARGET_FILES = $(MODULES:%=$(EBIN)/%.$(EMULATOR))
 
-EXTRA_FILES = install_SUITE_data/install_bin
+EXTRA_FILES = install_SUITE_data/install_bin $(ERL_XML_TARGET)
 
 # ----------------------------------------------------
 # Release directory specification
@@ -65,6 +68,10 @@ install_SUITE_data/install_bin: ../../make/install_bin
 	rm -f $@
 	cp -p $< $@
 
+$(ERL_XML_TARGET): $(ERL_XML)
+	rm -f $@
+	cp -p $< $@
+
 clean:
 	rm -f $(TARGET_FILES) $(EXTRA_FILES)
 	rm -f core *~
@@ -79,10 +86,10 @@ include $(ERL_TOP)/make/otp_release_targets.mk
 release_spec:
 
 release_tests_spec: opt
-	$(INSTALL_DIR) $(RELSYSDIR)
+	$(INSTALL_DIR) "$(RELSYSDIR)"
 	$(INSTALL_DATA) system.spec system.dynspec system.spec.vxworks \
-		$(ERL_FILES) $(TARGET_FILES) $(RELSYSDIR)
-	chmod -f -R u+w $(RELSYSDIR)
-	tar cf - *_SUITE_data utils | (cd $(RELSYSDIR); tar xf -)
+		$(ERL_FILES) $(TARGET_FILES) "$(RELSYSDIR)"
+	chmod -R u+w "$(RELSYSDIR)"
+	tar cf - *_SUITE_data utils | (cd "$(RELSYSDIR)"; tar xf -)
 
 release_docs_spec:
diff --git a/erts/test/autoimport_SUITE.erl b/erts/test/autoimport_SUITE.erl
new file mode 100644
index 0000000000..71ed5204b1
--- /dev/null
+++ b/erts/test/autoimport_SUITE.erl
@@ -0,0 +1,196 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 1998-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%% Purpose: Test erlang.xml re autoimports
+-module(autoimport_SUITE).
+
+-include_lib("test_server/include/test_server.hrl").
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
+	 autoimports/1]).
+-define(TEST_TIMEOUT, ?t:seconds(180)).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [autoimports].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
+init_per_testcase(_Func, Config) ->
+    Dog = test_server:timetrap(?TEST_TIMEOUT),
+    [{watchdog, Dog} | Config].
+
+end_per_testcase(_Func, Config) ->
+    Dog = ?config(watchdog, Config),
+    catch test_server:timetrap_cancel(Dog),
+    ok.
+
+
+autoimports(suite) ->
+    [];
+autoimports(doc) ->
+    ["Check that erlang.xml documents autoimports correctly"];
+autoimports(Config) when is_list(Config) ->
+    ?line XML = filename:join([?config(data_dir,Config),"erlang.xml"]),
+    ?line case xml(XML) of
+	      [] ->
+		  ok;
+	      List ->
+		  lists:foreach(fun({[],F,A}) ->
+					io:format("erlang:~s/~p is wrongly "
+						  "documented as ~s/~p~n",
+						  [F,A,F,A]);
+				   ({"erlang",F,A}) ->
+					io:format("~s/~p is wrongly "
+						  "documented as "
+						  "erlang:~s/~p~n",
+						  [F,A,F,A])
+				end,List),
+		  ?t:fail({wrong_autoimports,List})
+	  end.
+
+%%
+%% Ugly chunk of code to heuristically analyze the erlang.xml
+%% documentation file. Don't view this as an example...
+%%
+
+xml(XMLFile) ->
+    {ok,File} = file:open(XMLFile,[read]),
+    xskip_to_funcs(file:read_line(File),File),
+    DocData = xloop(file:read_line(File),File),
+    true = DocData =/= [],
+    file:close(File),
+    analyze(DocData).
+
+%% Skip lines up to and including the <funcs> tag.
+xskip_to_funcs({ok,Line},File) ->
+    case re:run(Line,"\\<funcs\\>",[{capture,none}]) of
+	nomatch ->
+	    xskip_to_funcs(file:read_line(File),File);
+	match ->
+	    ok
+    end.
+
+xloop({ok,Line},File) ->
+    case re:run(Line,"\\<name\\>",[{capture,none}]) of
+	nomatch ->
+	    xloop(file:read_line(File),File);
+	match ->
+	    X = re:replace(Line,"\\).*",")",[{return,list}]),
+	    Y = re:replace(X,".*\\>","",[{return,list}]),
+	    Mod = get_module(Y),
+	    Rest1 = fstrip(Mod++":",Y),
+	    Func = get_function(Rest1),
+	    Rest2 = fstrip(Func++"(", Rest1),
+	    Argc = count_args(Rest2,1,0,false),
+	    [{Mod,Func,Argc} |
+	     xloop(file:read_line(File),File)]
+    end;
+xloop(_,_) ->
+    [].
+
+analyze([{[],Func,Arity}|T]) ->
+    case erl_internal:bif(list_to_atom(Func),Arity) of
+	true ->
+	    analyze(T);
+	false ->
+	    [{[],Func,Arity} |
+	     analyze(T)]
+    end;
+analyze([{"erlang",Func,Arity}|T]) ->
+    case erl_internal:bif(list_to_atom(Func),Arity) of
+	true ->
+	    [{"erlang",Func,Arity}|analyze(T)];
+	false ->
+	    analyze(T)
+    end;
+analyze([_|T]) ->
+   analyze(T);
+analyze([]) ->
+   [].
+
+
+count_args([],_,N,false) ->
+    N;
+count_args([],_,N,true) ->
+    N+1;
+count_args(_,0,N,true) ->
+    N+1;
+count_args(_,0,N,false) ->
+    N;
+count_args([Par|T],Level,N,Got) when (Par =:= 40) or
+				 (Par =:= 123) or (Par =:= 91) ->
+    count_args(T,Level+1,N,(Level =:= 1) or Got);
+count_args([41|T],1,N,true) ->
+    count_args(T,0,N+1,false);
+count_args([Par|T],Level,N, Got) when (Par =:= 41) or
+				 (Par =:= 125) or (Par =:= 93) ->
+    count_args(T,Level-1,N,Got);
+count_args([$,|T],1,N,true) ->
+    count_args(T,1,N+1,false);
+count_args([$ |T],A,B,C) ->
+    count_args(T,A,B,C);
+count_args([_|T],1,N,_) ->
+    count_args(T,1,N,true);
+count_args([_|T],A,B,C) ->
+    count_args(T,A,B,C).
+
+fstrip([],X) ->
+    X;
+fstrip(_,[]) ->
+    [];
+fstrip([H|T1],[H|T2]) ->
+    fstrip(T1,T2);
+fstrip(_,L) ->
+    L.
+
+get_module(X) ->
+    get_module(X,[]).
+get_module([],_) ->
+    [];
+get_module([$:|_],Acc) ->
+    lists:reverse(Acc);
+get_module([40|_],_) -> %(
+    [];
+get_module([H|T],Acc) ->
+    get_module(T,[H|Acc]).
+
+get_function(X) ->
+    get_function(X,[]).
+get_function([],_) ->
+    [];
+get_function([40|_],Acc) -> %(
+    lists:reverse(Acc);
+get_function([H|T],Acc) ->
+    get_function(T,[H|Acc]).
diff --git a/erts/test/autoimport_SUITE_data/dummy.txt b/erts/test/autoimport_SUITE_data/dummy.txt
new file mode 100644
index 0000000000..972643e527
--- /dev/null
+++ b/erts/test/autoimport_SUITE_data/dummy.txt
@@ -0,0 +1,19 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 1998-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%% Purpouse: Dummy
diff --git a/erts/test/erl_print_SUITE.erl b/erts/test/erl_print_SUITE.erl
index 3bb7d4d016..a49d8f069f 100644
--- a/erts/test/erl_print_SUITE.erl
+++ b/erts/test/erl_print_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2005-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2012. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -32,14 +32,35 @@
 
 -define(DEFAULT_TIMEOUT, ?t:minutes(10)).
 
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2]).
 
--export([erlang_display/1, integer/1, float/1, string/1, character/1, snprintf/1, quote/1]).
+-export([erlang_display/1, integer/1, float/1, 
+	 string/1, character/1, snprintf/1, quote/1]).
 
 -include_lib("test_server/include/test_server.hrl").
-    
-all(doc) -> [];
-all(suite) -> test_cases().
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    test_cases().
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 %%
 %%
@@ -47,14 +68,9 @@ all(suite) -> test_cases().
 %%
 %%
 
-test_cases() ->
-    [erlang_display,
-     integer,
-     float,
-     string,
-     character,
-     snprintf,
-     quote].
+test_cases() -> 
+    [erlang_display, integer, float, string, character,
+     snprintf, quote].
 
 erlang_display(doc) -> [];
 erlang_display(suite) -> [];
@@ -231,7 +247,7 @@ init_per_testcase(Case, Config) ->
     Dog = ?t:timetrap(?DEFAULT_TIMEOUT),
     [{testcase, Case}, {watchdog, Dog} |Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
@@ -287,7 +303,7 @@ read_case_data(Port, TestCase) ->
 	      {Port, {data, {eol, [?PID_MARKER | PidStr]}}} ->
 		  ?line ?t:format("Port program pid: ~s~n", [PidStr]),
 		  ?line CaseProc = self(),
-		  ?line list_to_integer(PidStr), % Sanity check
+		  ?line _ = list_to_integer(PidStr), % Sanity check
 		  spawn_opt(fun () ->
 				    port_prog_killer(CaseProc, PidStr)
 			    end,
diff --git a/erts/test/erl_print_SUITE_data/Makefile.src b/erts/test/erl_print_SUITE_data/Makefile.src
index 109d55e572..96d71c7a98 100644
--- a/erts/test/erl_print_SUITE_data/Makefile.src
+++ b/erts/test/erl_print_SUITE_data/Makefile.src
@@ -1,7 +1,7 @@
 # 
 # %CopyrightBegin%
 # 
-# Copyright Ericsson AB 2005-2009. All Rights Reserved.
+# Copyright Ericsson AB 2005-2011. All Rights Reserved.
 # 
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -20,23 +20,35 @@
 include @erts_lib_include_internal_generated@@[email protected]
 
 CC = @CC@
-CFLAGS = @ERTS_CFLAGS@
-LIBS = @ERTS_LIBS@
+CFLAGST = @ERTS_CFLAGS@
+LIBST = @ERTS_LIBS@
+CFLAGSF = @CFLAGS@
+LIBSF = @LIBS@
+CP=cp
+CHMOD=chmod
 
-EPTF_CFLAGS = -Wall $(CFLAGS) @DEFS@ -I@erts_lib_include_internal@ -I@erts_lib_include_internal_generated@
-EPTF_LIBS = $(LIBS) -L@erts_lib_internal_path@ -lerts_internal@type_marker@
+COMMON_CFLAGS = -Wall @DEFS@ -I@erts_lib_include_internal@ -I@erts_lib_include_internal_generated@
 
-EPTT_CFLAGS = -DTHREAD_SAFE $(ETHR_DEFS) $(EPTF_CFLAGS)
-EPTT_LIBS = $(LIBS) -L@erts_lib_internal_path@ -lerts_internal_r@type_marker@ $(ETHR_LIBS)
+EPTF_CFLAGS = $(CFLAGSF) $(COMMON_CFLAGS)
+EPTF_LIBS = -L@erts_lib_internal_path@ -lerts_internal@type_marker@ $(LIBSF)
+
+EPTT_CFLAGS = -DTHREAD_SAFE $(ETHR_DEFS) $(CFLAGST) $(COMMON_CFLAGS)
+EPTT_LIBS = -L@erts_lib_internal_path@ -lerts_internal_r@type_marker@ $(ETHR_LIBS) $(LIBST)
 
 GCC = .@DS@gccifier -CC"$(CC)"
 
-PROGS = erl_print_tests.@emu_threads@@exe@
+PROGS = erl_print_tests.true@exe@ erl_print_tests.false@exe@
 
 all: $(PROGS)
 
+@IFEQ@ (@cross@, yes)
+gccifier@exe@:
+	$(CP) ..@DS@utils@[email protected] gccifier@exe@
+	$(CHMOD) a+x gccifier@exe@
+@ELSE@
 gccifier@exe@: ..@DS@utils@[email protected]
-	$(CC) $(CFLAGS) -o gccifier@exe@ ..@DS@utils@[email protected] $(LIBS)
+	$(CC) $(CFLAGST) -o gccifier@exe@ ..@DS@utils@[email protected] $(LIBST)
+@ENDIF@
 
 erl_print_tests.false@exe@: gccifier@exe@ erl_print_tests.c
 	$(GCC) $(EPTF_CFLAGS) -o erl_print_tests.false@exe@ erl_print_tests.c $(EPTF_LIBS)
diff --git a/erts/test/erl_print_SUITE_data/erl_print_tests.c b/erts/test/erl_print_SUITE_data/erl_print_tests.c
index 28ce78f4e1..82b0c21132 100644
--- a/erts/test/erl_print_SUITE_data/erl_print_tests.c
+++ b/erts/test/erl_print_SUITE_data/erl_print_tests.c
@@ -44,8 +44,8 @@
 #endif
 
 #ifdef __WIN32__
-#define signed_long_long	LONGLONG
-#define unsigned_long_long	ULONGLONG
+#define signed_long_long	__int64
+#define unsigned_long_long	unsigned __int64
 #else
 #define signed_long_long	signed long long
 #define unsigned_long_long	unsigned long long
diff --git a/erts/test/erlc_SUITE.erl b/erts/test/erlc_SUITE.erl
index 437f020f99..ada09db975 100644
--- a/erts/test/erlc_SUITE.erl
+++ b/erts/test/erlc_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -20,14 +20,33 @@
 
 %% Tests the erlc command by compiling various types of files.
 
--export([all/1, compile_erl/1, compile_yecc/1, compile_script/1,
-	 compile_mib/1, good_citizen/1, deep_cwd/1]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, compile_erl/1,
+	 compile_yecc/1, compile_script/1,
+	 compile_mib/1, good_citizen/1, deep_cwd/1, arg_overflow/1]).
 
 -include_lib("test_server/include/test_server.hrl").
 
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     [compile_erl, compile_yecc, compile_script, compile_mib,
-     good_citizen, deep_cwd].
+     good_citizen, deep_cwd, arg_overflow].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
 %% Copy from erlc_SUITE_data/include/erl_test.hrl.
 
@@ -60,7 +79,7 @@ compile_erl(Config) when is_list(Config) ->
 
     ?line run(Config, Cmd, FileName, "-Werror",
 	      ["compile: warnings being treated as errors\$",
-	       "Warning: function foo/0 is unused\$",
+	       "function foo/0 is unused\$",
 	       "_ERROR_"]),
 
     %% Check a bad file.
@@ -135,7 +154,8 @@ compile_mib(Config) when is_list(Config) ->
 
     ?line BadFile = filename:join(SrcDir, "BAD-MIB.mib"),
     ?line run(Config, Cmd, BadFile, "",
-	      ["Error: syntax error before: mibs\$", "compilation_failed_ERROR_"]),
+	      ["BAD-MIB.mib: 1: syntax error before: mibs\$",
+	       "compilation_failed_ERROR_"]),
 
     %% Make sure that no -I option works.
 
@@ -189,12 +209,51 @@ deep_cwd_1(PrivDir) ->
     ?line true = filelib:is_file("test.beam"),
     ok.
 
+%% Test that a large number of command line switches does not
+%% overflow the argument buffer
+arg_overflow(Config) when is_list(Config) ->
+    ?line {SrcDir, _OutDir, Cmd} = get_cmd(Config),
+    ?line FileName = filename:join(SrcDir, "erl_test_ok.erl"),
+    %% Each -D option will be expanded to three arguments when
+    %% invoking 'erl'.
+    ?line NumDOptions = num_d_options(),
+    ?line Args = lists:flatten([ ["-D", integer_to_list(N, 36), "=1 "] ||
+            N <- lists:seq(1, NumDOptions) ]),
+    ?line run(Config, Cmd, FileName, Args,
+	      ["Warning: function foo/0 is unused\$",
+	       "_OK_"]),
+    ok.
+
+num_d_options() ->
+    case {os:type(),os:version()} of
+	{{win32,_},_} ->
+	    %% The maximum size of a command line in the command
+	    %% shell on Windows is 8191 characters.
+	    %% Each -D option is expanded to "@dv NN 1", i.e.
+	    %% 8 characters. (Numbers up to 1295 can be expressed
+	    %% as two 36-base digits.)
+	    1000;
+	{{unix,linux},Version} when Version < {2,6,23} ->
+	    %% On some older 64-bit versions of Linux, the maximum number
+	    %% of arguments is 16383.
+	    %% See: http://www.in-ulm.de/~mascheck/various/argmax/
+	    5440;
+	{{unix,darwin},{Major,_,_}} when Major >= 11 ->
+	    %% "getconf ARG_MAX" still reports 262144 (as in previous
+	    %% version of MacOS X), but the useful space seem to have
+	    %% shrunk significantly (or possibly the number of arguments).
+	    %% 7673
+	    7500;
+	{_,_} ->
+	    12000
+    end.
+
 erlc() ->
     case os:find_executable("erlc") of
 	false ->
 	    test_server:fail("Can't find erlc");
 	Erlc ->
-	    Erlc
+	    "\"" ++ Erlc ++ "\""
     end.
 	    
 %% Runs a command.
diff --git a/erts/test/erlexec_SUITE.erl b/erts/test/erlexec_SUITE.erl
index 164ce9faaf..0dfe6c2e5f 100644
--- a/erts/test/erlexec_SUITE.erl
+++ b/erts/test/erlexec_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2007-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2007-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -31,9 +31,11 @@
 
 -define(DEFAULT_TIMEOUT, ?t:minutes(1)).
 
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2]).
 
--export([args_file/1, evil_args_file/1, env/1, args_file_env/1, otp_7461/1, otp_7461_remote/1, otp_8209/1]).
+-export([args_file/1, evil_args_file/1, env/1, args_file_env/1, otp_7461/1, otp_7461_remote/1, otp_8209/1, zdbbl_dist_buf_busy_limit/1]).
 
 -include_lib("test_server/include/test_server.hrl").
     
@@ -43,7 +45,7 @@ init_per_testcase(Case, Config) ->
     SavedEnv = save_env(),
     [{testcase, Case}, {watchdog, Dog}, {erl_flags_env, SavedEnv} |Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog = ?config(watchdog, Config),
     SavedEnv = ?config(erl_flags_env, Config),
     restore_env(SavedEnv),
@@ -51,10 +53,26 @@ fin_per_testcase(_Case, Config) ->
     ?t:timetrap_cancel(Dog),
     ok.
 
-all(doc) -> [];
-all(suite) ->
-    [args_file, evil_args_file, env, args_file_env, otp_7461, otp_8209].
+suite() -> [{ct_hooks,[ts_install_cth]}].
 
+all() -> 
+    [args_file, evil_args_file, env, args_file_env,
+     otp_7461, otp_8209, zdbbl_dist_buf_busy_limit].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
 otp_8209(doc) ->
     ["Test that plain first argument does not "
@@ -330,6 +348,25 @@ otp_7461_remote([halt, Pid]) ->
     io:format("halt order from ~p to node ~p\n",[Pid,node()]),
     halt().
 
+zdbbl_dist_buf_busy_limit(doc) ->    
+    ["Check +zdbbl flag"];
+zdbbl_dist_buf_busy_limit(suite) ->
+    [];
+zdbbl_dist_buf_busy_limit(Config) when is_list(Config) ->
+    LimKB = 1122233,
+    LimB = LimKB*1024,
+    ?line {ok,[[PName]]} = init:get_argument(progname),
+    ?line SNameS = "erlexec_test_02",
+    ?line SName = list_to_atom(SNameS++"@"++
+                         hd(tl(string:tokens(atom_to_list(node()),"@")))),
+    ?line Cmd = PName ++ " -sname "++SNameS++" -setcookie "++
+        atom_to_list(erlang:get_cookie()) ++
+	" +zdbbl " ++ integer_to_list(LimKB),
+    ?line open_port({spawn,Cmd},[]),
+    ?line pong = loop_ping(SName,40),
+    ?line LimB = rpc:call(SName,erlang,system_info,[dist_buf_busy_limit]),
+    ?line ok = cleanup_node(SNameS, 10),
+    ok.
     
 
 %%
diff --git a/erts/test/erlexec_SUITE_data/Makefile.src b/erts/test/erlexec_SUITE_data/Makefile.src
index b751547b8f..970a905c32 100644
--- a/erts/test/erlexec_SUITE_data/Makefile.src
+++ b/erts/test/erlexec_SUITE_data/Makefile.src
@@ -20,6 +20,8 @@
 CC = @CC@
 CFLAGS = @ERTS_CFLAGS@
 LIBS = @ERTS_LIBS@
+CP=cp
+CHMOD=chmod
 
 ERLX_T_CFLAGS = -Wall $(ERLX_DEFS) $(CFLAGS) @DEFS@
 
@@ -29,8 +31,14 @@ PROGS = erlexec_tests@exe@
 
 all: $(PROGS)
 
+@IFEQ@ (@cross@, yes)
+gccifier@exe@:
+	$(CP) ..@DS@utils@[email protected] gccifier@exe@
+	$(CHMOD) a+x gccifier@exe@
+@ELSE@
 gccifier@exe@: ..@DS@utils@[email protected]
 	$(CC) $(CFLAGS) -o gccifier@exe@ ..@DS@utils@[email protected] $(LIBS)
+@ENDIF@
 
 erlexec_tests@exe@: gccifier@exe@ erlexec_tests.c
 	$(GCC) $(ERLX_T_CFLAGS) -o erlexec_tests@exe@ erlexec_tests.c
diff --git a/erts/test/erlexec_SUITE_data/erlexec_tests.c b/erts/test/erlexec_SUITE_data/erlexec_tests.c
index a49a0b21a8..1d1ca881d9 100644
--- a/erts/test/erlexec_SUITE_data/erlexec_tests.c
+++ b/erts/test/erlexec_SUITE_data/erlexec_tests.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2008-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2008-2010. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 /* Used by test case otp_7461 to spawn a child process with a given
@@ -22,7 +22,7 @@
  * Author: Sverker Eriksson
  */
 
-#if defined (__WIN32__) || defined(VXWORKS) || defined(_OSE_)
+#if defined (__WIN32__) || defined(VXWORKS)
 int main() {return 0;}
 
 #else /* UNIX only */
diff --git a/erts/test/ethread_SUITE.erl b/erts/test/ethread_SUITE.erl
index bbc79e9381..24075286a8 100644
--- a/erts/test/ethread_SUITE.erl
+++ b/erts/test/ethread_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2004-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2004-2012. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -31,27 +31,24 @@
 
 -define(DEFAULT_TIMEOUT, ?t:minutes(10)).
 
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2]).
 
 -export([create_join_thread/1,
 	 equal_tids/1,
 	 mutex/1,
 	 try_lock_mutex/1,
-	 recursive_mutex/1,
-	 time_now/1,
 	 cond_wait/1,
-	 cond_timedwait/1,
 	 broadcast/1,
 	 detached_thread/1,
 	 max_threads/1,
-	 forksafety/1,
-	 vfork/1,
 	 tsd/1,
 	 spinlock/1,
 	 rwspinlock/1,
 	 rwmutex/1,
 	 atomic/1,
-	 gate/1]).
+	 dw_atomic_massage/1]).
 
 -include_lib("test_server/include/test_server.hrl").
 
@@ -60,25 +57,36 @@ tests() ->
      equal_tids,
      mutex,
      try_lock_mutex,
-     recursive_mutex,
-     time_now,
      cond_wait,
-     cond_timedwait,
      broadcast,
      detached_thread,
      max_threads,
-     forksafety,
-     vfork,
      tsd,
      spinlock,
      rwspinlock,
      rwmutex,
      atomic,
-     gate].
+     dw_atomic_massage].
 
-all(doc) -> [];
-all(suite) -> tests().
+suite() -> [{ct_hooks,[ts_install_cth]}].
 
+all() -> 
+    tests().
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
 
 %%
 %%
@@ -114,53 +122,37 @@ try_lock_mutex(suite) ->
 try_lock_mutex(Config) ->
     run_case(Config, "try_lock_mutex", "").
 
-recursive_mutex(doc) ->
-    ["Tests recursive mutexes."];
-recursive_mutex(suite) ->
-    [];
-recursive_mutex(Config) ->
-    run_case(Config, "recursive_mutex", "").
-
-time_now(doc) ->
-    ["Tests ethr_time_now by comparing time values with Erlang."];
-time_now(suite) ->
-    [];
-time_now(Config) ->
-    run_case(Config, "time_now", "", fun (P) ->
-					     spawn_link(fun () ->
-								watchdog(P)
-							end)
-				     end).
-
-wd_dispatch(P) ->
-    receive
-	bye ->
-	    ?line true = port_command(P, "-1 "),
-	    ?line bye;
-	L when is_list(L) ->
-	    ?line true = port_command(P, L),
-	    ?line wd_dispatch(P)
-    end.
-
-watchdog(Port) ->
-    ?line process_flag(priority, max),
-    ?line receive after 500 -> ok end,
-
-    ?line random:seed(),
-    ?line true = port_command(Port, "0 "),
-    ?line lists:foreach(fun (T) ->
-				erlang:send_after(T,
-						  self(),
-						  integer_to_list(T)
-						  ++ " ")
-			end,
-			lists:usort(lists:map(fun (_) ->
-						      random:uniform(4500)+500
-					      end,
-					      lists:duplicate(50,0)))),
-    ?line erlang:send_after(5100, self(), bye),
-
-    wd_dispatch(Port).
+%% Remove dead code?
+
+% wd_dispatch(P) ->
+%     receive
+% 	bye ->
+% 	    ?line true = port_command(P, "-1 "),
+% 	    ?line bye;
+% 	L when is_list(L) ->
+% 	    ?line true = port_command(P, L),
+% 	    ?line wd_dispatch(P)
+%     end.
+% 
+% watchdog(Port) ->
+%     ?line process_flag(priority, max),
+%     ?line receive after 500 -> ok end,
+% 
+%     ?line random:seed(),
+%     ?line true = port_command(Port, "0 "),
+%     ?line lists:foreach(fun (T) ->
+% 				erlang:send_after(T,
+% 						  self(),
+% 						  integer_to_list(T)
+% 						  ++ " ")
+% 			end,
+% 			lists:usort(lists:map(fun (_) ->
+% 						      random:uniform(4500)+500
+% 					      end,
+% 					      lists:duplicate(50,0)))),
+%     ?line erlang:send_after(5100, self(), bye),
+% 
+%     wd_dispatch(Port).
 
 cond_wait(doc) ->
     ["Tests ethr_cond_wait with ethr_cond_signal and ethr_cond_broadcast."];
@@ -169,13 +161,6 @@ cond_wait(suite) ->
 cond_wait(Config) ->
     run_case(Config, "cond_wait", "").
 
-cond_timedwait(doc) ->
-    ["Tests ethr_cond_timedwait with ethr_cond_signal and ethr_cond_broadcast."];
-cond_timedwait(suite) ->
-    [];
-cond_timedwait(Config) ->
-    run_case(Config, "cond_timedwait", "").
-
 broadcast(doc) ->
     ["Tests that a ethr_cond_broadcast really wakes up all waiting threads"];
 broadcast(suite) ->
@@ -188,33 +173,30 @@ detached_thread(doc) ->
 detached_thread(suite) ->
     [];
 detached_thread(Config) ->
-    run_case(Config, "detached_thread", "").
+    case {os:type(), os:version()} of
+	{{unix,darwin}, {9, _, _}} ->
+	    %% For some reason pthread_create() crashes when more
+	    %% threads cannot be created, instead of returning an
+	    %% error code on our MacOS X Leopard machine...
+	    {skipped, "MacOS X Leopard cannot cope with this test..."};
+	_ ->
+	    run_case(Config, "detached_thread", "")
+    end.
 
 max_threads(doc) ->
     ["Tests maximum number of threads."];
 max_threads(suite) ->
     [];
 max_threads(Config) ->
-    run_case(Config, "max_threads", "").
-
-forksafety(doc) ->
-    ["Tests forksafety."];
-forksafety(suite) ->
-    [];
-forksafety(Config) ->
-    run_case(Config, "forksafety", "").
-
-vfork(doc) ->
-    ["Tests vfork with threads."];
-vfork(suite) ->
-    case ?t:os_type() of
-	{unix, osf1} ->
-	    {skip, "vfork() known to hang multi-threaded applications on osf1"};
+    case {os:type(), os:version()} of
+	{{unix,darwin}, {9, _, _}} ->
+	    %% For some reason pthread_create() crashes when more
+	    %% threads cannot be created, instead of returning an
+	    %% error code on our MacOS X Leopard machine...
+	    {skipped, "MacOS X Leopard cannot cope with this test..."};
 	_ ->
-	    []
-    end;
-vfork(Config) ->
-    run_case(Config, "vfork", "").
+	    run_case(Config, "max_threads", "")
+    end.
 
 tsd(doc) ->
     ["Tests thread specific data."];
@@ -251,12 +233,12 @@ atomic(suite) ->
 atomic(Config) ->
     run_case(Config, "atomic", "").
 
-gate(doc) ->
-    ["Tests gates."];
-gate(suite) ->
+dw_atomic_massage(doc) ->
+    ["Massage double word atomics"];
+dw_atomic_massage(suite) ->
     [];
-gate(Config) ->
-    run_case(Config, "gate", "").
+dw_atomic_massage(Config) ->
+    run_case(Config, "dw_atomic_massage", "").
 
 %%
 %%
@@ -264,11 +246,18 @@ gate(Config) ->
 %%
 %%
 
-init_per_testcase(_Case, Config) ->
-    Dog = ?t:timetrap(?DEFAULT_TIMEOUT),
-    [{watchdog, Dog}|Config].
+init_per_testcase(Case, Config) ->
+    case inet:gethostname() of
+	{ok,"fenris"} when Case == max_threads ->
+	    %% Cannot use os:type+os:version as not all
+	    %% solaris10 machines are buggy.
+	    {skip, "This machine is buggy"};
+	_Else ->
+	    Dog = ?t:timetrap(?DEFAULT_TIMEOUT),
+	    [{watchdog, Dog}|Config]
+    end.
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
@@ -324,7 +313,7 @@ read_case_data(Port, TestCase) ->
 	      {Port, {data, {eol, [?PID_MARKER | PidStr]}}} ->
 		  ?line ?t:format("Port program pid: ~s~n", [PidStr]),
 		  ?line CaseProc = self(),
-		  ?line list_to_integer(PidStr), % Sanity check
+		  ?line _ = list_to_integer(PidStr), % Sanity check
 		  spawn_opt(fun () ->
 				    port_prog_killer(CaseProc, PidStr)
 			    end,
diff --git a/erts/test/ethread_SUITE_data/Makefile.src b/erts/test/ethread_SUITE_data/Makefile.src
index 132b23344c..f93a31178b 100644
--- a/erts/test/ethread_SUITE_data/Makefile.src
+++ b/erts/test/ethread_SUITE_data/Makefile.src
@@ -23,6 +23,8 @@ include @erts_lib_include_internal_generated@@DS@erts_internal.mk
 CC = @CC@
 CFLAGS = @ERTS_CFLAGS@
 LIBS = @ERTS_LIBS@
+CP=cp
+CHMOD=chmod
 
 ETHR_T_CFLAGS = -Wall $(ETHR_DEFS) $(CFLAGS) @DEFS@ -I@erts_lib_include_internal@ -I@erts_lib_include_internal_generated@
 ETHR_T_LIBS = $(LIBS) -L@erts_lib_internal_path@ $(ETHR_LIBS) $(ERTS_INTERNAL_X_LIBS)
@@ -33,8 +35,14 @@ PROGS = ethread_tests@exe@
 
 all: $(PROGS)
 
+@IFEQ@ (@cross@, yes)
+gccifier@exe@:
+	$(CP) ..@DS@utils@[email protected] gccifier@exe@
+	$(CHMOD) a+x gccifier@exe@
+@ELSE@
 gccifier@exe@: ..@DS@utils@[email protected]
 	$(CC) $(CFLAGS) -o gccifier@exe@ ..@DS@utils@[email protected] $(LIBS)
+@ENDIF@
 
 ethread_tests@exe@: gccifier@exe@ ethread_tests.c
 	$(GCC) $(ETHR_T_CFLAGS) -o ethread_tests@exe@ ethread_tests.c -lerts_internal_r $(ETHR_T_LIBS)
diff --git a/erts/test/ethread_SUITE_data/ethread_tests.c b/erts/test/ethread_SUITE_data/ethread_tests.c
index f779f13c51..ed96ecdbd2 100644
--- a/erts/test/ethread_SUITE_data/ethread_tests.c
+++ b/erts/test/ethread_SUITE_data/ethread_tests.c
@@ -1,19 +1,19 @@
 /*
  * %CopyrightBegin%
- * 
- * Copyright Ericsson AB 2004-2009. All Rights Reserved.
- * 
+ *
+ * Copyright Ericsson AB 2004-2011. All Rights Reserved.
+ *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
  * compliance with the License. You should have received a copy of the
  * Erlang Public License along with this software. If not, it can be
  * retrieved online at http://www.erlang.org/.
- * 
+ *
  * Software distributed under the License is distributed on an "AS IS"
  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  * the License for the specific language governing rights and limitations
  * under the License.
- * 
+ *
  * %CopyrightEnd%
  */
 
@@ -82,6 +82,7 @@ static void
 print_eol(void)
 {
     fprintf(stderr, EOL);
+    fflush(stderr);
 }
 
 static void print_line(char *frmt,...)
@@ -90,10 +91,12 @@ static void print_line(char *frmt,...)
     print_eol();
 }
 
+#if 0 /* Currently not used; silence annoying warning... */
 static void print(char *frmt,...)
 {
     PRINT_VA_LIST(frmt);
 }
+#endif
 
 static void fail(char *frmt,...)
 {
@@ -197,8 +200,8 @@ create_join_thread_test(void)
     }
 
     for (i = 1; i <= CJTT_NO_THREADS; i++) {
-	int *tres;
-	res = ethr_thr_join(cjtt_tids[i], (void **) &tres);
+	void *tres;
+	res = ethr_thr_join(cjtt_tids[i], &tres);
 	ASSERT(res == 0);
 	ASSERT(tres == &cjtt_res[i]);
 	ASSERT(cjtt_res[i] == i);
@@ -216,8 +219,8 @@ create_join_thread_test(void)
 #define ETT_THREADS 100000
 
 static ethr_tid ett_tids[3];
-static ethr_mutex ett_mutex = ETHR_MUTEX_INITER;
-static ethr_cond ett_cond = ETHR_COND_INITER;
+static ethr_mutex ett_mutex;
+static ethr_cond ett_cond;
 static int ett_terminate;
 
 static void *
@@ -234,14 +237,12 @@ static void *
 ett_thread2(void *unused)
 {
     int res;
-    res = ethr_mutex_lock(&ett_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&ett_mutex);
     while (!ett_terminate) {
 	res = ethr_cond_wait(&ett_cond, &ett_mutex);
 	ASSERT(res == 0);
     }
-    res = ethr_mutex_unlock(&ett_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&ett_mutex);
     return NULL;
 }
 
@@ -250,6 +251,10 @@ equal_tids_test(void)
 {
     int res, i;
 
+    res = ethr_mutex_init(&ett_mutex);
+    ASSERT(res == 0);
+    res = ethr_cond_init(&ett_cond);
+    ASSERT(res == 0);
     ett_tids[0] = ethr_self();
     
     res = ethr_thr_create(&ett_tids[1], ett_thread, (void *) &ett_tids[1], NULL);
@@ -298,13 +303,10 @@ equal_tids_test(void)
 	ASSERT(res == 0);
     }
 
-    res = ethr_mutex_lock(&ett_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&ett_mutex);
     ett_terminate = 1;
-    res = ethr_cond_signal(&ett_cond);
-    ASSERT(res == 0);
-    res = ethr_mutex_unlock(&ett_mutex);
-    ASSERT(res == 0);
+    ethr_cond_signal(&ett_cond);
+    ethr_mutex_unlock(&ett_mutex);
     res = ethr_thr_join(ett_tids[1], NULL);
     ASSERT(res == 0);
 
@@ -321,17 +323,14 @@ equal_tids_test(void)
  * Tests mutexes.
  */
 
-static ethr_mutex mt_mutex = ETHR_MUTEX_INITER;
+static ethr_mutex mt_mutex;
 static int mt_data;
 
 void *
 mt_thread(void *unused)
 {
-    int res;
-
     print_line("Aux thread tries to lock mutex");
-    res = ethr_mutex_lock(&mt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&mt_mutex);
     print_line("Aux thread locked mutex");
 
     ASSERT(mt_data == 0);
@@ -345,8 +344,7 @@ mt_thread(void *unused)
 
     ASSERT(mt_data == 1);
 
-    res = ethr_mutex_unlock(&mt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&mt_mutex);
     print_line("Aux thread unlocked mutex");
 
     return NULL;
@@ -356,18 +354,18 @@ mt_thread(void *unused)
 static void
 mutex_test(void)
 {
-    int do_restart = 1;
     int res;
     ethr_tid tid;
 
-    print_line("Running test with statically initialized mutex");
+    print_line("Trying to initialize mutex");
+    res = ethr_mutex_init(&mt_mutex);
+    ASSERT(res == 0);
+    print_line("Initialized mutex");
 
- restart:
     mt_data = 0;
 
     print_line("Main thread tries to lock mutex");
-    res = ethr_mutex_lock(&mt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&mt_mutex);
     print_line("Main thread locked mutex");
 
     ASSERT(mt_data == 0);
@@ -383,8 +381,7 @@ mutex_test(void)
 
     ASSERT(mt_data == 0);
 
-    res = ethr_mutex_unlock(&mt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&mt_mutex);
     print_line("Main thread unlocked mutex");
 
     print_line("Main thread goes to sleep for 1 second");
@@ -392,8 +389,7 @@ mutex_test(void)
     print_line("Main thread woke up");
 
     print_line("Main thread tries to lock mutex");
-    res = ethr_mutex_lock(&mt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&mt_mutex);
     print_line("Main thread locked mutex");
 
     ASSERT(mt_data == 1);
@@ -404,8 +400,7 @@ mutex_test(void)
 
     ASSERT(mt_data == 1);
 
-    res = ethr_mutex_unlock(&mt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&mt_mutex);
     print_line("Main thread unlocked mutex");
 
     res = ethr_thr_join(tid, NULL);
@@ -416,20 +411,6 @@ mutex_test(void)
     ASSERT(res == 0);
     print_line("Main thread destroyed mutex");
 
-    if (do_restart) {
-	do_restart = 0;
-
-	print_line("Running test with dynamically initialized mutex");
-
-	print_line("Trying to initialize mutex");
-	res = ethr_mutex_init(&mt_mutex);
-	ASSERT(res == 0);
-	print_line("Initialized mutex");
-
-	goto restart;
-
-    }
-
 }
 
 /*
@@ -438,9 +419,9 @@ mutex_test(void)
  * Tests try lock mutex operation.
  */
 
-static ethr_mutex tlmt_mtx1 = ETHR_MUTEX_INITER;
-static ethr_mutex tlmt_mtx2 = ETHR_MUTEX_INITER;
-static ethr_cond tlmt_cnd2 = ETHR_COND_INITER;
+static ethr_mutex tlmt_mtx1;
+static ethr_mutex tlmt_mtx2;
+static ethr_cond tlmt_cnd2;
 
 static int tlmt_mtx1_locked;
 static int tlmt_mtx1_do_unlock;
@@ -450,32 +431,24 @@ tlmt_thread(void *unused)
 {
     int res;
 
-    res = ethr_mutex_lock(&tlmt_mtx1);
-    ASSERT(res == 0);
-    res = ethr_mutex_lock(&tlmt_mtx2);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&tlmt_mtx1);
+    ethr_mutex_lock(&tlmt_mtx2);
 
     tlmt_mtx1_locked = 1;
-    res = ethr_cond_signal(&tlmt_cnd2);
-    ASSERT(res == 0);
+    ethr_cond_signal(&tlmt_cnd2);
 
     while (!tlmt_mtx1_do_unlock) {
 	res = ethr_cond_wait(&tlmt_cnd2, &tlmt_mtx2);
 	ASSERT(res == 0 || res == EINTR);
     }
 
-    res = ethr_mutex_unlock(&tlmt_mtx2);
-    ASSERT(res == 0);
-    res = ethr_mutex_unlock(&tlmt_mtx1);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&tlmt_mtx2);
+    ethr_mutex_unlock(&tlmt_mtx1);
 
-    res = ethr_mutex_lock(&tlmt_mtx2);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&tlmt_mtx2);
     tlmt_mtx1_locked = 0;
-    res = ethr_cond_signal(&tlmt_cnd2);
-    ASSERT(res == 0);
-    res = ethr_mutex_unlock(&tlmt_mtx2);
-    ASSERT(res == 0);
+    ethr_cond_signal(&tlmt_cnd2);
+    ethr_mutex_unlock(&tlmt_mtx2);
 
     return NULL;
 }
@@ -486,48 +459,49 @@ try_lock_mutex_test(void)
     int i, res;
     ethr_tid tid;
 
+    res = ethr_mutex_init(&tlmt_mtx1);
+    ASSERT(res == 0);
+    res = ethr_mutex_init(&tlmt_mtx2);
+    ASSERT(res == 0);
+    res = ethr_cond_init(&tlmt_cnd2);
+    ASSERT(res == 0);
+
     tlmt_mtx1_locked = 0;
     tlmt_mtx1_do_unlock = 0;
 
     res = ethr_thr_create(&tid, tlmt_thread, NULL, NULL);
     ASSERT(res == 0);
 
-    res = ethr_mutex_lock(&tlmt_mtx2);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&tlmt_mtx2);
 
     while (!tlmt_mtx1_locked) {
 	res = ethr_cond_wait(&tlmt_cnd2, &tlmt_mtx2);
 	ASSERT(res == 0 || res == EINTR);
     }
 
-    res = ethr_mutex_unlock(&tlmt_mtx2);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&tlmt_mtx2);
 
     for (i = 0; i < 10; i++) {
 	res = ethr_mutex_trylock(&tlmt_mtx1);
 	ASSERT(res == EBUSY);
     }
 
-    res = ethr_mutex_lock(&tlmt_mtx2);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&tlmt_mtx2);
 
     tlmt_mtx1_do_unlock = 1;
-    res = ethr_cond_signal(&tlmt_cnd2);
-    ASSERT(res == 0);
+    ethr_cond_signal(&tlmt_cnd2);
 
     while (tlmt_mtx1_locked) {
 	res = ethr_cond_wait(&tlmt_cnd2, &tlmt_mtx2);
 	ASSERT(res == 0 || res == EINTR);
     }
 
-    res = ethr_mutex_unlock(&tlmt_mtx2);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&tlmt_mtx2);
 
     res = ethr_mutex_trylock(&tlmt_mtx1);
     ASSERT(res == 0);
 
-    res = ethr_mutex_unlock(&tlmt_mtx1);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&tlmt_mtx1);
 
     res = ethr_thr_join(tid, NULL);
     ASSERT(res == 0);
@@ -541,328 +515,88 @@ try_lock_mutex_test(void)
 }
 
 /*
- * The recursive mutex test case.
- *
- * Tests recursive mutexes.
- */
-
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-
-static ethr_mutex rmt_mutex
-#ifdef ETHR_REC_MUTEX_INITER 
-                            = ETHR_REC_MUTEX_INITER
-#endif
-                                                   ;
-static int rmt_data;
-
-void *
-rmt_thread(void *unused)
-{
-    int res;
-
-    print_line("Aux thread tries to lock mutex");
-    res = ethr_mutex_lock(&rmt_mutex);
-    ASSERT(res == 0);
-    print_line("Aux thread locked mutex");
-
-    ASSERT(rmt_data == 0);
-
-    rmt_data = 1;
-    print_line("Aux thread wrote");
-
-    print_line("Aux thread goes to sleep for 1 second");
-    do_sleep(1);
-    print_line("Aux thread woke up");
-
-    ASSERT(rmt_data == 1);
-
-    res = ethr_mutex_unlock(&rmt_mutex);
-    ASSERT(res == 0);
-    print_line("Aux thread unlocked mutex");
-
-    return NULL;
-}
-
-#endif
-
-static void
-recursive_mutex_test(void)
-{
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-    int do_restart = 1;
-    int res;
-    ethr_tid tid;
-
-#ifdef ETHR_REC_MUTEX_INITER
-    print_line("Running test with statically initialized mutex");
-#else
-    goto dynamic_init;
-#endif
-
- restart:
-    rmt_data = 0;
-
-    print_line("Main thread tries to lock mutex");
-    res = ethr_mutex_lock(&rmt_mutex);
-    ASSERT(res == 0);
-    print_line("Main thread locked mutex");
-
-    print_line("Main thread tries to lock mutex again");
-    res = ethr_mutex_lock(&rmt_mutex);
-    ASSERT(res == 0);
-    print_line("Main thread locked mutex again");
-
-    ASSERT(rmt_data == 0);
-
-    print_line("Main thread about to create aux thread");
-    res = ethr_thr_create(&tid, rmt_thread, NULL, NULL);
-    ASSERT(res == 0);
-    print_line("Main thread created aux thread");
-
-    print_line("Main thread goes to sleep for 1 second");
-    do_sleep(1);
-    print_line("Main thread woke up");
-
-    ASSERT(rmt_data == 0);
-
-    res = ethr_mutex_unlock(&rmt_mutex);
-    ASSERT(res == 0);
-    print_line("Main thread unlocked mutex");
-
-    print_line("Main thread goes to sleep for 1 second");
-    do_sleep(1);
-    print_line("Main thread woke up");
-
-    ASSERT(rmt_data == 0);
-
-    res = ethr_mutex_unlock(&rmt_mutex);
-    ASSERT(res == 0);
-    print_line("Main thread unlocked mutex again");
-
-    print_line("Main thread goes to sleep for 1 second");
-    do_sleep(1);
-    print_line("Main thread woke up");
-
-    print_line("Main thread tries to lock mutex");
-    res = ethr_mutex_lock(&rmt_mutex);
-    ASSERT(res == 0);
-    print_line("Main thread locked mutex");
-
-    ASSERT(rmt_data == 1);
-
-    print_line("Main thread goes to sleep for 1 second");
-    do_sleep(1);
-    print_line("Main thread woke up");
-
-    ASSERT(rmt_data == 1);
-
-    res = ethr_mutex_unlock(&rmt_mutex);
-    ASSERT(res == 0);
-    print_line("Main thread unlocked mutex");
-
-    res = ethr_thr_join(tid, NULL);
-    ASSERT(res == 0);
-    print_line("Main thread joined aux thread");
-
-    res = ethr_mutex_destroy(&rmt_mutex);
-    ASSERT(res == 0);
-    print_line("Main thread destroyed mutex");
-
-    if (do_restart) {
-#ifndef ETHR_REC_MUTEX_INITER
-    dynamic_init:
-#endif
-	do_restart = 0;
-
-	print_line("Running test with dynamically initialized mutex");
-
-	print_line("Trying to initialize mutex");
-	res = ethr_rec_mutex_init(&rmt_mutex);
-	ASSERT(res == 0);
-	print_line("Initialized mutex");
-
-	goto restart;
-    }
-
-#ifndef ETHR_REC_MUTEX_INITER
-    succeed("Static initializer for recursive mutexes not supported");
-#endif
-
-#else /* #ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT */
-    skip("Recursive mutexes not supported");
-#endif /* #ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT */
-}
-
-/*
- * The time now test.
- *
- * Tests ethr_time_now by comparing time values with Erlang.
- */
-#define TNT_MAX_TIME_DIFF 200000
-#define TNT_MAX_TIME_VALUES 52
-
-static void
-time_now_test(void)
-{
-    int scanf_res, time_now_res, i, no_values, max_abs_diff;
-    static ethr_timeval tv[TNT_MAX_TIME_VALUES];
-    static int ms[TNT_MAX_TIME_VALUES];
-
-    i = 0;
-    do {
-	ASSERT(i < TNT_MAX_TIME_VALUES);
-	scanf_res = scanf("%d", &ms[i]);
-	time_now_res = ethr_time_now(&tv[i]);
-	ASSERT(scanf_res == 1);
-	ASSERT(time_now_res == 0);
-#if 0
-	print_line("Got %d; %ld:%ld", ms[i], tv[i].tv_sec, tv[i].tv_nsec);
-#endif
-	i++;
-    } while (ms[i-1] >= 0);
-
-    no_values = i-1;
-
-    ASSERT(ms[0] == 0);
-
-    print_line("TNT_MAX_TIME_DIFF = %d (us)", TNT_MAX_TIME_DIFF);
-
-    max_abs_diff = 0;
-
-    for (i = 1; i < no_values; i++) {
-	long diff;
-	long tn_us;
-	long e_us;
-
-	tn_us = (tv[i].tv_sec - tv[0].tv_sec) * 1000000;
-	tn_us += (tv[i].tv_nsec - tv[0].tv_nsec)/1000;
-
-	e_us = ms[i]*1000;
-	
-	diff = e_us - tn_us;
-
-	print_line("Erlang time = %ld us; ethr_time_now = %ld us; diff %ld us",
-		   e_us, tn_us, diff);
-
-	if (max_abs_diff < abs((int) diff)) {
-	    max_abs_diff = abs((int) diff);
-	}
-
-	ASSERT(e_us - TNT_MAX_TIME_DIFF <= tn_us);
-	ASSERT(tn_us <= e_us + TNT_MAX_TIME_DIFF);
-    }
-
-    print_line("Max absolute diff = %d us", max_abs_diff);
-    succeed("Max absolute diff = %d us", max_abs_diff);
-}
-
-/*
  * The cond wait test case.
  *
  * Tests ethr_cond_wait with ethr_cond_signal and ethr_cond_broadcast.
  */
 
 
-static ethr_mutex cwt_mutex = ETHR_MUTEX_INITER;
-static ethr_cond cwt_cond = ETHR_COND_INITER;
+static ethr_mutex cwt_mutex;
+static ethr_cond cwt_cond;
 static int cwt_counter;
 
 void *
-cwt_thread(void *is_timedwait_test_ptr)
+cwt_thread(void *unused)
 {
-    int use_timedwait = *((int *) is_timedwait_test_ptr);
     int res;
 
-    res = ethr_mutex_lock(&cwt_mutex);
-    ASSERT(res == 0);
-
-    if (use_timedwait) {
-	ethr_timeval tv;
-	res = ethr_time_now(&tv);
-	ASSERT(res == 0);
-	tv.tv_sec += 3600; /* Make sure we won't time out */
+    ethr_mutex_lock(&cwt_mutex);
 
-	do {
-	    res = ethr_cond_timedwait(&cwt_cond, &cwt_mutex, &tv);
-	} while (res == EINTR);
-	ASSERT(res == 0);
-    }
-    else {
-	do {
-	    res = ethr_cond_wait(&cwt_cond, &cwt_mutex);
-	} while (res == EINTR);
-	ASSERT(res == 0);
-    }
+    do {
+	res = ethr_cond_wait(&cwt_cond, &cwt_mutex);
+    } while (res == EINTR);
+    ASSERT(res == 0);
 
     cwt_counter++;
 
-    res = ethr_mutex_unlock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&cwt_mutex);
 
     return NULL;
 }
 
 static void
-cond_wait_test(int is_timedwait_test)
+cond_wait_test(void)
 {
-    int do_restart = !is_timedwait_test;
     ethr_tid tid1, tid2;
     int res;
 
-    if (!is_timedwait_test)
-	print_line("Running test with statically initialized mutex and cond");
+    res = ethr_mutex_init(&cwt_mutex);
+    ASSERT(res == 0);
+    res = ethr_cond_init(&cwt_cond);
+    ASSERT(res == 0);
 
- restart:
     /* Wake with signal */
 
     cwt_counter = 0;
 
-    res = ethr_thr_create(&tid1, cwt_thread, (void *) &is_timedwait_test, NULL);
+    res = ethr_thr_create(&tid1, cwt_thread, NULL, NULL);
     ASSERT(res == 0);
-    res = ethr_thr_create(&tid2, cwt_thread, (void *) &is_timedwait_test, NULL);
+    res = ethr_thr_create(&tid2, cwt_thread, NULL, NULL);
     ASSERT(res == 0);
 
     do_sleep(1); /* Make sure threads waits on cond var */
 
-    res = ethr_mutex_lock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&cwt_mutex);
 
-    res = ethr_cond_signal(&cwt_cond); /* Wake one thread */
-    ASSERT(res == 0);
+    ethr_cond_signal(&cwt_cond); /* Wake one thread */
 
     do_sleep(1); /* Make sure awakened thread waits on mutex */
 
     ASSERT(cwt_counter == 0);
 
-    res = ethr_mutex_unlock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&cwt_mutex);
 
     do_sleep(1);  /* Let awakened thread proceed */
 
-    res = ethr_mutex_lock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&cwt_mutex);
 
     ASSERT(cwt_counter == 1);
 
-    res = ethr_cond_signal(&cwt_cond); /* Wake the other thread */
-    ASSERT(res == 0);
+    ethr_cond_signal(&cwt_cond); /* Wake the other thread */
 
     do_sleep(1); /* Make sure awakened thread waits on mutex */
 
     ASSERT(cwt_counter == 1);
 
-    res = ethr_mutex_unlock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&cwt_mutex);
 
     do_sleep(1);  /* Let awakened thread proceed */
 
-    res = ethr_mutex_lock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&cwt_mutex);
 
     ASSERT(cwt_counter == 2);
 
-    res = ethr_mutex_unlock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&cwt_mutex);
 
     res = ethr_thr_join(tid1, NULL);
     ASSERT(res == 0);
@@ -875,35 +609,30 @@ cond_wait_test(int is_timedwait_test)
 
     cwt_counter = 0;
 
-    res = ethr_thr_create(&tid1, cwt_thread, (void *) &is_timedwait_test, NULL);
+    res = ethr_thr_create(&tid1, cwt_thread, NULL, NULL);
     ASSERT(res == 0);
-    res = ethr_thr_create(&tid2, cwt_thread, (void *) &is_timedwait_test, NULL);
+    res = ethr_thr_create(&tid2, cwt_thread, NULL, NULL);
     ASSERT(res == 0);
 
     do_sleep(1); /* Make sure threads waits on cond var */
 
-    res = ethr_mutex_lock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&cwt_mutex);
 
-    res = ethr_cond_broadcast(&cwt_cond); /* Wake the threads */
-    ASSERT(res == 0);
+    ethr_cond_broadcast(&cwt_cond); /* Wake the threads */
 
     do_sleep(1); /* Make sure awakened threads wait on mutex */
 
     ASSERT(cwt_counter == 0);
 
-    res = ethr_mutex_unlock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&cwt_mutex);
 
     do_sleep(1);  /* Let awakened threads proceed */
 
-    res = ethr_mutex_lock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&cwt_mutex);
 
     ASSERT(cwt_counter == 2);
 
-    res = ethr_mutex_unlock(&cwt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&cwt_mutex);
 
     res = ethr_thr_join(tid1, NULL);
     ASSERT(res == 0);
@@ -916,113 +645,6 @@ cond_wait_test(int is_timedwait_test)
     res = ethr_cond_destroy(&cwt_cond);
     ASSERT(res == 0);
 
-    if (do_restart) {
-	do_restart = 0;
-	res = ethr_mutex_init(&cwt_mutex);
-	ASSERT(res == 0);
-	res = ethr_cond_init(&cwt_cond);
-	ASSERT(res == 0);
-	print_line("Running test with dynamically initialized mutex and cond");
-	goto restart;
-    }
-}
-
-/*
- * The cond timedwait test case.
- *
- * Tests ethr_cond_timedwait with ethr_cond_signal and ethr_cond_broadcast.
- */
-
-#define CTWT_MAX_TIME_DIFF 100000
-
-static long
-ctwt_check_timeout(long to)
-{
-    int res;
-    ethr_timeval tva, tvb;
-    long diff, abs_diff;
-
-    res = ethr_time_now(&tva);
-    ASSERT(res == 0);
-
-    tva.tv_sec += to / 1000;
-    tva.tv_nsec += (to % 1000) * 1000000;
-    if (tva.tv_nsec >= 1000000000) {
-	tva.tv_sec++;
-	tva.tv_nsec -= 1000000000;
-	ASSERT(tva.tv_nsec < 1000000000);
-    }
-
-    do {
-	res = ethr_cond_timedwait(&cwt_cond, &cwt_mutex, &tva);
-    } while (res == EINTR);
-    ASSERT(res == ETIMEDOUT);
-
-    res = ethr_time_now(&tvb);
-    ASSERT(res == 0);
-
-    diff = (tvb.tv_sec - tva.tv_sec) * 1000000;
-    diff += (tvb.tv_nsec - tva.tv_nsec)/1000;
-
-    print("Timeout=%ld; ", to);
-    print("tva.tv_sec=%ld tva.tv_nsec=%ld; ", tva.tv_sec, tva.tv_nsec);
-    print("tvb.tv_sec=%ld tvb.tv_nsec=%ld; ", tvb.tv_sec, tvb.tv_nsec);
-    print_line("diff (tvb - tva) = %ld us", diff);
-
-    abs_diff = (long) abs((int) diff);
-
-    ASSERT(CTWT_MAX_TIME_DIFF >= abs_diff);
-    return abs_diff;
-}
-
-static void
-cond_timedwait_test(void)
-{
-    int do_restart = 1;
-    long abs_diff, max_abs_diff = 0;
-    int res;
-
-#define CTWT_UPD_MAX_DIFF if (abs_diff > max_abs_diff) max_abs_diff = abs_diff;
-
-    print_line("Running test with statically initialized mutex and cond");
-
-    print_line("CTWT_MAX_TIME_DIFF=%d", CTWT_MAX_TIME_DIFF);
-
- restart:
-
-    res = ethr_mutex_lock(&cwt_mutex);
-    ASSERT(res == 0);
-
-    abs_diff = ctwt_check_timeout(300);
-    CTWT_UPD_MAX_DIFF;
-    abs_diff = ctwt_check_timeout(700);
-    CTWT_UPD_MAX_DIFF;
-    abs_diff = ctwt_check_timeout(1000);
-    CTWT_UPD_MAX_DIFF;
-    abs_diff = ctwt_check_timeout(2300);
-    CTWT_UPD_MAX_DIFF;
-    abs_diff = ctwt_check_timeout(5100);
-    CTWT_UPD_MAX_DIFF;
-
-    res = ethr_mutex_unlock(&cwt_mutex);
-    ASSERT(res == 0);
-
-    cond_wait_test(1);
-
-    if (do_restart) {
-	do_restart = 0;
-	res = ethr_mutex_init(&cwt_mutex);
-	ASSERT(res == 0);
-	res = ethr_cond_init(&cwt_cond);
-	ASSERT(res == 0);
-	print_line("Running test with dynamically initialized mutex and cond");
-	goto restart;
-    }
-
-    print_line("Max absolute diff = %d us", max_abs_diff);
-    succeed("Max absolute diff = %d us", max_abs_diff);
-
-#undef CTWT_UPD_MAX_DIFF
 }
 
 /*
@@ -1037,9 +659,9 @@ cond_timedwait_test(void)
 static int bct_woken = 0;
 static int bct_waiting = 0;
 static int bct_done = 0;
-static ethr_mutex bct_mutex = ETHR_MUTEX_INITER;
-static ethr_cond bct_cond = ETHR_COND_INITER;
-static ethr_cond bct_cntrl_cond = ETHR_COND_INITER;
+static ethr_mutex bct_mutex;
+static ethr_cond bct_cond;
+static ethr_cond bct_cntrl_cond;
 
 
 static void *
@@ -1047,30 +669,24 @@ bct_thread(void *unused)
 {
     int res;
 
-    res = ethr_mutex_lock(&bct_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&bct_mutex);
 
     while (!bct_done) {
 
 	bct_waiting++;
-	if (bct_waiting == BCT_THREADS) {
-	    res = ethr_cond_signal(&bct_cntrl_cond);
-	    ASSERT(res == 0);
-	}
+	if (bct_waiting == BCT_THREADS)
+	    ethr_cond_signal(&bct_cntrl_cond);
 	do {
 	    res = ethr_cond_wait(&bct_cond, &bct_mutex);
 	} while (res == EINTR);
 	ASSERT(res == 0);
 	bct_woken++;
-	if (bct_woken == BCT_THREADS) {
-	    res = ethr_cond_signal(&bct_cntrl_cond);
-	    ASSERT(res == 0);
-	}
+	if (bct_woken == BCT_THREADS)
+	    ethr_cond_signal(&bct_cntrl_cond);
 
     }
 
-    res = ethr_mutex_unlock(&bct_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&bct_mutex);
 
     return NULL;
 }
@@ -1081,14 +697,20 @@ broadcast_test(void)
     int res, i;
     ethr_tid tid[BCT_THREADS];
 
+    res = ethr_mutex_init(&bct_mutex);
+    ASSERT(res == 0);
+    res = ethr_cond_init(&bct_cntrl_cond);
+    ASSERT(res == 0);
+    res = ethr_cond_init(&bct_cond);
+    ASSERT(res == 0);
+
     for (i = 0; i < BCT_THREADS; i++) {
 	res = ethr_thr_create(&tid[i], bct_thread, NULL, NULL);
 	ASSERT(res == 0);
 
     }
 
-    res = ethr_mutex_lock(&bct_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&bct_mutex);
 
     for (i = 0; i < BCT_NO_OF_WAITS; i++) {
 
@@ -1101,8 +723,7 @@ broadcast_test(void)
 	bct_woken = 0;
 
 	/* Wake all threads */
-	res = ethr_cond_broadcast(&bct_cond);
-	ASSERT(res == 0);
+	ethr_cond_broadcast(&bct_cond);
 
 	while (bct_woken != BCT_THREADS) {
 	    res = ethr_cond_wait(&bct_cntrl_cond, &bct_mutex);
@@ -1114,13 +735,11 @@ broadcast_test(void)
     bct_done = 1;
 
     /* Wake all threads */
-    res = ethr_cond_broadcast(&bct_cond);
-    ASSERT(res == 0);
+    ethr_cond_broadcast(&bct_cond);
 
-    res = ethr_mutex_unlock(&bct_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&bct_mutex);
 
-    for (i = 0; i < BCT_THREADS - 1; i++) {
+    for (i = 0; i < BCT_THREADS; i++) {
 	res = ethr_thr_join(tid[i], NULL);
 	ASSERT(res == 0);
     }
@@ -1143,26 +762,22 @@ broadcast_test(void)
 #define DT_THREADS (50*1024)
 #define DT_BATCH_SIZE 64
 
-static ethr_mutex dt_mutex = ETHR_MUTEX_INITER;
-static ethr_cond dt_cond = ETHR_COND_INITER;
+static ethr_mutex dt_mutex;
+static ethr_cond dt_cond;
 static int dt_count;
 static int dt_limit;
 
 static void *
 dt_thread(void *unused)
 {
-    int res;
-
-    res = ethr_mutex_lock(&dt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&dt_mutex);
 
     dt_count++;
 
     if (dt_count >= dt_limit)
 	ethr_cond_signal(&dt_cond);
 
-    res = ethr_mutex_unlock(&dt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&dt_mutex);
 
     return NULL;
 }
@@ -1174,6 +789,11 @@ detached_thread_test(void)
     ethr_tid tid[DT_BATCH_SIZE];
     int i, j, res;
 
+    res = ethr_mutex_init(&dt_mutex);
+    ASSERT(res == 0);
+    res = ethr_cond_init(&dt_cond);
+    ASSERT(res == 0);
+
     thr_opts.detached = 1;
     dt_count = 0;
     dt_limit = 0;
@@ -1187,14 +807,12 @@ detached_thread_test(void)
 	    ASSERT(res == 0);
 	}
 
-	res = ethr_mutex_lock(&dt_mutex);
-	ASSERT(res == 0);
+	ethr_mutex_lock(&dt_mutex);
 	while (dt_count < dt_limit) {
 	    res = ethr_cond_wait(&dt_cond, &dt_mutex);
 	    ASSERT(res == 0 || res == EINTR);
 	}
-	res = ethr_mutex_unlock(&dt_mutex);
-	ASSERT(res == 0);
+	ethr_mutex_unlock(&dt_mutex);
 
 	print_line("dt_count = %d", dt_count);
     }
@@ -1209,10 +827,11 @@ detached_thread_test(void)
  * Tests
  */
 #define MTT_TIMES 10
+#define MTT_HARD_LIMIT (80000)
 
 static int mtt_terminate;
-static ethr_mutex mtt_mutex = ETHR_MUTEX_INITER;
-static ethr_cond mtt_cond = ETHR_COND_INITER;
+static ethr_mutex mtt_mutex;
+static ethr_cond mtt_cond;
 static char mtt_string[22*MTT_TIMES]; /* 22 is enough for ", %d" */
 
 
@@ -1220,16 +839,14 @@ void *mtt_thread(void *unused)
 {
     int res;
 
-    res = ethr_mutex_lock(&mtt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&mtt_mutex);
 
     while (!mtt_terminate) {
 	res = ethr_cond_wait(&mtt_cond, &mtt_mutex);
 	ASSERT(res == 0 || res == EINTR);
     }
 
-    res = ethr_mutex_unlock(&mtt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&mtt_mutex);
 
     return NULL; 
 }
@@ -1251,30 +868,33 @@ mtt_create_join_threads(void)
     while (1) {
 	if (ix >= no_tids) {
 	    no_tids += 100;
+	    if (no_tids > MTT_HARD_LIMIT) {
+		print_line("Hit the hard limit on number of threads (%d)!", 
+			   MTT_HARD_LIMIT);
+		break;
+	    }
 	    tids = (ethr_tid *) realloc((void *)tids, sizeof(ethr_tid)*no_tids);
 	    ASSERT(tids);
 	}
 	res = ethr_thr_create(&tids[ix], mtt_thread, NULL, NULL);
-	if (res != 0)
+	if (res != 0) {
 	    break;
+	}
 	ix++;
-    } while (res == 0);
+    }
 
     no_threads = ix;
 
     print_line("%d = ethr_thr_create()", res);
     print_line("Number of created threads: %d", no_threads);
 
-    res = ethr_mutex_lock(&mtt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_lock(&mtt_mutex);
 
     mtt_terminate = 1;
 
-    res = ethr_cond_broadcast(&mtt_cond);
-    ASSERT(res == 0);
+    ethr_cond_broadcast(&mtt_cond);
 
-    res = ethr_mutex_unlock(&mtt_mutex);
-    ASSERT(res == 0);
+    ethr_mutex_unlock(&mtt_mutex);
 
     while (ix) {
 	res = ethr_thr_join(tids[--ix], NULL);
@@ -1292,9 +912,14 @@ mtt_create_join_threads(void)
 static void
 max_threads_test(void)
 {
-    int no_threads[MTT_TIMES], i, up, down, eq;
+    int no_threads[MTT_TIMES], i, up, down, eq, res;
     char *str;
 
+    res = ethr_mutex_init(&mtt_mutex);
+    ASSERT(res == 0);
+    res = ethr_cond_init(&mtt_cond);
+    ASSERT(res == 0);
+
     for (i = 0; i < MTT_TIMES; i++) {
 	no_threads[i] = mtt_create_join_threads();
     }
@@ -1326,272 +951,6 @@ max_threads_test(void)
 
 }
 
-
-/*
- * The forksafety test case.
- *
- * Tests forksafety.
- */
-#ifdef __WIN32__
-#define NO_FORK_PRESENT
-#endif
-
-#ifndef NO_FORK_PRESENT
-
-static ethr_mutex ft_test_inner_mutex = ETHR_MUTEX_INITER;
-static ethr_mutex ft_test_outer_mutex = ETHR_MUTEX_INITER;
-static ethr_mutex ft_go_mutex = ETHR_MUTEX_INITER;
-static ethr_cond ft_go_cond = ETHR_COND_INITER;
-static int ft_go;
-static int ft_have_forked;
-
-static void *
-ft_thread(void *unused)
-{
-    int res;
-
-    res = ethr_mutex_lock(&ft_test_outer_mutex);
-    ASSERT(res == 0);
-
-    res = ethr_mutex_lock(&ft_go_mutex);
-    ASSERT(res == 0);
-
-    ft_go = 1;
-    res = ethr_cond_signal(&ft_go_cond);
-    ASSERT(res == 0);
-    res = ethr_mutex_unlock(&ft_go_mutex);
-    ASSERT(res == 0);
-
-    do_sleep(1);
-    ASSERT(!ft_have_forked);
-
-    res = ethr_mutex_lock(&ft_test_inner_mutex);
-    ASSERT(res == 0);
-
-    res = ethr_mutex_unlock(&ft_test_inner_mutex);
-    ASSERT(res == 0);
-
-    do_sleep(1);
-    ASSERT(!ft_have_forked);
-
-    res = ethr_mutex_unlock(&ft_test_outer_mutex);
-    ASSERT(res == 0);
-
-    do_sleep(1);
-    ASSERT(ft_have_forked);
-    
-
-    return NULL;
-}
-
-#endif /* #ifndef NO_FORK_PRESENT */
-
-static void
-forksafety_test(void)
-{
-#ifdef NO_FORK_PRESENT
-    skip("No fork() present; nothing to test");
-#elif defined(DEBUG)
-    skip("Doesn't work in debug build");
-#else
-    char snd_msg[] = "ok, bye!";
-    char rec_msg[sizeof(snd_msg)*2];
-    ethr_tid tid;
-    int res;
-    int fds[2];
-
-
-    res = ethr_mutex_set_forksafe(&ft_test_inner_mutex);
-    if (res == ENOTSUP) {
-	skip("Forksafety not supported on this platform!");
-    }
-    ASSERT(res == 0);
-    res = ethr_mutex_set_forksafe(&ft_test_outer_mutex);
-    ASSERT(res == 0);
-
-
-    res = pipe(fds);
-    ASSERT(res == 0);
-
-    ft_go = 0;
-    ft_have_forked = 0;
-
-    res = ethr_mutex_lock(&ft_go_mutex);
-    ASSERT(res == 0);
-    
-    res = ethr_thr_create(&tid, ft_thread, NULL, NULL);
-    ASSERT(res == 0);
-
-    do {
-	res = ethr_cond_wait(&ft_go_cond, &ft_go_mutex);
-    } while (res == EINTR || !ft_go);
-    ASSERT(res == 0);
-
-    res = ethr_mutex_unlock(&ft_go_mutex);
-    ASSERT(res == 0);
-
-    res = fork();
-    ft_have_forked = 1;
-    if (res == 0) {
-	close(fds[0]);
-	res = ethr_mutex_lock(&ft_test_outer_mutex);
-	if (res != 0)
-	    _exit(1);
-	res = ethr_mutex_lock(&ft_test_inner_mutex);
-	if (res != 0)
-	    _exit(1);
-	res = ethr_mutex_unlock(&ft_test_inner_mutex);
-	if (res != 0)
-	    _exit(1);
-	res = ethr_mutex_unlock(&ft_test_outer_mutex);
-	if (res != 0)
-	    _exit(1);
-
-	res = ethr_mutex_destroy(&ft_test_inner_mutex);
-	if (res != 0)
-	    _exit(1);
-	res = ethr_mutex_destroy(&ft_test_outer_mutex);
-	if (res != 0)
-	    _exit(1);
-
-	res = (int) write(fds[1], (void *) snd_msg, sizeof(snd_msg));
-	if (res != sizeof(snd_msg))
-	    _exit(1);
-	close(fds[1]);
-	_exit(0);
-    }
-    ASSERT(res > 0);
-    close(fds[1]);
-
-    res = (int) read(fds[0], (void *) rec_msg, sizeof(rec_msg));
-    ASSERT(res == (int) sizeof(snd_msg));
-    ASSERT(strcmp(snd_msg, rec_msg) == 0);
-
-    close(fds[0]);
-#endif
-}
-
-
-/*
- * The vfork test case.
- *
- * Tests vfork with threads.
- */
-
-#ifdef __WIN32__
-#define NO_VFORK_PRESENT
-#endif
-
-#ifndef NO_VFORK_PRESENT
-
-#undef vfork
-
-static ethr_mutex vt_mutex = ETHR_MUTEX_INITER;
-
-static void *
-vt_thread(void *vprog)
-{
-    char *prog = (char *) vprog;
-    int res;
-    char snd_msg[] = "ok, bye!";
-    char rec_msg[sizeof(snd_msg)*2];
-    int fds[2];
-    char closefd[20];
-    char writefd[20];
-
-    res = pipe(fds);
-    ASSERT(res == 0);
-
-    res = sprintf(closefd, "%d", fds[0]);
-    ASSERT(res <= 20);
-    res = sprintf(writefd, "%d", fds[1]);
-    ASSERT(res <= 20);
-
-    print("parent: About to vfork and execute ");
-    print("execlp(\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", NULL)",
-	  prog, prog, "vfork", "exec", snd_msg, closefd, writefd);
-    print_line(" in child");
-    res = vfork();
-    if (res == 0) {
-	execlp(prog, prog, "vfork", "exec", snd_msg, closefd, writefd, NULL);
-	_exit(1);
-    }
-    ASSERT(res > 0);
-
-    print_line("parent: I'm back");
-
-    close(fds[1]);
-
-    res = (int) read(fds[0], (void *) rec_msg, sizeof(rec_msg));
-    print_line("parent: %d = read()", res);
-    print_line("parent: rec_msg=\"%s\"", rec_msg);
-    ASSERT(res == (int) sizeof(snd_msg));
-    ASSERT(strcmp(snd_msg, rec_msg) == 0);
-
-    close(fds[0]);
-
-    return NULL;
-}
-
-#endif /* #ifndef NO_VFORK_PRESENT */
-
-static void
-vfork_test(int argc, char *argv[])
-{
-#ifdef NO_VFORK_PRESENT
-    skip("No vfork() present; nothing to test");
-#else
-    int res;
-    ethr_tid tid;
-
-    if (argc == 6 && strcmp("exec", argv[2]) == 0) {
-	/* We are child after vfork() and execlp() ... */
-
-	char *snd_msg;
-	int closefd;
-	int writefd;
-
-	snd_msg = argv[3];
-	closefd = atoi(argv[4]);
-	writefd = atoi(argv[5]);
-
-	print_line("child: snd_msg=\"%s\"; closefd=%d writefd=%d",
-		   snd_msg, closefd, writefd);
-
-	close(closefd);
-
-	res = (int) write(writefd, (void *) snd_msg, strlen(snd_msg)+1);
-	print_line("child: %d = write()", res);
-	if (res != strlen(snd_msg)+1)
-	    exit(1);
-	close(writefd);
-	print_line("child: bye");
-	exit(0);
-    }
-    ASSERT(argc == 2);
-
-    res = ethr_mutex_set_forksafe(&vt_mutex);
-    ASSERT(res == 0 || res == ENOTSUP);
-    res = ethr_mutex_lock(&vt_mutex);
-    ASSERT(res == 0);
-
-    res = ethr_thr_create(&tid, vt_thread, (void *) argv[0], NULL);
-    ASSERT(res == 0);
-
-    do_sleep(1);
-
-    res = ethr_mutex_unlock(&vt_mutex);
-    ASSERT(res == 0);
-
-    res = ethr_thr_join(tid, NULL);
-    ASSERT(res == 0);
-
-    res = ethr_mutex_destroy(&vt_mutex);
-    ASSERT(res == 0);
-#endif
-}
-
-
 /*
  * The tsd test case.
  *
@@ -1651,11 +1010,8 @@ static int st_data;
 void *
 st_thread(void *unused)
 {
-    int res;
-
     print_line("Aux thread tries to lock spinlock");
-    res = ethr_spin_lock(&st_spinlock);
-    ASSERT(res == 0);
+    ethr_spin_lock(&st_spinlock);
     print_line("Aux thread locked spinlock");
 
     ASSERT(st_data == 0);
@@ -1669,8 +1025,7 @@ st_thread(void *unused)
 
     ASSERT(st_data == 1);
 
-    res = ethr_spin_unlock(&st_spinlock);
-    ASSERT(res == 0);
+    ethr_spin_unlock(&st_spinlock);
     print_line("Aux thread unlocked spinlock");
 
     return NULL;
@@ -1691,8 +1046,7 @@ spinlock_test(void)
     st_data = 0;
 
     print_line("Main thread tries to lock spinlock");
-    res = ethr_spin_lock(&st_spinlock);
-    ASSERT(res == 0);
+    ethr_spin_lock(&st_spinlock);
     print_line("Main thread locked spinlock");
 
     ASSERT(st_data == 0);
@@ -1708,8 +1062,7 @@ spinlock_test(void)
 
     ASSERT(st_data == 0);
 
-    res = ethr_spin_unlock(&st_spinlock);
-    ASSERT(res == 0);
+    ethr_spin_unlock(&st_spinlock);
     print_line("Main thread unlocked spinlock");
 
     print_line("Main thread goes to sleep for 1 second");
@@ -1717,8 +1070,7 @@ spinlock_test(void)
     print_line("Main thread woke up");
 
     print_line("Main thread tries to lock spinlock");
-    res = ethr_spin_lock(&st_spinlock);
-    ASSERT(res == 0);
+    ethr_spin_lock(&st_spinlock);
     print_line("Main thread locked spinlock");
 
     ASSERT(st_data == 1);
@@ -1729,8 +1081,7 @@ spinlock_test(void)
 
     ASSERT(st_data == 1);
 
-    res = ethr_spin_unlock(&st_spinlock);
-    ASSERT(res == 0);
+    ethr_spin_unlock(&st_spinlock);
     print_line("Main thread unlocked spinlock");
 
     res = ethr_thr_join(tid, NULL);
@@ -1757,23 +1108,19 @@ void *
 rwst_thread(void *unused)
 {
     int data;
-    int res;
 
     print_line("Aux thread tries to read lock rwspinlock");
-    res = ethr_read_lock(&rwst_rwspinlock);
-    ASSERT(res == 0);
+    ethr_read_lock(&rwst_rwspinlock);
     print_line("Aux thread read locked rwspinlock");
 
     ASSERT(rwst_data == 4711);
 
     print_line("Aux thread tries to read unlock rwspinlock");
-    res = ethr_read_unlock(&rwst_rwspinlock);
-    ASSERT(res == 0);
+    ethr_read_unlock(&rwst_rwspinlock);
     print_line("Aux thread read unlocked rwspinlock");
 
     print_line("Aux thread tries to write lock rwspinlock");
-    res = ethr_write_lock(&rwst_rwspinlock);
-    ASSERT(res == 0);
+    ethr_write_lock(&rwst_rwspinlock);
     print_line("Aux thread write locked rwspinlock");
 
     data = ++rwst_data;
@@ -1787,8 +1134,7 @@ rwst_thread(void *unused)
     ++rwst_data;
 
     print_line("Aux thread tries to write unlock rwspinlock");
-    res = ethr_write_unlock(&rwst_rwspinlock);
-    ASSERT(res == 0);
+    ethr_write_unlock(&rwst_rwspinlock);
     print_line("Aux thread write unlocked rwspinlock");
 
     return NULL;
@@ -1810,8 +1156,7 @@ rwspinlock_test(void)
     rwst_data = 4711;
 
     print_line("Main thread tries to read lock rwspinlock");
-    res = ethr_read_lock(&rwst_rwspinlock);
-    ASSERT(res == 0);
+    ethr_read_lock(&rwst_rwspinlock);
     print_line("Main thread read locked rwspinlock");
 
     ASSERT(rwst_data == 4711);
@@ -1828,13 +1173,11 @@ rwspinlock_test(void)
     ASSERT(rwst_data == 4711);
 
     print_line("Main thread tries to read unlock rwspinlock");
-    res = ethr_read_unlock(&rwst_rwspinlock);
-    ASSERT(res == 0);
+    ethr_read_unlock(&rwst_rwspinlock);
     print_line("Main thread read unlocked rwspinlock");
 
     print_line("Main thread tries to write lock rwspinlock");
-    res = ethr_write_lock(&rwst_rwspinlock);
-    ASSERT(res == 0);
+    ethr_write_lock(&rwst_rwspinlock);
     print_line("Main thread write locked rwspinlock");
 
     data = ++rwst_data;
@@ -1847,8 +1190,7 @@ rwspinlock_test(void)
     ++rwst_data;
 
     print_line("Main thread tries to write unlock rwspinlock");
-    res = ethr_write_unlock(&rwst_rwspinlock);
-    ASSERT(res == 0);
+    ethr_write_unlock(&rwst_rwspinlock);
     print_line("Main thread write unlocked rwspinlock");
 
     res = ethr_thr_join(tid, NULL);
@@ -1875,23 +1217,19 @@ void *
 rwmt_thread(void *unused)
 {
     int data;
-    int res;
 
     print_line("Aux thread tries to read lock rwmutex");
-    res = ethr_rwmutex_rlock(&rwmt_rwmutex);
-    ASSERT(res == 0);
+    ethr_rwmutex_rlock(&rwmt_rwmutex);
     print_line("Aux thread read locked rwmutex");
 
     ASSERT(rwmt_data == 4711);
 
     print_line("Aux thread tries to read unlock rwmutex");
-    res = ethr_rwmutex_runlock(&rwmt_rwmutex);
-    ASSERT(res == 0);
+    ethr_rwmutex_runlock(&rwmt_rwmutex);
     print_line("Aux thread read unlocked rwmutex");
 
     print_line("Aux thread tries to write lock rwmutex");
-    res = ethr_rwmutex_rwlock(&rwmt_rwmutex);
-    ASSERT(res == 0);
+    ethr_rwmutex_rwlock(&rwmt_rwmutex);
     print_line("Aux thread write locked rwmutex");
 
     data = ++rwmt_data;
@@ -1905,8 +1243,7 @@ rwmt_thread(void *unused)
     ++rwmt_data;
 
     print_line("Aux thread tries to write unlock rwmutex");
-    res = ethr_rwmutex_rwunlock(&rwmt_rwmutex);
-    ASSERT(res == 0);
+    ethr_rwmutex_rwunlock(&rwmt_rwmutex);
     print_line("Aux thread write unlocked rwmutex");
 
     return NULL;
@@ -1928,8 +1265,7 @@ rwmutex_test(void)
     rwmt_data = 4711;
 
     print_line("Main thread tries to read lock rwmutex");
-    res = ethr_rwmutex_rlock(&rwmt_rwmutex);
-    ASSERT(res == 0);
+    ethr_rwmutex_rlock(&rwmt_rwmutex);
     print_line("Main thread read locked rwmutex");
 
     ASSERT(rwmt_data == 4711);
@@ -1946,13 +1282,11 @@ rwmutex_test(void)
     ASSERT(rwmt_data == 4711);
 
     print_line("Main thread tries to read unlock rwmutex");
-    res = ethr_rwmutex_runlock(&rwmt_rwmutex);
-    ASSERT(res == 0);
+    ethr_rwmutex_runlock(&rwmt_rwmutex);
     print_line("Main thread read unlocked rwmutex");
 
     print_line("Main thread tries to write lock rwmutex");
-    res = ethr_rwmutex_rwlock(&rwmt_rwmutex);
-    ASSERT(res == 0);
+    ethr_rwmutex_rwlock(&rwmt_rwmutex);
     print_line("Main thread write locked rwmutex");
 
     data = ++rwmt_data;
@@ -1965,8 +1299,7 @@ rwmutex_test(void)
     ++rwmt_data;
 
     print_line("Main thread tries to write unlock rwmutex");
-    res = ethr_rwmutex_rwunlock(&rwmt_rwmutex);
-    ASSERT(res == 0);
+    ethr_rwmutex_rwunlock(&rwmt_rwmutex);
     print_line("Main thread write unlocked rwmutex");
 
     res = ethr_thr_join(tid, NULL);
@@ -1985,6 +1318,9 @@ rwmutex_test(void)
  * Tests atomics.
  */
 
+#define AT_AINT32_MAX 0x7fffffff
+#define AT_AINT32_MIN 0x80000000
+
 #define AT_THREADS 4
 #define AT_ITER 10000
 
@@ -1995,72 +1331,475 @@ static ethr_atomic_t at_go;
 static ethr_atomic_t at_done;
 static ethr_atomic_t at_data;
 
+#define AT_TEST_INIT(T, A, B) \
+do { \
+    ethr_ ## A ## _init ## B(&A, 17); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == 17); \
+} while (0)
+
+#define AT_TEST_SET(T, A, B) \
+do { \
+    ethr_ ## A ## _set ## B(&A, 4711); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \
+} while (0)
+
+#define AT_TEST_XCHG(T, A, B) \
+do { \
+    ethr_ ## A ## _set ## B(&A, 4711); \
+    ASSERT(ethr_ ## A ## _xchg ## B(&A, 17) == 4711); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == 17); \
+} while (0)
+
+#define AT_TEST_CMPXCHG(T, A, B) \
+do { \
+    ethr_ ## A ## _set ## B(&A, 4711); \
+    ASSERT(ethr_ ## A ## _cmpxchg ## B(&A, 17, 33) == 4711); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \
+    ASSERT(ethr_ ## A ## _cmpxchg ## B(&A, 17, 4711) == 4711); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == 17); \
+} while (0)
+
+#define AT_TEST_ADD_READ(T, A, B) \
+do { \
+    T var_ = AT_AINT32_MAX; \
+    var_ += 4711; \
+    ethr_ ## A ## _set ## B(&A, AT_AINT32_MAX); \
+    ASSERT(ethr_ ## A ## _add_read ## B(&A, 4711) == var_); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \
+    var_ = AT_AINT32_MIN; \
+    var_ -= 4711; \
+    ethr_ ## A ## _set ## B(&A, AT_AINT32_MIN); \
+    ASSERT(ethr_ ## A ## _add_read ## B(&A, -4711) == var_); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \
+    ethr_ ## A ## _set ## B(&A, 4711); \
+    ASSERT(ethr_ ## A ## _add_read ## B(&A, 10) == 4721); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == 4721); \
+} while (0)
+
+#define AT_TEST_ADD(T, A, B) \
+do { \
+    T var_ = AT_AINT32_MAX; \
+    var_ += 4711; \
+    ethr_ ## A ## _set ## B(&A, AT_AINT32_MAX); \
+    ethr_ ## A ## _add ## B(&A, 4711); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \
+    var_ = AT_AINT32_MIN; \
+    var_ -= 4711; \
+    ethr_ ## A ## _set ## B(&A, AT_AINT32_MIN); \
+    ethr_ ## A ## _add ## B(&A, -4711); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \
+    ethr_ ## A ## _set ## B(&A, 11); \
+    ethr_ ## A ## _add ## B(&A, 4700); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \
+} while (0)
+
+#define AT_TEST_INC_READ(T, A, B) \
+do { \
+    T var_ = AT_AINT32_MAX; \
+    var_++; \
+    ethr_ ## A ## _set ## B(&A, AT_AINT32_MAX); \
+    ASSERT(ethr_ ## A ## _inc_read ## B(&A) == var_); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \
+    ethr_ ## A ## _set ## B(&A, 4710); \
+    ASSERT(ethr_ ## A ## _inc_read ## B(&A) == 4711); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \
+} while (0)
+
+#define AT_TEST_DEC_READ(T, A, B) \
+do { \
+    T var_ = AT_AINT32_MIN; \
+    var_--; \
+    ethr_ ## A ## _set ## B(&A, AT_AINT32_MIN); \
+    ASSERT(ethr_ ## A ## _dec_read ## B(&A) == var_); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \
+    ethr_ ## A ## _set ## B(&A, 17); \
+    ASSERT(ethr_ ## A ## _dec_read ## B(&A) == 16); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == 16); \
+} while (0)
+
+
+#define AT_TEST_INC(T, A, B) \
+do { \
+    T var_ = AT_AINT32_MAX; \
+    var_++; \
+    ethr_ ## A ## _set ## B(&A, AT_AINT32_MAX); \
+    ethr_ ## A ## _inc ## B(&A); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \
+    ethr_ ## A ## _set ## B(&A, 4710); \
+    ethr_ ## A ## _inc ## B(&A); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == 4711); \
+} while (0)
+
+#define AT_TEST_DEC(T, A, B) \
+do { \
+    T var_ = AT_AINT32_MIN; \
+    var_--; \
+    ethr_ ## A ## _set ## B(&A, AT_AINT32_MIN); \
+    ethr_ ## A ## _dec ## B(&A); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == var_); \
+    ethr_ ## A ## _set ## B(&A, 17); \
+    ethr_ ## A ## _dec ## B(&A); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == 16); \
+} while (0)
+
+#define AT_TEST_READ_BAND(T, A, B) \
+do { \
+    ethr_ ## A ## _set ## B(&A, 0x13131313); \
+    ASSERT(ethr_ ## A ## _read_band ## B(&A, 0x31313131) == 0x13131313); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == 0x11111111); \
+} while (0)
+
+#define AT_TEST_READ_BOR(T, A, B) \
+do { \
+    ethr_ ## A ## _set ## B(&A, 0x11111111); \
+    ASSERT(ethr_ ## A ## _read_bor ## B(&A, 0x23232323) == 0x11111111); \
+    ASSERT(ethr_ ## A ## _read ## B(&A) == 0x33333333); \
+} while (0)
+
+
+static void
+atomic_basic_test(void)
+{
+    /*
+     * Verify that each op does what it is expected
+     * to do for at least one input.
+     */
+    ethr_atomic32_t atomic32;
+    ethr_atomic_t atomic;
+
+    print_line("AT_AINT32_MAX=%d",AT_AINT32_MAX);
+    print_line("AT_AINT32_MIN=%d",AT_AINT32_MIN);
+
+    AT_TEST_INIT(ethr_sint32_t, atomic32, );
+    AT_TEST_SET(ethr_sint32_t, atomic32, );
+    AT_TEST_XCHG(ethr_sint32_t, atomic32, );
+    AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, );
+    AT_TEST_ADD_READ(ethr_sint32_t, atomic32, );
+    AT_TEST_ADD(ethr_sint32_t, atomic32, );
+    AT_TEST_INC_READ(ethr_sint32_t, atomic32, );
+    AT_TEST_DEC_READ(ethr_sint32_t, atomic32, );
+    AT_TEST_INC(ethr_sint32_t, atomic32, );
+    AT_TEST_DEC(ethr_sint32_t, atomic32, );
+    AT_TEST_READ_BAND(ethr_sint32_t, atomic32, );
+    AT_TEST_READ_BOR(ethr_sint32_t, atomic32, );
+
+    AT_TEST_INIT(ethr_sint32_t, atomic32, _acqb);
+    AT_TEST_SET(ethr_sint32_t, atomic32, _acqb);
+    AT_TEST_XCHG(ethr_sint32_t, atomic32, _acqb);
+    AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _acqb);
+    AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _acqb);
+    AT_TEST_ADD(ethr_sint32_t, atomic32, _acqb);
+    AT_TEST_INC_READ(ethr_sint32_t, atomic32, _acqb);
+    AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _acqb);
+    AT_TEST_INC(ethr_sint32_t, atomic32, _acqb);
+    AT_TEST_DEC(ethr_sint32_t, atomic32, _acqb);
+    AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _acqb);
+    AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _acqb);
+
+    AT_TEST_INIT(ethr_sint32_t, atomic32, _relb);
+    AT_TEST_SET(ethr_sint32_t, atomic32, _relb);
+    AT_TEST_XCHG(ethr_sint32_t, atomic32, _relb);
+    AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _relb);
+    AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _relb);
+    AT_TEST_ADD(ethr_sint32_t, atomic32, _relb);
+    AT_TEST_INC_READ(ethr_sint32_t, atomic32, _relb);
+    AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _relb);
+    AT_TEST_INC(ethr_sint32_t, atomic32, _relb);
+    AT_TEST_DEC(ethr_sint32_t, atomic32, _relb);
+    AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _relb);
+    AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _relb);
+
+    AT_TEST_INIT(ethr_sint32_t, atomic32, _rb);
+    AT_TEST_SET(ethr_sint32_t, atomic32, _rb);
+    AT_TEST_XCHG(ethr_sint32_t, atomic32, _rb);
+    AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _rb);
+    AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _rb);
+    AT_TEST_ADD(ethr_sint32_t, atomic32, _rb);
+    AT_TEST_INC_READ(ethr_sint32_t, atomic32, _rb);
+    AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _rb);
+    AT_TEST_INC(ethr_sint32_t, atomic32, _rb);
+    AT_TEST_DEC(ethr_sint32_t, atomic32, _rb);
+    AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _rb);
+    AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _rb);
+
+    AT_TEST_INIT(ethr_sint32_t, atomic32, _wb);
+    AT_TEST_SET(ethr_sint32_t, atomic32, _wb);
+    AT_TEST_XCHG(ethr_sint32_t, atomic32, _wb);
+    AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _wb);
+    AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _wb);
+    AT_TEST_ADD(ethr_sint32_t, atomic32, _wb);
+    AT_TEST_INC_READ(ethr_sint32_t, atomic32, _wb);
+    AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _wb);
+    AT_TEST_INC(ethr_sint32_t, atomic32, _wb);
+    AT_TEST_DEC(ethr_sint32_t, atomic32, _wb);
+    AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _wb);
+    AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _wb);
+
+    AT_TEST_INIT(ethr_sint32_t, atomic32, _mb);
+    AT_TEST_SET(ethr_sint32_t, atomic32, _mb);
+    AT_TEST_XCHG(ethr_sint32_t, atomic32, _mb);
+    AT_TEST_CMPXCHG(ethr_sint32_t, atomic32, _mb);
+    AT_TEST_ADD_READ(ethr_sint32_t, atomic32, _mb);
+    AT_TEST_ADD(ethr_sint32_t, atomic32, _mb);
+    AT_TEST_INC_READ(ethr_sint32_t, atomic32, _mb);
+    AT_TEST_DEC_READ(ethr_sint32_t, atomic32, _mb);
+    AT_TEST_INC(ethr_sint32_t, atomic32, _mb);
+    AT_TEST_DEC(ethr_sint32_t, atomic32, _mb);
+    AT_TEST_READ_BAND(ethr_sint32_t, atomic32, _mb);
+    AT_TEST_READ_BOR(ethr_sint32_t, atomic32, _mb);
+
+    AT_TEST_INIT(ethr_sint_t, atomic, );
+    AT_TEST_SET(ethr_sint_t, atomic, );
+    AT_TEST_XCHG(ethr_sint_t, atomic, );
+    AT_TEST_CMPXCHG(ethr_sint_t, atomic, );
+    AT_TEST_ADD_READ(ethr_sint_t, atomic, );
+    AT_TEST_ADD(ethr_sint_t, atomic, );
+    AT_TEST_INC_READ(ethr_sint_t, atomic, );
+    AT_TEST_DEC_READ(ethr_sint_t, atomic, );
+    AT_TEST_INC(ethr_sint_t, atomic, );
+    AT_TEST_DEC(ethr_sint_t, atomic, );
+    AT_TEST_READ_BAND(ethr_sint_t, atomic, );
+    AT_TEST_READ_BOR(ethr_sint_t, atomic, );
+
+    AT_TEST_INIT(ethr_sint_t, atomic, _acqb);
+    AT_TEST_SET(ethr_sint_t, atomic, _acqb);
+    AT_TEST_XCHG(ethr_sint_t, atomic, _acqb);
+    AT_TEST_CMPXCHG(ethr_sint_t, atomic, _acqb);
+    AT_TEST_ADD_READ(ethr_sint_t, atomic, _acqb);
+    AT_TEST_ADD(ethr_sint_t, atomic, _acqb);
+    AT_TEST_INC_READ(ethr_sint_t, atomic, _acqb);
+    AT_TEST_DEC_READ(ethr_sint_t, atomic, _acqb);
+    AT_TEST_INC(ethr_sint_t, atomic, _acqb);
+    AT_TEST_DEC(ethr_sint_t, atomic, _acqb);
+    AT_TEST_READ_BAND(ethr_sint_t, atomic, _acqb);
+    AT_TEST_READ_BOR(ethr_sint_t, atomic, _acqb);
+
+    AT_TEST_INIT(ethr_sint_t, atomic, _relb);
+    AT_TEST_SET(ethr_sint_t, atomic, _relb);
+    AT_TEST_XCHG(ethr_sint_t, atomic, _relb);
+    AT_TEST_CMPXCHG(ethr_sint_t, atomic, _relb);
+    AT_TEST_ADD_READ(ethr_sint_t, atomic, _relb);
+    AT_TEST_ADD(ethr_sint_t, atomic, _relb);
+    AT_TEST_INC_READ(ethr_sint_t, atomic, _relb);
+    AT_TEST_DEC_READ(ethr_sint_t, atomic, _relb);
+    AT_TEST_INC(ethr_sint_t, atomic, _relb);
+    AT_TEST_DEC(ethr_sint_t, atomic, _relb);
+    AT_TEST_READ_BAND(ethr_sint_t, atomic, _relb);
+    AT_TEST_READ_BOR(ethr_sint_t, atomic, _relb);
+
+    AT_TEST_INIT(ethr_sint_t, atomic, _rb);
+    AT_TEST_SET(ethr_sint_t, atomic, _rb);
+    AT_TEST_XCHG(ethr_sint_t, atomic, _rb);
+    AT_TEST_CMPXCHG(ethr_sint_t, atomic, _rb);
+    AT_TEST_ADD_READ(ethr_sint_t, atomic, _rb);
+    AT_TEST_ADD(ethr_sint_t, atomic, _rb);
+    AT_TEST_INC_READ(ethr_sint_t, atomic, _rb);
+    AT_TEST_DEC_READ(ethr_sint_t, atomic, _rb);
+    AT_TEST_INC(ethr_sint_t, atomic, _rb);
+    AT_TEST_DEC(ethr_sint_t, atomic, _rb);
+    AT_TEST_READ_BAND(ethr_sint_t, atomic, _rb);
+    AT_TEST_READ_BOR(ethr_sint_t, atomic, _rb);
+
+    AT_TEST_INIT(ethr_sint_t, atomic, _wb);
+    AT_TEST_SET(ethr_sint_t, atomic, _wb);
+    AT_TEST_XCHG(ethr_sint_t, atomic, _wb);
+    AT_TEST_CMPXCHG(ethr_sint_t, atomic, _wb);
+    AT_TEST_ADD_READ(ethr_sint_t, atomic, _wb);
+    AT_TEST_ADD(ethr_sint_t, atomic, _wb);
+    AT_TEST_INC_READ(ethr_sint_t, atomic, _wb);
+    AT_TEST_DEC_READ(ethr_sint_t, atomic, _wb);
+    AT_TEST_INC(ethr_sint_t, atomic, _wb);
+    AT_TEST_DEC(ethr_sint_t, atomic, _wb);
+    AT_TEST_READ_BAND(ethr_sint_t, atomic, _wb);
+    AT_TEST_READ_BOR(ethr_sint_t, atomic, _wb);
+
+    AT_TEST_INIT(ethr_sint_t, atomic, _mb);
+    AT_TEST_SET(ethr_sint_t, atomic, _mb);
+    AT_TEST_XCHG(ethr_sint_t, atomic, _mb);
+    AT_TEST_CMPXCHG(ethr_sint_t, atomic, _mb);
+    AT_TEST_ADD_READ(ethr_sint_t, atomic, _mb);
+    AT_TEST_ADD(ethr_sint_t, atomic, _mb);
+    AT_TEST_INC_READ(ethr_sint_t, atomic, _mb);
+    AT_TEST_DEC_READ(ethr_sint_t, atomic, _mb);
+    AT_TEST_INC(ethr_sint_t, atomic, _mb);
+    AT_TEST_DEC(ethr_sint_t, atomic, _mb);
+    AT_TEST_READ_BAND(ethr_sint_t, atomic, _mb);
+    AT_TEST_READ_BOR(ethr_sint_t, atomic, _mb);
+
+    /* Double word */
+    {
+	ethr_dw_atomic_t dw_atomic;
+	ethr_dw_sint_t dw0, dw1;
+	dw0.sint[0] = 4711;
+	dw0.sint[1] = 4712;
+
+	/* init */
+	ethr_dw_atomic_init(&dw_atomic, &dw0);
+	ethr_dw_atomic_read(&dw_atomic, &dw1);
+	ETHR_ASSERT(dw1.sint[0] == 4711);
+	ETHR_ASSERT(dw1.sint[1] == 4712);
+	
+	/* set */
+	dw0.sint[0] = 42;
+	dw0.sint[1] = ~((ethr_sint_t) 0);
+	ethr_dw_atomic_set(&dw_atomic, &dw0);
+	ethr_dw_atomic_read(&dw_atomic, &dw1);
+	ASSERT(dw1.sint[0] == 42);
+	ASSERT(dw1.sint[1] == ~((ethr_sint_t) 0));
+
+	/* cmpxchg */
+	dw0.sint[0] = 17;
+	dw0.sint[1] = 18;
+	dw1.sint[0] = 19;
+	dw1.sint[1] = 20;
+	ASSERT(!ethr_dw_atomic_cmpxchg(&dw_atomic, &dw1, &dw0));
+	ethr_dw_atomic_read(&dw_atomic, &dw0);
+	ASSERT(dw0.sint[0] == 42);
+	ASSERT(dw0.sint[1] == ~((ethr_sint_t) 0));
+
+	ASSERT(ethr_dw_atomic_cmpxchg(&dw_atomic, &dw1, &dw0));
+
+	ethr_dw_atomic_read(&dw_atomic, &dw0);
+	ASSERT(dw0.sint[0] == 19);
+	ASSERT(dw0.sint[1] == 20);
+    }
+}
+
+
+#define AT_DW_MIN 12
+#define AT_DW_MAX 42
+#define AT_DW_THREADS (AT_DW_MAX - AT_DW_MIN + 1)
+
+#define AT_DW_LOOPS 200000
+#define AT_DW_R_LOOPS 10
+
+ethr_dw_atomic_t at_dw_atomic;
+
+void
+at_dw_valid(ethr_dw_sint_t *dw)
+{
+    int i;
+    char c;
+    char *cp;
+
+    ASSERT(dw->sint[0] == dw->sint[1]);
+
+    cp = (char *) &dw->sint[0];
+    c = cp[0];
+
+    ASSERT(AT_DW_MIN <= c && c <= AT_DW_MAX);
+
+    for (i = 0; i < sizeof(ethr_sint_t); i++)
+	ASSERT(c == cp[i]);
+}
+
+void *
+at_dw_thr(void *vval)
+{
+    int l, r;
+    ethr_sint_t val = (ethr_sint_t) vval;
+    ethr_dw_sint_t dw;
+    ethr_dw_sint_t my_dw;
+
+    my_dw.sint[0] = val;
+    my_dw.sint[1] = val;
+
+    ethr_dw_atomic_set(&at_dw_atomic, &my_dw);
+    for (l = 0; l < AT_DW_LOOPS; l++) {
+	for (r = 0; r < AT_DW_R_LOOPS; r++) {
+	    ethr_dw_atomic_read(&at_dw_atomic, &dw);
+	    at_dw_valid(&dw);
+	}
+	ethr_dw_atomic_set(&at_dw_atomic, &my_dw);
+	for (r = 0; r < AT_DW_R_LOOPS; r++) {
+	    ethr_dw_atomic_read(&at_dw_atomic, &dw);
+	    at_dw_valid(&dw);
+	}
+	dw.sint[0] = 0;
+	dw.sint[1] = 0;
+	while (1) {
+	    if (ethr_dw_atomic_cmpxchg(&at_dw_atomic, &my_dw, &dw))
+		break;
+	}
+    }
+}
+
+static void
+dw_atomic_massage_test(void)
+{
+    int i, res;
+    ethr_tid tid[AT_DW_THREADS];
+    ethr_thr_opts thr_opts = ETHR_THR_OPTS_DEFAULT_INITER;
+    ethr_dw_sint_t dw;
+
+    dw.sint[0] = dw.sint[1] = 0;
+
+    ethr_dw_atomic_init(&at_dw_atomic, &dw);
+
+    for (i = AT_DW_MIN; i <= AT_DW_MAX; i++) {
+	ethr_sint_t val;
+	memset(&val, i, sizeof(ethr_sint_t));
+	res = ethr_thr_create(&tid[i-AT_DW_MIN], at_dw_thr, (void *) val, &thr_opts);
+	ASSERT(res == 0);
+    }
+    for (i = AT_DW_MIN; i <= AT_DW_MAX; i++) {
+	res = ethr_thr_join(tid[i-AT_DW_MIN], NULL);
+	ASSERT(res == 0);
+    }
+}
+
 void *
 at_thread(void *unused)
 {
-    int res, i;
+    int i;
     long val, go;
-    
 
-    res = ethr_atomic_inctest(&at_ready, &val);
-    ASSERT(res == 0);
+    val = ethr_atomic_inc_read(&at_ready);
     ASSERT(val > 0);
     ASSERT(val <= AT_THREADS);
 
     do {
-	res = ethr_atomic_read(&at_go, &go);
-	ASSERT(res == 0);
+	go = ethr_atomic_read(&at_go);
     } while (!go);
 
     for (i = 0; i < AT_ITER; i++) {
-	res = ethr_atomic_or_old(&at_data, at_set_val, &val);
-	ASSERT(res == 0);
+	val = ethr_atomic_read_bor(&at_data, at_set_val);
 	ASSERT(val >= (i == 0 ? 0 : at_set_val) + (long) 4711);
 	ASSERT(val <= at_max_val);
 
-	res = ethr_atomic_and_old(&at_data, ~at_rm_val, &val);
-	ASSERT(res == 0);
+	val = ethr_atomic_read_band(&at_data, ~at_rm_val);
 	ASSERT(val >= at_set_val + (long) 4711);
 	ASSERT(val <= at_max_val);
 
-	res = ethr_atomic_read(&at_data, &val);
-	ASSERT(res == 0);
+	val = ethr_atomic_read(&at_data);
 	ASSERT(val >= at_set_val + (long) 4711);
 	ASSERT(val <= at_max_val);
 
-	res = ethr_atomic_inctest(&at_data, &val);
-	ASSERT(res == 0);
+	val = ethr_atomic_inc_read(&at_data);
 	ASSERT(val > at_set_val + (long) 4711);
 	ASSERT(val <= at_max_val);
 
-	res = ethr_atomic_dectest(&at_data, &val);
-	ASSERT(res == 0);
+	val = ethr_atomic_dec_read(&at_data);
 	ASSERT(val >= at_set_val + (long) 4711);
 	ASSERT(val <= at_max_val);
 
-	res = ethr_atomic_inc(&at_data);
-	ASSERT(res == 0);
+	ethr_atomic_inc(&at_data);
 
-	res = ethr_atomic_dec(&at_data);
-	ASSERT(res == 0);
+	ethr_atomic_dec(&at_data);
 
-	res = ethr_atomic_addtest(&at_data, (long) 4711, &val);
-	ASSERT(res == 0);
+	val = ethr_atomic_add_read(&at_data, (long) 4711);
 	ASSERT(val >= at_set_val + (long) 2*4711);
 	ASSERT(val <= at_max_val);
 
-	res = ethr_atomic_add(&at_data, (long) -4711);
-	ASSERT(res == 0);
+	ethr_atomic_add(&at_data, (long) -4711);
 	ASSERT(val >= at_set_val + (long) 4711);
 	ASSERT(val <= at_max_val);
     }
 
-    res = ethr_atomic_inc(&at_done);
-    ASSERT(res == 0);
+    ethr_atomic_inc(&at_done);
     return NULL;
 }
 
-
 static void
 atomic_test(void)
 {
@@ -2069,14 +1808,15 @@ atomic_test(void)
     ethr_tid tid[AT_THREADS];
     ethr_thr_opts thr_opts = ETHR_THR_OPTS_DEFAULT_INITER;
 
-    if (sizeof(long) > 4) {
+    atomic_basic_test();
+
+#if ETHR_SIZEOF_PTR > 4
 	at_rm_val = ((long) 1) << 57;
 	at_set_val = ((long) 1) << 60;
-    }
-    else {
+#else
 	at_rm_val = ((long) 1) << 27;
 	at_set_val = ((long) 1) << 30;
-    }
+#endif
 
     at_max_val = at_set_val + at_rm_val + ((long) AT_THREADS + 1) * 4711;
     data_init = at_rm_val + (long) 4711;
@@ -2085,22 +1825,15 @@ atomic_test(void)
     thr_opts.detached = 1;
 
     print_line("Initializing");
-    res = ethr_atomic_init(&at_ready, 0);
-    ASSERT(res == 0);
-    res = ethr_atomic_init(&at_go, 0);
-    ASSERT(res == 0);
-    res = ethr_atomic_init(&at_done, data_init);
-    ASSERT(res == 0);
-    res = ethr_atomic_init(&at_data, data_init);
-    ASSERT(res == 0);
+    ethr_atomic_init(&at_ready, 0);
+    ethr_atomic_init(&at_go, 0);
+    ethr_atomic_init(&at_done, data_init);
+    ethr_atomic_init(&at_data, data_init);
 
-    res = ethr_atomic_read(&at_data, &val);
-    ASSERT(res == 0);
+    val = ethr_atomic_read(&at_data);
     ASSERT(val == data_init);
-    res = ethr_atomic_set(&at_done, 0);
-    ASSERT(res == 0);
-    res = ethr_atomic_read(&at_done, &val);
-    ASSERT(res == 0);
+    ethr_atomic_set(&at_done, 0);
+    val = ethr_atomic_read(&at_done);
     ASSERT(val == 0);
 
     print_line("Creating threads");
@@ -2111,31 +1844,27 @@ atomic_test(void)
 
     print_line("Waiting for threads to ready up");
     do {
-	res = ethr_atomic_read(&at_ready, &val);
-	ASSERT(res == 0);
+	val = ethr_atomic_read(&at_ready);
 	ASSERT(val >= 0);
 	ASSERT(val <= AT_THREADS);
     } while (val != AT_THREADS);
 
     print_line("Letting threads loose");
-    res = ethr_atomic_xchg(&at_go, 17, &val);
-    ASSERT(res == 0);
+    val = ethr_atomic_xchg(&at_go, 17);
     ASSERT(val == 0);
-    res = ethr_atomic_read(&at_go, &val);
-    ASSERT(res == 0);
+    val = ethr_atomic_read(&at_go);
     ASSERT(val == 17);
 
 
     print_line("Waiting for threads to finish");
     do {
-	res = ethr_atomic_read(&at_done, &val);
-	ASSERT(res == 0);
+	val = ethr_atomic_read(&at_done);
 	ASSERT(val >= 0);
 	ASSERT(val <= AT_THREADS);
     } while (val != AT_THREADS);
 
     print_line("Checking result");
-    res = ethr_atomic_read(&at_data, &val);
+    val = ethr_atomic_read(&at_data);
     ASSERT(res == 0);
     ASSERT(val == data_final);
     print_line("Result ok");
@@ -2143,190 +1872,6 @@ atomic_test(void)
 }
 
 
-/*
- * The gate test case.
- *
- * Tests gates.
- */
-
-#define GT_THREADS 10
-
-static ethr_atomic_t gt_wait1;
-static ethr_atomic_t gt_wait2;
-static ethr_atomic_t gt_done;
-
-static ethr_gate gt_gate1;
-static ethr_gate gt_gate2;
-
-void *
-gt_thread(void *thr_no)
-{
-    int no = (int)(long) thr_no;
-    int swait = no % 2 == 0;
-    int res;
-    long done;
-
-
-    do {
-
-	res = ethr_atomic_inc(&gt_wait1);
-	ASSERT(res == 0);
-
-	if (swait)
-	    res = ethr_gate_swait(&gt_gate1, INT_MAX);
-	else
-	    res = ethr_gate_wait(&gt_gate1);
-	ASSERT(res == 0);
-
-	res = ethr_atomic_dec(&gt_wait1);
-	ASSERT(res == 0);
-
-	res = ethr_atomic_inc(&gt_wait2);
-	ASSERT(res == 0);
-
-	if (swait)
-	    res = ethr_gate_swait(&gt_gate2, INT_MAX);
-	else
-	    res = ethr_gate_wait(&gt_gate2);
-	ASSERT(res == 0);
-
-	res = ethr_atomic_dec(&gt_wait2);
-	ASSERT(res == 0);
-
-	res = ethr_atomic_read(&gt_done, &done);
-	ASSERT(res == 0);
-    } while (!done);
-    return NULL;
-}
-
-
-static void
-gate_test(void)
-{
-    long val;
-    int res, i;
-    ethr_tid tid[GT_THREADS];
-
-    print_line("Initializing");
-    res = ethr_atomic_init(&gt_wait1, 0);
-    ASSERT_EQ(res, 0, "%d");
-    res = ethr_atomic_init(&gt_wait2, 0);
-    ASSERT_EQ(res, 0, "%d");
-    res = ethr_atomic_init(&gt_done, 0);
-    ASSERT_EQ(res, 0, "%d");
-    res = ethr_gate_init(&gt_gate1);
-    ASSERT_EQ(res, 0, "%d");
-    res = ethr_gate_init(&gt_gate2);
-    ASSERT_EQ(res, 0, "%d");
-
-    print_line("Creating threads");
-    for (i = 0; i < GT_THREADS; i++) {
-	res = ethr_thr_create(&tid[i], gt_thread, (void *) i, NULL);
-	ASSERT_EQ(res, 0, "%d");
-    }
-
-    print_line("Waiting for threads to ready up");
-    do {
-	res = ethr_atomic_read(&gt_wait1, &val);
-	ASSERT_EQ(res, 0, "%d");
-	ASSERT(0 <= val && val <= GT_THREADS);
-    } while (val != GT_THREADS);
-
-    print_line("Testing");
-
-    res = ethr_gate_let_through(&gt_gate1, 8);
-    ASSERT_EQ(res, 0, "%d");
-
-    WAIT_UNTIL_LIM((res = ethr_atomic_read(&gt_wait2, &val),
-		    (res != 0 || val == 8)),
-		   60);
-
-    res = ethr_atomic_read(&gt_wait1, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, GT_THREADS - 8, "%ld");
-
-    res = ethr_atomic_read(&gt_wait2, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, 8, "%ld");
-    
-    res = ethr_gate_let_through(&gt_gate2, 4);
-    ASSERT_EQ(res, 0, "%d");
-
-    WAIT_UNTIL_LIM((res = ethr_atomic_read(&gt_wait2, &val),
-		    (res != 0 || val == 4)),
-		   60);
-
-    res = ethr_atomic_read(&gt_wait1, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, GT_THREADS - 4, "%ld");
-
-    res = ethr_atomic_read(&gt_wait2, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, 4, "%ld");
-    
-    res = ethr_gate_let_through(&gt_gate1, GT_THREADS);
-    ASSERT_EQ(res, 0, "%d");
-
-    WAIT_UNTIL_LIM((res = ethr_atomic_read(&gt_wait2, &val),
-		    (res != 0 || val == GT_THREADS)),
-		   60);
-    res = ethr_atomic_read(&gt_wait1, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, 0, "%ld");
-
-    res = ethr_atomic_read(&gt_wait2, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, GT_THREADS, "%ld");
-    
-    res = ethr_gate_let_through(&gt_gate2, GT_THREADS);
-    ASSERT_EQ(res, 0, "%d");
-
-    WAIT_UNTIL_LIM((res = ethr_atomic_read(&gt_wait2, &val),
-		    (res != 0 || val == 4)),
-		   60);
-    res = ethr_atomic_read(&gt_wait1, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, GT_THREADS - 4, "%ld");
-
-    res = ethr_atomic_read(&gt_wait2, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, 4, "%ld");
-    
-    res = ethr_atomic_set(&gt_done, 1);
-    ASSERT_EQ(res, 0, "%d");
-
-    res = ethr_gate_let_through(&gt_gate2, GT_THREADS);
-    ASSERT_EQ(res, 0, "%d");
-    res = ethr_gate_let_through(&gt_gate1, GT_THREADS - 4);
-    ASSERT_EQ(res, 0, "%d");
-
-    WAIT_UNTIL_LIM(((res = ethr_atomic_read(&gt_wait1, &val)) != 0
-		    || (val == 0
-			&& ((res = ethr_atomic_read(&gt_wait2, &val)) != 0
-			    || val == 0))),
-		   60);
-
-    res = ethr_atomic_read(&gt_wait1, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, 0, "%ld");
-
-    res = ethr_atomic_read(&gt_wait2, &val);
-    ASSERT_EQ(res, 0, "%d");
-    ASSERT_EQ(val, 0, "%ld");
-    
-    print_line("Joining threads");
-    for (i = 0; i < GT_THREADS; i++) {
-	res = ethr_thr_join(tid[i], NULL);
-	ASSERT_EQ(res, 0, "%d");
-    }
-
-    res = ethr_gate_destroy(&gt_gate1);
-    ASSERT_EQ(res, 0, "%d");
-    res = ethr_gate_destroy(&gt_gate2);
-    ASSERT_EQ(res, 0, "%d");
-
-}
-
 #endif /* #ifndef ETHR_NO_THREAD_LIB */
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
@@ -2342,14 +1887,12 @@ main(int argc, char *argv[])
 #ifndef ETHR_NO_THREAD_LIB
     {
 	char *testcase;
-	int res;
 
 	send_my_pid();
 
 	testcase = argv[1];
-	res = ethr_init(NULL);
 
-	if (res != 0)
+	if (ethr_init(NULL) != 0 || ethr_late_init(NULL) != 0)
 	    fail("Failed to initialize the ethread library");
 
 	if (strcmp(testcase, "create_join_thread") == 0)
@@ -2360,24 +1903,14 @@ main(int argc, char *argv[])
 	    mutex_test();
 	else if (strcmp(testcase, "try_lock_mutex") == 0)
 	    try_lock_mutex_test();
-	else if (strcmp(testcase, "recursive_mutex") == 0)
-	    recursive_mutex_test();
-	else if (strcmp(testcase, "time_now") == 0)
-	    time_now_test();
 	else if (strcmp(testcase, "cond_wait") == 0)
-	    cond_wait_test(0);
-	else if (strcmp(testcase, "cond_timedwait") == 0)
-	    cond_timedwait_test();
+	    cond_wait_test();
 	else if (strcmp(testcase, "broadcast") == 0)
 	    broadcast_test();
 	else if (strcmp(testcase, "detached_thread") == 0)
 	    detached_thread_test();
 	else if (strcmp(testcase, "max_threads") == 0)
 	    max_threads_test();
-	else if (strcmp(testcase, "forksafety") == 0)
-	    forksafety_test();
-	else if (strcmp(testcase, "vfork") == 0)
-	    vfork_test(argc, argv);
 	else if (strcmp(testcase, "tsd") == 0)
 	    tsd_test();
 	else if (strcmp(testcase, "spinlock") == 0)
@@ -2388,8 +1921,8 @@ main(int argc, char *argv[])
 	    rwmutex_test();
 	else if (strcmp(testcase, "atomic") == 0)
 	    atomic_test();
-	else if (strcmp(testcase, "gate") == 0)
-	    gate_test();
+	else if (strcmp(testcase, "dw_atomic_massage") == 0)
+	    dw_atomic_massage_test();
 	else
 	    skip("Test case \"%s\" not implemented yet", testcase);
 
diff --git a/erts/test/install_SUITE.erl b/erts/test/install_SUITE.erl
index e14790bc1b..559d95007c 100644
--- a/erts/test/install_SUITE.erl
+++ b/erts/test/install_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2010. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -29,8 +29,9 @@
 
 %-define(line_trace, 1).
 
--export([all/1, init_per_suite/1, end_per_suite/1,
-	 init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2, 
+	 init_per_suite/1, end_per_suite/1,
+	 init_per_testcase/2, end_per_testcase/2]).
 
 -export([bin_default/1,
 	 bin_default_dirty/1,
@@ -64,27 +65,32 @@
 	       erlang_bindir = "",
 	       bindir_symlinks = ""}).
 
-need_symlink_cases() ->
-    [bin_unreachable_absolute,
-     bin_unreachable_relative,
-     bin_same_dir,
-     bin_ok_symlink,
-     bin_dirname_fail,
+need_symlink_cases() -> 
+    [bin_unreachable_absolute, bin_unreachable_relative,
+     bin_same_dir, bin_ok_symlink, bin_dirname_fail,
      bin_no_use_dirname_fail].
 
-dont_need_symlink_cases() ->
-    [bin_default,
-     bin_default_dirty,
-     bin_outside_eprfx,
-     bin_outside_eprfx_dirty,
-     bin_not_abs,
-     bin_unreasonable_path,
-     'bin white space',
+dont_need_symlink_cases() -> 
+    [bin_default, bin_default_dirty, bin_outside_eprfx,
+     bin_outside_eprfx_dirty, bin_not_abs,
+     bin_unreasonable_path, 'bin white space',
      bin_no_srcfile].
 
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     dont_need_symlink_cases() ++ need_symlink_cases().
 
+groups() -> 
+    [].
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 %%
 %% The test cases
 %%
@@ -446,7 +452,7 @@ bin_dirname_fail(Config) when is_list(Config) ->
     ?line Be = Bs,
     ?line EBs = "/opt/lib/erlang/otp/bin",
     ?line EBe = EBs,
-    ?line CMDPRFX = "PATH="++?config(data_dir,Config)++":"++os:getenv("PATH"),
+    ?line CMDPRFX = "PATH=\""++?config(data_dir,Config)++":"++os:getenv("PATH")++"\"",
     ChkRes = fun (Res, #inst{test_prefix = TP,
 			     destdir = D,
 			     extra_prefix = EP,
@@ -479,7 +485,7 @@ bin_no_use_dirname_fail(Config) when is_list(Config) ->
     ?line EBs = "/opt/lib/erlang/otp/bin",
     ?line EBe = EBs,
     ?line RP = "../lib/erlang/otp/bin",
-    ?line CMDPRFX = "PATH="++?config(data_dir,Config)++":"++os:getenv("PATH"),
+    ?line CMDPRFX = "PATH=\""++?config(data_dir,Config)++":"++os:getenv("PATH")++"\"",
     ChkRes = fun (Res, #inst{test_prefix = TP,
 			     destdir = D,
 			     extra_prefix = EP,
@@ -585,7 +591,7 @@ init_per_testcase_aux(true, _OsType, Case, Config) ->
      {test_dir, make_dirs(?config(priv_dir, Config), atom_to_list(Case))}
      | Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
diff --git a/erts/test/nt_SUITE.erl b/erts/test/nt_SUITE.erl
index 530fb55270..f9bd15a0ce 100644
--- a/erts/test/nt_SUITE.erl
+++ b/erts/test/nt_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1998-2010. All Rights Reserved.
+%% Copyright Ericsson AB 1998-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -22,7 +22,9 @@
 -include_lib("test_server/include/test_server.hrl").
 -include_lib("kernel/include/file.hrl").
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,nt/1,handle_eventlog/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,init_per_testcase/2,
+	 end_per_testcase/2,nt/1,handle_eventlog/2,
 	 middleman/1,service_basic/1, service_env/1, user_env/1, synced/1, 
 	 service_prio/1, 
 	 logout/1, debug/1, restart/1, restart_always/1,stopaction/1,
@@ -31,20 +33,38 @@
 
 -define(TEST_SERVICES, [1,2,3,4,5,6,7,8,9,10,11]).
 
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     case os:type() of
-	{win32,nt} ->
-	    [nt, service_basic, service_env, user_env, synced, service_prio, 
-	     logout, debug,
-	     restart, restart_always, stopaction];
-	_ -> [nt] %%% Just to give a little hint why they are skipped...
+	{win32, nt} ->
+	    [nt, service_basic, service_env, user_env, synced,
+	     service_prio, logout, debug, restart, restart_always,
+	     stopaction];
+	_ -> [nt]
     end.
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 init_per_testcase(_Func, Config) ->
     Dog = test_server:timetrap(?TEST_TIMEOUT),
     [{watchdog, Dog} | Config].
 
-fin_per_testcase(_Func, Config) ->
+end_per_testcase(_Func, Config) ->
     lists:foreach(fun(X) -> 
 			  catch remove_service("test_service_" ++ 
 					 integer_to_list(X)) end,
@@ -470,12 +490,12 @@ middleman(Waitfor) ->
 match_event(_X, []) ->
     nomatch;
 match_event({Time,Cat,Fac,Sev,Mes},[{Pid,Ref,{Cat,Fac,Sev,MesRE}} | Tail]) ->
-    case regexp:match(Mes,MesRE) of
-	{match,_,_} ->
+    case re:run(Mes,MesRE,[{capture,none}]) of
+	match ->
 	    %%io:format("Match!~n"),
 	    {ok,{Pid,Ref,Time,Mes},Tail};
-	_Z ->
-	    %%io:format("No match (~p)~n",[_Z]),
+	nomatch ->
+	    %%io:format("No match~n"),
 	    case match_event({Time,Cat,Fac,Sev,Mes},Tail) of
 		{ok,X,Rest} ->
 		    {ok,X,[{Pid,Ref,{Cat,Fac,Sev,MesRE}} | Rest]};
diff --git a/erts/test/otp_SUITE.erl b/erts/test/otp_SUITE.erl
index 425ad31782..332733e075 100644
--- a/erts/test/otp_SUITE.erl
+++ b/erts/test/otp_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2000-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2000-2012. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,7 +19,8 @@
 
 -module(otp_SUITE).
 
--export([all/1,init_per_suite/1,end_per_suite/1]).
+-export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2,
+	 init_per_suite/1,end_per_suite/1]).
 -export([undefined_functions/1,deprecated_not_in_obsolete/1,
 	 obsolete_but_not_deprecated/1,call_to_deprecated/1,
          call_to_size_1/1,strong_components/1]).
@@ -28,10 +29,22 @@
 
 -import(lists, [filter/2,foldl/3,foreach/2]).
 
-all(suite) ->
-    [undefined_functions,deprecated_not_in_obsolete,
-     obsolete_but_not_deprecated,call_to_deprecated,
-     call_to_size_1,strong_components].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [undefined_functions, deprecated_not_in_obsolete,
+     obsolete_but_not_deprecated, call_to_deprecated,
+     call_to_size_1, strong_components].
+
+groups() -> 
+    [].
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 init_per_suite(Config) ->
     Dog = test_server:timetrap(?t:minutes(10)),
@@ -75,21 +88,25 @@ undefined_functions(Config) when is_list(Config) ->
     ?line Undef1 = hipe_filter(Undef0),
     ?line Undef2 = ssl_crypto_filter(Undef1),
     ?line Undef3 = edoc_filter(Undef2),
-    ?line Undef = eunit_filter(Undef3),
-    ?line Undef = megaco_filter(Undef),
+    Undef4 = eunit_filter(Undef3),
+    Undef5 = dialyzer_filter(Undef4),
+    Undef = wx_filter(Undef5),
 
     case Undef of
 	[] -> ok;
 	_ ->
+	    Fd = open_log(Config, "undefined_functions"),
 	    foreach(fun ({MFA1,MFA2}) ->
 			    io:format("~s calls undefined ~s",
-				      [format_mfa(MFA1),format_mfa(MFA2)])
+				      [format_mfa(Server, MFA1),
+				       format_mfa(MFA2)]),
+			    io:format(Fd, "~s ~s\n",
+				      [format_mfa(Server, MFA1),
+				       format_mfa(MFA2)])
 		    end, Undef),
+	    close_log(Fd),
 	    ?line ?t:fail({length(Undef),undefined_functions_in_otp})
-
-    end,
-
-    ok.
+    end.
 
 hipe_filter(Undef) ->
     case erlang:system_info(hipe_architecture) of
@@ -137,6 +154,9 @@ ssl_crypto_filter(Undef) ->
 	{{error,bad_name},{error,bad_name}} ->
 	    filter(fun({_,{ssl,_,_}}) -> false;
 		      ({_,{crypto,_,_}}) -> false;
+		      ({_,{ssh,_,_}}) -> false;
+		      ({_,{ssh_connection,_,_}}) -> false;
+		      ({_,{ssh_sftp,_,_}}) -> false;
 		      (_) -> true
 		   end, Undef);
 	{_,_} -> Undef
@@ -154,34 +174,33 @@ eunit_filter(Undef) ->
 	      (_) -> true
 	   end, Undef).
 
-megaco_filter(Undef) ->
-    %% Intentional calls to undefined functions.
-    filter(fun({{megaco_compact_text_encoder,encode_action_reply,3},
-		{megaco_compact_text_encoder_v3,encode_action_reply,2}}) -> false;
-	      ({{megaco_compact_text_encoder,encode_action_request,3},
-		{megaco_compact_text_encoder_v3,encode_action_request,2}}) -> false;
-	      ({{megaco_compact_text_encoder,encode_action_requests,3},
-		{megaco_compact_text_encoder_v3,encode_action_requests,2}}) -> false;
-	      ({{megaco_compact_text_encoder,encode_command_request,3},
-		{megaco_compact_text_encoder_v3,encode_command_request,2}}) -> false;
-	      ({{megaco_compact_text_encoder,encode_message,3},
-		{megaco_compact_text_encoder_v3,encode_message,2}}) -> false;
-	      ({{megaco_compact_text_encoder,encode_transaction,3},
-		{megaco_compact_text_encoder_v3,encode_transaction,2}}) -> false;
-	      ({{megaco_pretty_text_encoder,encode_action_reply,3},
-		{megaco_pretty_text_encoder_v3,encode_action_reply,2}}) -> false;
-	      ({{megaco_pretty_text_encoder,encode_action_request,3},
-		{megaco_pretty_text_encoder_v3,encode_action_request,2}}) -> false;
-	      ({{megaco_pretty_text_encoder,encode_action_requests,3},
-		{megaco_pretty_text_encoder_v3,encode_action_requests,2}}) -> false;
-	      ({{megaco_pretty_text_encoder,encode_command_request,3},
-		{megaco_pretty_text_encoder_v3,encode_command_request,2}}) -> false;
-	      ({{megaco_pretty_text_encoder,encode_message,3},
-		{megaco_pretty_text_encoder_v3,encode_message,2}}) -> false;
-	      ({{megaco_pretty_text_encoder,encode_transaction,3},
-		{megaco_pretty_text_encoder_v3,encode_transaction,2}}) -> false;
-	      (_) -> true
-	   end, Undef).
+dialyzer_filter(Undef) ->
+    case code:lib_dir(dialyzer) of
+	{error,bad_name} ->
+	    filter(fun({_,{dialyzer_callgraph,_,_}}) -> false;
+		      ({_,{dialyzer_codeserver,_,_}}) -> false;
+		      ({_,{dialyzer_contracts,_,_}}) -> false;
+		      ({_,{dialyzer_cl_parse,_,_}}) -> false;
+		      ({_,{dialyzer_plt,_,_}}) -> false;
+		      ({_,{dialyzer_succ_typings,_,_}}) -> false;
+		      ({_,{dialyzer_utils,_,_}}) -> false;
+		      (_) -> true
+		   end, Undef);
+	_ -> Undef
+    end.
+
+wx_filter(Undef) ->
+    case code:lib_dir(wx) of
+	{error,bad_name} ->
+	    filter(fun({_,{MaybeWxModule,_,_}}) ->
+			   case atom_to_list(MaybeWxModule) of
+			       "wx"++_ -> false;
+			       _ -> true
+			   end
+		   end, Undef);
+	_ -> Undef
+    end.
+				   
 
 deprecated_not_in_obsolete(Config) when is_list(Config) ->
     ?line Server = ?config(xref_server, Config),
@@ -198,7 +217,10 @@ deprecated_not_in_obsolete(Config) when is_list(Config) ->
 	_ ->
 	    io:put_chars("The following functions have -deprecated() attributes,\n"
 			 "but are not listed in otp_internal:obsolete/3.\n"),
-	    ?line print_mfas(L),
+	    print_mfas(group_leader(), Server, L),
+	    Fd = open_log(Config, "deprecated_not_obsolete"),
+	    print_mfas(Fd, Server, L),
+	    close_log(Fd),
 	    ?line ?t:fail({length(L),deprecated_but_not_obsolete})
     end.
 
@@ -219,11 +241,13 @@ obsolete_but_not_deprecated(Config) when is_list(Config) ->
 	    io:put_chars("The following functions are listed "
 			 "in otp_internal:obsolete/3,\n"
 			 "but don't have -deprecated() attributes.\n"),
-	    ?line print_mfas(L),
+	    print_mfas(group_leader(), Server, L),
+	    Fd = open_log(Config, "obsolete_not_deprecated"),
+	    print_mfas(Fd, Server, L),
+	    close_log(Fd),
 	    ?line ?t:fail({length(L),obsolete_but_not_deprecated})
     end.
     
-    
 call_to_deprecated(Config) when is_list(Config) ->
     Server = ?config(xref_server, Config),
     ?line {ok,DeprecatedCalls} = xref:q(Server, "strict(E || DF)"),
@@ -288,11 +312,28 @@ strong_components(Config) when is_list(Config) ->
 %%% Common help functions.
 %%%
     
-
-print_mfas([MFA|T]) ->
-    io:format("~s\n", [format_mfa(MFA)]),
-    print_mfas(T);
-print_mfas([]) -> ok.
+print_mfas(Fd, Server, MFAs) ->
+    [io:format(Fd, "~s\n", [format_mfa(Server, MFA)]) || MFA <- MFAs],
+    ok.
 
 format_mfa({M,F,A}) ->
     lists:flatten(io_lib:format("~s:~s/~p", [M,F,A])).
+
+format_mfa(Server, MFA) ->
+    MFAString = format_mfa(MFA),
+    AQ = "(App)" ++ MFAString,
+    AppPrefix = case xref:q(Server, AQ) of
+		    {ok,[App]} -> "[" ++ atom_to_list(App) ++ "]";
+		    _ -> ""
+		end,
+    AppPrefix ++ MFAString.
+
+open_log(Config, Name) ->
+    PrivDir = ?config(priv_dir, Config),
+    RunDir = filename:dirname(filename:dirname(PrivDir)),
+    Path = filename:join(RunDir, "system_"++Name++".log"),
+    {ok,Fd} = file:open(Path, [write]),
+    Fd.
+
+close_log(Fd) ->
+    ok = file:close(Fd).
diff --git a/erts/test/run_erl_SUITE.erl b/erts/test/run_erl_SUITE.erl
index efeafbad8c..6838c8924d 100644
--- a/erts/test/run_erl_SUITE.erl
+++ b/erts/test/run_erl_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2005-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2005-2011. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -19,7 +19,9 @@
 
 -module(run_erl_SUITE).
 
--export([all/1,init_per_testcase/2,fin_per_testcase/2,
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2,
+	 init_per_testcase/2,end_per_testcase/2,
 	 basic/1,heavy/1,heavier/1,defunct/1]).
 -export([ping_me_back/1]).
 
@@ -29,13 +31,31 @@ init_per_testcase(_Case, Config) ->
     Dog = ?t:timetrap(?t:minutes(2)),
     [{watchdog, Dog}|Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
 
-all(suite) ->
-    [basic,heavy,heavier,defunct].
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [basic, heavy, heavier, defunct].
+
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
 
 basic(Config) when is_list(Config) ->
     case os:type() of
@@ -230,7 +250,7 @@ defunct_2(Config, Perl) ->
     ?line ok = file:make_dir(LogDir),
     ?line Pipe = LogDir ++ "/",
     ?line RunErl = os:find_executable(run_erl),
-    ?line Cmd = Perl ++ " " ++ RunErlTest ++ " " ++ RunErl ++ " " ++
+    ?line Cmd = Perl ++ " " ++ RunErlTest ++ " \"" ++ RunErl ++ "\" " ++
 	Defuncter ++ " " ++ Pipe ++ " " ++ LogDir,
     ?line io:format("~p", [Cmd]),
     ?line Res = os:cmd(Cmd),
diff --git a/erts/test/system.spec b/erts/test/system.spec
index 9bfe2dbcf8..e0561ba0b2 100644
--- a/erts/test/system.spec
+++ b/erts/test/system.spec
@@ -1 +1 @@
-{topcase, {dir, "../system_test"}}.
+{suites,"../system_test",all}.
diff --git a/erts/test/utils/gccifier.c b/erts/test/utils/gccifier.c
index 64de764260..a1019f9a72 100644
--- a/erts/test/utils/gccifier.c
+++ b/erts/test/utils/gccifier.c
@@ -73,17 +73,23 @@ save_arg(args_t *args, char *arg1, ...)
 	    args->vec = (char **) (args->no
 				   ? realloc((void *) args->vec,
 					     (sizeof(char *)
-					      *(args->no + ARGS_INCR + 1)))
+					      *(args->no + ARGS_INCR + 2)))
 				   : malloc((sizeof(char *)
-					     *(args->no + ARGS_INCR + 1))));
+					     *(args->no + ARGS_INCR + 2))));
 	    if (!args->vec)
 		enomem();
 	    args->no += ARGS_INCR;
 	}
+	if (carg == arg1) {
+	  args->vec[args->ix++] = "\"";
+	  args->chars++;
+	}
 	args->vec[args->ix++] = carg;
 	args->chars += strlen(carg);
 	carg = va_arg(argp, char *);
     }
+    args->vec[args->ix++] = "\"";
+    args->chars++;
     args->vec[args->ix++] = " ";
     args->chars++;
     va_end(argp);
@@ -231,6 +237,9 @@ main(int argc, char *argv[])
 	    CHECK_FIRST_LINK_ARG;
 	    save_arg(&link_args, "-libpath:", arg, NULL);
 	}
+	else if (strcmp("-link",arg) == 0) {
+	  CHECK_FIRST_LINK_ARG;
+	}
 #endif /* #ifdef __WIN32__ */
 	else if (is_prefix("-l", &arg)) {
 	    CHECK_FIRST_LINK_ARG;
diff --git a/erts/emulator/pcre/Makefile b/erts/test/utils/gccifier.sh
index 72eea01130..4c6ba35db2 100644..100755
--- a/erts/emulator/pcre/Makefile
+++ b/erts/test/utils/gccifier.sh
@@ -1,26 +1,26 @@
-# 
+#!/bin/sh
+#
 # %CopyrightBegin%
-# 
-# Copyright Ericsson AB 2008-2009. All Rights Reserved.
-# 
+#
+# Copyright Ericsson AB 2005-2009. All Rights Reserved.
+#
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
 # compliance with the License. You should have received a copy of the
 # Erlang Public License along with this software. If not, it can be
 # retrieved online at http://www.erlang.org/.
-# 
+#
 # Software distributed under the License is distributed on an "AS IS"
 # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 # the License for the specific language governing rights and limitations
 # under the License.
-# 
-# %CopyrightEnd%
-# 
 #
-# Invoke with GNU make or clearmake -C gnu.
+# %CopyrightEnd%
 #
 
-include $(ERL_TOP)/make/run_make.mk
-
-table:
-	$(MAKE) -f $(TARGET)/Makefile $@
\ No newline at end of file
+CC=`echo "$1" | sed -e "s/-CC//"`
+shift
+echo "->"
+echo "$CC $*"
+$CC $*
+echo ""
diff --git a/erts/test/z_SUITE.erl b/erts/test/z_SUITE.erl
index 8faddeb0d3..78968ed405 100644
--- a/erts/test/z_SUITE.erl
+++ b/erts/test/z_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%% Copyright Ericsson AB 2008-2012. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -35,26 +35,45 @@
 
 -define(DEFAULT_TIMEOUT, ?t:minutes(5)).
 
--export([all/1, init_per_testcase/2, fin_per_testcase/2]).
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+	 init_per_group/2,end_per_group/2, 
+	 init_per_testcase/2, end_per_testcase/2]).
 
 -export([search_for_core_files/1, core_files/1]).
 
--include_lib("test_server/include/test_server.hrl").
+-include_lib("common_test/include/ct.hrl").
     
 
 init_per_testcase(Case, Config) ->
     Dog = ?t:timetrap(?DEFAULT_TIMEOUT),
     [{testcase, Case}, {watchdog, Dog} |Config].
 
-fin_per_testcase(_Case, Config) ->
+end_per_testcase(_Case, Config) ->
     Dog = ?config(watchdog, Config),
     ?t:timetrap_cancel(Dog),
     ok.
 
-all(doc) -> [];
-all(suite) ->
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
     [core_files].
 
+groups() -> 
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+end_per_group(_GroupName, Config) ->
+    Config.
+
+
 
 core_files(doc) ->
     [];
@@ -147,9 +166,12 @@ core_search_conf(RunByTS, DBTop, XDir) ->
 
 file_inspect(#core_search_conf{file = File}, Core) ->
     FRes0 = os:cmd(File ++ " " ++ Core),
-    FRes = case regexp:match(FRes0, Core) of
-	       {match, S, E} ->
+    FRes = case string:str(FRes0, Core) of
+	       0 ->
+		   FRes0;
+	       S ->
 		   L = length(FRes0),
+		   E = length(Core),
 		   case S of
 		       1 ->
 			   lists:sublist(FRes0, E+1, L+1);
@@ -159,19 +181,13 @@ file_inspect(#core_search_conf{file = File}, Core) ->
 			       " "
 			       ++
 			       lists:sublist(FRes0, E+1, L+1)
-		   end;
-	       _ -> FRes0
+		   end
 	   end,
-    case regexp:match(FRes, "[Tt][Ee][Xx][Tt]") of
+    case re:run(FRes, "text|ascii", [caseless,{capture,none}]) of
+	match ->
+	    not_a_core;
 	nomatch ->
-	    case regexp:match(FRes, "[Aa][Ss][Cc][Ii][Ii]") of
-		nomatch ->
-		    probably_a_core;
-		_ ->
-		    not_a_core
-	    end;
-	_ ->
-	    not_a_core
+	    probably_a_core
     end.
 
 mk_readable(F) ->
@@ -220,8 +236,8 @@ format_core(#core_search_conf{file = false}, Core, Ignore) ->
 	      [Ignore, Core] ++ mod_time_list(Core));
 format_core(#core_search_conf{file = File}, Core, Ignore) ->
     FRes = str_strip(os:cmd(File ++ " " ++ Core)),
-    case catch regexp:match(FRes, Core) of
-	{match, _, _} ->
+    case catch re:run(FRes, Core, [caseless,{capture,none}]) of
+	match ->
 	    io:format("  ~s~s " ++ time_fstr() ++ "~n",
 		      [Ignore, FRes] ++ mod_time_list(Core));
 	_ ->
@@ -253,6 +269,8 @@ core_file_search(#core_search_conf{search_dir = Base,
 				     core_cand(Conf, Core, Cores);
 				 "core." ++ _ ->
 				     core_cand(Conf, Core, Cores);
+				 Bin when is_binary(Bin) -> %Icky filename; ignore
+				     Cores;
 				 BName ->
 				     case lists:suffix(".core", BName) of
 					 true -> core_cand(Conf, Core, Cores);
diff --git a/erts/vsn.mk b/erts/vsn.mk
index 4e6571c70b..da0f03b24f 100644
--- a/erts/vsn.mk
+++ b/erts/vsn.mk
@@ -1,7 +1,7 @@
 # 
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 1997-2010. All Rights Reserved.
+# Copyright Ericsson AB 1997-2012. All Rights Reserved.
 #
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -17,8 +17,8 @@
 # %CopyrightEnd%
 # 
 
-VSN = 5.7.5.2
-SYSTEM_VSN = R13B04
+VSN = 5.9.2
+SYSTEM_VSN = R15B02
 
 # Port number 4365 in 4.2
 # Port number 4366 in 4.3